diff options
| -rw-r--r-- | Cargo.lock | 16 | ||||
| -rw-r--r-- | Cargo.toml | 5 | ||||
| -rw-r--r-- | docs/architecture.md | 13 | ||||
| -rw-r--r-- | helix-core/Cargo.toml | 5 | ||||
| -rw-r--r-- | helix-core/src/syntax.rs | 36 | ||||
| -rw-r--r-- | helix-syntax/Cargo.toml | 21 | ||||
| -rw-r--r-- | helix-syntax/README.md | 13 | ||||
| -rw-r--r-- | helix-syntax/src/lib.rs | 31 | ||||
| -rw-r--r-- | helix-term/Cargo.toml | 4 | ||||
| -rw-r--r-- | helix-term/build.rs | 5 | ||||
| -rw-r--r-- | helix-term/src/grammars.rs (renamed from helix-syntax/build.rs) | 63 | ||||
| -rw-r--r-- | helix-term/src/lib.rs | 1 |
12 files changed, 72 insertions, 141 deletions
@@ -358,11 +358,12 @@ dependencies = [ name = "helix-core" version = "0.6.0" dependencies = [ + "anyhow", "arc-swap", "chrono", "encoding_rs", "etcetera", - "helix-syntax", + "libloading", "log", "once_cell", "quickcheck", @@ -416,21 +417,11 @@ dependencies = [ ] [[package]] -name = "helix-syntax" -version = "0.6.0" -dependencies = [ - "anyhow", - "cc", - "libloading", - "threadpool", - "tree-sitter", -] - -[[package]] name = "helix-term" version = "0.6.0" dependencies = [ "anyhow", + "cc", "chrono", "content_inspector", "crossterm", @@ -454,6 +445,7 @@ dependencies = [ "serde_json", "signal-hook", "signal-hook-tokio", + "threadpool", "tokio", "tokio-stream", "toml", @@ -4,7 +4,6 @@ members = [ "helix-view", "helix-term", "helix-tui", - "helix-syntax", "helix-lsp", "helix-dap", "xtask", @@ -14,10 +13,6 @@ default-members = [ "helix-term" ] -# Build helix-syntax in release mode to make the code path faster in development. -# [profile.dev.package."helix-syntax"] -# opt-level = 3 - [profile.dev] split-debuginfo = "unpacked" diff --git a/docs/architecture.md b/docs/architecture.md index 40e01745..3c743eca 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,11 +1,10 @@ -| Crate | Description | -| ----------- | ----------- | -| helix-core | Core editing primitives, functional. | -| helix-syntax | Tree-sitter grammars | -| helix-lsp | Language server client | -| helix-view | UI abstractions for use in backends, imperative shell. | -| helix-term | Terminal UI | +| Crate | Description | +| ----------- | ----------- | +| helix-core | Core editing primitives, functional. | +| helix-lsp | Language server client | +| helix-view | UI abstractions for use in backends, imperative shell. | +| helix-term | Terminal UI | | helix-tui | TUI primitives, forked from tui-rs, inspired by Cursive | diff --git a/helix-core/Cargo.toml b/helix-core/Cargo.toml index 6682c37f..5582d38b 100644 --- a/helix-core/Cargo.toml +++ b/helix-core/Cargo.toml @@ -13,8 +13,6 @@ include = ["src/**/*", "README.md"] [features] [dependencies] -helix-syntax = { version = "0.6", path = "../helix-syntax" } - ropey = "1.3" smallvec = "1.8" smartstring = "1.0.0" @@ -40,5 +38,8 @@ encoding_rs = "0.8" chrono = { version = "0.4", default-features = false, features = ["alloc", "std"] } +libloading = "0.7" +anyhow = "1" + [dev-dependencies] quickcheck = { version = "1", default-features = false } diff --git a/helix-core/src/syntax.rs b/helix-core/src/syntax.rs index 53d20da3..0f7d224d 100644 --- a/helix-core/src/syntax.rs +++ b/helix-core/src/syntax.rs @@ -7,7 +7,9 @@ use crate::{ Rope, RopeSlice, Tendril, }; -pub use helix_syntax::get_language; +use anyhow::{Context, Result}; +use libloading::{Library, Symbol}; +use tree_sitter::Language; use arc_swap::{ArcSwap, Guard}; use slotmap::{DefaultKey as LayerId, HopSlotMap}; @@ -25,6 +27,34 @@ use std::{ use once_cell::sync::{Lazy, OnceCell}; use serde::{Deserialize, Serialize}; +#[cfg(unix)] +pub const DYLIB_EXTENSION: &str = "so"; + +#[cfg(windows)] +pub const DYLIB_EXTENSION: &str = "dll"; + +fn replace_dashes_with_underscores(name: &str) -> String { + name.replace('-', "_") +} + +pub fn get_language(runtime_path: &std::path::Path, name: &str) -> Result<Language> { + let name = name.to_ascii_lowercase(); + let mut library_path = runtime_path.join("grammars").join(&name); + library_path.set_extension(DYLIB_EXTENSION); + + let library = unsafe { Library::new(&library_path) } + .with_context(|| format!("Error opening dynamic library {:?}", &library_path))?; + let language_fn_name = format!("tree_sitter_{}", replace_dashes_with_underscores(&name)); + let language = unsafe { + let language_fn: Symbol<unsafe extern "C" fn() -> Language> = library + .get(language_fn_name.as_bytes()) + .with_context(|| format!("Failed to load symbol {}", language_fn_name))?; + language_fn() + }; + std::mem::forget(library); + Ok(language) +} + fn deserialize_regex<'de, D>(deserializer: D) -> Result<Option<Regex>, D::Error> where D: serde::Deserializer<'de>, @@ -426,7 +456,7 @@ impl LanguageConfiguration { &injections_query, &locals_query, ) - .unwrap(); // TODO: avoid panic + .unwrap_or_else(|query_error| panic!("Could not parse queries for language {:?}. Are your grammars out of sync? Try running 'hx --fetch-grammars' and 'hx --build-grammars'. This query could not be parsed: {:?}", self.language_id, query_error)); config.configure(scopes); Some(Arc::new(config)) @@ -2023,7 +2053,7 @@ mod test { ); let loader = Loader::new(Configuration { language: vec![] }); - let language = get_language(&crate::RUNTIME_DIR, "Rust").unwrap(); + let language = get_language("Rust").unwrap(); let query = Query::new(language, query_str).unwrap(); let textobject = TextObjectQuery { query }; diff --git a/helix-syntax/Cargo.toml b/helix-syntax/Cargo.toml deleted file mode 100644 index 855839be..00000000 --- a/helix-syntax/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -name = "helix-syntax" -version = "0.6.0" -authors = ["Blaž Hrastnik <blaz@mxxn.io>"] -edition = "2021" -license = "MPL-2.0" -description = "Tree-sitter grammars support" -categories = ["editor"] -repository = "https://github.com/helix-editor/helix" -homepage = "https://helix-editor.com" -include = ["src/**/*", "languages/**/*", "build.rs", "!**/docs/**/*", "!**/test/**/*", "!**/examples/**/*", "!**/build/**/*"] - -[dependencies] -tree-sitter = "0.20" -libloading = "0.7" -anyhow = "1" - -[build-dependencies] -cc = { version = "1" } -threadpool = { version = "1.0" } -anyhow = "1" diff --git a/helix-syntax/README.md b/helix-syntax/README.md deleted file mode 100644 index bba2197a..00000000 --- a/helix-syntax/README.md +++ /dev/null @@ -1,13 +0,0 @@ -helix-syntax -============ - -Syntax highlighting for helix, (shallow) submodules resides here. - -Differences from nvim-treesitter --------------------------------- - -As the syntax are commonly ported from -<https://github.com/nvim-treesitter/nvim-treesitter>. - -Note that we do not support the custom `#any-of` predicate which is -supported by neovim so one needs to change it to `#match` with regex. diff --git a/helix-syntax/src/lib.rs b/helix-syntax/src/lib.rs deleted file mode 100644 index b0ec48d8..00000000 --- a/helix-syntax/src/lib.rs +++ /dev/null @@ -1,31 +0,0 @@ -use anyhow::{Context, Result}; -use libloading::{Library, Symbol}; -use tree_sitter::Language; - -fn replace_dashes_with_underscores(name: &str) -> String { - name.replace('-', "_") -} -#[cfg(unix)] -const DYLIB_EXTENSION: &str = "so"; - -#[cfg(windows)] -const DYLIB_EXTENSION: &str = "dll"; - -pub fn get_language(runtime_path: &std::path::Path, name: &str) -> Result<Language> { - let name = name.to_ascii_lowercase(); - let mut library_path = runtime_path.join("grammars").join(&name); - // TODO: duplicated under build - library_path.set_extension(DYLIB_EXTENSION); - - let library = unsafe { Library::new(&library_path) } - .with_context(|| format!("Error opening dynamic library {:?}", &library_path))?; - let language_fn_name = format!("tree_sitter_{}", replace_dashes_with_underscores(&name)); - let language = unsafe { - let language_fn: Symbol<unsafe extern "C" fn() -> Language> = library - .get(language_fn_name.as_bytes()) - .with_context(|| format!("Failed to load symbol {}", language_fn_name))?; - language_fn() - }; - std::mem::forget(library); - Ok(language) -} diff --git a/helix-term/Cargo.toml b/helix-term/Cargo.toml index 9f7821f6..93d50d7e 100644 --- a/helix-term/Cargo.toml +++ b/helix-term/Cargo.toml @@ -66,5 +66,9 @@ grep-searcher = "0.1.8" # Remove once retain_mut lands in stable rust retain_mut = "0.1.7" +# compiling grammars +cc = { version = "1" } +threadpool = { version = "1.0" } + [target.'cfg(not(windows))'.dependencies] # https://github.com/vorner/signal-hook/issues/100 signal-hook-tokio = { version = "0.3", features = ["futures-v0_3"] } diff --git a/helix-term/build.rs b/helix-term/build.rs index b5d62b28..7303041c 100644 --- a/helix-term/build.rs +++ b/helix-term/build.rs @@ -14,5 +14,10 @@ fn main() { None => env!("CARGO_PKG_VERSION").into(), }; + println!( + "cargo:rustc-env=BUILD_TARGET={}", + std::env::var("TARGET").unwrap() + ); + println!("cargo:rustc-env=VERSION_AND_GIT_HASH={}", version); } diff --git a/helix-syntax/build.rs b/helix-term/src/grammars.rs index fa8be8b3..6a4910a3 100644 --- a/helix-syntax/build.rs +++ b/helix-term/src/grammars.rs @@ -6,11 +6,13 @@ use std::{ process::Command, }; -use std::sync::mpsc::channel; +use helix_core::syntax::DYLIB_EXTENSION; -fn collect_tree_sitter_dirs(ignore: &[String]) -> Result<Vec<String>> { +const BUILD_TARGET: &str = env!("BUILD_TARGET"); + +pub fn collect_tree_sitter_dirs(ignore: &[String]) -> Result<Vec<String>> { let mut dirs = Vec::new(); - let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("languages"); + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../helix-syntax/languages"); for entry in fs::read_dir(path)? { let entry = entry?; @@ -32,12 +34,6 @@ fn collect_tree_sitter_dirs(ignore: &[String]) -> Result<Vec<String>> { Ok(dirs) } -#[cfg(unix)] -const DYLIB_EXTENSION: &str = "so"; - -#[cfg(windows)] -const DYLIB_EXTENSION: &str = "dll"; - fn build_library(src_path: &Path, language: &str) -> Result<()> { let header_path = src_path; // let grammar_path = src_path.join("grammar.json"); @@ -65,7 +61,12 @@ fn build_library(src_path: &Path, language: &str) -> Result<()> { return Ok(()); } let mut config = cc::Build::new(); - config.cpp(true).opt_level(2).cargo_metadata(false); + config + .cpp(true) + .opt_level(2) + .cargo_metadata(false) + .host(BUILD_TARGET) + .target(BUILD_TARGET); let compiler = config.get_compiler(); let mut command = Command::new(compiler.path()); command.current_dir(src_path); @@ -148,9 +149,10 @@ fn mtime(path: &Path) -> Result<SystemTime> { Ok(fs::metadata(path)?.modified()?) } -fn build_dir(dir: &str, language: &str) { +pub fn build_dir(dir: &str, language: &str) { println!("Build language {}", language); - if PathBuf::from("languages") + if PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../helix-syntax/languages") .join(dir) .read_dir() .unwrap() @@ -158,49 +160,16 @@ fn build_dir(dir: &str, language: &str) { .is_none() { eprintln!( - "The directory {} is empty, you probably need to use 'git submodule update --init --recursive'?", + "The directory {} is empty, you probably need to use './scripts/grammars sync'?", dir ); std::process::exit(1); } let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("languages") + .join("../helix-syntax/languages") .join(dir) .join("src"); build_library(&path, language).unwrap(); } - -fn main() { - let ignore = vec![ - "tree-sitter-typescript".to_string(), - "tree-sitter-ocaml".to_string(), - ]; - let dirs = collect_tree_sitter_dirs(&ignore).unwrap(); - - let mut n_jobs = 0; - let pool = threadpool::Builder::new().build(); // by going through the builder, it'll use num_cpus - let (tx, rx) = channel(); - - for dir in dirs { - let tx = tx.clone(); - n_jobs += 1; - - pool.execute(move || { - let language = &dir.strip_prefix("tree-sitter-").unwrap(); - build_dir(&dir, language); - - // report progress - tx.send(1).unwrap(); - }); - } - pool.join(); - // drop(tx); - assert_eq!(rx.try_iter().sum::<usize>(), n_jobs); - - build_dir("tree-sitter-typescript/tsx", "tsx"); - build_dir("tree-sitter-typescript/typescript", "typescript"); - build_dir("tree-sitter-ocaml/ocaml", "ocaml"); - build_dir("tree-sitter-ocaml/interface", "ocaml-interface") -} diff --git a/helix-term/src/lib.rs b/helix-term/src/lib.rs index fc8e934e..22747998 100644 --- a/helix-term/src/lib.rs +++ b/helix-term/src/lib.rs @@ -7,6 +7,7 @@ pub mod commands; pub mod compositor; pub mod config; pub mod health; +pub mod grammars; pub mod job; pub mod keymap; pub mod ui; |