migrate grammar fetching/building code into helix-loader crate

This is a rather large refactor that moves most of the code for
loading, fetching, and building grammars into a new helix-loader
module. This works well with the [[grammars]] syntax for
languages.toml defined earlier: we only have to depend on the types
for GrammarConfiguration in helix-loader and can leave all the
[[language]] entries for helix-core.

9 files changed, 18 insertions, 321 deletions

diff --git a/helix-term/Cargo.toml b/helix-term/Cargo.toml
index 93d50d7e..86d72561 100644
--- a/helix-term/Cargo.toml
+++ b/helix-term/Cargo.toml
@@ -26,6 +26,7 @@ helix-core = { version = "0.6", path = "../helix-core" }
 helix-view = { version = "0.6", path = "../helix-view" }
 helix-lsp = { version = "0.6", path = "../helix-lsp" }
 helix-dap = { version = "0.6", path = "../helix-dap" }
+helix-loader = { version = "0.6", path = "../helix-loader" }
 
 anyhow = "1"
 once_cell = "1.10"
diff --git a/helix-term/build.rs b/helix-term/build.rs
index 7303041c..b5d62b28 100644
--- a/helix-term/build.rs
+++ b/helix-term/build.rs
@@ -14,10 +14,5 @@ fn main() {
         None => env!("CARGO_PKG_VERSION").into(),
     };
 
-    println!(
-        "cargo:rustc-env=BUILD_TARGET={}",
-        std::env::var("TARGET").unwrap()
-    );
-
     println!("cargo:rustc-env=VERSION_AND_GIT_HASH={}", version);
 }
diff --git a/helix-term/src/application.rs b/helix-term/src/application.rs
index e885bc49..269ce13d 100644
--- a/helix-term/src/application.rs
+++ b/helix-term/src/application.rs
@@ -61,10 +61,10 @@ impl Application {
         let mut compositor = Compositor::new()?;
         let size = compositor.size();
 
-        let conf_dir = helix_core::config_dir();
+        let conf_dir = helix_loader::config_dir();
 
         let theme_loader =
-            std::sync::Arc::new(theme::Loader::new(&conf_dir, &helix_core::runtime_dir()));
+            std::sync::Arc::new(theme::Loader::new(&conf_dir, &helix_loader::runtime_dir()));
 
         let true_color = config.editor.true_color || crate::true_color();
         let theme = config
@@ -109,7 +109,7 @@ impl Application {
         compositor.push(editor_view);
 
         if args.load_tutor {
-            let path = helix_core::runtime_dir().join("tutor.txt");
+            let path = helix_loader::runtime_dir().join("tutor.txt");
             editor.open(path, Action::VerticalSplit)?;
             // Unset path to prevent accidentally saving to the original tutor file.
             doc_mut!(editor).set_path(None)?;
diff --git a/helix-term/src/commands/typed.rs b/helix-term/src/commands/typed.rs
index 4cc996d6..3301d148 100644
--- a/helix-term/src/commands/typed.rs
+++ b/helix-term/src/commands/typed.rs
@@ -828,7 +828,7 @@ fn tutor(
     _args: &[Cow<str>],
     _event: PromptEvent,
 ) -> anyhow::Result<()> {
-    let path = helix_core::runtime_dir().join("tutor.txt");
+    let path = helix_loader::runtime_dir().join("tutor.txt");
     cx.editor.open(path, Action::Replace)?;
     // Unset path to prevent accidentally saving to the original tutor file.
     doc_mut!(cx.editor).set_path(None)?;
diff --git a/helix-term/src/grammars.rs b/helix-term/src/grammars.rs
deleted file mode 100644
index 2e0be4bc..00000000
--- a/helix-term/src/grammars.rs
+++ /dev/null
@@ -1,296 +0,0 @@
-use anyhow::{anyhow, Context, Result};
-use std::fs;
-use std::time::SystemTime;
-use std::{
-    path::{Path, PathBuf},
-    process::Command,
-    sync::mpsc::channel,
-};
-
-use helix_core::syntax::{GrammarConfiguration, GrammarSelection, GrammarSource, DYLIB_EXTENSION};
-
-const BUILD_TARGET: &str = env!("BUILD_TARGET");
-const REMOTE_NAME: &str = "origin";
-
-pub fn fetch_grammars() -> Result<()> {
-    run_parallel(get_grammar_configs(), fetch_grammar, "fetch")
-}
-
-pub fn build_grammars() -> Result<()> {
-    run_parallel(get_grammar_configs(), build_grammar, "build")
-}
-
-fn run_parallel<F>(grammars: Vec<GrammarConfiguration>, job: F, action: &'static str) -> Result<()>
-where
-    F: Fn(GrammarConfiguration) -> Result<()> + std::marker::Send + 'static + Copy,
-{
-    let mut n_jobs = 0;
-    let pool = threadpool::Builder::new().build();
-    let (tx, rx) = channel();
-
-    for grammar in grammars {
-        let tx = tx.clone();
-        n_jobs += 1;
-
-        pool.execute(move || {
-            let grammar_id = grammar.grammar_id.clone();
-            job(grammar).unwrap_or_else(|err| {
-                eprintln!("Failed to {} grammar '{}'\n{}", action, grammar_id, err)
-            });
-
-            // report progress
-            tx.send(1).unwrap();
-        });
-    }
-    pool.join();
-
-    if rx.try_iter().sum::<usize>() == n_jobs {
-        Ok(())
-    } else {
-        Err(anyhow!("Failed to {} some grammar(s).", action))
-    }
-}
-
-fn fetch_grammar(grammar: GrammarConfiguration) -> Result<()> {
-    if let GrammarSource::Git { remote, revision } = grammar.source {
-        let grammar_dir = helix_core::runtime_dir()
-            .join("grammars/sources")
-            .join(grammar.grammar_id.clone());
-
-        fs::create_dir_all(grammar_dir.clone()).expect("Could not create grammar directory");
-
-        // create the grammar dir contains a git directory
-        if !grammar_dir.join(".git").is_dir() {
-            git(&grammar_dir, ["init"])?;
-        }
-
-        // ensure the remote matches the configured remote
-        if get_remote_url(&grammar_dir).map_or(true, |s| s.trim_end() != remote) {
-            set_remote(&grammar_dir, &remote)?;
-        }
-
-        // ensure the revision matches the configured revision
-        if get_revision(&grammar_dir).map_or(true, |s| s.trim_end() != revision) {
-            // Fetch the exact revision from the remote.
-            // Supported by server-side git since v2.5.0 (July 2015),
-            // enabled by default on major git hosts.
-            git(&grammar_dir, ["fetch", REMOTE_NAME, &revision])?;
-            git(&grammar_dir, ["checkout", &revision])?;
-
-            println!(
-                "Grammar '{}' checked out at '{}'.",
-                grammar.grammar_id, revision
-            );
-            Ok(())
-        } else {
-            println!("Grammar '{}' is already up to date.", grammar.grammar_id);
-            Ok(())
-        }
-    } else {
-        println!("Skipping local grammar '{}'", grammar.grammar_id);
-        Ok(())
-    }
-}
-
-// Sets the remote for a repository to the given URL, creating the remote if
-// it does not yet exist.
-fn set_remote(repository: &Path, remote_url: &str) -> Result<String> {
-    git(repository, ["remote", "set-url", REMOTE_NAME, remote_url])
-        .or_else(|_| git(repository, ["remote", "add", REMOTE_NAME, remote_url]))
-}
-
-fn get_remote_url(repository: &Path) -> Option<String> {
-    git(repository, ["remote", "get-url", REMOTE_NAME]).ok()
-}
-
-fn get_revision(repository: &Path) -> Option<String> {
-    git(repository, ["rev-parse", "HEAD"]).ok()
-}
-
-// A wrapper around 'git' commands which returns stdout in success and a
-// helpful error message showing the command, stdout, and stderr in error.
-fn git<I, S>(repository: &Path, args: I) -> Result<String>
-where
-    I: IntoIterator<Item = S>,
-    S: AsRef<std::ffi::OsStr>,
-{
-    let output = Command::new("git")
-        .args(args)
-        .current_dir(repository)
-        .output()?;
-
-    if output.status.success() {
-        Ok(String::from_utf8_lossy(&output.stdout).into_owned())
-    } else {
-        // TODO: figure out how to display the git command using `args`
-        Err(anyhow!(
-            "Git command failed.\nStdout: {}\nStderr: {}",
-            String::from_utf8_lossy(&output.stdout),
-            String::from_utf8_lossy(&output.stderr),
-        ))
-    }
-}
-
-fn build_grammar(grammar: GrammarConfiguration) -> Result<()> {
-    let grammar_dir = if let GrammarSource::Local { ref path } = grammar.source {
-        PathBuf::from(path)
-    } else {
-        helix_core::runtime_dir()
-            .join("grammars/sources")
-            .join(grammar.grammar_id.clone())
-    };
-
-    grammar_dir.read_dir().with_context(|| {
-        format!(
-            "The directory {:?} is empty, you probably need to use 'hx --fetch-grammars'?",
-            grammar_dir
-        )
-    })?;
-
-    let path = match grammar.path {
-        Some(ref subpath) => grammar_dir.join(subpath),
-        None => grammar_dir,
-    }
-    .join("src");
-
-    build_tree_sitter_library(&path, grammar)
-}
-
-// Returns the set of grammar configurations the user requests.
-// Grammars are configured in the default and user `languages.toml` and are
-// merged. The `grammar_selection` key of the config is then used to filter
-// down all grammars into a subset of the user's choosing.
-fn get_grammar_configs() -> Vec<GrammarConfiguration> {
-    let config = helix_core::config::user_syntax_loader().expect("Could not parse languages.toml");
-
-    match config.grammar_selection {
-        Some(GrammarSelection::Only(selections)) => config
-            .grammar
-            .into_iter()
-            .filter(|grammar| selections.contains(&grammar.grammar_id))
-            .collect(),
-        Some(GrammarSelection::Except(rejections)) => config
-            .grammar
-            .into_iter()
-            .filter(|grammar| !rejections.contains(&grammar.grammar_id))
-            .collect(),
-        None => config.grammar,
-    }
-}
-
-fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) -> Result<()> {
-    let header_path = src_path;
-    let parser_path = src_path.join("parser.c");
-    let mut scanner_path = src_path.join("scanner.c");
-
-    let scanner_path = if scanner_path.exists() {
-        Some(scanner_path)
-    } else {
-        scanner_path.set_extension("cc");
-        if scanner_path.exists() {
-            Some(scanner_path)
-        } else {
-            None
-        }
-    };
-    let parser_lib_path = helix_core::runtime_dir().join("../runtime/grammars");
-    let mut library_path = parser_lib_path.join(grammar.grammar_id.clone());
-    library_path.set_extension(DYLIB_EXTENSION);
-
-    let recompile = needs_recompile(&library_path, &parser_path, &scanner_path)
-        .context("Failed to compare source and binary timestamps")?;
-
-    if !recompile {
-        println!("Grammar '{}' is already built.", grammar.grammar_id);
-        return Ok(());
-    }
-
-    println!("Building grammar '{}'", grammar.grammar_id);
-
-    let mut config = cc::Build::new();
-    config
-        .cpp(true)
-        .opt_level(2)
-        .cargo_metadata(false)
-        .host(BUILD_TARGET)
-        .target(BUILD_TARGET);
-    let compiler = config.get_compiler();
-    let mut command = Command::new(compiler.path());
-    command.current_dir(src_path);
-    for (key, value) in compiler.env() {
-        command.env(key, value);
-    }
-
-    if cfg!(windows) {
-        command
-            .args(&["/nologo", "/LD", "/I"])
-            .arg(header_path)
-            .arg("/Od")
-            .arg("/utf-8");
-        if let Some(scanner_path) = scanner_path.as_ref() {
-            command.arg(scanner_path);
-        }
-
-        command
-            .arg(parser_path)
-            .arg("/link")
-            .arg(format!("/out:{}", library_path.to_str().unwrap()));
-    } else {
-        command
-            .arg("-shared")
-            .arg("-fPIC")
-            .arg("-fno-exceptions")
-            .arg("-g")
-            .arg("-I")
-            .arg(header_path)
-            .arg("-o")
-            .arg(&library_path)
-            .arg("-O2");
-        if let Some(scanner_path) = scanner_path.as_ref() {
-            if scanner_path.extension() == Some("c".as_ref()) {
-                command.arg("-xc").arg("-std=c99").arg(scanner_path);
-            } else {
-                command.arg(scanner_path);
-            }
-        }
-        command.arg("-xc").arg(parser_path);
-        if cfg!(all(unix, not(target_os = "macos"))) {
-            command.arg("-Wl,-z,relro,-z,now");
-        }
-    }
-
-    let output = command.output().context("Failed to execute C compiler")?;
-    if !output.status.success() {
-        return Err(anyhow!(
-            "Parser compilation failed.\nStdout: {}\nStderr: {}",
-            String::from_utf8_lossy(&output.stdout),
-            String::from_utf8_lossy(&output.stderr)
-        ));
-    }
-
-    Ok(())
-}
-
-fn needs_recompile(
-    lib_path: &Path,
-    parser_c_path: &Path,
-    scanner_path: &Option<PathBuf>,
-) -> Result<bool> {
-    if !lib_path.exists() {
-        return Ok(true);
-    }
-    let lib_mtime = mtime(lib_path)?;
-    if mtime(parser_c_path)? > lib_mtime {
-        return Ok(true);
-    }
-    if let Some(scanner_path) = scanner_path {
-        if mtime(scanner_path)? > lib_mtime {
-            return Ok(true);
-        }
-    }
-    Ok(false)
-}
-
-fn mtime(path: &Path) -> Result<SystemTime> {
-    Ok(fs::metadata(path)?.modified()?)
-}
diff --git a/helix-term/src/health.rs b/helix-term/src/health.rs
index 5ef20d93..f13d35f0 100644
--- a/helix-term/src/health.rs
+++ b/helix-term/src/health.rs
@@ -1,8 +1,6 @@
 use crossterm::style::{Color, Print, Stylize};
-use helix_core::{
-    config::{default_syntax_loader, user_syntax_loader},
-    syntax::load_runtime_file,
-};
+use helix_core::config::{default_syntax_loader, user_syntax_loader};
+use helix_loader::grammar::load_runtime_file;
 
 #[derive(Copy, Clone)]
 pub enum TsFeature {
@@ -43,10 +41,10 @@ impl TsFeature {
 
 /// Display general diagnostics.
 pub fn general() {
-    let config_file = helix_core::config_file();
-    let lang_file = helix_core::lang_config_file();
-    let log_file = helix_core::log_file();
-    let rt_dir = helix_core::runtime_dir();
+    let config_file = helix_loader::config_file();
+    let lang_file = helix_loader::lang_config_file();
+    let log_file = helix_loader::log_file();
+    let rt_dir = helix_loader::runtime_dir();
 
     if config_file.exists() {
         println!("Config file: {}", config_file.display());
diff --git a/helix-term/src/lib.rs b/helix-term/src/lib.rs
index 22747998..fc8e934e 100644
--- a/helix-term/src/lib.rs
+++ b/helix-term/src/lib.rs
@@ -7,7 +7,6 @@ pub mod commands;
 pub mod compositor;
 pub mod config;
 pub mod health;
-pub mod grammars;
 pub mod job;
 pub mod keymap;
 pub mod ui;
diff --git a/helix-term/src/main.rs b/helix-term/src/main.rs
index 67f88b4e..a69e121b 100644
--- a/helix-term/src/main.rs
+++ b/helix-term/src/main.rs
@@ -40,7 +40,7 @@ fn main() -> Result<()> {
 
 #[tokio::main]
 async fn main_impl() -> Result<i32> {
-    let logpath = helix_core::log_file();
+    let logpath = helix_loader::log_file();
     let parent = logpath.parent().unwrap();
     if !parent.exists() {
         std::fs::create_dir_all(parent).ok();
@@ -105,21 +105,21 @@ FLAGS:
     }
 
     if args.fetch_grammars {
-        helix_term::grammars::fetch_grammars()?;
+        helix_loader::grammar::fetch_grammars()?;
         return Ok(0);
     }
 
     if args.build_grammars {
-        helix_term::grammars::build_grammars()?;
+        helix_loader::grammar::build_grammars()?;
         return Ok(0);
     }
 
-    let conf_dir = helix_core::config_dir();
+    let conf_dir = helix_loader::config_dir();
     if !conf_dir.exists() {
         std::fs::create_dir_all(&conf_dir).ok();
     }
 
-    let config = match std::fs::read_to_string(helix_core::config_file()) {
+    let config = match std::fs::read_to_string(helix_loader::config_file()) {
         Ok(config) => toml::from_str(&config)
             .map(merge_keys)
             .unwrap_or_else(|err| {
diff --git a/helix-term/src/ui/mod.rs b/helix-term/src/ui/mod.rs
index d46de2d3..6299a473 100644
--- a/helix-term/src/ui/mod.rs
+++ b/helix-term/src/ui/mod.rs
@@ -218,9 +218,9 @@ pub mod completers {
     }
 
     pub fn theme(_editor: &Editor, input: &str) -> Vec<Completion> {
-        let mut names = theme::Loader::read_names(&helix_core::runtime_dir().join("themes"));
+        let mut names = theme::Loader::read_names(&helix_loader::runtime_dir().join("themes"));
         names.extend(theme::Loader::read_names(
-            &helix_core::config_dir().join("themes"),
+            &helix_loader::config_dir().join("themes"),
         ));
         names.push("default".into());
         names.push("base16_default".into());