aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMichael Davis2022-02-14 02:08:28 +0000
committerBlaž Hrastnik2022-03-10 08:31:57 +0000
commit00b2d616eb6e35f4223e97beced0226cd146b92b (patch)
tree79cf6f239fd27e3f800b5f8b464e0ab7ccfe8e20
parent8330f6af202f0a9f681d94c7aec586cf5416e0c9 (diff)
implement build_grammars and fetch_grammars
build_grammars adapts the functionality that previously came from helix-syntax to be used at runtime from the command line flags. fetch_grammars wraps command-line git to perform the same actions previously done in the scripts in #1560.
-rw-r--r--helix-term/src/grammars.rs205
-rw-r--r--helix-term/src/main.rs10
2 files changed, 167 insertions, 48 deletions
diff --git a/helix-term/src/grammars.rs b/helix-term/src/grammars.rs
index 6a4910a3..7a7a4c18 100644
--- a/helix-term/src/grammars.rs
+++ b/helix-term/src/grammars.rs
@@ -4,39 +4,170 @@ use std::time::SystemTime;
use std::{
path::{Path, PathBuf},
process::Command,
+ sync::mpsc::channel,
};
-use helix_core::syntax::DYLIB_EXTENSION;
+use helix_core::syntax::{GrammarConfiguration, GrammarSource, DYLIB_EXTENSION};
const BUILD_TARGET: &str = env!("BUILD_TARGET");
+const REMOTE_NAME: &str = "origin";
-pub fn collect_tree_sitter_dirs(ignore: &[String]) -> Result<Vec<String>> {
- let mut dirs = Vec::new();
- let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../helix-syntax/languages");
+pub fn fetch_grammars() -> Result<()> {
+ run_parallel(get_grammar_configs(), fetch_grammar, "fetch")
+}
+
+pub fn build_grammars() -> Result<()> {
+ run_parallel(get_grammar_configs(), build_grammar, "build")
+}
+
+fn run_parallel<F>(grammars: Vec<GrammarConfiguration>, job: F, action: &'static str) -> Result<()>
+where
+ F: Fn(GrammarConfiguration) -> Result<()> + std::marker::Send + 'static + Copy,
+{
+ let mut n_jobs = 0;
+ let pool = threadpool::Builder::new().build();
+ let (tx, rx) = channel();
+
+ for grammar in grammars {
+ let tx = tx.clone();
+ n_jobs += 1;
+
+ pool.execute(move || {
+ let grammar_id = grammar.grammar_id.clone();
+ job(grammar).unwrap_or_else(|err| {
+ eprintln!("Failed to {} grammar '{}'\n{}", action, grammar_id, err)
+ });
+
+ // report progress
+ tx.send(1).unwrap();
+ });
+ }
+ pool.join();
+
+ if rx.try_iter().sum::<usize>() == n_jobs {
+ Ok(())
+ } else {
+ Err(anyhow!("Failed to {} some grammar(s).", action))
+ }
+}
- for entry in fs::read_dir(path)? {
- let entry = entry?;
- let path = entry.path();
+fn fetch_grammar(grammar: GrammarConfiguration) -> Result<()> {
+ if let GrammarSource::Git { remote, revision } = grammar.source {
+ let grammar_dir = helix_core::runtime_dir()
+ .join("grammars/sources")
+ .join(grammar.grammar_id.clone());
- if !entry.file_type()?.is_dir() {
- continue;
+ fs::create_dir_all(grammar_dir.clone()).expect("Could not create grammar directory");
+
+ // create the grammar dir contains a git directory
+ if !grammar_dir.join(".git").is_dir() {
+ git(&grammar_dir, ["init"])?;
+ }
+
+ // ensure the remote matches the configured remote
+ if get_remote_url(&grammar_dir).map_or(true, |s| s.trim_end() != remote) {
+ set_remote(&grammar_dir, &remote)?;
}
- let dir = path.file_name().unwrap().to_str().unwrap().to_string();
+ // ensure the revision matches the configured revision
+ if get_revision(&grammar_dir).map_or(true, |s| s.trim_end() != revision) {
+ // Fetch the exact revision from the remote.
+ // Supported by server-side git since v2.5.0 (July 2015),
+ // enabled by default on major git hosts.
+ git(&grammar_dir, ["fetch", REMOTE_NAME, &revision])?;
+ git(&grammar_dir, ["checkout", &revision])?;
- // filter ignores
- if ignore.contains(&dir) {
- continue;
+ println!(
+ "Grammar '{}' checked out at '{}'.",
+ grammar.grammar_id, revision
+ );
+ Ok(())
+ } else {
+ println!("Grammar '{}' is already up to date.", grammar.grammar_id);
+ Ok(())
}
- dirs.push(dir)
+ } else {
+ println!("Skipping local grammar '{}'", grammar.grammar_id);
+ Ok(())
}
+}
- Ok(dirs)
+// Sets the remote for a repository to the given URL, creating the remote if
+// it does not yet exist.
+fn set_remote(repository: &Path, remote_url: &str) -> Result<String> {
+ git(repository, ["remote", "set-url", REMOTE_NAME, remote_url])
+ .or_else(|_| git(repository, ["remote", "add", REMOTE_NAME, remote_url]))
}
-fn build_library(src_path: &Path, language: &str) -> Result<()> {
+fn get_remote_url(repository: &Path) -> Option<String> {
+ git(repository, ["remote", "get-url", REMOTE_NAME]).ok()
+}
+
+fn get_revision(repository: &Path) -> Option<String> {
+ git(repository, ["rev-parse", "HEAD"]).ok()
+}
+
+// A wrapper around 'git' commands which returns stdout in success and a
+// helpful error message showing the command, stdout, and stderr in error.
+fn git<I, S>(repository: &Path, args: I) -> Result<String>
+where
+ I: IntoIterator<Item = S>,
+ S: AsRef<std::ffi::OsStr>,
+{
+ let output = Command::new("git")
+ .args(args)
+ .current_dir(repository)
+ .output()?;
+
+ if output.status.success() {
+ Ok(String::from_utf8_lossy(&output.stdout).into_owned())
+ } else {
+ // TODO: figure out how to display the git command using `args`
+ Err(anyhow!(
+ "Git command failed.\nStdout: {}\nStderr: {}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr),
+ ))
+ }
+}
+
+fn build_grammar(grammar: GrammarConfiguration) -> Result<()> {
+ let grammar_dir = if let GrammarSource::Local { ref path } = grammar.source {
+ PathBuf::from(path)
+ } else {
+ helix_core::runtime_dir()
+ .join("grammars/sources")
+ .join(grammar.grammar_id.clone())
+ };
+
+ grammar_dir.read_dir().with_context(|| {
+ format!(
+ "The directory {:?} is empty, you probably need to use 'hx --fetch-grammars'?",
+ grammar_dir
+ )
+ })?;
+
+ let path = match grammar.path {
+ Some(ref subpath) => grammar_dir.join(subpath),
+ None => grammar_dir,
+ }
+ .join("src");
+
+ build_tree_sitter_library(&path, grammar)
+}
+
+// Returns the set of grammar configurations the user requests.
+// Grammars are configured in the default and user `languages.toml` and are
+// merged. The `grammar_selection` key of the config is then used to filter
+// down all grammars into a subset of the user's choosing.
+fn get_grammar_configs() -> Vec<GrammarConfiguration> {
+ let config = helix_core::config::user_syntax_loader().expect("Could not parse languages.toml");
+
+ config.grammar
+}
+
+fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) -> Result<()> {
let header_path = src_path;
- // let grammar_path = src_path.join("grammar.json");
let parser_path = src_path.join("parser.c");
let mut scanner_path = src_path.join("scanner.c");
@@ -50,16 +181,20 @@ fn build_library(src_path: &Path, language: &str) -> Result<()> {
None
}
};
- let parser_lib_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../runtime/grammars");
- let mut library_path = parser_lib_path.join(language);
+ let parser_lib_path = helix_core::runtime_dir().join("../runtime/grammars");
+ let mut library_path = parser_lib_path.join(grammar.grammar_id.clone());
library_path.set_extension(DYLIB_EXTENSION);
let recompile = needs_recompile(&library_path, &parser_path, &scanner_path)
- .with_context(|| "Failed to compare source and binary timestamps")?;
+ .context("Failed to compare source and binary timestamps")?;
if !recompile {
+ println!("Grammar '{}' is already built.", grammar.grammar_id);
return Ok(());
}
+
+ println!("Building grammar '{}'", grammar.grammar_id);
+
let mut config = cc::Build::new();
config
.cpp(true)
@@ -112,9 +247,7 @@ fn build_library(src_path: &Path, language: &str) -> Result<()> {
}
}
- let output = command
- .output()
- .with_context(|| "Failed to execute C compiler")?;
+ let output = command.output().context("Failed to execute C compiler")?;
if !output.status.success() {
return Err(anyhow!(
"Parser compilation failed.\nStdout: {}\nStderr: {}",
@@ -125,6 +258,7 @@ fn build_library(src_path: &Path, language: &str) -> Result<()> {
Ok(())
}
+
fn needs_recompile(
lib_path: &Path,
parser_c_path: &Path,
@@ -148,28 +282,3 @@ fn needs_recompile(
fn mtime(path: &Path) -> Result<SystemTime> {
Ok(fs::metadata(path)?.modified()?)
}
-
-pub fn build_dir(dir: &str, language: &str) {
- println!("Build language {}", language);
- if PathBuf::from(env!("CARGO_MANIFEST_DIR"))
- .join("../helix-syntax/languages")
- .join(dir)
- .read_dir()
- .unwrap()
- .next()
- .is_none()
- {
- eprintln!(
- "The directory {} is empty, you probably need to use './scripts/grammars sync'?",
- dir
- );
- std::process::exit(1);
- }
-
- let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
- .join("../helix-syntax/languages")
- .join(dir)
- .join("src");
-
- build_library(&path, language).unwrap();
-}
diff --git a/helix-term/src/main.rs b/helix-term/src/main.rs
index 4f73c9b4..67f88b4e 100644
--- a/helix-term/src/main.rs
+++ b/helix-term/src/main.rs
@@ -104,6 +104,16 @@ FLAGS:
std::process::exit(0);
}
+ if args.fetch_grammars {
+ helix_term::grammars::fetch_grammars()?;
+ return Ok(0);
+ }
+
+ if args.build_grammars {
+ helix_term::grammars::build_grammars()?;
+ return Ok(0);
+ }
+
let conf_dir = helix_core::config_dir();
if !conf_dir.exists() {
std::fs::create_dir_all(&conf_dir).ok();