diff options
| -rw-r--r-- | Cargo.lock | 13 | ||||
| -rw-r--r-- | helix-core/Cargo.toml | 1 | ||||
| -rw-r--r-- | helix-core/src/language_mode.rs | 993 | ||||
| -rw-r--r-- | helix-core/src/lib.rs | 1 | ||||
| -rw-r--r-- | helix-syntax/build.rs | 5 | ||||
| -rw-r--r-- | helix-term/Cargo.toml | 2 | ||||
| -rw-r--r-- | helix-term/src/editor.rs | 215 | ||||
| -rw-r--r-- | helix-term/src/main.rs | 1 | ||||
| -rw-r--r-- | helix-term/src/theme.rs | 59 |
9 files changed, 1150 insertions, 140 deletions
@@ -372,6 +372,7 @@ dependencies = [ "ropey", "smallvec", "tendril", + "tree-sitter", "unicode-segmentation", "unicode-width", ] @@ -396,8 +397,6 @@ dependencies = [ "helix-syntax", "num_cpus", "smol", - "tree-sitter", - "tree-sitter-highlight", "tui", ] @@ -720,16 +719,6 @@ dependencies = [ ] [[package]] -name = "tree-sitter-highlight" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6e3c800ac0db1562a045a4893cbbd7c484eb93426cae5632f9e5d24dd588cd1" -dependencies = [ - "regex", - "tree-sitter", -] - -[[package]] name = "tui" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" diff --git a/helix-core/Cargo.toml b/helix-core/Cargo.toml index 9bd82e52..23eccdc2 100644 --- a/helix-core/Cargo.toml +++ b/helix-core/Cargo.toml @@ -14,3 +14,4 @@ tendril = { git = "https://github.com/servo/tendril" } unicode-segmentation = "1.6.0" unicode-width = "0.1.7" # slab = "0.4.2" +tree-sitter = "0.16.1" diff --git a/helix-core/src/language_mode.rs b/helix-core/src/language_mode.rs new file mode 100644 index 00000000..3f104be4 --- /dev/null +++ b/helix-core/src/language_mode.rs @@ -0,0 +1,993 @@ +pub struct LanguageMode { + parser: Parser, +} + +impl LanguageMode { + // buffer, grammar, config, grammars, sync_timeout? + pub fn new() -> Self { + unimplemented!() + // make a new root layer + // track markers of injections + // + // track scope_descriptor: a Vec of scopes for item in tree + // + // fetch grammar for parser based on language string + // update root layer + } + + // fn buffer_changed -> call layer.update(range, new_text) on root layer and then all marker layers + + // call this on transaction.apply() -> buffer_changed(changes) + // + // fn parse(language, old_tree, ranges) + // + // fn tree() -> Tree + // + // <!--update_for_injection(grammar)--> + + // Highlighting + // fn highlight_iter() -> iterates over all the scopes + // on_tokenize + // on_change_highlighting + + // Commenting + // comment_strings_for_pos + // is_commented + + // Indentation + // suggested_indent_for_line_at_buffer_row + // suggested_indent_for_buffer_row + // indent_level_for_line + + // TODO: Folding + + // Syntax APIs + // get_syntax_node_containing_range -> + // ... + // get_syntax_node_at_pos + // buffer_range_for_scope_at_pos +} + +pub struct LanguageLayer { + // mode +// grammar +// depth +// tree: Tree, +} + +impl LanguageLayer { + // fn highlight_iter() -> same as Mode but for this layer. Mode composits these + // fn buffer_changed + // fn update(range) + // fn update_injections() +} + +// -- refactored from tree-sitter-highlight to be able to retain state +// TODO: add seek() to iter + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::{iter, mem, ops, str, usize}; +use tree_sitter::{ + Language, Node, Parser, Point, Query, QueryCaptures, QueryCursor, QueryError, QueryMatch, + Range, Tree, +}; + +const CANCELLATION_CHECK_INTERVAL: usize = 100; +const BUFFER_HTML_RESERVE_CAPACITY: usize = 10 * 1024; +const BUFFER_LINES_RESERVE_CAPACITY: usize = 1000; + +/// Indicates which highlight should be applied to a region of source code. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub struct Highlight(pub usize); + +/// Represents the reason why syntax highlighting failed. +#[derive(Debug, PartialEq, Eq)] +pub enum Error { + Cancelled, + InvalidLanguage, + Unknown, +} + +/// Represents a single step in rendering a syntax-highlighted document. +#[derive(Copy, Clone, Debug)] +pub enum HighlightEvent { + Source { start: usize, end: usize }, + HighlightStart(Highlight), + HighlightEnd, +} + +/// Contains the data neeeded to higlight code written in a particular language. +/// +/// This struct is immutable and can be shared between threads. +pub struct HighlightConfiguration { + pub language: Language, + pub query: Query, + combined_injections_query: Option<Query>, + locals_pattern_index: usize, + highlights_pattern_index: usize, + highlight_indices: Vec<Option<Highlight>>, + non_local_variable_patterns: Vec<bool>, + injection_content_capture_index: Option<u32>, + injection_language_capture_index: Option<u32>, + local_scope_capture_index: Option<u32>, + local_def_capture_index: Option<u32>, + local_def_value_capture_index: Option<u32>, + local_ref_capture_index: Option<u32>, +} + +/// Performs syntax highlighting, recognizing a given list of highlight names. +/// +/// For the best performance `Highlighter` values should be reused between +/// syntax highlighting calls. A separate highlighter is needed for each thread that +/// is performing highlighting. +pub struct Highlighter { + parser: Parser, + cursors: Vec<QueryCursor>, +} + +#[derive(Debug)] +struct LocalDef<'a> { + name: &'a str, + value_range: ops::Range<usize>, + highlight: Option<Highlight>, +} + +#[derive(Debug)] +struct LocalScope<'a> { + inherits: bool, + range: ops::Range<usize>, + local_defs: Vec<LocalDef<'a>>, +} + +struct HighlightIter<'a, F> +where + F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a, +{ + source: &'a [u8], + byte_offset: usize, + highlighter: &'a mut Highlighter, + injection_callback: F, + cancellation_flag: Option<&'a AtomicUsize>, + layers: Vec<HighlightIterLayer<'a>>, + iter_count: usize, + next_event: Option<HighlightEvent>, + last_highlight_range: Option<(usize, usize, usize)>, +} + +struct HighlightIterLayer<'a> { + _tree: Tree, + cursor: QueryCursor, + captures: iter::Peekable<QueryCaptures<'a, &'a [u8]>>, + config: &'a HighlightConfiguration, + highlight_end_stack: Vec<usize>, + scope_stack: Vec<LocalScope<'a>>, + ranges: Vec<Range>, + depth: usize, +} + +impl Highlighter { + pub fn new() -> Self { + Highlighter { + parser: Parser::new(), + cursors: Vec::new(), + } + } + + pub fn parser(&mut self) -> &mut Parser { + &mut self.parser + } + + /// Iterate over the highlighted regions for a given slice of source code. + pub fn highlight<'a>( + &'a mut self, + config: &'a HighlightConfiguration, + source: &'a [u8], + cancellation_flag: Option<&'a AtomicUsize>, + mut injection_callback: impl FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a, + ) -> Result<impl Iterator<Item = Result<HighlightEvent, Error>> + 'a, Error> { + let layers = HighlightIterLayer::new( + source, + self, + cancellation_flag, + &mut injection_callback, + config, + 0, + vec![Range { + start_byte: 0, + end_byte: usize::MAX, + start_point: Point::new(0, 0), + end_point: Point::new(usize::MAX, usize::MAX), + }], + )?; + assert_ne!(layers.len(), 0); + let mut result = HighlightIter { + source, + byte_offset: 0, + injection_callback, + cancellation_flag, + highlighter: self, + iter_count: 0, + layers: layers, + next_event: None, + last_highlight_range: None, + }; + result.sort_layers(); + Ok(result) + } +} + +impl HighlightConfiguration { + /// Creates a `HighlightConfiguration` for a given `Language` and set of highlighting + /// queries. + /// + /// # Parameters + /// + /// * `language` - The Tree-sitter `Language` that should be used for parsing. + /// * `highlights_query` - A string containing tree patterns for syntax highlighting. This + /// should be non-empty, otherwise no syntax highlights will be added. + /// * `injections_query` - A string containing tree patterns for injecting other languages + /// into the document. This can be empty if no injections are desired. + /// * `locals_query` - A string containing tree patterns for tracking local variable + /// definitions and references. This can be empty if local variable tracking is not needed. + /// + /// Returns a `HighlightConfiguration` that can then be used with the `highlight` method. + pub fn new( + language: Language, + highlights_query: &str, + injection_query: &str, + locals_query: &str, + ) -> Result<Self, QueryError> { + // Concatenate the query strings, keeping track of the start offset of each section. + let mut query_source = String::new(); + query_source.push_str(injection_query); + let locals_query_offset = query_source.len(); + query_source.push_str(locals_query); + let highlights_query_offset = query_source.len(); + query_source.push_str(highlights_query); + + // Construct a single query by concatenating the three query strings, but record the + // range of pattern indices that belong to each individual string. + let mut query = Query::new(language, &query_source)?; + let mut locals_pattern_index = 0; + let mut highlights_pattern_index = 0; + for i in 0..(query.pattern_count()) { + let pattern_offset = query.start_byte_for_pattern(i); + if pattern_offset < highlights_query_offset { + if pattern_offset < highlights_query_offset { + highlights_pattern_index += 1; + } + if pattern_offset < locals_query_offset { + locals_pattern_index += 1; + } + } + } + + // Construct a separate query just for dealing with the 'combined injections'. + // Disable the combined injection patterns in the main query. + let mut combined_injections_query = Query::new(language, injection_query)?; + let mut has_combined_queries = false; + for pattern_index in 0..locals_pattern_index { + let settings = query.property_settings(pattern_index); + if settings.iter().any(|s| &*s.key == "injection.combined") { + has_combined_queries = true; + query.disable_pattern(pattern_index); + } else { + combined_injections_query.disable_pattern(pattern_index); + } + } + let combined_injections_query = if has_combined_queries { + Some(combined_injections_query) + } else { + None + }; + + // Find all of the highlighting patterns that are disabled for nodes that + // have been identified as local variables. + let non_local_variable_patterns = (0..query.pattern_count()) + .map(|i| { + query + .property_predicates(i) + .iter() + .any(|(prop, positive)| !*positive && prop.key.as_ref() == "local") + }) + .collect(); + + // Store the numeric ids for all of the special captures. + let mut injection_content_capture_index = None; + let mut injection_language_capture_index = None; + let mut local_def_capture_index = None; + let mut local_def_value_capture_index = None; + let mut local_ref_capture_index = None; + let mut local_scope_capture_index = None; + for (i, name) in query.capture_names().iter().enumerate() { + let i = Some(i as u32); + match name.as_str() { + "injection.content" => injection_content_capture_index = i, + "injection.language" => injection_language_capture_index = i, + "local.definition" => local_def_capture_index = i, + "local.definition-value" => local_def_value_capture_index = i, + "local.reference" => local_ref_capture_index = i, + "local.scope" => local_scope_capture_index = i, + _ => {} + } + } + + let highlight_indices = vec![None; query.capture_names().len()]; + Ok(HighlightConfiguration { + language, + query, + combined_injections_query, + locals_pattern_index, + highlights_pattern_index, + highlight_indices, + non_local_variable_patterns, + injection_content_capture_index, + injection_language_capture_index, + local_def_capture_index, + local_def_value_capture_index, + local_ref_capture_index, + local_scope_capture_index, + }) + } + + /// Get a slice containing all of the highlight names used in the configuration. + pub fn names(&self) -> &[String] { + self.query.capture_names() + } + + /// Set the list of recognized highlight names. + /// + /// Tree-sitter syntax-highlighting queries specify highlights in the form of dot-separated + /// highlight names like `punctuation.bracket` and `function.method.builtin`. Consumers of + /// these queries can choose to recognize highlights with different levels of specificity. + /// For example, the string `function.builtin` will match against `function.method.builtin` + /// and `function.builtin.constructor`, but will not match `function.method`. + /// + /// When highlighting, results are returned as `Highlight` values, which contain the index + /// of the matched highlight this list of highlight names. + pub fn configure(&mut self, recognized_names: &[String]) { + let mut capture_parts = Vec::new(); + self.highlight_indices.clear(); + self.highlight_indices + .extend(self.query.capture_names().iter().map(move |capture_name| { + capture_parts.clear(); + capture_parts.extend(capture_name.split('.')); + + let mut best_index = None; + let mut best_match_len = 0; + for (i, recognized_name) in recognized_names.iter().enumerate() { + let mut len = 0; + let mut matches = true; + for part in recognized_name.split('.') { + len += 1; + if !capture_parts.contains(&part) { + matches = false; + break; + } + } + if matches && len > best_match_len { + best_index = Some(i); + best_match_len = len; + } + } + best_index.map(Highlight) + })); + } +} + +impl<'a> HighlightIterLayer<'a> { + /// Create a new 'layer' of highlighting for this document. + /// + /// In the even that the new layer contains "combined injections" (injections where multiple + /// disjoint ranges are parsed as one syntax tree), these will be eagerly processed and + /// added to the returned vector. + fn new<F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a>( + source: &'a [u8], + highlighter: &mut Highlighter, + cancellation_flag: Option<&'a AtomicUsize>, + injection_callback: &mut F, + mut config: &'a HighlightConfiguration, + mut depth: usize, + mut ranges: Vec<Range>, + ) -> Result<Vec<Self>, Error> { + let mut result = Vec::with_capacity(1); + let mut queue = Vec::new(); + loop { + if highlighter.parser.set_included_ranges(&ranges).is_ok() { + highlighter + .parser + .set_language(config.language) + .map_err(|_| Error::InvalidLanguage)?; + + unsafe { highlighter.parser.set_cancellation_flag(cancellation_flag) }; + let tree = highlighter + .parser + .parse(source, None) + .ok_or(Error::Cancelled)?; + unsafe { highlighter.parser.set_cancellation_flag(None) }; + let mut cursor = highlighter.cursors.pop().unwrap_or(QueryCursor::new()); + + // Process combined injections. + if let Some(combined_injections_query) = &config.combined_injections_query { + let mut injections_by_pattern_index = + vec![(None, Vec::new(), false); combined_injections_query.pattern_count()]; + let matches = + cursor.matches(combined_injections_query, tree.root_node(), |n: Node| { + &source[n.byte_range()] + }); + for mat in matches { + let entry = &mut injections_by_pattern_index[mat.pattern_index]; + let (language_name, content_node, include_children) = + injection_for_match(config, combined_injections_query, &mat, source); + if language_name.is_some() { + entry.0 = language_name; + } + if let Some(content_node) = content_node { + entry.1.push(content_node); + } + entry.2 = include_children; + } + for (lang_name, content_nodes, includes_children) in injections_by_pattern_index + { + if let (Some(lang_name), false) = (lang_name, content_nodes.is_empty()) { + if let Some(next_config) = (injection_callback)(lang_name) { + let ranges = Self::intersect_ranges( + &ranges, + &content_nodes, + includes_children, + ); + if !ranges.is_empty() { + queue.push((next_config, depth + 1, ranges)); + } + } + } + } + } + + // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which + // prevents them from being moved. But both of these values are really just + // pointers, so it's actually ok to move them. + let tree_ref = unsafe { mem::transmute::<_, &'static Tree>(&tree) }; + let cursor_ref = + unsafe { mem::transmute::<_, &'static mut QueryCursor>(&mut cursor) }; + let captures = cursor_ref + .captures(&config.query, tree_ref.root_node(), move |n: Node| { + &source[n.byte_range()] + }) + .peekable(); + + result.push(HighlightIterLayer { + highlight_end_stack: Vec::new(), + scope_stack: vec![LocalScope { + inherits: false, + range: 0..usize::MAX, + local_defs: Vec::new(), + }], + cursor, + depth, + _tree: tree, + captures, + config, + ranges, + }); + } + + if queue.is_empty() { + break; + } else { + let (next_config, next_depth, next_ranges) = queue.remove(0); + config = next_config; + depth = next_depth; + ranges = next_ranges; + } + } + + Ok(result) + } + + // Compute the ranges that should be included when parsing an injection. + // This takes into account three things: + // * `parent_ranges` - The ranges must all fall within the *current* layer's ranges. + // * `nodes` - Every injection takes place within a set of nodes. The injection ranges + // are the ranges of those nodes. + // * `includes_children` - For some injections, the content nodes' children should be + // excluded from the nested document, so that only the content nodes' *own* content + // is reparsed. For other injections, the content nodes' entire ranges should be + // reparsed, including the ranges of their children. + fn intersect_ranges( + parent_ranges: &[Range], + nodes: &[Node], + includes_children: bool, + ) -> Vec<Range> { + let mut cursor = nodes[0].walk(); + let mut result = Vec::new(); + let mut parent_range_iter = parent_ranges.iter(); + let mut parent_range = parent_range_iter + .next() + .expect("Layers should only be constructed with non-empty ranges vectors"); + for node in nodes.iter() { + let mut preceding_range = Range { + start_byte: 0, + start_point: Point::new(0, 0), + end_byte: node.start_byte(), + end_point: node.start_position(), + }; + let following_range = Range { + start_byte: node.end_byte(), + start_point: node.end_position(), + end_byte: usize::MAX, + end_point: Point::new(usize::MAX, usize::MAX), + }; + + for excluded_range in node + .children(&mut cursor) + .filter_map(|child| { + if includes_children { + None + } else { + Some(child.range()) + } + }) + .chain([following_range].iter().cloned()) + { + let mut range = Range { + start_byte: preceding_range.end_byte, + start_point: preceding_range.end_point, + end_byte: excluded_range.start_byte, + end_point: excluded_range.start_point, + }; + preceding_range = excluded_range; + + if range.end_byte < parent_range.start_byte { + continue; + } + + while parent_range.start_byte <= range.end_byte { + if parent_range.end_byte > range.start_byte { + if range.start_byte < parent_range.start_byte { + range.start_byte = parent_range.start_byte; + range.start_point = parent_range.start_point; + } + + if parent_range.end_byte < range.end_byte { + if range.start_byte < parent_range.end_byte { + result.push(Range { + start_byte: range.start_byte, + start_point: range.start_point, + end_byte: parent_range.end_byte, + end_point: parent_range.end_point, + }); + } + range.start_byte = parent_range.end_byte; + range.start_point = parent_range.end_point; + } else { + if range.start_byte < range.end_byte { + result.push(range); + } + break; + } + } + + if let Some(next_range) = parent_range_iter.next() { + parent_range = next_range; + } else { + return result; + } + } + } + } + result + } + + // First, sort scope boundaries by their byte offset in the document. At a + // given position, emit scope endings before scope beginnings. Finally, emit + // scope boundaries from deeper layers first. + fn sort_key(&mut self) -> Option<(usize, bool, isize)> { + let depth = -(self.depth as isize); + let next_start = self + .captures + .peek() + .map(|(m, i)| m.captures[*i].node.start_byte()); + let next_end = self.highlight_end_stack.last().cloned(); + match (next_start, next_end) { + (Some(start), Some(end)) => { + if start < end { + Some((start, true, depth)) + } else { + Some((end, false, depth)) + } + } + (Some(i), None) => Some((i, true, depth)), + (None, Some(j)) => Some((j, false, depth)), + _ => None, + } + } +} + +impl<'a, F> HighlightIter<'a, F> +where + F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a, +{ + fn emit_event( + &mut self, + offset: usize, + event: Option<HighlightEvent>, + ) -> Option<Result<HighlightEvent, Error>> { + let result; + if self.byte_offset < offset { + result = Some(Ok(HighlightEvent::Source { + start: self.byte_offset, + end: offset, + })); + self.byte_offset = offset; + self.next_event = event; + } else { + result = event.map(Ok); + } + self.sort_layers(); + result + } + + fn sort_layers(&mut self) { + while !self.layers.is_empty() { + if let Some(sort_key) = self.layers[0].sort_key() { + let mut i = 0; + while i + 1 < self.layers.len() { + if let Some(next_offset) = self.layers[i + 1].sort_key() { + if next_offset < sort_key { + i += 1; + continue; + } + } + break; + } + if i > 0 { + &self.layers[0..(i + 1)].rotate_left(1); + } + break; + } else { + let layer = self.layers.remove(0); + self.highlighter.cursors.push(layer.cursor); + } + } + } + + fn insert_layer(&mut self, mut layer: HighlightIterLayer<'a>) { + if let Some(sort_key) = layer.sort_key() { + let mut i = 1; + while i < self.layers.len() { + if let Some(sort_key_i) = self.layers[i].sort_key() { + if sort_key_i > sort_key { + self.layers.insert(i, layer); + return; + } + i += 1; + } else { + self.layers.remove(i); + } + } + self.layers.push(layer); + } + } +} + +impl<'a, F> Iterator for HighlightIter<'a, F> +where + F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a, +{ + type Item = Result<HighlightEvent, Error>; + + fn next(&mut self) -> Option<Self::Item> { + 'main: loop { + // If we've already determined the next highlight boundary, just return it. + if let Some(e) = self.next_event.take() { + return Some(Ok(e)); + } + + // Periodically check for cancellation, returning `Cancelled` error if the + // cancellation flag was flipped. + if let Some(cancellation_flag) = self.cancellation_flag { + self.iter_count += 1; + if self.iter_count >= CANCELLATION_CHECK_INTERVAL { + self.iter_count = 0; + if cancellation_flag.load(Ordering::Relaxed) != 0 { + return Some(Err(Error::Cancelled)); + } + } + } + + // If none of the layers have any more highlight boundaries, terminate. + if self.layers.is_empty() { + return if self.byte_offset < self.source.len() { + let result = Some(Ok(HighlightEvent::Source { + start: self.byte_offset, + end: self.source.len(), + })); + self.byte_offset = self.source.len(); + result + } else { + None + }; + } + + // Get the next capture from whichever layer has the earliest highlight boundary. + let range; + let layer = &mut self.layers[0]; + if let Some((next_match, capture_index)) = layer.captures.peek() { + let next_capture = next_match.captures[*capture_index]; + range = next_capture.node.byte_range(); + + // If any previous highlight ends before this node starts, then before + // processing this capture, emit the source code up until the end of the + // previous highlight, and an end event for that highlight. + if let Some(end_byte) = layer.highlight_end_stack.last().cloned() { + if end_byte <= range.start { + layer.highlight_end_stack.pop(); + return self.emit_event(end_byte, Some(HighlightEvent::HighlightEnd)); + } + } + } + // If there are no more captures, then emit any remaining highlight end events. + // And if there are none of those, then just advance to the end of the document. + else if let Some(end_byte) = layer.highlight_end_stack.last().cloned() { + layer.highlight_end_stack.pop(); + return self.emit_event(end_byte, Some(HighlightEvent::HighlightEnd)); + } else { + return self.emit_event(self.source.len(), None); + }; + + let (mut match_, capture_index) = layer.captures.next().unwrap(); + let mut capture = match_.captures[capture_index]; + + // If this capture represents an injection, then process the injection. + if match_.pattern_index < layer.config.locals_pattern_index { + let (language_name, content_node, include_children) = + injection_for_match(&layer.config, &layer.config.query, &match_, &self.source); + + // Explicitly remove this match so that none of its other captures will remain + // in the stream of captures. + match_.remove(); + + // If a language is found with the given name, then add a new language layer + // to the highlighted document. + if let (Some(language_name), Some(content_node)) = (language_name, content_node) { + if let Some(config) = (self.injection_callback)(language_name) { + let ranges = HighlightIterLayer::intersect_ranges( + &self.layers[0].ranges, + &[content_node], + include_children, + ); + if !ranges.is_empty() { + match HighlightIterLayer::new( + self.source, + self.highlighter, + self.cancellation_flag, + &mut self.injection_callback, + config, + self.layers[0].depth + 1, + ranges, + ) { + Ok(layers) => { + for layer in layers { + self.insert_layer(layer); + } + } + Err(e) => return Some(Err(e)), + } + } + } + } + + self.sort_layers(); + continue 'main; + } + + // Remove from the local scope stack any local scopes that have already ended. + while range.start > layer.scope_stack.last().unwrap().range.end { + layer.scope_stack.pop(); + } + + // If this capture is for tracking local variables, then process the + // local variable info. + let mut reference_highlight = None; + let mut definition_highlight = None; + while match_.pattern_index < layer.config.highlights_pattern_index { + // If the node represents a local scope, push a new local scope onto + // the scope stack. + if Some(capture.index) == layer.config.local_scope_capture_index { + definition_highlight = None; + let mut scope = LocalScope { + inherits: true, + range: range.clone(), + local_defs: Vec::new(), + }; + for prop in layer.config.query.property_settings(match_.pattern_index) { + match prop.key.as_ref() { + "local.scope-inherits" => { + scope.inherits = + prop.value.as_ref().map_or(true, |r| r.as_ref() == "true"); + } + _ => {} + } + } + layer.scope_stack.push(scope); + } + // If the node represents a definition, add a new definition to the + // local scope at the top of the scope stack. + else if Some(capture.index) == layer.config.local_def_capture_index { + reference_highlight = None; + definition_highlight = None; + let scope = layer.scope_stack.last_mut().unwrap(); + + let mut value_range = 0..0; + for capture in match_.captures { + if Some(capture.index) == layer.config.local_def_value_capture_index { + value_range = capture.node.byte_range(); + } + } + + if let Ok(name) = str::from_utf8(&self.source[range.clone()]) { + scope.local_defs.push(LocalDef { + name, + value_range, + highlight: None, + }); + definition_highlight = + scope.local_defs.last_mut().map(|s| &mut s.highlight); + } + } + // If the node represents a reference, then try to find the corresponding + // definition in the scope stack. + else if Some(capture.index) == layer.config.local_ref_capture_index { + if definition_highlight.is_none() { + definition_highlight = None; + if let Ok(name) = str::from_utf8(&self.source[range.clone()]) { + for scope in layer.scope_stack.iter().rev() { + if let Some(highlight) = + scope.local_defs.iter().rev().find_map(|def| { + if def.name == name && range.start >= def.value_range.end { + Some(def.highlight) + } else { + None + } + }) + { + reference_highlight = highlight; + break; + } + if !scope.inherits { + break; + } + } + } + } + } + + // Continue processing any additional matches for the same node. + if let Some((next_match, next_capture_index)) = layer.captures.peek() { + let next_capture = next_match.captures[*next_capture_index]; + if next_capture.node == capture.node { + capture = next_capture; + match_ = layer.captures.next().unwrap().0; + continue; + } + } + + self.sort_layers(); + continue 'main; + } + + // Otherwise, this capture must represent a highlight. + // If this exact range has already been highlighted by an earlier pattern, or by + // a different layer, then skip over this one. + if let Some((last_start, last_end, last_depth)) = self.last_highlight_range { + if range.start == last_start && range.end == last_end && layer.depth < last_depth { + self.sort_layers(); + continue 'main; + } + } + + // If the current node was found to be a local variable, then skip over any + // highlighting patterns that are disabled for local variables. + if definition_highlight.is_some() || reference_highlight.is_some() { + while layer.config.non_local_variable_patterns[match_.pattern_index] { + if let Some((next_match, next_capture_index)) = layer.captures.peek() { + let next_capture = next_match.captures[*next_capture_index]; + if next_capture.node == capture.node { + capture = next_capture; + match_ = layer.captures.next().unwrap().0; + continue; + } + } + + self.sort_layers(); + continue 'main; + } + } + + // Once a highlighting pattern is found for the current node, skip over + // any later highlighting patterns that also match this node. Captures + // for a given node are ordered by pattern index, so these subsequent + // captures are guaranteed to be for highlighting, not injections or + // local variables. + while let Some((next_match, next_capture_index)) = layer.captures.peek() { + let next_capture = next_match.captures[*next_capture_index]; + if next_capture.node == capture.node { + layer.captures.next(); + } else { + break; + } + } + + let current_highlight = layer.config.highlight_indices[capture.index as usize]; + + // If this node represents a local definition, then store the current + // highlight value on the local scope entry representing this node. + if let Some(definition_highlight) = definition_highlight { + *definition_highlight = current_highlight; + } + + // Emit a scope start event and push the node's end position to the stack. + if let Some(highlight) = reference_highlight.or(current_highlight) { + self.last_highlight_range = Some((range.start, range.end, layer.depth)); + layer.highlight_end_stack.push(range.end); + return self + .emit_event(range.start, Some(HighlightEvent::HighlightStart(highlight))); + } + + self.sort_layers(); + } + } +} + +fn injection_for_match<'a>( + config: &HighlightConfiguration, + query: &'a Query, + query_match: &QueryMatch<'a>, + source: &'a [u8], +) -> (Option<&'a str>, Option<Node<'a>>, bool) { + let content_capture_index = config.injection_content_capture_index; + let language_capture_index = config.injection_language_capture_index; + + let mut language_name = None; + let mut content_node = None; + for capture in query_match.captures { + let index = Some(capture.index); + if index == language_capture_index { + language_name = capture.node.utf8_text(source).ok(); + } else if index == content_capture_index { + content_node = Some(capture.node); + } + } + + let mut include_children = false; + for prop in query.property_settings(query_match.pattern_index) { + match prop.key.as_ref() { + // In addition to specifying the language name via the text of a + // captured node, it can also be hard-coded via a `#set!` predicate + // that sets the injection.language key. + "injection.language" => { + if language_name.is_none() { + language_name = prop.value.as_ref().map(|s| s.as_ref()) + } + } + + // By default, injections do not include the *children* of an + // `injection.content` node - only the ranges that belong to the + // node itself. This can be changed using a `#set!` predicate that + // sets the `injection.include-children` key. + "injection.include-children" => include_children = true, + _ => {} + } + } + + (language_name, content_node, include_children) +} + +fn shrink_and_clear<T>(vec: &mut Vec<T>, capacity: usize) { + if vec.len() > capacity { + vec.truncate(capacity); + vec.shrink_to_fit(); + } + vec.clear(); +} diff --git a/helix-core/src/lib.rs b/helix-core/src/lib.rs index da00b9ae..6ab40694 100644 --- a/helix-core/src/lib.rs +++ b/helix-core/src/lib.rs @@ -1,6 +1,7 @@ #![allow(unused)] pub mod commands; pub mod graphemes; +pub mod language_mode; mod selection; pub mod state; mod transaction; diff --git a/helix-syntax/build.rs b/helix-syntax/build.rs index 217ba9fa..6b721c11 100644 --- a/helix-syntax/build.rs +++ b/helix-syntax/build.rs @@ -1,7 +1,4 @@ -use cc::Build; - -use std::io::{BufWriter, Read, Write}; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::{env, fs}; fn get_opt_level() -> u32 { diff --git a/helix-term/Cargo.toml b/helix-term/Cargo.toml index dcb01c1d..ce3add5c 100644 --- a/helix-term/Cargo.toml +++ b/helix-term/Cargo.toml @@ -24,5 +24,3 @@ crossterm = { version = "0.17", features = ["event-stream"] } smol = "1" num_cpus = "1.13.0" tui = { version = "0.10.0", default-features = false, features = ["crossterm"] } -tree-sitter = "0.16.1" -tree-sitter-highlight = "0.2.0" diff --git a/helix-term/src/editor.rs b/helix-term/src/editor.rs index f7abc806..af18842d 100644 --- a/helix-term/src/editor.rs +++ b/helix-term/src/editor.rs @@ -1,5 +1,21 @@ -use crate::{keymap, Args}; +use crate::{keymap, theme::Theme, Args}; +use helix_core::{ + language_mode::{HighlightConfiguration, HighlightEvent, Highlighter}, + state::coords_at_pos, + state::Mode, + State, +}; + +use std::{ + io::{self, stdout, Write}, + path::PathBuf, + time::Duration, +}; + +use smol::prelude::*; + use anyhow::Error; + use crossterm::{ cursor, cursor::position, @@ -9,17 +25,7 @@ use crossterm::{ terminal::{self, disable_raw_mode, enable_raw_mode}, }; -use helix_core::{state::coords_at_pos, state::Mode, State}; -use smol::prelude::*; -use std::collections::HashMap; -use std::io::{self, stdout, Write}; -use std::path::PathBuf; -use std::time::Duration; - -use tui::backend::CrosstermBackend; -use tui::buffer::Buffer as Surface; -use tui::layout::Rect; -use tui::style::Style; +use tui::{backend::CrosstermBackend, buffer::Buffer as Surface, layout::Rect, style::Style}; type Terminal = tui::Terminal<CrosstermBackend<std::io::Stdout>>; @@ -31,7 +37,10 @@ pub struct Editor { first_line: u16, size: (u16, u16), surface: Surface, - theme: HashMap<&'static str, Style>, + theme: Theme, + highlighter: Highlighter, + highlight_config: HighlightConfiguration, + highlight_names: Vec<String>, } impl Editor { @@ -42,43 +51,60 @@ impl Editor { let size = terminal::size().unwrap(); let area = Rect::new(0, 0, size.0, size.1); - use tui::style::Color; - let theme = hashmap! { - "attribute" => Style::default().fg(Color::Rgb(219, 191, 239)), // lilac - "keyword" => Style::default().fg(Color::Rgb(236, 205, 186)), // almond - "punctuation" => Style::default().fg(Color::Rgb(164, 160, 232)), // lavender - "punctuation.delimiter" => Style::default().fg(Color::Rgb(164, 160, 232)), // lavender - "operator" => Style::default().fg(Color::Rgb(219, 191, 239)), // lilac - "property" => Style::default().fg(Color::Rgb(164, 160, 232)), // lavender - "variable.parameter" => Style::default().fg(Color::Rgb(164, 160, 232)), // lavender - // TODO distinguish type from type.builtin? - "type" => Style::default().fg(Color::Rgb(255, 255, 255)), // white - "type.builtin" => Style::default().fg(Color::Rgb(255, 255, 255)), // white - "constructor" => Style::default().fg(Color::Rgb(219, 191, 239)), // lilac - "function" => Style::default().fg(Color::Rgb(255, 255, 255)), // white - "function.macro" => Style::default().fg(Color::Rgb(219, 191, 239)), // lilac - "comment" => Style::default().fg(Color::Rgb(105, 124, 129)), // sirocco - "variable.builtin" => Style::default().fg(Color::Rgb(159, 242, 143)), // mint - "constant" => Style::default().fg(Color::Rgb(255, 255, 255)), // white - "constant.builtin" => Style::default().fg(Color::Rgb(255, 255, 255)), // white - "string" => Style::default().fg(Color::Rgb(204, 204, 204)), // silver - "escape" => Style::default().fg(Color::Rgb(239, 186, 93)), // honey - // used for lifetimes - "label" => Style::default().fg(Color::Rgb(239, 186, 93)), // honey - - // TODO: diferentiate number builtin - // TODO: diferentiate doc comment - // TODO: variable as lilac - // TODO: mod/use statements as white - // TODO: mod stuff as chamoise - // TODO: add "(scoped_identifier) @path" for std::mem:: - // - // concat (ERROR) @syntax-error and "MISSING ;" selectors for errors - - "module" => Style::default().fg(Color::Rgb(255, 0, 0)), // white - "variable" => Style::default().fg(Color::Rgb(255, 0, 0)), // white - "function.builtin" => Style::default().fg(Color::Rgb(255, 0, 0)), // white - }; + let highlight_names: Vec<String> = [ + "attribute", + "constant.builtin", + "constant", + "function.builtin", + "function.macro", + "function", + "keyword", + "operator", + "property", + "punctuation", + "comment", + "escape", + "label", + // "punctuation.bracket", + "punctuation.delimiter", + "string", + "string.special", + "tag", + "type", + "type.builtin", + "constructor", + "variable", + "variable.builtin", + "variable.parameter", + "path", + ] + .iter() + .map(|s| s.to_string()) + .collect(); + + // let mut parser = tree_sitter::Parser::new(); + // parser.set_language(language).unwrap(); + // let tree = parser.parse(source_code, None).unwrap(); + + let language = helix_syntax::get_language(&helix_syntax::LANG::Rust); + + let highlighter = Highlighter::new(); + + let mut highlight_config = HighlightConfiguration::new( + language, + &std::fs::read_to_string( + "../helix-syntax/languages/tree-sitter-rust/queries/highlights.scm", + ) + .unwrap(), + &std::fs::read_to_string( + "../helix-syntax/languages/tree-sitter-rust/queries/injections.scm", + ) + .unwrap(), + "", // locals.scm + ) + .unwrap(); + + highlight_config.configure(&highlight_names); let mut editor = Editor { terminal, @@ -86,7 +112,11 @@ impl Editor { first_line: 0, size, surface: Surface::empty(area), - theme, + theme: Theme::default(), + // TODO; move to state + highlighter, + highlight_config, + highlight_names, }; if let Some(file) = args.files.pop() { @@ -108,72 +138,16 @@ impl Editor { let mut surface = Surface::empty(area); let mut stdout = stdout(); - // - use tree_sitter_highlight::{HighlightConfiguration, HighlightEvent, Highlighter}; - - let highlight_names: Vec<String> = [ - "attribute", - "constant.builtin", - "constant", - "function.builtin", - "function.macro", - "function", - "keyword", - "operator", - "property", - "punctuation", - "comment", - "escape", - "label", - // "punctuation.bracket", - "punctuation.delimiter", - "string", - "string.special", - "tag", - "type", - "type.builtin", - "constructor", - "variable", - "variable.builtin", - "variable.parameter", - "path", - ] - .iter() - .cloned() - .map(String::from) - .collect(); - - let language = helix_syntax::get_language(&helix_syntax::LANG::Rust); - // let mut parser = tree_sitter::Parser::new(); - // parser.set_language(language).unwrap(); - // let tree = parser.parse(source_code, None).unwrap(); - - let mut highlighter = Highlighter::new(); - - let mut config = HighlightConfiguration::new( - language, - &std::fs::read_to_string( - "../helix-syntax/languages/tree-sitter-rust/queries/highlights.scm", - ) - .unwrap(), - &std::fs::read_to_string( - "../helix-syntax/languages/tree-sitter-rust/queries/injections.scm", - ) - .unwrap(), - "", // locals.scm - ) - .unwrap(); - - config.configure(&highlight_names); - - // TODO: inefficient, should feed chunks.iter() to tree_sitter.parse_with(|offset, - // pos|) + // TODO: inefficient, should feed chunks.iter() to tree_sitter.parse_with(|offset, pos|) let source_code = state.doc.to_string(); // TODO: cache highlight results // TODO: only recalculate when state.doc is actually modified - let highlights = highlighter - .highlight(&config, source_code.as_bytes(), None, |_| None) + let highlights = self + .highlighter + .highlight(&self.highlight_config, source_code.as_bytes(), None, |_| { + None + }) .unwrap(); let mut spans = Vec::new(); @@ -186,12 +160,10 @@ impl Editor { for event in highlights { match event.unwrap() { HighlightEvent::HighlightStart(span) => { - // eprintln!("highlight style started: {:?}", highlight_names[span.0]); spans.push(span); } HighlightEvent::HighlightEnd => { spans.pop(); - // eprintln!("highlight style ended"); } HighlightEvent::Source { start, end } => { // TODO: filter out spans out of viewport for now.. @@ -204,17 +176,16 @@ impl Editor { use helix_core::graphemes::{grapheme_width, RopeGraphemes}; use tui::style::Color; - let style = match spans.first() { - Some(span) => self - .theme - .get(highlight_names[span.0].as_str()) - .map(|style| *style) - .unwrap_or(Style::default().fg(Color::Rgb(0, 0, 255))), + let style = match spans.first() { + Some(span) => self.theme.get(self.highlight_names[span.0].as_str()), None => Style::default().fg(Color::Rgb(164, 160, 232)), // lavender - // None => Style::default().fg(Color::Rgb(219, 191, 239)), // lilac }; + // TODO: we could render the text to a surface, then cache that, that + // way if only the selection/cursor changes we can copy from cache + // and paint the new cursor. + // iterate over range char by char for grapheme in RopeGraphemes::new(&text) { // TODO: track current char_index diff --git a/helix-term/src/main.rs b/helix-term/src/main.rs index e3c16c14..d6c6c499 100644 --- a/helix-term/src/main.rs +++ b/helix-term/src/main.rs @@ -3,6 +3,7 @@ mod macros; mod editor; mod keymap; +mod theme; use editor::Editor; diff --git a/helix-term/src/theme.rs b/helix-term/src/theme.rs new file mode 100644 index 00000000..0353f0e6 --- /dev/null +++ b/helix-term/src/theme.rs @@ -0,0 +1,59 @@ +use std::collections::HashMap; +use tui::style::{Color, Style}; + +/// Color theme for syntax highlighting. +pub struct Theme { + mapping: HashMap<&'static str, Style>, +} + +impl Default for Theme { + fn default() -> Self { + let mapping = hashmap! { + "attribute" => Style::default().fg(Color::Rgb(219, 191, 239)), // lilac + "keyword" => Style::default().fg(Color::Rgb(236, 205, 186)), // almond + "punctuation" => Style::default().fg(Color::Rgb(164, 160, 232)), // lavender + "punctuation.delimiter" => Style::default().fg(Color::Rgb(164, 160, 232)), // lavender + "operator" => Style::default().fg(Color::Rgb(219, 191, 239)), // lilac + "property" => Style::default().fg(Color::Rgb(164, 160, 232)), // lavender + "variable.parameter" => Style::default().fg(Color::Rgb(164, 160, 232)), // lavender + // TODO distinguish type from type.builtin? + "type" => Style::default().fg(Color::Rgb(255, 255, 255)), // white + "type.builtin" => Style::default().fg(Color::Rgb(255, 255, 255)), // white + "constructor" => Style::default().fg(Color::Rgb(219, 191, 239)), // lilac + "function" => Style::default().fg(Color::Rgb(255, 255, 255)), // white + "function.macro" => Style::default().fg(Color::Rgb(219, 191, 239)), // lilac + "comment" => Style::default().fg(Color::Rgb(105, 124, 129)), // sirocco + "variable.builtin" => Style::default().fg(Color::Rgb(159, 242, 143)), // mint + "constant" => Style::default().fg(Color::Rgb(255, 255, 255)), // white + "constant.builtin" => Style::default().fg(Color::Rgb(255, 255, 255)), // white + "string" => Style::default().fg(Color::Rgb(204, 204, 204)), // silver + "escape" => Style::default().fg(Color::Rgb(239, 186, 93)), // honey + // used for lifetimes + "label" => Style::default().fg(Color::Rgb(239, 186, 93)), // honey + + // TODO: diferentiate number builtin + // TODO: diferentiate doc comment + // TODO: variable as lilac + // TODO: mod/use statements as white + // TODO: mod stuff as chamoise + // TODO: add "(scoped_identifier) @path" for std::mem:: + // + // concat (ERROR) @syntax-error and "MISSING ;" selectors for errors + + "module" => Style::default().fg(Color::Rgb(255, 0, 0)), // white + "variable" => Style::default().fg(Color::Rgb(255, 0, 0)), // white + "function.builtin" => Style::default().fg(Color::Rgb(255, 0, 0)), // white + }; + + Self { mapping } + } +} + +impl Theme { + pub fn get(&self, scope: &str) -> Style { + self.mapping + .get(scope) + .copied() + .unwrap_or_else(|| Style::default().fg(Color::Rgb(0, 0, 255))) + } +} |