aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--Cargo.lock13
-rw-r--r--helix-core/Cargo.toml1
-rw-r--r--helix-core/src/language_mode.rs993
-rw-r--r--helix-core/src/lib.rs1
-rw-r--r--helix-syntax/build.rs5
-rw-r--r--helix-term/Cargo.toml2
-rw-r--r--helix-term/src/editor.rs215
-rw-r--r--helix-term/src/main.rs1
-rw-r--r--helix-term/src/theme.rs59
9 files changed, 1150 insertions, 140 deletions
diff --git a/Cargo.lock b/Cargo.lock
index 6334ee8b..2919b53e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -372,6 +372,7 @@ dependencies = [
"ropey",
"smallvec",
"tendril",
+ "tree-sitter",
"unicode-segmentation",
"unicode-width",
]
@@ -396,8 +397,6 @@ dependencies = [
"helix-syntax",
"num_cpus",
"smol",
- "tree-sitter",
- "tree-sitter-highlight",
"tui",
]
@@ -720,16 +719,6 @@ dependencies = [
]
[[package]]
-name = "tree-sitter-highlight"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6e3c800ac0db1562a045a4893cbbd7c484eb93426cae5632f9e5d24dd588cd1"
-dependencies = [
- "regex",
- "tree-sitter",
-]
-
-[[package]]
name = "tui"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/helix-core/Cargo.toml b/helix-core/Cargo.toml
index 9bd82e52..23eccdc2 100644
--- a/helix-core/Cargo.toml
+++ b/helix-core/Cargo.toml
@@ -14,3 +14,4 @@ tendril = { git = "https://github.com/servo/tendril" }
unicode-segmentation = "1.6.0"
unicode-width = "0.1.7"
# slab = "0.4.2"
+tree-sitter = "0.16.1"
diff --git a/helix-core/src/language_mode.rs b/helix-core/src/language_mode.rs
new file mode 100644
index 00000000..3f104be4
--- /dev/null
+++ b/helix-core/src/language_mode.rs
@@ -0,0 +1,993 @@
+pub struct LanguageMode {
+ parser: Parser,
+}
+
+impl LanguageMode {
+ // buffer, grammar, config, grammars, sync_timeout?
+ pub fn new() -> Self {
+ unimplemented!()
+ // make a new root layer
+ // track markers of injections
+ //
+ // track scope_descriptor: a Vec of scopes for item in tree
+ //
+ // fetch grammar for parser based on language string
+ // update root layer
+ }
+
+ // fn buffer_changed -> call layer.update(range, new_text) on root layer and then all marker layers
+
+ // call this on transaction.apply() -> buffer_changed(changes)
+ //
+ // fn parse(language, old_tree, ranges)
+ //
+ // fn tree() -> Tree
+ //
+ // <!--update_for_injection(grammar)-->
+
+ // Highlighting
+ // fn highlight_iter() -> iterates over all the scopes
+ // on_tokenize
+ // on_change_highlighting
+
+ // Commenting
+ // comment_strings_for_pos
+ // is_commented
+
+ // Indentation
+ // suggested_indent_for_line_at_buffer_row
+ // suggested_indent_for_buffer_row
+ // indent_level_for_line
+
+ // TODO: Folding
+
+ // Syntax APIs
+ // get_syntax_node_containing_range ->
+ // ...
+ // get_syntax_node_at_pos
+ // buffer_range_for_scope_at_pos
+}
+
+pub struct LanguageLayer {
+ // mode
+// grammar
+// depth
+// tree: Tree,
+}
+
+impl LanguageLayer {
+ // fn highlight_iter() -> same as Mode but for this layer. Mode composits these
+ // fn buffer_changed
+ // fn update(range)
+ // fn update_injections()
+}
+
+// -- refactored from tree-sitter-highlight to be able to retain state
+// TODO: add seek() to iter
+
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::{iter, mem, ops, str, usize};
+use tree_sitter::{
+ Language, Node, Parser, Point, Query, QueryCaptures, QueryCursor, QueryError, QueryMatch,
+ Range, Tree,
+};
+
+const CANCELLATION_CHECK_INTERVAL: usize = 100;
+const BUFFER_HTML_RESERVE_CAPACITY: usize = 10 * 1024;
+const BUFFER_LINES_RESERVE_CAPACITY: usize = 1000;
+
+/// Indicates which highlight should be applied to a region of source code.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub struct Highlight(pub usize);
+
+/// Represents the reason why syntax highlighting failed.
+#[derive(Debug, PartialEq, Eq)]
+pub enum Error {
+ Cancelled,
+ InvalidLanguage,
+ Unknown,
+}
+
+/// Represents a single step in rendering a syntax-highlighted document.
+#[derive(Copy, Clone, Debug)]
+pub enum HighlightEvent {
+ Source { start: usize, end: usize },
+ HighlightStart(Highlight),
+ HighlightEnd,
+}
+
+/// Contains the data neeeded to higlight code written in a particular language.
+///
+/// This struct is immutable and can be shared between threads.
+pub struct HighlightConfiguration {
+ pub language: Language,
+ pub query: Query,
+ combined_injections_query: Option<Query>,
+ locals_pattern_index: usize,
+ highlights_pattern_index: usize,
+ highlight_indices: Vec<Option<Highlight>>,
+ non_local_variable_patterns: Vec<bool>,
+ injection_content_capture_index: Option<u32>,
+ injection_language_capture_index: Option<u32>,
+ local_scope_capture_index: Option<u32>,
+ local_def_capture_index: Option<u32>,
+ local_def_value_capture_index: Option<u32>,
+ local_ref_capture_index: Option<u32>,
+}
+
+/// Performs syntax highlighting, recognizing a given list of highlight names.
+///
+/// For the best performance `Highlighter` values should be reused between
+/// syntax highlighting calls. A separate highlighter is needed for each thread that
+/// is performing highlighting.
+pub struct Highlighter {
+ parser: Parser,
+ cursors: Vec<QueryCursor>,
+}
+
+#[derive(Debug)]
+struct LocalDef<'a> {
+ name: &'a str,
+ value_range: ops::Range<usize>,
+ highlight: Option<Highlight>,
+}
+
+#[derive(Debug)]
+struct LocalScope<'a> {
+ inherits: bool,
+ range: ops::Range<usize>,
+ local_defs: Vec<LocalDef<'a>>,
+}
+
+struct HighlightIter<'a, F>
+where
+ F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
+{
+ source: &'a [u8],
+ byte_offset: usize,
+ highlighter: &'a mut Highlighter,
+ injection_callback: F,
+ cancellation_flag: Option<&'a AtomicUsize>,
+ layers: Vec<HighlightIterLayer<'a>>,
+ iter_count: usize,
+ next_event: Option<HighlightEvent>,
+ last_highlight_range: Option<(usize, usize, usize)>,
+}
+
+struct HighlightIterLayer<'a> {
+ _tree: Tree,
+ cursor: QueryCursor,
+ captures: iter::Peekable<QueryCaptures<'a, &'a [u8]>>,
+ config: &'a HighlightConfiguration,
+ highlight_end_stack: Vec<usize>,
+ scope_stack: Vec<LocalScope<'a>>,
+ ranges: Vec<Range>,
+ depth: usize,
+}
+
+impl Highlighter {
+ pub fn new() -> Self {
+ Highlighter {
+ parser: Parser::new(),
+ cursors: Vec::new(),
+ }
+ }
+
+ pub fn parser(&mut self) -> &mut Parser {
+ &mut self.parser
+ }
+
+ /// Iterate over the highlighted regions for a given slice of source code.
+ pub fn highlight<'a>(
+ &'a mut self,
+ config: &'a HighlightConfiguration,
+ source: &'a [u8],
+ cancellation_flag: Option<&'a AtomicUsize>,
+ mut injection_callback: impl FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
+ ) -> Result<impl Iterator<Item = Result<HighlightEvent, Error>> + 'a, Error> {
+ let layers = HighlightIterLayer::new(
+ source,
+ self,
+ cancellation_flag,
+ &mut injection_callback,
+ config,
+ 0,
+ vec![Range {
+ start_byte: 0,
+ end_byte: usize::MAX,
+ start_point: Point::new(0, 0),
+ end_point: Point::new(usize::MAX, usize::MAX),
+ }],
+ )?;
+ assert_ne!(layers.len(), 0);
+ let mut result = HighlightIter {
+ source,
+ byte_offset: 0,
+ injection_callback,
+ cancellation_flag,
+ highlighter: self,
+ iter_count: 0,
+ layers: layers,
+ next_event: None,
+ last_highlight_range: None,
+ };
+ result.sort_layers();
+ Ok(result)
+ }
+}
+
+impl HighlightConfiguration {
+ /// Creates a `HighlightConfiguration` for a given `Language` and set of highlighting
+ /// queries.
+ ///
+ /// # Parameters
+ ///
+ /// * `language` - The Tree-sitter `Language` that should be used for parsing.
+ /// * `highlights_query` - A string containing tree patterns for syntax highlighting. This
+ /// should be non-empty, otherwise no syntax highlights will be added.
+ /// * `injections_query` - A string containing tree patterns for injecting other languages
+ /// into the document. This can be empty if no injections are desired.
+ /// * `locals_query` - A string containing tree patterns for tracking local variable
+ /// definitions and references. This can be empty if local variable tracking is not needed.
+ ///
+ /// Returns a `HighlightConfiguration` that can then be used with the `highlight` method.
+ pub fn new(
+ language: Language,
+ highlights_query: &str,
+ injection_query: &str,
+ locals_query: &str,
+ ) -> Result<Self, QueryError> {
+ // Concatenate the query strings, keeping track of the start offset of each section.
+ let mut query_source = String::new();
+ query_source.push_str(injection_query);
+ let locals_query_offset = query_source.len();
+ query_source.push_str(locals_query);
+ let highlights_query_offset = query_source.len();
+ query_source.push_str(highlights_query);
+
+ // Construct a single query by concatenating the three query strings, but record the
+ // range of pattern indices that belong to each individual string.
+ let mut query = Query::new(language, &query_source)?;
+ let mut locals_pattern_index = 0;
+ let mut highlights_pattern_index = 0;
+ for i in 0..(query.pattern_count()) {
+ let pattern_offset = query.start_byte_for_pattern(i);
+ if pattern_offset < highlights_query_offset {
+ if pattern_offset < highlights_query_offset {
+ highlights_pattern_index += 1;
+ }
+ if pattern_offset < locals_query_offset {
+ locals_pattern_index += 1;
+ }
+ }
+ }
+
+ // Construct a separate query just for dealing with the 'combined injections'.
+ // Disable the combined injection patterns in the main query.
+ let mut combined_injections_query = Query::new(language, injection_query)?;
+ let mut has_combined_queries = false;
+ for pattern_index in 0..locals_pattern_index {
+ let settings = query.property_settings(pattern_index);
+ if settings.iter().any(|s| &*s.key == "injection.combined") {
+ has_combined_queries = true;
+ query.disable_pattern(pattern_index);
+ } else {
+ combined_injections_query.disable_pattern(pattern_index);
+ }
+ }
+ let combined_injections_query = if has_combined_queries {
+ Some(combined_injections_query)
+ } else {
+ None
+ };
+
+ // Find all of the highlighting patterns that are disabled for nodes that
+ // have been identified as local variables.
+ let non_local_variable_patterns = (0..query.pattern_count())
+ .map(|i| {
+ query
+ .property_predicates(i)
+ .iter()
+ .any(|(prop, positive)| !*positive && prop.key.as_ref() == "local")
+ })
+ .collect();
+
+ // Store the numeric ids for all of the special captures.
+ let mut injection_content_capture_index = None;
+ let mut injection_language_capture_index = None;
+ let mut local_def_capture_index = None;
+ let mut local_def_value_capture_index = None;
+ let mut local_ref_capture_index = None;
+ let mut local_scope_capture_index = None;
+ for (i, name) in query.capture_names().iter().enumerate() {
+ let i = Some(i as u32);
+ match name.as_str() {
+ "injection.content" => injection_content_capture_index = i,
+ "injection.language" => injection_language_capture_index = i,
+ "local.definition" => local_def_capture_index = i,
+ "local.definition-value" => local_def_value_capture_index = i,
+ "local.reference" => local_ref_capture_index = i,
+ "local.scope" => local_scope_capture_index = i,
+ _ => {}
+ }
+ }
+
+ let highlight_indices = vec![None; query.capture_names().len()];
+ Ok(HighlightConfiguration {
+ language,
+ query,
+ combined_injections_query,
+ locals_pattern_index,
+ highlights_pattern_index,
+ highlight_indices,
+ non_local_variable_patterns,
+ injection_content_capture_index,
+ injection_language_capture_index,
+ local_def_capture_index,
+ local_def_value_capture_index,
+ local_ref_capture_index,
+ local_scope_capture_index,
+ })
+ }
+
+ /// Get a slice containing all of the highlight names used in the configuration.
+ pub fn names(&self) -> &[String] {
+ self.query.capture_names()
+ }
+
+ /// Set the list of recognized highlight names.
+ ///
+ /// Tree-sitter syntax-highlighting queries specify highlights in the form of dot-separated
+ /// highlight names like `punctuation.bracket` and `function.method.builtin`. Consumers of
+ /// these queries can choose to recognize highlights with different levels of specificity.
+ /// For example, the string `function.builtin` will match against `function.method.builtin`
+ /// and `function.builtin.constructor`, but will not match `function.method`.
+ ///
+ /// When highlighting, results are returned as `Highlight` values, which contain the index
+ /// of the matched highlight this list of highlight names.
+ pub fn configure(&mut self, recognized_names: &[String]) {
+ let mut capture_parts = Vec::new();
+ self.highlight_indices.clear();
+ self.highlight_indices
+ .extend(self.query.capture_names().iter().map(move |capture_name| {
+ capture_parts.clear();
+ capture_parts.extend(capture_name.split('.'));
+
+ let mut best_index = None;
+ let mut best_match_len = 0;
+ for (i, recognized_name) in recognized_names.iter().enumerate() {
+ let mut len = 0;
+ let mut matches = true;
+ for part in recognized_name.split('.') {
+ len += 1;
+ if !capture_parts.contains(&part) {
+ matches = false;
+ break;
+ }
+ }
+ if matches && len > best_match_len {
+ best_index = Some(i);
+ best_match_len = len;
+ }
+ }
+ best_index.map(Highlight)
+ }));
+ }
+}
+
+impl<'a> HighlightIterLayer<'a> {
+ /// Create a new 'layer' of highlighting for this document.
+ ///
+ /// In the even that the new layer contains "combined injections" (injections where multiple
+ /// disjoint ranges are parsed as one syntax tree), these will be eagerly processed and
+ /// added to the returned vector.
+ fn new<F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a>(
+ source: &'a [u8],
+ highlighter: &mut Highlighter,
+ cancellation_flag: Option<&'a AtomicUsize>,
+ injection_callback: &mut F,
+ mut config: &'a HighlightConfiguration,
+ mut depth: usize,
+ mut ranges: Vec<Range>,
+ ) -> Result<Vec<Self>, Error> {
+ let mut result = Vec::with_capacity(1);
+ let mut queue = Vec::new();
+ loop {
+ if highlighter.parser.set_included_ranges(&ranges).is_ok() {
+ highlighter
+ .parser
+ .set_language(config.language)
+ .map_err(|_| Error::InvalidLanguage)?;
+
+ unsafe { highlighter.parser.set_cancellation_flag(cancellation_flag) };
+ let tree = highlighter
+ .parser
+ .parse(source, None)
+ .ok_or(Error::Cancelled)?;
+ unsafe { highlighter.parser.set_cancellation_flag(None) };
+ let mut cursor = highlighter.cursors.pop().unwrap_or(QueryCursor::new());
+
+ // Process combined injections.
+ if let Some(combined_injections_query) = &config.combined_injections_query {
+ let mut injections_by_pattern_index =
+ vec![(None, Vec::new(), false); combined_injections_query.pattern_count()];
+ let matches =
+ cursor.matches(combined_injections_query, tree.root_node(), |n: Node| {
+ &source[n.byte_range()]
+ });
+ for mat in matches {
+ let entry = &mut injections_by_pattern_index[mat.pattern_index];
+ let (language_name, content_node, include_children) =
+ injection_for_match(config, combined_injections_query, &mat, source);
+ if language_name.is_some() {
+ entry.0 = language_name;
+ }
+ if let Some(content_node) = content_node {
+ entry.1.push(content_node);
+ }
+ entry.2 = include_children;
+ }
+ for (lang_name, content_nodes, includes_children) in injections_by_pattern_index
+ {
+ if let (Some(lang_name), false) = (lang_name, content_nodes.is_empty()) {
+ if let Some(next_config) = (injection_callback)(lang_name) {
+ let ranges = Self::intersect_ranges(
+ &ranges,
+ &content_nodes,
+ includes_children,
+ );
+ if !ranges.is_empty() {
+ queue.push((next_config, depth + 1, ranges));
+ }
+ }
+ }
+ }
+ }
+
+ // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which
+ // prevents them from being moved. But both of these values are really just
+ // pointers, so it's actually ok to move them.
+ let tree_ref = unsafe { mem::transmute::<_, &'static Tree>(&tree) };
+ let cursor_ref =
+ unsafe { mem::transmute::<_, &'static mut QueryCursor>(&mut cursor) };
+ let captures = cursor_ref
+ .captures(&config.query, tree_ref.root_node(), move |n: Node| {
+ &source[n.byte_range()]
+ })
+ .peekable();
+
+ result.push(HighlightIterLayer {
+ highlight_end_stack: Vec::new(),
+ scope_stack: vec![LocalScope {
+ inherits: false,
+ range: 0..usize::MAX,
+ local_defs: Vec::new(),
+ }],
+ cursor,
+ depth,
+ _tree: tree,
+ captures,
+ config,
+ ranges,
+ });
+ }
+
+ if queue.is_empty() {
+ break;
+ } else {
+ let (next_config, next_depth, next_ranges) = queue.remove(0);
+ config = next_config;
+ depth = next_depth;
+ ranges = next_ranges;
+ }
+ }
+
+ Ok(result)
+ }
+
+ // Compute the ranges that should be included when parsing an injection.
+ // This takes into account three things:
+ // * `parent_ranges` - The ranges must all fall within the *current* layer's ranges.
+ // * `nodes` - Every injection takes place within a set of nodes. The injection ranges
+ // are the ranges of those nodes.
+ // * `includes_children` - For some injections, the content nodes' children should be
+ // excluded from the nested document, so that only the content nodes' *own* content
+ // is reparsed. For other injections, the content nodes' entire ranges should be
+ // reparsed, including the ranges of their children.
+ fn intersect_ranges(
+ parent_ranges: &[Range],
+ nodes: &[Node],
+ includes_children: bool,
+ ) -> Vec<Range> {
+ let mut cursor = nodes[0].walk();
+ let mut result = Vec::new();
+ let mut parent_range_iter = parent_ranges.iter();
+ let mut parent_range = parent_range_iter
+ .next()
+ .expect("Layers should only be constructed with non-empty ranges vectors");
+ for node in nodes.iter() {
+ let mut preceding_range = Range {
+ start_byte: 0,
+ start_point: Point::new(0, 0),
+ end_byte: node.start_byte(),
+ end_point: node.start_position(),
+ };
+ let following_range = Range {
+ start_byte: node.end_byte(),
+ start_point: node.end_position(),
+ end_byte: usize::MAX,
+ end_point: Point::new(usize::MAX, usize::MAX),
+ };
+
+ for excluded_range in node
+ .children(&mut cursor)
+ .filter_map(|child| {
+ if includes_children {
+ None
+ } else {
+ Some(child.range())
+ }
+ })
+ .chain([following_range].iter().cloned())
+ {
+ let mut range = Range {
+ start_byte: preceding_range.end_byte,
+ start_point: preceding_range.end_point,
+ end_byte: excluded_range.start_byte,
+ end_point: excluded_range.start_point,
+ };
+ preceding_range = excluded_range;
+
+ if range.end_byte < parent_range.start_byte {
+ continue;
+ }
+
+ while parent_range.start_byte <= range.end_byte {
+ if parent_range.end_byte > range.start_byte {
+ if range.start_byte < parent_range.start_byte {
+ range.start_byte = parent_range.start_byte;
+ range.start_point = parent_range.start_point;
+ }
+
+ if parent_range.end_byte < range.end_byte {
+ if range.start_byte < parent_range.end_byte {
+ result.push(Range {
+ start_byte: range.start_byte,
+ start_point: range.start_point,
+ end_byte: parent_range.end_byte,
+ end_point: parent_range.end_point,
+ });
+ }
+ range.start_byte = parent_range.end_byte;
+ range.start_point = parent_range.end_point;
+ } else {
+ if range.start_byte < range.end_byte {
+ result.push(range);
+ }
+ break;
+ }
+ }
+
+ if let Some(next_range) = parent_range_iter.next() {
+ parent_range = next_range;
+ } else {
+ return result;
+ }
+ }
+ }
+ }
+ result
+ }
+
+ // First, sort scope boundaries by their byte offset in the document. At a
+ // given position, emit scope endings before scope beginnings. Finally, emit
+ // scope boundaries from deeper layers first.
+ fn sort_key(&mut self) -> Option<(usize, bool, isize)> {
+ let depth = -(self.depth as isize);
+ let next_start = self
+ .captures
+ .peek()
+ .map(|(m, i)| m.captures[*i].node.start_byte());
+ let next_end = self.highlight_end_stack.last().cloned();
+ match (next_start, next_end) {
+ (Some(start), Some(end)) => {
+ if start < end {
+ Some((start, true, depth))
+ } else {
+ Some((end, false, depth))
+ }
+ }
+ (Some(i), None) => Some((i, true, depth)),
+ (None, Some(j)) => Some((j, false, depth)),
+ _ => None,
+ }
+ }
+}
+
+impl<'a, F> HighlightIter<'a, F>
+where
+ F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
+{
+ fn emit_event(
+ &mut self,
+ offset: usize,
+ event: Option<HighlightEvent>,
+ ) -> Option<Result<HighlightEvent, Error>> {
+ let result;
+ if self.byte_offset < offset {
+ result = Some(Ok(HighlightEvent::Source {
+ start: self.byte_offset,
+ end: offset,
+ }));
+ self.byte_offset = offset;
+ self.next_event = event;
+ } else {
+ result = event.map(Ok);
+ }
+ self.sort_layers();
+ result
+ }
+
+ fn sort_layers(&mut self) {
+ while !self.layers.is_empty() {
+ if let Some(sort_key) = self.layers[0].sort_key() {
+ let mut i = 0;
+ while i + 1 < self.layers.len() {
+ if let Some(next_offset) = self.layers[i + 1].sort_key() {
+ if next_offset < sort_key {
+ i += 1;
+ continue;
+ }
+ }
+ break;
+ }
+ if i > 0 {
+ &self.layers[0..(i + 1)].rotate_left(1);
+ }
+ break;
+ } else {
+ let layer = self.layers.remove(0);
+ self.highlighter.cursors.push(layer.cursor);
+ }
+ }
+ }
+
+ fn insert_layer(&mut self, mut layer: HighlightIterLayer<'a>) {
+ if let Some(sort_key) = layer.sort_key() {
+ let mut i = 1;
+ while i < self.layers.len() {
+ if let Some(sort_key_i) = self.layers[i].sort_key() {
+ if sort_key_i > sort_key {
+ self.layers.insert(i, layer);
+ return;
+ }
+ i += 1;
+ } else {
+ self.layers.remove(i);
+ }
+ }
+ self.layers.push(layer);
+ }
+ }
+}
+
+impl<'a, F> Iterator for HighlightIter<'a, F>
+where
+ F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
+{
+ type Item = Result<HighlightEvent, Error>;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ 'main: loop {
+ // If we've already determined the next highlight boundary, just return it.
+ if let Some(e) = self.next_event.take() {
+ return Some(Ok(e));
+ }
+
+ // Periodically check for cancellation, returning `Cancelled` error if the
+ // cancellation flag was flipped.
+ if let Some(cancellation_flag) = self.cancellation_flag {
+ self.iter_count += 1;
+ if self.iter_count >= CANCELLATION_CHECK_INTERVAL {
+ self.iter_count = 0;
+ if cancellation_flag.load(Ordering::Relaxed) != 0 {
+ return Some(Err(Error::Cancelled));
+ }
+ }
+ }
+
+ // If none of the layers have any more highlight boundaries, terminate.
+ if self.layers.is_empty() {
+ return if self.byte_offset < self.source.len() {
+ let result = Some(Ok(HighlightEvent::Source {
+ start: self.byte_offset,
+ end: self.source.len(),
+ }));
+ self.byte_offset = self.source.len();
+ result
+ } else {
+ None
+ };
+ }
+
+ // Get the next capture from whichever layer has the earliest highlight boundary.
+ let range;
+ let layer = &mut self.layers[0];
+ if let Some((next_match, capture_index)) = layer.captures.peek() {
+ let next_capture = next_match.captures[*capture_index];
+ range = next_capture.node.byte_range();
+
+ // If any previous highlight ends before this node starts, then before
+ // processing this capture, emit the source code up until the end of the
+ // previous highlight, and an end event for that highlight.
+ if let Some(end_byte) = layer.highlight_end_stack.last().cloned() {
+ if end_byte <= range.start {
+ layer.highlight_end_stack.pop();
+ return self.emit_event(end_byte, Some(HighlightEvent::HighlightEnd));
+ }
+ }
+ }
+ // If there are no more captures, then emit any remaining highlight end events.
+ // And if there are none of those, then just advance to the end of the document.
+ else if let Some(end_byte) = layer.highlight_end_stack.last().cloned() {
+ layer.highlight_end_stack.pop();
+ return self.emit_event(end_byte, Some(HighlightEvent::HighlightEnd));
+ } else {
+ return self.emit_event(self.source.len(), None);
+ };
+
+ let (mut match_, capture_index) = layer.captures.next().unwrap();
+ let mut capture = match_.captures[capture_index];
+
+ // If this capture represents an injection, then process the injection.
+ if match_.pattern_index < layer.config.locals_pattern_index {
+ let (language_name, content_node, include_children) =
+ injection_for_match(&layer.config, &layer.config.query, &match_, &self.source);
+
+ // Explicitly remove this match so that none of its other captures will remain
+ // in the stream of captures.
+ match_.remove();
+
+ // If a language is found with the given name, then add a new language layer
+ // to the highlighted document.
+ if let (Some(language_name), Some(content_node)) = (language_name, content_node) {
+ if let Some(config) = (self.injection_callback)(language_name) {
+ let ranges = HighlightIterLayer::intersect_ranges(
+ &self.layers[0].ranges,
+ &[content_node],
+ include_children,
+ );
+ if !ranges.is_empty() {
+ match HighlightIterLayer::new(
+ self.source,
+ self.highlighter,
+ self.cancellation_flag,
+ &mut self.injection_callback,
+ config,
+ self.layers[0].depth + 1,
+ ranges,
+ ) {
+ Ok(layers) => {
+ for layer in layers {
+ self.insert_layer(layer);
+ }
+ }
+ Err(e) => return Some(Err(e)),
+ }
+ }
+ }
+ }
+
+ self.sort_layers();
+ continue 'main;
+ }
+
+ // Remove from the local scope stack any local scopes that have already ended.
+ while range.start > layer.scope_stack.last().unwrap().range.end {
+ layer.scope_stack.pop();
+ }
+
+ // If this capture is for tracking local variables, then process the
+ // local variable info.
+ let mut reference_highlight = None;
+ let mut definition_highlight = None;
+ while match_.pattern_index < layer.config.highlights_pattern_index {
+ // If the node represents a local scope, push a new local scope onto
+ // the scope stack.
+ if Some(capture.index) == layer.config.local_scope_capture_index {
+ definition_highlight = None;
+ let mut scope = LocalScope {
+ inherits: true,
+ range: range.clone(),
+ local_defs: Vec::new(),
+ };
+ for prop in layer.config.query.property_settings(match_.pattern_index) {
+ match prop.key.as_ref() {
+ "local.scope-inherits" => {
+ scope.inherits =
+ prop.value.as_ref().map_or(true, |r| r.as_ref() == "true");
+ }
+ _ => {}
+ }
+ }
+ layer.scope_stack.push(scope);
+ }
+ // If the node represents a definition, add a new definition to the
+ // local scope at the top of the scope stack.
+ else if Some(capture.index) == layer.config.local_def_capture_index {
+ reference_highlight = None;
+ definition_highlight = None;
+ let scope = layer.scope_stack.last_mut().unwrap();
+
+ let mut value_range = 0..0;
+ for capture in match_.captures {
+ if Some(capture.index) == layer.config.local_def_value_capture_index {
+ value_range = capture.node.byte_range();
+ }
+ }
+
+ if let Ok(name) = str::from_utf8(&self.source[range.clone()]) {
+ scope.local_defs.push(LocalDef {
+ name,
+ value_range,
+ highlight: None,
+ });
+ definition_highlight =
+ scope.local_defs.last_mut().map(|s| &mut s.highlight);
+ }
+ }
+ // If the node represents a reference, then try to find the corresponding
+ // definition in the scope stack.
+ else if Some(capture.index) == layer.config.local_ref_capture_index {
+ if definition_highlight.is_none() {
+ definition_highlight = None;
+ if let Ok(name) = str::from_utf8(&self.source[range.clone()]) {
+ for scope in layer.scope_stack.iter().rev() {
+ if let Some(highlight) =
+ scope.local_defs.iter().rev().find_map(|def| {
+ if def.name == name && range.start >= def.value_range.end {
+ Some(def.highlight)
+ } else {
+ None
+ }
+ })
+ {
+ reference_highlight = highlight;
+ break;
+ }
+ if !scope.inherits {
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ // Continue processing any additional matches for the same node.
+ if let Some((next_match, next_capture_index)) = layer.captures.peek() {
+ let next_capture = next_match.captures[*next_capture_index];
+ if next_capture.node == capture.node {
+ capture = next_capture;
+ match_ = layer.captures.next().unwrap().0;
+ continue;
+ }
+ }
+
+ self.sort_layers();
+ continue 'main;
+ }
+
+ // Otherwise, this capture must represent a highlight.
+ // If this exact range has already been highlighted by an earlier pattern, or by
+ // a different layer, then skip over this one.
+ if let Some((last_start, last_end, last_depth)) = self.last_highlight_range {
+ if range.start == last_start && range.end == last_end && layer.depth < last_depth {
+ self.sort_layers();
+ continue 'main;
+ }
+ }
+
+ // If the current node was found to be a local variable, then skip over any
+ // highlighting patterns that are disabled for local variables.
+ if definition_highlight.is_some() || reference_highlight.is_some() {
+ while layer.config.non_local_variable_patterns[match_.pattern_index] {
+ if let Some((next_match, next_capture_index)) = layer.captures.peek() {
+ let next_capture = next_match.captures[*next_capture_index];
+ if next_capture.node == capture.node {
+ capture = next_capture;
+ match_ = layer.captures.next().unwrap().0;
+ continue;
+ }
+ }
+
+ self.sort_layers();
+ continue 'main;
+ }
+ }
+
+ // Once a highlighting pattern is found for the current node, skip over
+ // any later highlighting patterns that also match this node. Captures
+ // for a given node are ordered by pattern index, so these subsequent
+ // captures are guaranteed to be for highlighting, not injections or
+ // local variables.
+ while let Some((next_match, next_capture_index)) = layer.captures.peek() {
+ let next_capture = next_match.captures[*next_capture_index];
+ if next_capture.node == capture.node {
+ layer.captures.next();
+ } else {
+ break;
+ }
+ }
+
+ let current_highlight = layer.config.highlight_indices[capture.index as usize];
+
+ // If this node represents a local definition, then store the current
+ // highlight value on the local scope entry representing this node.
+ if let Some(definition_highlight) = definition_highlight {
+ *definition_highlight = current_highlight;
+ }
+
+ // Emit a scope start event and push the node's end position to the stack.
+ if let Some(highlight) = reference_highlight.or(current_highlight) {
+ self.last_highlight_range = Some((range.start, range.end, layer.depth));
+ layer.highlight_end_stack.push(range.end);
+ return self
+ .emit_event(range.start, Some(HighlightEvent::HighlightStart(highlight)));
+ }
+
+ self.sort_layers();
+ }
+ }
+}
+
+fn injection_for_match<'a>(
+ config: &HighlightConfiguration,
+ query: &'a Query,
+ query_match: &QueryMatch<'a>,
+ source: &'a [u8],
+) -> (Option<&'a str>, Option<Node<'a>>, bool) {
+ let content_capture_index = config.injection_content_capture_index;
+ let language_capture_index = config.injection_language_capture_index;
+
+ let mut language_name = None;
+ let mut content_node = None;
+ for capture in query_match.captures {
+ let index = Some(capture.index);
+ if index == language_capture_index {
+ language_name = capture.node.utf8_text(source).ok();
+ } else if index == content_capture_index {
+ content_node = Some(capture.node);
+ }
+ }
+
+ let mut include_children = false;
+ for prop in query.property_settings(query_match.pattern_index) {
+ match prop.key.as_ref() {
+ // In addition to specifying the language name via the text of a
+ // captured node, it can also be hard-coded via a `#set!` predicate
+ // that sets the injection.language key.
+ "injection.language" => {
+ if language_name.is_none() {
+ language_name = prop.value.as_ref().map(|s| s.as_ref())
+ }
+ }
+
+ // By default, injections do not include the *children* of an
+ // `injection.content` node - only the ranges that belong to the
+ // node itself. This can be changed using a `#set!` predicate that
+ // sets the `injection.include-children` key.
+ "injection.include-children" => include_children = true,
+ _ => {}
+ }
+ }
+
+ (language_name, content_node, include_children)
+}
+
+fn shrink_and_clear<T>(vec: &mut Vec<T>, capacity: usize) {
+ if vec.len() > capacity {
+ vec.truncate(capacity);
+ vec.shrink_to_fit();
+ }
+ vec.clear();
+}
diff --git a/helix-core/src/lib.rs b/helix-core/src/lib.rs
index da00b9ae..6ab40694 100644
--- a/helix-core/src/lib.rs
+++ b/helix-core/src/lib.rs
@@ -1,6 +1,7 @@
#![allow(unused)]
pub mod commands;
pub mod graphemes;
+pub mod language_mode;
mod selection;
pub mod state;
mod transaction;
diff --git a/helix-syntax/build.rs b/helix-syntax/build.rs
index 217ba9fa..6b721c11 100644
--- a/helix-syntax/build.rs
+++ b/helix-syntax/build.rs
@@ -1,7 +1,4 @@
-use cc::Build;
-
-use std::io::{BufWriter, Read, Write};
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
use std::{env, fs};
fn get_opt_level() -> u32 {
diff --git a/helix-term/Cargo.toml b/helix-term/Cargo.toml
index dcb01c1d..ce3add5c 100644
--- a/helix-term/Cargo.toml
+++ b/helix-term/Cargo.toml
@@ -24,5 +24,3 @@ crossterm = { version = "0.17", features = ["event-stream"] }
smol = "1"
num_cpus = "1.13.0"
tui = { version = "0.10.0", default-features = false, features = ["crossterm"] }
-tree-sitter = "0.16.1"
-tree-sitter-highlight = "0.2.0"
diff --git a/helix-term/src/editor.rs b/helix-term/src/editor.rs
index f7abc806..af18842d 100644
--- a/helix-term/src/editor.rs
+++ b/helix-term/src/editor.rs
@@ -1,5 +1,21 @@
-use crate::{keymap, Args};
+use crate::{keymap, theme::Theme, Args};
+use helix_core::{
+ language_mode::{HighlightConfiguration, HighlightEvent, Highlighter},
+ state::coords_at_pos,
+ state::Mode,
+ State,
+};
+
+use std::{
+ io::{self, stdout, Write},
+ path::PathBuf,
+ time::Duration,
+};
+
+use smol::prelude::*;
+
use anyhow::Error;
+
use crossterm::{
cursor,
cursor::position,
@@ -9,17 +25,7 @@ use crossterm::{
terminal::{self, disable_raw_mode, enable_raw_mode},
};
-use helix_core::{state::coords_at_pos, state::Mode, State};
-use smol::prelude::*;
-use std::collections::HashMap;
-use std::io::{self, stdout, Write};
-use std::path::PathBuf;
-use std::time::Duration;
-
-use tui::backend::CrosstermBackend;
-use tui::buffer::Buffer as Surface;
-use tui::layout::Rect;
-use tui::style::Style;
+use tui::{backend::CrosstermBackend, buffer::Buffer as Surface, layout::Rect, style::Style};
type Terminal = tui::Terminal<CrosstermBackend<std::io::Stdout>>;
@@ -31,7 +37,10 @@ pub struct Editor {
first_line: u16,
size: (u16, u16),
surface: Surface,
- theme: HashMap<&'static str, Style>,
+ theme: Theme,
+ highlighter: Highlighter,
+ highlight_config: HighlightConfiguration,
+ highlight_names: Vec<String>,
}
impl Editor {
@@ -42,43 +51,60 @@ impl Editor {
let size = terminal::size().unwrap();
let area = Rect::new(0, 0, size.0, size.1);
- use tui::style::Color;
- let theme = hashmap! {
- "attribute" => Style::default().fg(Color::Rgb(219, 191, 239)), // lilac
- "keyword" => Style::default().fg(Color::Rgb(236, 205, 186)), // almond
- "punctuation" => Style::default().fg(Color::Rgb(164, 160, 232)), // lavender
- "punctuation.delimiter" => Style::default().fg(Color::Rgb(164, 160, 232)), // lavender
- "operator" => Style::default().fg(Color::Rgb(219, 191, 239)), // lilac
- "property" => Style::default().fg(Color::Rgb(164, 160, 232)), // lavender
- "variable.parameter" => Style::default().fg(Color::Rgb(164, 160, 232)), // lavender
- // TODO distinguish type from type.builtin?
- "type" => Style::default().fg(Color::Rgb(255, 255, 255)), // white
- "type.builtin" => Style::default().fg(Color::Rgb(255, 255, 255)), // white
- "constructor" => Style::default().fg(Color::Rgb(219, 191, 239)), // lilac
- "function" => Style::default().fg(Color::Rgb(255, 255, 255)), // white
- "function.macro" => Style::default().fg(Color::Rgb(219, 191, 239)), // lilac
- "comment" => Style::default().fg(Color::Rgb(105, 124, 129)), // sirocco
- "variable.builtin" => Style::default().fg(Color::Rgb(159, 242, 143)), // mint
- "constant" => Style::default().fg(Color::Rgb(255, 255, 255)), // white
- "constant.builtin" => Style::default().fg(Color::Rgb(255, 255, 255)), // white
- "string" => Style::default().fg(Color::Rgb(204, 204, 204)), // silver
- "escape" => Style::default().fg(Color::Rgb(239, 186, 93)), // honey
- // used for lifetimes
- "label" => Style::default().fg(Color::Rgb(239, 186, 93)), // honey
-
- // TODO: diferentiate number builtin
- // TODO: diferentiate doc comment
- // TODO: variable as lilac
- // TODO: mod/use statements as white
- // TODO: mod stuff as chamoise
- // TODO: add "(scoped_identifier) @path" for std::mem::
- //
- // concat (ERROR) @syntax-error and "MISSING ;" selectors for errors
-
- "module" => Style::default().fg(Color::Rgb(255, 0, 0)), // white
- "variable" => Style::default().fg(Color::Rgb(255, 0, 0)), // white
- "function.builtin" => Style::default().fg(Color::Rgb(255, 0, 0)), // white
- };
+ let highlight_names: Vec<String> = [
+ "attribute",
+ "constant.builtin",
+ "constant",
+ "function.builtin",
+ "function.macro",
+ "function",
+ "keyword",
+ "operator",
+ "property",
+ "punctuation",
+ "comment",
+ "escape",
+ "label",
+ // "punctuation.bracket",
+ "punctuation.delimiter",
+ "string",
+ "string.special",
+ "tag",
+ "type",
+ "type.builtin",
+ "constructor",
+ "variable",
+ "variable.builtin",
+ "variable.parameter",
+ "path",
+ ]
+ .iter()
+ .map(|s| s.to_string())
+ .collect();
+
+ // let mut parser = tree_sitter::Parser::new();
+ // parser.set_language(language).unwrap();
+ // let tree = parser.parse(source_code, None).unwrap();
+
+ let language = helix_syntax::get_language(&helix_syntax::LANG::Rust);
+
+ let highlighter = Highlighter::new();
+
+ let mut highlight_config = HighlightConfiguration::new(
+ language,
+ &std::fs::read_to_string(
+ "../helix-syntax/languages/tree-sitter-rust/queries/highlights.scm",
+ )
+ .unwrap(),
+ &std::fs::read_to_string(
+ "../helix-syntax/languages/tree-sitter-rust/queries/injections.scm",
+ )
+ .unwrap(),
+ "", // locals.scm
+ )
+ .unwrap();
+
+ highlight_config.configure(&highlight_names);
let mut editor = Editor {
terminal,
@@ -86,7 +112,11 @@ impl Editor {
first_line: 0,
size,
surface: Surface::empty(area),
- theme,
+ theme: Theme::default(),
+ // TODO; move to state
+ highlighter,
+ highlight_config,
+ highlight_names,
};
if let Some(file) = args.files.pop() {
@@ -108,72 +138,16 @@ impl Editor {
let mut surface = Surface::empty(area);
let mut stdout = stdout();
- //
- use tree_sitter_highlight::{HighlightConfiguration, HighlightEvent, Highlighter};
-
- let highlight_names: Vec<String> = [
- "attribute",
- "constant.builtin",
- "constant",
- "function.builtin",
- "function.macro",
- "function",
- "keyword",
- "operator",
- "property",
- "punctuation",
- "comment",
- "escape",
- "label",
- // "punctuation.bracket",
- "punctuation.delimiter",
- "string",
- "string.special",
- "tag",
- "type",
- "type.builtin",
- "constructor",
- "variable",
- "variable.builtin",
- "variable.parameter",
- "path",
- ]
- .iter()
- .cloned()
- .map(String::from)
- .collect();
-
- let language = helix_syntax::get_language(&helix_syntax::LANG::Rust);
- // let mut parser = tree_sitter::Parser::new();
- // parser.set_language(language).unwrap();
- // let tree = parser.parse(source_code, None).unwrap();
-
- let mut highlighter = Highlighter::new();
-
- let mut config = HighlightConfiguration::new(
- language,
- &std::fs::read_to_string(
- "../helix-syntax/languages/tree-sitter-rust/queries/highlights.scm",
- )
- .unwrap(),
- &std::fs::read_to_string(
- "../helix-syntax/languages/tree-sitter-rust/queries/injections.scm",
- )
- .unwrap(),
- "", // locals.scm
- )
- .unwrap();
-
- config.configure(&highlight_names);
-
- // TODO: inefficient, should feed chunks.iter() to tree_sitter.parse_with(|offset,
- // pos|)
+ // TODO: inefficient, should feed chunks.iter() to tree_sitter.parse_with(|offset, pos|)
let source_code = state.doc.to_string();
// TODO: cache highlight results
// TODO: only recalculate when state.doc is actually modified
- let highlights = highlighter
- .highlight(&config, source_code.as_bytes(), None, |_| None)
+ let highlights = self
+ .highlighter
+ .highlight(&self.highlight_config, source_code.as_bytes(), None, |_| {
+ None
+ })
.unwrap();
let mut spans = Vec::new();
@@ -186,12 +160,10 @@ impl Editor {
for event in highlights {
match event.unwrap() {
HighlightEvent::HighlightStart(span) => {
- // eprintln!("highlight style started: {:?}", highlight_names[span.0]);
spans.push(span);
}
HighlightEvent::HighlightEnd => {
spans.pop();
- // eprintln!("highlight style ended");
}
HighlightEvent::Source { start, end } => {
// TODO: filter out spans out of viewport for now..
@@ -204,17 +176,16 @@ impl Editor {
use helix_core::graphemes::{grapheme_width, RopeGraphemes};
use tui::style::Color;
- let style = match spans.first() {
- Some(span) => self
- .theme
- .get(highlight_names[span.0].as_str())
- .map(|style| *style)
- .unwrap_or(Style::default().fg(Color::Rgb(0, 0, 255))),
+ let style = match spans.first() {
+ Some(span) => self.theme.get(self.highlight_names[span.0].as_str()),
None => Style::default().fg(Color::Rgb(164, 160, 232)), // lavender
- // None => Style::default().fg(Color::Rgb(219, 191, 239)), // lilac
};
+ // TODO: we could render the text to a surface, then cache that, that
+ // way if only the selection/cursor changes we can copy from cache
+ // and paint the new cursor.
+
// iterate over range char by char
for grapheme in RopeGraphemes::new(&text) {
// TODO: track current char_index
diff --git a/helix-term/src/main.rs b/helix-term/src/main.rs
index e3c16c14..d6c6c499 100644
--- a/helix-term/src/main.rs
+++ b/helix-term/src/main.rs
@@ -3,6 +3,7 @@
mod macros;
mod editor;
mod keymap;
+mod theme;
use editor::Editor;
diff --git a/helix-term/src/theme.rs b/helix-term/src/theme.rs
new file mode 100644
index 00000000..0353f0e6
--- /dev/null
+++ b/helix-term/src/theme.rs
@@ -0,0 +1,59 @@
+use std::collections::HashMap;
+use tui::style::{Color, Style};
+
+/// Color theme for syntax highlighting.
+pub struct Theme {
+ mapping: HashMap<&'static str, Style>,
+}
+
+impl Default for Theme {
+ fn default() -> Self {
+ let mapping = hashmap! {
+ "attribute" => Style::default().fg(Color::Rgb(219, 191, 239)), // lilac
+ "keyword" => Style::default().fg(Color::Rgb(236, 205, 186)), // almond
+ "punctuation" => Style::default().fg(Color::Rgb(164, 160, 232)), // lavender
+ "punctuation.delimiter" => Style::default().fg(Color::Rgb(164, 160, 232)), // lavender
+ "operator" => Style::default().fg(Color::Rgb(219, 191, 239)), // lilac
+ "property" => Style::default().fg(Color::Rgb(164, 160, 232)), // lavender
+ "variable.parameter" => Style::default().fg(Color::Rgb(164, 160, 232)), // lavender
+ // TODO distinguish type from type.builtin?
+ "type" => Style::default().fg(Color::Rgb(255, 255, 255)), // white
+ "type.builtin" => Style::default().fg(Color::Rgb(255, 255, 255)), // white
+ "constructor" => Style::default().fg(Color::Rgb(219, 191, 239)), // lilac
+ "function" => Style::default().fg(Color::Rgb(255, 255, 255)), // white
+ "function.macro" => Style::default().fg(Color::Rgb(219, 191, 239)), // lilac
+ "comment" => Style::default().fg(Color::Rgb(105, 124, 129)), // sirocco
+ "variable.builtin" => Style::default().fg(Color::Rgb(159, 242, 143)), // mint
+ "constant" => Style::default().fg(Color::Rgb(255, 255, 255)), // white
+ "constant.builtin" => Style::default().fg(Color::Rgb(255, 255, 255)), // white
+ "string" => Style::default().fg(Color::Rgb(204, 204, 204)), // silver
+ "escape" => Style::default().fg(Color::Rgb(239, 186, 93)), // honey
+ // used for lifetimes
+ "label" => Style::default().fg(Color::Rgb(239, 186, 93)), // honey
+
+ // TODO: diferentiate number builtin
+ // TODO: diferentiate doc comment
+ // TODO: variable as lilac
+ // TODO: mod/use statements as white
+ // TODO: mod stuff as chamoise
+ // TODO: add "(scoped_identifier) @path" for std::mem::
+ //
+ // concat (ERROR) @syntax-error and "MISSING ;" selectors for errors
+
+ "module" => Style::default().fg(Color::Rgb(255, 0, 0)), // white
+ "variable" => Style::default().fg(Color::Rgb(255, 0, 0)), // white
+ "function.builtin" => Style::default().fg(Color::Rgb(255, 0, 0)), // white
+ };
+
+ Self { mapping }
+ }
+}
+
+impl Theme {
+ pub fn get(&self, scope: &str) -> Style {
+ self.mapping
+ .get(scope)
+ .copied()
+ .unwrap_or_else(|| Style::default().fg(Color::Rgb(0, 0, 255)))
+ }
+}