Indentation rework (#1562)

* WIP: Rework indentation system

* Add ComplexNode for context-aware indentation (including a proof of concept for assignment statements in rust)

* Add switch statements to Go indents.toml (fixes the second half of issue #1523)
Remove commented-out code

* Migrate all existing indentation queries.
Add more options to ComplexNode and use them to improve C/C++ indentation.

* Add comments & replace Option<Vec<_>> with Vec<_>

* Add more detailed documentation for tree-sitter indentation

* Improve code style in indent.rs

* Use tree-sitter queries for indentation instead of TOML config.
Migrate existing indent queries.

* Add documentation for the new indent queries.
Change xtask docgen to look for indents.scm instead of indents.toml

* Improve code style in indent.rs.
Fix an issue with the rust indent query.

* Move indentation test sources to separate files.
Add `#not-kind-eq?`, `#same-line?` and `#not-same-line` custom predicates.
Improve the rust and c indent queries.

* Fix indent test.
Improve rust indent queries.

* Move indentation tests to integration test folder.

* Improve code style in indent.rs.
Reuse tree-sitter cursors for indentation queries.

* Migrate HCL indent query

* Replace custom loading in indent tests with a designated languages.toml

* Update indent query file name for --health command.

* Fix single-space formatting in indent queries.

* Add explanation for unwrapping.

Co-authored-by: Triton171 <triton0171@gmail.com>

4 files changed, 187 insertions, 0 deletions

diff --git a/helix-core/tests/data/indent/indent.rs b/helix-core/tests/data/indent/indent.rs
new file mode 120000
index 00000000..2ac16cf9
--- /dev/null
+++ b/helix-core/tests/data/indent/indent.rs
@@ -0,0 +1 @@
+../../../src/indent.rs
\ No newline at end of file
diff --git a/helix-core/tests/data/indent/languages.toml b/helix-core/tests/data/indent/languages.toml
new file mode 100644
index 00000000..f9cef494
--- /dev/null
+++ b/helix-core/tests/data/indent/languages.toml
@@ -0,0 +1,13 @@
+# This languages.toml should contain definitions for all languages for which we have indent tests
+[[language]]
+name = "rust"
+scope = "source.rust"
+injection-regex = "rust"
+file-types = ["rs"]
+comment-token = "//"
+roots = ["Cargo.toml", "Cargo.lock"]
+indent = { tab-width = 4, unit = "    " }
+
+[[grammar]]
+name = "rust"
+source = { git = "https://github.com/tree-sitter/tree-sitter-rust", rev = "a360da0a29a19c281d08295a35ecd0544d2da211" }
diff --git a/helix-core/tests/data/indent/rust.rs b/helix-core/tests/data/indent/rust.rs
new file mode 100644
index 00000000..010745e0
--- /dev/null
+++ b/helix-core/tests/data/indent/rust.rs
@@ -0,0 +1,105 @@
+use std::{
+    io::{self, stdout, Stdout, Write},
+    path::PathBuf,
+    sync::Arc,
+    time::Duration,
+};
+mod test {
+    fn hello_world() {
+        1 + 1;
+
+        let does_indentation_work = 1;
+
+        let mut really_long_variable_name_using_up_the_line =
+            really_long_fn_that_should_definitely_go_on_the_next_line();
+        really_long_variable_name_using_up_the_line =
+            really_long_fn_that_should_definitely_go_on_the_next_line();
+        really_long_variable_name_using_up_the_line |=
+            really_long_fn_that_should_definitely_go_on_the_next_line();
+
+        let (
+            a_long_variable_name_in_this_tuple,
+            b_long_variable_name_in_this_tuple,
+            c_long_variable_name_in_this_tuple,
+            d_long_variable_name_in_this_tuple,
+            e_long_variable_name_in_this_tuple,
+        ): (usize, usize, usize, usize, usize) =
+            if really_long_fn_that_should_definitely_go_on_the_next_line() {
+                (
+                    03294239434,
+                    1213412342314,
+                    21231234134,
+                    834534234549898789,
+                    9879234234543853457,
+                )
+            } else {
+                (0, 1, 2, 3, 4)
+            };
+
+        let test_function = function_with_param(this_param,
+            that_param
+        );
+
+        let test_function = function_with_param(
+            this_param,
+            that_param
+        );
+
+        let test_function = function_with_proper_indent(param1,
+            param2,
+        );
+
+        let selection = Selection::new(
+            changes
+                .clone()
+                .map(|(start, end, text): (usize, usize, Option<Tendril>)| {
+                    let len = text.map(|text| text.len()).unwrap() - 1; // minus newline
+                    let pos = start + len;
+                    Range::new(pos, pos)
+                })
+                .collect(),
+            0,
+        );
+
+        return;
+    }
+}
+
+impl<A, D> MyTrait<A, D> for YourType
+where
+    A: TraitB + TraitC,
+    D: TraitE + TraitF,
+{
+
+}
+#[test]
+//
+match test {
+    Some(a) => 1,
+    None => {
+        unimplemented!()
+    }
+}
+std::panic::set_hook(Box::new(move |info| {
+    hook(info);
+}));
+
+{ { {
+    1
+}}}
+
+pub fn change<I>(document: &Document, changes: I) -> Self
+where
+    I: IntoIterator<Item = Change> + ExactSizeIterator,
+{
+    [
+        1,
+        2,
+        3,
+    ];
+    (
+        1,
+        2
+    );
+    true
+}
diff --git a/helix-core/tests/indent.rs b/helix-core/tests/indent.rs
new file mode 100644
index 00000000..ff04d05f
--- /dev/null
+++ b/helix-core/tests/indent.rs
@@ -0,0 +1,68 @@
+use helix_core::{
+    indent::{treesitter_indent_for_pos, IndentStyle},
+    syntax::Loader,
+    Syntax,
+};
+use std::path::PathBuf;
+
+#[test]
+fn test_treesitter_indent_rust() {
+    test_treesitter_indent("rust.rs", "source.rust");
+}
+#[test]
+fn test_treesitter_indent_rust_2() {
+    test_treesitter_indent("indent.rs", "source.rust");
+    // TODO Use commands.rs as indentation test.
+    // Currently this fails because we can't align the parameters of a closure yet
+    // test_treesitter_indent("commands.rs", "source.rust");
+}
+
+fn test_treesitter_indent(file_name: &str, lang_scope: &str) {
+    let mut test_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    test_dir.push("tests/data/indent");
+
+    let mut test_file = test_dir.clone();
+    test_file.push(file_name);
+    let test_file = std::fs::File::open(test_file).unwrap();
+    let doc = ropey::Rope::from_reader(test_file).unwrap();
+
+    let mut config_file = test_dir;
+    config_file.push("languages.toml");
+    let config = std::fs::read(config_file).unwrap();
+    let config = toml::from_slice(&config).unwrap();
+    let loader = Loader::new(config);
+
+    // set runtime path so we can find the queries
+    let mut runtime = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    runtime.push("../runtime");
+    std::env::set_var("HELIX_RUNTIME", runtime.to_str().unwrap());
+
+    let language_config = loader.language_config_for_scope(lang_scope).unwrap();
+    let highlight_config = language_config.highlight_config(&[]).unwrap();
+    let syntax = Syntax::new(&doc, highlight_config, std::sync::Arc::new(loader));
+    let indent_query = language_config.indent_query().unwrap();
+    let text = doc.slice(..);
+
+    for i in 0..doc.len_lines() {
+        let line = text.line(i);
+        if let Some(pos) = helix_core::find_first_non_whitespace_char(line) {
+            let suggested_indent = treesitter_indent_for_pos(
+                indent_query,
+                &syntax,
+                &IndentStyle::Spaces(4),
+                text,
+                i,
+                text.line_to_char(i) + pos,
+                false,
+            )
+            .unwrap();
+            assert!(
+                line.get_slice(..pos).map_or(false, |s| s == suggested_indent),
+                "Wrong indentation on line {}:\n\"{}\" (original line)\n\"{}\" (suggested indentation)\n",
+                i+1,
+                line.slice(..line.len_chars()-1),
+                suggested_indent,
+            );
+        }
+    }
+}