Merge branch 'master' into cursor-shape-new

10 files changed, 522 insertions, 204 deletions

diff --git a/helix-core/Cargo.toml b/helix-core/Cargo.toml
index 0a2a56d9..3c11ec76 100644
--- a/helix-core/Cargo.toml
+++ b/helix-core/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "helix-core"
-version = "0.5.0"
+version = "0.6.0"
 authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
 edition = "2021"
 license = "MPL-2.0"
@@ -13,7 +13,7 @@ include = ["src/**/*", "README.md"]
 [features]
 
 [dependencies]
-helix-syntax = { version = "0.5", path = "../helix-syntax" }
+helix-syntax = { version = "0.6", path = "../helix-syntax" }
 
 ropey = "1.3"
 smallvec = "1.7"
@@ -23,7 +23,7 @@ unicode-width = "0.1"
 unicode-general-category = "0.4"
 # slab = "0.4.2"
 tree-sitter = "0.20"
-once_cell = "1.8"
+once_cell = "1.9"
 arc-swap = "1"
 regex = "1"
 
@@ -35,6 +35,7 @@ toml = "0.5"
 similar = "2.1"
 
 etcetera = "0.3"
+encoding_rs = "0.8"
 
 chrono = { version = "0.4", default-features = false, features = ["alloc", "std"] }
 
diff --git a/helix-core/src/auto_pairs.rs b/helix-core/src/auto_pairs.rs
index c037afef..1b3de6ea 100644
--- a/helix-core/src/auto_pairs.rs
+++ b/helix-core/src/auto_pairs.rs
@@ -1,7 +1,7 @@
 //! When typing the opening character of one of the possible pairs defined below,
 //! this module provides the functionality to insert the paired closing character.
 
-use crate::{Range, Rope, Selection, Tendril, Transaction};
+use crate::{movement::Direction, Range, Rope, Selection, Tendril, Transaction};
 use log::debug;
 use smallvec::SmallVec;
 
@@ -30,7 +30,6 @@ const CLOSE_BEFORE: &str = ")]}'\":;,> \n\r\u{000B}\u{000C}\u{0085}\u{2028}\u{20
 
 // [TODO]
 // * delete implementation where it erases the whole bracket (|) -> |
-// * do not reduce to cursors; use whole selections, and surround with pair
 // * change to multi character pairs to handle cases like placing the cursor in the
 //   middle of triple quotes, and more exotic pairs like Jinja's {% %}
 
@@ -38,20 +37,18 @@ const CLOSE_BEFORE: &str = ")]}'\":;,> \n\r\u{000B}\u{000C}\u{0085}\u{2028}\u{20
 pub fn hook(doc: &Rope, selection: &Selection, ch: char) -> Option<Transaction> {
     debug!("autopairs hook selection: {:#?}", selection);
 
-    let cursors = selection.clone().cursors(doc.slice(..));
-
     for &(open, close) in PAIRS {
         if open == ch {
             if open == close {
-                return Some(handle_same(doc, &cursors, open, CLOSE_BEFORE, OPEN_BEFORE));
+                return Some(handle_same(doc, selection, open, CLOSE_BEFORE, OPEN_BEFORE));
             } else {
-                return Some(handle_open(doc, &cursors, open, close, CLOSE_BEFORE));
+                return Some(handle_open(doc, selection, open, close, CLOSE_BEFORE));
             }
         }
 
         if close == ch {
             // && char_at pos == close
-            return Some(handle_close(doc, &cursors, open, close));
+            return Some(handle_close(doc, selection, open, close));
         }
     }
 
@@ -66,6 +63,36 @@ fn prev_char(doc: &Rope, pos: usize) -> Option<char> {
     doc.get_char(pos - 1)
 }
 
+/// calculate what the resulting range should be for an auto pair insertion
+fn get_next_range(
+    start_range: &Range,
+    offset: usize,
+    typed_char: char,
+    len_inserted: usize,
+) -> Range {
+    let end_head = start_range.head + offset + typed_char.len_utf8();
+
+    let end_anchor = match (start_range.len(), start_range.direction()) {
+        // if we have a zero width cursor, it shifts to the same number
+        (0, _) => end_head,
+
+        // if we are inserting for a regular one-width cursor, the anchor
+        // moves with the head
+        (1, Direction::Forward) => end_head - 1,
+        (1, Direction::Backward) => end_head + 1,
+
+        // if we are appending, the anchor stays where it is; only offset
+        // for multiple range insertions
+        (_, Direction::Forward) => start_range.anchor + offset,
+
+        // when we are inserting in front of a selection, we need to move
+        // the anchor over by however many characters were inserted overall
+        (_, Direction::Backward) => start_range.anchor + offset + len_inserted,
+    };
+
+    Range::new(end_anchor, end_head)
+}
+
 fn handle_open(
     doc: &Rope,
     selection: &Selection,
@@ -74,36 +101,32 @@ fn handle_open(
     close_before: &str,
 ) -> Transaction {
     let mut end_ranges = SmallVec::with_capacity(selection.len());
-
     let mut offs = 0;
 
     let transaction = Transaction::change_by_selection(doc, selection, |start_range| {
-        let start_head = start_range.head;
-
-        let next = doc.get_char(start_head);
-        let end_head = start_head + offs + open.len_utf8();
-
-        let end_anchor = if start_range.is_empty() {
-            end_head
-        } else {
-            start_range.anchor + offs
-        };
-
-        end_ranges.push(Range::new(end_anchor, end_head));
+        let cursor = start_range.cursor(doc.slice(..));
+        let next_char = doc.get_char(cursor);
+        let len_inserted;
 
-        match next {
+        let change = match next_char {
             Some(ch) if !close_before.contains(ch) => {
-                offs += open.len_utf8();
-                (start_head, start_head, Some(Tendril::from_char(open)))
+                len_inserted = open.len_utf8();
+                (cursor, cursor, Some(Tendril::from_char(open)))
             }
             // None | Some(ch) if close_before.contains(ch) => {}
             _ => {
                 // insert open & close
                 let pair = Tendril::from_iter([open, close]);
-                offs += open.len_utf8() + close.len_utf8();
-                (start_head, start_head, Some(pair))
+                len_inserted = open.len_utf8() + close.len_utf8();
+                (cursor, cursor, Some(pair))
             }
-        }
+        };
+
+        let next_range = get_next_range(start_range, offs, open, len_inserted);
+        end_ranges.push(next_range);
+        offs += len_inserted;
+
+        change
     });
 
     let t = transaction.with_selection(Selection::new(end_ranges, selection.primary_index()));
@@ -117,28 +140,28 @@ fn handle_close(doc: &Rope, selection: &Selection, _open: char, close: char) ->
     let mut offs = 0;
 
     let transaction = Transaction::change_by_selection(doc, selection, |start_range| {
-        let start_head = start_range.head;
-        let next = doc.get_char(start_head);
-        let end_head = start_head + offs + close.len_utf8();
+        let cursor = start_range.cursor(doc.slice(..));
+        let next_char = doc.get_char(cursor);
+        let mut len_inserted = 0;
 
-        let end_anchor = if start_range.is_empty() {
-            end_head
+        let change = if next_char == Some(close) {
+            // return transaction that moves past close
+            (cursor, cursor, None) // no-op
         } else {
-            start_range.anchor + offs
+            len_inserted += close.len_utf8();
+            (cursor, cursor, Some(Tendril::from_char(close)))
         };
 
-        end_ranges.push(Range::new(end_anchor, end_head));
+        let next_range = get_next_range(start_range, offs, close, len_inserted);
+        end_ranges.push(next_range);
+        offs += len_inserted;
 
-        if next == Some(close) {
-            // return transaction that moves past close
-            (start_head, start_head, None) // no-op
-        } else {
-            offs += close.len_utf8();
-            (start_head, start_head, Some(Tendril::from_char(close)))
-        }
+        change
     });
 
-    transaction.with_selection(Selection::new(end_ranges, selection.primary_index()))
+    let t = transaction.with_selection(Selection::new(end_ranges, selection.primary_index()));
+    debug!("auto pair transaction: {:#?}", t);
+    t
 }
 
 /// handle cases where open and close is the same, or in triples ("""docstring""")
@@ -154,42 +177,41 @@ fn handle_same(
     let mut offs = 0;
 
     let transaction = Transaction::change_by_selection(doc, selection, |start_range| {
-        let start_head = start_range.head;
-        let end_head = start_head + offs + token.len_utf8();
+        let cursor = start_range.cursor(doc.slice(..));
+        let mut len_inserted = 0;
 
-        // if selection, retain anchor, if cursor, move over
-        let end_anchor = if start_range.is_empty() {
-            end_head
-        } else {
-            start_range.anchor + offs
-        };
+        let next_char = doc.get_char(cursor);
+        let prev_char = prev_char(doc, cursor);
 
-        end_ranges.push(Range::new(end_anchor, end_head));
-
-        let next = doc.get_char(start_head);
-        let prev = prev_char(doc, start_head);
-
-        if next == Some(token) {
+        let change = if next_char == Some(token) {
             //  return transaction that moves past close
-            (start_head, start_head, None) // no-op
+            (cursor, cursor, None) // no-op
         } else {
             let mut pair = Tendril::with_capacity(2 * token.len_utf8() as u32);
             pair.push_char(token);
 
             // for equal pairs, don't insert both open and close if either
             // side has a non-pair char
-            if (next.is_none() || close_before.contains(next.unwrap()))
-                && (prev.is_none() || open_before.contains(prev.unwrap()))
+            if (next_char.is_none() || close_before.contains(next_char.unwrap()))
+                && (prev_char.is_none() || open_before.contains(prev_char.unwrap()))
             {
                 pair.push_char(token);
             }
 
-            offs += pair.len();
-            (start_head, start_head, Some(pair))
-        }
+            len_inserted += pair.len();
+            (cursor, cursor, Some(pair))
+        };
+
+        let next_range = get_next_range(start_range, offs, token, len_inserted);
+        end_ranges.push(next_range);
+        offs += len_inserted;
+
+        change
     });
 
-    transaction.with_selection(Selection::new(end_ranges, selection.primary_index()))
+    let t = transaction.with_selection(Selection::new(end_ranges, selection.primary_index()));
+    debug!("auto pair transaction: {:#?}", t);
+    t
 }
 
 #[cfg(test)]
@@ -252,7 +274,20 @@ mod test {
             &Selection::single(1, 0),
             PAIRS,
             |open, close| format!("{}{}", open, close),
-            &Selection::single(1, 1),
+            &Selection::single(2, 1),
+        );
+    }
+
+    /// [] -> append ( -> ([])
+    #[test]
+    fn test_append_blank() {
+        test_hooks_with_pairs(
+            // this is what happens when you have a totally blank document and then append
+            &Rope::from("\n\n"),
+            &Selection::single(0, 2),
+            PAIRS,
+            |open, close| format!("\n{}{}\n", open, close),
+            &Selection::single(0, 3),
         );
     }
 
@@ -276,26 +311,50 @@ mod test {
                 )
             },
             &Selection::new(
-                smallvec!(Range::point(1), Range::point(4), Range::point(7),),
+                smallvec!(Range::new(2, 1), Range::new(5, 4), Range::new(8, 7),),
                 0,
             ),
         );
     }
 
-    // [TODO] broken until it works with selections
     /// fo[o] -> append ( -> fo[o(])
-    #[ignore]
     #[test]
     fn test_append() {
         test_hooks_with_pairs(
-            &Rope::from("foo"),
+            &Rope::from("foo\n"),
             &Selection::single(2, 4),
-            PAIRS,
-            |open, close| format!("foo{}{}", open, close),
+            differing_pairs(),
+            |open, close| format!("foo{}{}\n", open, close),
             &Selection::single(2, 5),
         );
     }
 
+    /// fo[o]                fo[o(])
+    /// fo[o] -> append ( -> fo[o(])
+    /// fo[o]                fo[o(])
+    #[test]
+    fn test_append_multi() {
+        test_hooks_with_pairs(
+            &Rope::from("foo\nfoo\nfoo\n"),
+            &Selection::new(
+                smallvec!(Range::new(2, 4), Range::new(6, 8), Range::new(10, 12)),
+                0,
+            ),
+            differing_pairs(),
+            |open, close| {
+                format!(
+                    "foo{open}{close}\nfoo{open}{close}\nfoo{open}{close}\n",
+                    open = open,
+                    close = close
+                )
+            },
+            &Selection::new(
+                smallvec!(Range::new(2, 5), Range::new(8, 11), Range::new(14, 17)),
+                0,
+            ),
+        );
+    }
+
     /// ([]) -> insert ) -> ()[]
     #[test]
     fn test_insert_close_inside_pair() {
@@ -307,7 +366,23 @@ mod test {
                 &Selection::single(2, 1),
                 *close,
                 &doc,
-                &Selection::point(2),
+                &Selection::single(3, 2),
+            );
+        }
+    }
+
+    /// [(]) -> append ) -> [()]
+    #[test]
+    fn test_append_close_inside_pair() {
+        for (open, close) in PAIRS {
+            let doc = Rope::from(format!("{}{}\n", open, close));
+
+            test_hooks(
+                &doc,
+                &Selection::single(0, 2),
+                *close,
+                &doc,
+                &Selection::single(0, 3),
             );
         }
     }
@@ -323,8 +398,33 @@ mod test {
         );
 
         let expected_sel = Selection::new(
-            // smallvec!(Range::new(3, 2), Range::new(6, 5), Range::new(9, 8),),
-            smallvec!(Range::point(2), Range::point(5), Range::point(8),),
+            smallvec!(Range::new(3, 2), Range::new(6, 5), Range::new(9, 8),),
+            0,
+        );
+
+        for (open, close) in PAIRS {
+            let doc = Rope::from(format!(
+                "{open}{close}\n{open}{close}\n{open}{close}\n",
+                open = open,
+                close = close
+            ));
+
+            test_hooks(&doc, &sel, *close, &doc, &expected_sel);
+        }
+    }
+
+    /// [(])                [()]
+    /// [(]) -> append ) -> [()]
+    /// [(])                [()]
+    #[test]
+    fn test_append_close_inside_pair_multi_cursor() {
+        let sel = Selection::new(
+            smallvec!(Range::new(0, 2), Range::new(3, 5), Range::new(6, 8),),
+            0,
+        );
+
+        let expected_sel = Selection::new(
+            smallvec!(Range::new(0, 3), Range::new(3, 6), Range::new(6, 9),),
             0,
         );
 
@@ -343,7 +443,7 @@ mod test {
     #[test]
     fn test_insert_open_inside_pair() {
         let sel = Selection::single(2, 1);
-        let expected_sel = Selection::point(2);
+        let expected_sel = Selection::single(3, 2);
 
         for (open, close) in differing_pairs() {
             let doc = Rope::from(format!("{}{}", open, close));
@@ -357,11 +457,49 @@ mod test {
         }
     }
 
+    /// [word(]) -> append ( -> [word((]))
+    #[test]
+    fn test_append_open_inside_pair() {
+        let sel = Selection::single(0, 6);
+        let expected_sel = Selection::single(0, 7);
+
+        for (open, close) in differing_pairs() {
+            let doc = Rope::from(format!("word{}{}", open, close));
+            let expected_doc = Rope::from(format!(
+                "word{open}{open}{close}{close}",
+                open = open,
+                close = close
+            ));
+
+            test_hooks(&doc, &sel, *open, &expected_doc, &expected_sel);
+        }
+    }
+
     /// ([]) -> insert " -> ("[]")
     #[test]
     fn test_insert_nested_open_inside_pair() {
         let sel = Selection::single(2, 1);
-        let expected_sel = Selection::point(2);
+        let expected_sel = Selection::single(3, 2);
+
+        for (outer_open, outer_close) in differing_pairs() {
+            let doc = Rope::from(format!("{}{}", outer_open, outer_close,));
+
+            for (inner_open, inner_close) in matching_pairs() {
+                let expected_doc = Rope::from(format!(
+                    "{}{}{}{}",
+                    outer_open, inner_open, inner_close, outer_close
+                ));
+
+                test_hooks(&doc, &sel, *inner_open, &expected_doc, &expected_sel);
+            }
+        }
+    }
+
+    /// [(]) -> append " -> [("]")
+    #[test]
+    fn test_append_nested_open_inside_pair() {
+        let sel = Selection::single(0, 2);
+        let expected_sel = Selection::single(0, 3);
 
         for (outer_open, outer_close) in differing_pairs() {
             let doc = Rope::from(format!("{}{}", outer_open, outer_close,));
@@ -385,21 +523,44 @@ mod test {
             &Selection::single(1, 0),
             PAIRS,
             |open, _| format!("{}word", open),
-            &Selection::point(1),
+            &Selection::single(2, 1),
         )
     }
 
-    // [TODO] broken until it works with selections
     /// [wor]d -> insert ( -> ([wor]d
     #[test]
-    #[ignore]
     fn test_insert_open_with_selection() {
         test_hooks_with_pairs(
             &Rope::from("word"),
-            &Selection::single(0, 4),
+            &Selection::single(3, 0),
             PAIRS,
             |open, _| format!("{}word", open),
-            &Selection::single(1, 5),
+            &Selection::single(4, 1),
+        )
+    }
+
+    /// [wor]d -> append ) -> [wor)]d
+    #[test]
+    fn test_append_close_inside_non_pair_with_selection() {
+        let sel = Selection::single(0, 4);
+        let expected_sel = Selection::single(0, 5);
+
+        for (_, close) in PAIRS {
+            let doc = Rope::from("word");
+            let expected_doc = Rope::from(format!("wor{}d", close));
+            test_hooks(&doc, &sel, *close, &expected_doc, &expected_sel);
+        }
+    }
+
+    /// foo[ wor]d -> insert ( -> foo([) wor]d
+    #[test]
+    fn test_insert_open_trailing_word_with_selection() {
+        test_hooks_with_pairs(
+            &Rope::from("foo word"),
+            &Selection::single(7, 3),
+            differing_pairs(),
+            |open, close| format!("foo{}{} word", open, close),
+            &Selection::single(9, 4),
         )
     }
 
@@ -413,7 +574,7 @@ mod test {
     fn test_insert_open_after_non_pair() {
         let doc = Rope::from("word");
         let sel = Selection::single(5, 4);
-        let expected_sel = Selection::point(5);
+        let expected_sel = Selection::single(6, 5);
 
         test_hooks_with_pairs(
             &doc,
@@ -431,4 +592,18 @@ mod test {
             &expected_sel,
         );
     }
+
+    /// appending with only a cursor should stay a cursor
+    ///
+    /// [] -> append to end "foo -> "foo[]"
+    #[test]
+    fn test_append_single_cursor() {
+        test_hooks_with_pairs(
+            &Rope::from("\n"),
+            &Selection::single(0, 1),
+            PAIRS,
+            |open, close| format!("{}{}\n", open, close),
+            &Selection::single(1, 2),
+        );
+    }
 }
diff --git a/helix-core/src/diagnostic.rs b/helix-core/src/diagnostic.rs
index 4fcf51c9..210ad639 100644
--- a/helix-core/src/diagnostic.rs
+++ b/helix-core/src/diagnostic.rs
@@ -1,12 +1,19 @@
 //! LSP diagnostic utility types.
+use serde::{Deserialize, Serialize};
 
 /// Describes the severity level of a [`Diagnostic`].
-#[derive(Debug, Clone, Copy, Eq, PartialEq)]
+#[derive(Debug, Clone, Copy, Eq, PartialEq, PartialOrd, Ord, Deserialize, Serialize)]
 pub enum Severity {
-    Error,
-    Warning,
-    Info,
     Hint,
+    Info,
+    Warning,
+    Error,
+}
+
+impl Default for Severity {
+    fn default() -> Self {
+        Self::Hint
+    }
 }
 
 /// A range of `char`s within the text.
diff --git a/helix-core/src/indent.rs b/helix-core/src/indent.rs
index b6f5081a..1fc2b8a5 100644
--- a/helix-core/src/indent.rs
+++ b/helix-core/src/indent.rs
@@ -1,6 +1,5 @@
 use crate::{
     chars::{char_is_line_ending, char_is_whitespace},
-    find_first_non_whitespace_char,
     syntax::{IndentQuery, LanguageConfiguration, Syntax},
     tree_sitter::Node,
     Rope, RopeSlice,
@@ -174,8 +173,7 @@ pub fn auto_detect_indent_style(document_text: &Rope) -> Option<IndentStyle> {
 
 /// To determine indentation of a newly inserted line, figure out the indentation at the last col
 /// of the previous line.
-#[allow(dead_code)]
-fn indent_level_for_line(line: RopeSlice, tab_width: usize) -> usize {
+pub fn indent_level_for_line(line: RopeSlice, tab_width: usize) -> usize {
     let mut len = 0;
     for ch in line.chars() {
         match ch {
@@ -210,10 +208,15 @@ fn get_highest_syntax_node_at_bytepos(syntax: &Syntax, pos: usize) -> Option<Nod
     Some(node)
 }
 
-fn calculate_indentation(query: &IndentQuery, node: Option<Node>, newline: bool) -> usize {
-    // NOTE: can't use contains() on query because of comparing Vec<String> and &str
-    // https://doc.rust-lang.org/std/vec/struct.Vec.html#method.contains
-
+/// Calculate the indentation at a given treesitter node.
+/// If newline is false, then any "indent" nodes on the line are ignored ("outdent" still applies).
+/// This is because the indentation is only increased starting at the second line of the node.
+fn calculate_indentation(
+    query: &IndentQuery,
+    node: Option<Node>,
+    line: usize,
+    newline: bool,
+) -> usize {
     let mut increment: isize = 0;
 
     let mut node = match node {
@@ -221,70 +224,45 @@ fn calculate_indentation(query: &IndentQuery, node: Option<Node>, newline: bool)
         None => return 0,
     };
 
-    let mut prev_start = node.start_position().row;
-
-    // if we're calculating indentation for a brand new line then the current node will become the
-    // parent node. We need to take it's indentation level into account too.
-    let node_kind = node.kind();
-    if newline && query.indent.contains(node_kind) {
-        increment += 1;
-    }
-
-    while let Some(parent) = node.parent() {
-        let parent_kind = parent.kind();
-        let start = parent.start_position().row;
-
-        // detect deeply nested indents in the same line
-        // .map(|a| {       <-- ({ is two scopes
-        //     let len = 1; <-- indents one level
-        // })               <-- }) is two scopes
-        let starts_same_line = start == prev_start;
-
-        if query.outdent.contains(node.kind()) && !starts_same_line {
-            // we outdent by skipping the rules for the current level and jumping up
-            // node = parent;
-            increment -= 1;
-            // continue;
+    let mut current_line = line;
+    let mut consider_indent = newline;
+    let mut increment_from_line: isize = 0;
+
+    loop {
+        let node_kind = node.kind();
+        let start = node.start_position().row;
+        if current_line != start {
+            // Indent/dedent by at most one per line:
+            // .map(|a| {       <-- ({ is two scopes
+            //     let len = 1; <-- indents one level
+            // })               <-- }) is two scopes
+            if consider_indent || increment_from_line < 0 {
+                increment += increment_from_line.signum();
+            }
+            increment_from_line = 0;
+            current_line = start;
+            consider_indent = true;
         }
 
-        if query.indent.contains(parent_kind) // && not_first_or_last_sibling
-            && !starts_same_line
-        {
-            // println!("is_scope {}", parent_kind);
-            prev_start = start;
-            increment += 1
+        if query.outdent.contains(node_kind) {
+            increment_from_line -= 1;
+        }
+        if query.indent.contains(node_kind) {
+            increment_from_line += 1;
         }
 
-        // if last_scope && increment > 0 && ...{ ignore }
-
-        node = parent;
+        if let Some(parent) = node.parent() {
+            node = parent;
+        } else {
+            break;
+        }
+    }
+    if consider_indent || increment_from_line < 0 {
+        increment += increment_from_line.signum();
     }
-
     increment.max(0) as usize
 }
 
-#[allow(dead_code)]
-fn suggested_indent_for_line(
-    language_config: &LanguageConfiguration,
-    syntax: Option<&Syntax>,
-    text: RopeSlice,
-    line_num: usize,
-    _tab_width: usize,
-) -> usize {
-    if let Some(start) = find_first_non_whitespace_char(text.line(line_num)) {
-        return suggested_indent_for_pos(
-            Some(language_config),
-            syntax,
-            text,
-            start + text.line_to_char(line_num),
-            false,
-        );
-    };
-
-    // if the line is blank, indent should be zero
-    0
-}
-
 // TODO: two usecases: if we are triggering this for a new, blank line:
 // - it should return 0 when mass indenting stuff
 // - it should look up the wrapper node and count it too when we press o/O
@@ -293,23 +271,20 @@ pub fn suggested_indent_for_pos(
     syntax: Option<&Syntax>,
     text: RopeSlice,
     pos: usize,
+    line: usize,
     new_line: bool,
-) -> usize {
+) -> Option<usize> {
     if let (Some(query), Some(syntax)) = (
         language_config.and_then(|config| config.indent_query()),
         syntax,
     ) {
         let byte_start = text.char_to_byte(pos);
         let node = get_highest_syntax_node_at_bytepos(syntax, byte_start);
-
-        // let config = load indentation query config from Syntax(should contain language_config)
-
         // TODO: special case for comments
         // TODO: if preserve_leading_whitespace
-        calculate_indentation(query, node, new_line)
+        Some(calculate_indentation(query, node, line, new_line))
     } else {
-        // TODO: heuristics for non-tree sitter grammars
-        0
+        None
     }
 }
 
@@ -442,6 +417,7 @@ where
         );
 
         let doc = Rope::from(doc);
+        use crate::diagnostic::Severity;
         use crate::syntax::{
             Configuration, IndentationConfiguration, LanguageConfiguration, Loader,
         };
@@ -459,6 +435,8 @@ where
                 roots: vec![],
                 comment_token: None,
                 auto_format: false,
+                diagnostic_severity: Severity::Warning,
+                tree_sitter_library: None,
                 language_server: None,
                 indent: Some(IndentationConfiguration {
                     tab_width: 4,
@@ -482,14 +460,23 @@ where
 
         for i in 0..doc.len_lines() {
             let line = text.line(i);
-            let indent = indent_level_for_line(line, tab_width);
-            assert_eq!(
-                suggested_indent_for_line(&language_config, Some(&syntax), text, i, tab_width),
-                indent,
-                "line {}: {}",
-                i,
-                line
-            );
+            if let Some(pos) = crate::find_first_non_whitespace_char(line) {
+                let indent = indent_level_for_line(line, tab_width);
+                assert_eq!(
+                    suggested_indent_for_pos(
+                        Some(&language_config),
+                        Some(&syntax),
+                        text,
+                        text.line_to_char(i) + pos,
+                        i,
+                        false
+                    ),
+                    Some(indent),
+                    "line {}: \"{}\"",
+                    i,
+                    line
+                );
+            }
         }
     }
 }
diff --git a/helix-core/src/lib.rs b/helix-core/src/lib.rs
index 92a59f31..7fd23b97 100644
--- a/helix-core/src/lib.rs
+++ b/helix-core/src/lib.rs
@@ -1,3 +1,5 @@
+pub use encoding_rs as encoding;
+
 pub mod auto_pairs;
 pub mod chars;
 pub mod comment;
@@ -37,8 +39,14 @@ pub fn find_first_non_whitespace_char(line: RopeSlice) -> Option<usize> {
     line.chars().position(|ch| !ch.is_whitespace())
 }
 
-/// Find `.git` root.
-pub fn find_root(root: Option<&str>) -> Option<std::path::PathBuf> {
+/// Find project root.
+///
+/// Order of detection:
+/// * Top-most folder containing a root marker in current git repository
+/// * Git repostory root if no marker detected
+/// * Top-most folder containing a root marker if not git repository detected
+/// * Current working directory as fallback
+pub fn find_root(root: Option<&str>, root_markers: &[String]) -> Option<std::path::PathBuf> {
     let current_dir = std::env::current_dir().expect("unable to determine current directory");
 
     let root = match root {
@@ -50,16 +58,30 @@ pub fn find_root(root: Option<&str>) -> Option<std::path::PathBuf> {
                 current_dir.join(root)
             }
         }
-        None => current_dir,
+        None => current_dir.clone(),
     };
 
+    let mut top_marker = None;
     for ancestor in root.ancestors() {
-        // TODO: also use defined roots if git isn't found
+        for marker in root_markers {
+            if ancestor.join(marker).exists() {
+                top_marker = Some(ancestor);
+                break;
+            }
+        }
+        // don't go higher than repo
         if ancestor.join(".git").is_dir() {
-            return Some(ancestor.to_path_buf());
+            // Use workspace if detected from marker
+            return Some(top_marker.unwrap_or(ancestor).to_path_buf());
         }
     }
-    None
+
+    // In absence of git repo, use workspace if detected
+    if top_marker.is_some() {
+        top_marker.map(|a| a.to_path_buf())
+    } else {
+        Some(current_dir)
+    }
 }
 
 pub fn runtime_dir() -> std::path::PathBuf {
diff --git a/helix-core/src/match_brackets.rs b/helix-core/src/match_brackets.rs
index cd554005..0189dedd 100644
--- a/helix-core/src/match_brackets.rs
+++ b/helix-core/src/match_brackets.rs
@@ -11,7 +11,7 @@ const PAIRS: &[(char, char)] = &[
     ('\"', '\"'),
 ];
 
-// limit matching pairs to only ( ) { } [ ] < >
+// limit matching pairs to only ( ) { } [ ] < > ' ' " "
 
 // Returns the position of the matching bracket under cursor.
 //
diff --git a/helix-core/src/movement.rs b/helix-core/src/movement.rs
index 01a8f890..47fe6827 100644
--- a/helix-core/src/movement.rs
+++ b/helix-core/src/movement.rs
@@ -307,8 +307,6 @@ fn reached_target(target: WordMotionTarget, prev_ch: char, next_ch: char) -> boo
 
 #[cfg(test)]
 mod test {
-    use std::array::{self, IntoIter};
-
     use ropey::Rope;
 
     use super::*;
@@ -360,7 +358,7 @@ mod test {
             ((Direction::Backward, 999usize), (0, 0)), // |This is a simple alphabetic line
         ];
 
-        for ((direction, amount), coordinates) in IntoIter::new(moves_and_expected_coordinates) {
+        for ((direction, amount), coordinates) in moves_and_expected_coordinates {
             range = move_horizontally(slice, range, direction, amount, Movement::Move);
             assert_eq!(coords_at_pos(slice, range.head), coordinates.into())
         }
@@ -374,7 +372,7 @@ mod test {
 
         let mut range = Range::point(position);
 
-        let moves_and_expected_coordinates = IntoIter::new([
+        let moves_and_expected_coordinates = [
             ((Direction::Forward, 11usize), (1, 1)), // Multiline\nt|ext sample\n...
             ((Direction::Backward, 1usize), (1, 0)), // Multiline\n|text sample\n...
             ((Direction::Backward, 5usize), (0, 5)), // Multi|line\ntext sample\n...
@@ -384,7 +382,7 @@ mod test {
             ((Direction::Backward, 0usize), (0, 3)), // Mul|tiline\ntext sample\n...
             ((Direction::Forward, 999usize), (5, 0)), // ...and whitespaced\n|
             ((Direction::Forward, 999usize), (5, 0)), // ...and whitespaced\n|
-        ]);
+        ];
 
         for ((direction, amount), coordinates) in moves_and_expected_coordinates {
             range = move_horizontally(slice, range, direction, amount, Movement::Move);
@@ -402,11 +400,11 @@ mod test {
         let mut range = Range::point(position);
         let original_anchor = range.anchor;
 
-        let moves = IntoIter::new([
+        let moves = [
             (Direction::Forward, 1usize),
             (Direction::Forward, 5usize),
             (Direction::Backward, 3usize),
-        ]);
+        ];
 
         for (direction, amount) in moves {
             range = move_horizontally(slice, range, direction, amount, Movement::Extend);
@@ -420,7 +418,7 @@ mod test {
         let slice = text.slice(..);
         let position = pos_at_coords(slice, (0, 0).into(), true);
         let mut range = Range::point(position);
-        let moves_and_expected_coordinates = IntoIter::new([
+        let moves_and_expected_coordinates = [
             ((Direction::Forward, 1usize), (1, 0)),
             ((Direction::Forward, 2usize), (3, 0)),
             ((Direction::Forward, 1usize), (4, 0)),
@@ -430,7 +428,7 @@ mod test {
             ((Direction::Backward, 0usize), (4, 0)),
             ((Direction::Forward, 5), (5, 0)),
             ((Direction::Forward, 999usize), (5, 0)),
-        ]);
+        ];
 
         for ((direction, amount), coordinates) in moves_and_expected_coordinates {
             range = move_vertically(slice, range, direction, amount, Movement::Move);
@@ -450,7 +448,7 @@ mod test {
             H,
             V,
         }
-        let moves_and_expected_coordinates = IntoIter::new([
+        let moves_and_expected_coordinates = [
             // Places cursor at the end of line
             ((Axis::H, Direction::Forward, 8usize), (0, 8)),
             // First descent preserves column as the target line is wider
@@ -463,7 +461,7 @@ mod test {
             ((Axis::V, Direction::Backward, 999usize), (0, 8)),
             ((Axis::V, Direction::Forward, 4usize), (4, 8)),
             ((Axis::V, Direction::Forward, 999usize), (5, 0)),
-        ]);
+        ];
 
         for ((axis, direction, amount), coordinates) in moves_and_expected_coordinates {
             range = match axis {
@@ -489,7 +487,7 @@ mod test {
             H,
             V,
         }
-        let moves_and_expected_coordinates = IntoIter::new([
+        let moves_and_expected_coordinates = [
             // Places cursor at the fourth kana.
             ((Axis::H, Direction::Forward, 4), (0, 4)),
             // Descent places cursor at the 4th character.
@@ -498,7 +496,7 @@ mod test {
             ((Axis::H, Direction::Backward, 1usize), (1, 3)),
             // Jumping back up 1 line.
             ((Axis::V, Direction::Backward, 1usize), (0, 3)),
-        ]);
+        ];
 
         for ((axis, direction, amount), coordinates) in moves_and_expected_coordinates {
             range = match axis {
@@ -530,7 +528,7 @@ mod test {
 
     #[test]
     fn test_behaviour_when_moving_to_start_of_next_words() {
-        let tests = array::IntoIter::new([
+        let tests = [
             ("Basic forward motion stops at the first space",
                 vec![(1, Range::new(0, 0), Range::new(0, 6))]),
             (" Starting from a boundary advances the anchor",
@@ -604,7 +602,7 @@ mod test {
                 vec![
                     (1, Range::new(0, 0), Range::new(0, 6)),
                 ]),
-        ]);
+        ];
 
         for (sample, scenario) in tests {
             for (count, begin, expected_end) in scenario.into_iter() {
@@ -616,7 +614,7 @@ mod test {
 
     #[test]
     fn test_behaviour_when_moving_to_start_of_next_long_words() {
-        let tests = array::IntoIter::new([
+        let tests = [
             ("Basic forward motion stops at the first space",
                 vec![(1, Range::new(0, 0), Range::new(0, 6))]),
             (" Starting from a boundary advances the anchor",
@@ -688,7 +686,7 @@ mod test {
                 vec![
                     (1, Range::new(0, 0), Range::new(0, 8)),
                 ]),
-        ]);
+        ];
 
         for (sample, scenario) in tests {
             for (count, begin, expected_end) in scenario.into_iter() {
@@ -700,7 +698,7 @@ mod test {
 
     #[test]
     fn test_behaviour_when_moving_to_start_of_previous_words() {
-        let tests = array::IntoIter::new([
+        let tests = [
             ("Basic backward motion from the middle of a word",
                 vec![(1, Range::new(3, 3), Range::new(4, 0))]),
 
@@ -773,7 +771,7 @@ mod test {
                 vec![
                     (1, Range::new(0, 6), Range::new(6, 0)),
                 ]),
-        ]);
+        ];
 
         for (sample, scenario) in tests {
             for (count, begin, expected_end) in scenario.into_iter() {
@@ -785,7 +783,7 @@ mod test {
 
     #[test]
     fn test_behaviour_when_moving_to_start_of_previous_long_words() {
-        let tests = array::IntoIter::new([
+        let tests = [
             (
                 "Basic backward motion from the middle of a word",
                 vec![(1, Range::new(3, 3), Range::new(4, 0))],
@@ -870,7 +868,7 @@ mod test {
                 vec![
                     (1, Range::new(0, 8), Range::new(8, 0)),
                 ]),
-        ]);
+        ];
 
         for (sample, scenario) in tests {
             for (count, begin, expected_end) in scenario.into_iter() {
@@ -882,7 +880,7 @@ mod test {
 
     #[test]
     fn test_behaviour_when_moving_to_end_of_next_words() {
-        let tests = array::IntoIter::new([
+        let tests = [
             ("Basic forward motion from the start of a word to the end of it",
                 vec![(1, Range::new(0, 0), Range::new(0, 5))]),
             ("Basic forward motion from the end of a word to the end of the next",
@@ -954,7 +952,7 @@ mod test {
                 vec![
                     (1, Range::new(0, 0), Range::new(0, 5)),
                 ]),
-        ]);
+        ];
 
         for (sample, scenario) in tests {
             for (count, begin, expected_end) in scenario.into_iter() {
@@ -966,7 +964,7 @@ mod test {
 
     #[test]
     fn test_behaviour_when_moving_to_end_of_previous_words() {
-        let tests = array::IntoIter::new([
+        let tests = [
             ("Basic backward motion from the middle of a word",
                 vec![(1, Range::new(9, 9), Range::new(10, 5))]),
             ("Starting from after boundary retreats the anchor",
@@ -1036,7 +1034,7 @@ mod test {
                 vec![
                     (1, Range::new(0, 10), Range::new(10, 4)),
                 ]),
-        ]);
+        ];
 
         for (sample, scenario) in tests {
             for (count, begin, expected_end) in scenario.into_iter() {
@@ -1048,7 +1046,7 @@ mod test {
 
     #[test]
     fn test_behaviour_when_moving_to_end_of_next_long_words() {
-        let tests = array::IntoIter::new([
+        let tests = [
             ("Basic forward motion from the start of a word to the end of it",
                 vec![(1, Range::new(0, 0), Range::new(0, 5))]),
             ("Basic forward motion from the end of a word to the end of the next",
@@ -1118,7 +1116,7 @@ mod test {
                 vec![
                     (1, Range::new(0, 0), Range::new(0, 7)),
                 ]),
-        ]);
+        ];
 
         for (sample, scenario) in tests {
             for (count, begin, expected_end) in scenario.into_iter() {
diff --git a/helix-core/src/object.rs b/helix-core/src/object.rs
index 717c5994..21fa24fb 100644
--- a/helix-core/src/object.rs
+++ b/helix-core/src/object.rs
@@ -1,7 +1,5 @@
 use crate::{Range, RopeSlice, Selection, Syntax};
 
-// TODO: to contract_selection we'd need to store the previous ranges before expand.
-// Maybe just contract to the first child node?
 pub fn expand_selection(syntax: &Syntax, text: RopeSlice, selection: &Selection) -> Selection {
     let tree = syntax.tree();
 
@@ -34,3 +32,30 @@ pub fn expand_selection(syntax: &Syntax, text: RopeSlice, selection: &Selection)
         }
     })
 }
+
+pub fn shrink_selection(syntax: &Syntax, text: RopeSlice, selection: &Selection) -> Selection {
+    let tree = syntax.tree();
+
+    selection.clone().transform(|range| {
+        let from = text.char_to_byte(range.from());
+        let to = text.char_to_byte(range.to());
+
+        let descendant = match tree.root_node().descendant_for_byte_range(from, to) {
+            // find first child, if not possible, fallback to the node that contains selection
+            Some(descendant) => match descendant.child(0) {
+                Some(child) => child,
+                None => descendant,
+            },
+            None => return range,
+        };
+
+        let from = text.byte_to_char(descendant.start_byte());
+        let to = text.byte_to_char(descendant.end_byte());
+
+        if range.head < range.anchor {
+            Range::new(to, from)
+        } else {
+            Range::new(from, to)
+        }
+    })
+}
diff --git a/helix-core/src/selection.rs b/helix-core/src/selection.rs
index 116a1c7c..1515c4fc 100644
--- a/helix-core/src/selection.rs
+++ b/helix-core/src/selection.rs
@@ -7,6 +7,7 @@ use crate::{
         ensure_grapheme_boundary_next, ensure_grapheme_boundary_prev, next_grapheme_boundary,
         prev_grapheme_boundary,
     },
+    movement::Direction,
     Assoc, ChangeSet, RopeSlice,
 };
 use smallvec::{smallvec, SmallVec};
@@ -82,6 +83,13 @@ impl Range {
         std::cmp::max(self.anchor, self.head)
     }
 
+    /// Total length of the range.
+    #[inline]
+    #[must_use]
+    pub fn len(&self) -> usize {
+        self.to() - self.from()
+    }
+
     /// The (inclusive) range of lines that the range overlaps.
     #[inline]
     #[must_use]
@@ -102,6 +110,27 @@ impl Range {
         self.anchor == self.head
     }
 
+    /// `Direction::Backward` when head < anchor.
+    /// `Direction::Backward` otherwise.
+    #[inline]
+    #[must_use]
+    pub fn direction(&self) -> Direction {
+        if self.head < self.anchor {
+            Direction::Backward
+        } else {
+            Direction::Forward
+        }
+    }
+
+    // flips the direction of the selection
+    pub fn flip(&self) -> Self {
+        Self {
+            anchor: self.head,
+            head: self.anchor,
+            horiz: self.horiz,
+        }
+    }
+
     /// Check two ranges for overlap.
     #[must_use]
     pub fn overlaps(&self, other: &Self) -> bool {
@@ -111,6 +140,11 @@ impl Range {
         self.from() == other.from() || (self.to() > other.from() && other.to() > self.from())
     }
 
+    #[inline]
+    pub fn contains_range(&self, other: &Self) -> bool {
+        self.from() <= other.from() && self.to() >= other.to()
+    }
+
     pub fn contains(&self, pos: usize) -> bool {
         self.from() <= pos && pos < self.to()
     }
@@ -515,6 +549,39 @@ impl Selection {
     pub fn len(&self) -> usize {
         self.ranges.len()
     }
+
+    // returns true if self ⊇ other
+    pub fn contains(&self, other: &Selection) -> bool {
+        // can't contain other if it is larger
+        if other.len() > self.len() {
+            return false;
+        }
+
+        let (mut iter_self, mut iter_other) = (self.iter(), other.iter());
+        let (mut ele_self, mut ele_other) = (iter_self.next(), iter_other.next());
+
+        loop {
+            match (ele_self, ele_other) {
+                (Some(ra), Some(rb)) => {
+                    if !ra.contains_range(rb) {
+                        // `self` doesn't contain next element from `other`, advance `self`, we need to match all from `other`
+                        ele_self = iter_self.next();
+                    } else {
+                        // matched element from `other`, advance `other`
+                        ele_other = iter_other.next();
+                    };
+                }
+                (None, Some(_)) => {
+                    // exhausted `self`, we can't match the reminder of `other`
+                    return false;
+                }
+                (_, None) => {
+                    // no elements from `other` left to match, `self` contains `other`
+                    return true;
+                }
+            }
+        }
+    }
 }
 
 impl<'a> IntoIterator for &'a Selection {
@@ -953,4 +1020,30 @@ mod test {
             &["", "abcd", "efg", "rs", "xyz"]
         );
     }
+    #[test]
+    fn test_selection_contains() {
+        fn contains(a: Vec<(usize, usize)>, b: Vec<(usize, usize)>) -> bool {
+            let sela = Selection::new(a.iter().map(|a| Range::new(a.0, a.1)).collect(), 0);
+            let selb = Selection::new(b.iter().map(|b| Range::new(b.0, b.1)).collect(), 0);
+            sela.contains(&selb)
+        }
+
+        // exact match
+        assert!(contains(vec!((1, 1)), vec!((1, 1))));
+
+        // larger set contains smaller
+        assert!(contains(vec!((1, 1), (2, 2), (3, 3)), vec!((2, 2))));
+
+        // multiple matches
+        assert!(contains(vec!((1, 1), (2, 2)), vec!((1, 1), (2, 2))));
+
+        // smaller set can't contain bigger
+        assert!(!contains(vec!((1, 1)), vec!((1, 1), (2, 2))));
+
+        assert!(contains(
+            vec!((1, 1), (2, 4), (5, 6), (7, 9), (10, 13)),
+            vec!((3, 4), (7, 9))
+        ));
+        assert!(!contains(vec!((1, 1), (5, 6)), vec!((1, 6))));
+    }
 }
diff --git a/helix-core/src/syntax.rs b/helix-core/src/syntax.rs
index ef35fc75..5d37c219 100644
--- a/helix-core/src/syntax.rs
+++ b/helix-core/src/syntax.rs
@@ -1,5 +1,6 @@
 use crate::{
     chars::char_is_line_ending,
+    diagnostic::Severity,
     regex::Regex,
     transaction::{ChangeSet, Operation},
     Rope, RopeSlice, Tendril,
@@ -63,6 +64,10 @@ pub struct LanguageConfiguration {
 
     #[serde(default)]
     pub auto_format: bool,
+    #[serde(default)]
+    pub diagnostic_severity: Severity,
+
+    pub tree_sitter_library: Option<String>, // tree-sitter library name, defaults to language_id
 
     // content_regex
     #[serde(default, skip_serializing, deserialize_with = "deserialize_regex")]
@@ -189,9 +194,14 @@ impl LanguageConfiguration {
         if highlights_query.is_empty() {
             None
         } else {
-            let language = get_language(&crate::RUNTIME_DIR, &self.language_id)
-                .map_err(|e| log::info!("{}", e))
-                .ok()?;
+            let language = get_language(
+                &crate::RUNTIME_DIR,
+                self.tree_sitter_library
+                    .as_deref()
+                    .unwrap_or(&self.language_id),
+            )
+            .map_err(|e| log::info!("{}", e))
+            .ok()?;
             let config = HighlightConfiguration::new(
                 language,
                 &highlights_query,