rework positioning/rendering and enable softwrap/virtual text (#5420)

* rework positioning/rendering, enables softwrap/virtual text

This commit is a large rework of the core text positioning and
rendering code in helix to remove the assumption that on-screen
columns/lines correspond to text columns/lines.

A generic `DocFormatter` is introduced that positions graphemes on
and is used both for rendering and for movements/scrolling.
Both virtual text support (inline, grapheme overlay and multi-line)
and a capable softwrap implementation is included.

fix picker highlight

cleanup doc formatter, use word bondaries for wrapping

make visual vertical movement a seperate commnad

estimate line gutter width to improve performance

cache cursor position

cleanup and optimize doc formatter

cleanup documentation

fix typos

Co-authored-by: Daniel Hines <d4hines@gmail.com>

update documentation

fix panic in last_visual_line funciton

improve soft-wrap documentation

add extend_visual_line_up/down commands

fix non-visual vertical movement

streamline virtual text highlighting, add softwrap indicator

fix cursor position if softwrap is disabled

improve documentation of text_annotations module

avoid crashes if view anchor is out of bounds

fix: consider horizontal offset when traslation char_idx -> vpos

improve default configuration

fix: mixed up horizontal and vertical offset

reset view position after config reload

apply suggestions from review

disabled softwrap for very small screens to avoid endless spin

fix wrap_indicator setting

fix bar cursor disappearring on the EOF character

add keybinding for linewise vertical movement

fix: inconsistent gutter highlights

improve virtual text API

make scope idx lookup more ergonomic

allow overlapping overlays

correctly track char_pos for virtual text

adjust configuration

deprecate old position fucntions

fix infinite loop in highlight lookup

fix gutter style

fix formatting

document max-line-width interaction with softwrap

change wrap-indicator example to use empty string

fix: rare panic when view is in invalid state (bis)

* Apply suggestions from code review

Co-authored-by: Michael Davis <mcarsondavis@gmail.com>

* improve documentation for positoning functions

* simplify tests

* fix documentation of Grapheme::width

* Apply suggestions from code review

Co-authored-by: Michael Davis <mcarsondavis@gmail.com>

* add explicit drop invocation

* Add explicit MoveFn type alias

* add docuntation to Editor::cursor_cache

* fix a few typos

* explain use of allow(deprecated)

* make gj and gk extend in select mode

* remove unneded debug and TODO

* mark tab_width_at #[inline]

* add fast-path to move_vertically_visual in case softwrap is disabled

* rename first_line to first_visual_line

* simplify duplicate if/else

---------

Co-authored-by: Michael Davis <mcarsondavis@gmail.com>

1 files changed, 180 insertions, 22 deletions

diff --git a/helix-core/src/movement.rs b/helix-core/src/movement.rs
index 278375e8..11c12a6f 100644
--- a/helix-core/src/movement.rs
+++ b/helix-core/src/movement.rs
@@ -4,16 +4,19 @@ use ropey::iter::Chars;
 use tree_sitter::{Node, QueryCursor};
 
 use crate::{
+    char_idx_at_visual_offset,
     chars::{categorize_char, char_is_line_ending, CharCategory},
+    doc_formatter::TextFormat,
     graphemes::{
         next_grapheme_boundary, nth_next_grapheme_boundary, nth_prev_grapheme_boundary,
         prev_grapheme_boundary,
     },
     line_ending::rope_is_line_ending,
-    pos_at_visual_coords,
+    position::char_idx_at_visual_block_offset,
     syntax::LanguageConfiguration,
+    text_annotations::TextAnnotations,
     textobject::TextObject,
-    visual_coords_at_pos, Position, Range, RopeSlice,
+    visual_offset_from_block, Range, RopeSlice,
 };
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
@@ -34,7 +37,8 @@ pub fn move_horizontally(
     dir: Direction,
     count: usize,
     behaviour: Movement,
-    _: usize,
+    _: &TextFormat,
+    _: &mut TextAnnotations,
 ) -> Range {
     let pos = range.cursor(slice);
 
@@ -48,35 +52,116 @@ pub fn move_horizontally(
     range.put_cursor(slice, new_pos, behaviour == Movement::Extend)
 }
 
+pub fn move_vertically_visual(
+    slice: RopeSlice,
+    range: Range,
+    dir: Direction,
+    count: usize,
+    behaviour: Movement,
+    text_fmt: &TextFormat,
+    annotations: &mut TextAnnotations,
+) -> Range {
+    if !text_fmt.soft_wrap {
+        move_vertically(slice, range, dir, count, behaviour, text_fmt, annotations);
+    }
+    annotations.clear_line_annotations();
+    let pos = range.cursor(slice);
+
+    // Compute the current position's 2d coordinates.
+    let (visual_pos, block_off) = visual_offset_from_block(slice, pos, pos, text_fmt, annotations);
+    let new_col = range
+        .old_visual_position
+        .map_or(visual_pos.col as u32, |(_, col)| col);
+
+    // Compute the new position.
+    let mut row_off = match dir {
+        Direction::Forward => count as isize,
+        Direction::Backward => -(count as isize),
+    };
+
+    // TODO how to handle inline annotations that span an entire visual line (very unlikely).
+
+    // Compute visual offset relative to block start to avoid trasversing the block twice
+    row_off += visual_pos.row as isize;
+    let new_pos = char_idx_at_visual_offset(
+        slice,
+        block_off,
+        row_off,
+        new_col as usize,
+        text_fmt,
+        annotations,
+    )
+    .0;
+
+    // Special-case to avoid moving to the end of the last non-empty line.
+    if behaviour == Movement::Extend && slice.line(slice.char_to_line(new_pos)).len_chars() == 0 {
+        return range;
+    }
+
+    let mut new_range = range.put_cursor(slice, new_pos, behaviour == Movement::Extend);
+    new_range.old_visual_position = Some((0, new_col));
+    new_range
+}
+
 pub fn move_vertically(
     slice: RopeSlice,
     range: Range,
     dir: Direction,
     count: usize,
     behaviour: Movement,
-    tab_width: usize,
+    text_fmt: &TextFormat,
+    annotations: &mut TextAnnotations,
 ) -> Range {
+    annotations.clear_line_annotations();
     let pos = range.cursor(slice);
+    let line_idx = slice.char_to_line(pos);
+    let line_start = slice.line_to_char(line_idx);
 
     // Compute the current position's 2d coordinates.
-    let Position { row, col } = visual_coords_at_pos(slice, pos, tab_width);
-    let horiz = range.horiz.unwrap_or(col as u32);
+    let visual_pos = visual_offset_from_block(slice, line_start, pos, text_fmt, annotations).0;
+    let (mut new_row, new_col) = range
+        .old_visual_position
+        .map_or((visual_pos.row as u32, visual_pos.col as u32), |pos| pos);
+    new_row = new_row.max(visual_pos.row as u32);
+    let line_idx = slice.char_to_line(pos);
 
     // Compute the new position.
-    let new_row = match dir {
-        Direction::Forward => (row + count).min(slice.len_lines().saturating_sub(1)),
-        Direction::Backward => row.saturating_sub(count),
+    let mut new_line_idx = match dir {
+        Direction::Forward => line_idx.saturating_add(count),
+        Direction::Backward => line_idx.saturating_sub(count),
     };
-    let new_col = col.max(horiz as usize);
-    let new_pos = pos_at_visual_coords(slice, Position::new(new_row, new_col), tab_width);
+
+    let line = if new_line_idx >= slice.len_lines() - 1 {
+        // there is no line terminator for the last line
+        // so the logic below is not necessary here
+        new_line_idx = slice.len_lines() - 1;
+        slice
+    } else {
+        // char_idx_at_visual_block_offset returns a one-past-the-end index
+        // in case it reaches the end of the slice
+        // to avoid moving to the nextline in that case the line terminator is removed from the line
+        let new_line_end = prev_grapheme_boundary(slice, slice.line_to_char(new_line_idx + 1));
+        slice.slice(..new_line_end)
+    };
+
+    let new_line_start = line.line_to_char(new_line_idx);
+
+    let (new_pos, _) = char_idx_at_visual_block_offset(
+        line,
+        new_line_start,
+        new_row as usize,
+        new_col as usize,
+        text_fmt,
+        annotations,
+    );
 
     // Special-case to avoid moving to the end of the last non-empty line.
-    if behaviour == Movement::Extend && slice.line(new_row).len_chars() == 0 {
+    if behaviour == Movement::Extend && slice.line(new_line_idx).len_chars() == 0 {
         return range;
     }
 
     let mut new_range = range.put_cursor(slice, new_pos, behaviour == Movement::Extend);
-    new_range.horiz = Some(horiz);
+    new_range.old_visual_position = Some((new_row, new_col));
     new_range
 }
 
@@ -473,7 +558,16 @@ mod test {
         assert_eq!(
             coords_at_pos(
                 slice,
-                move_vertically(slice, range, Direction::Forward, 1, Movement::Move, 4).head
+                move_vertically_visual(
+                    slice,
+                    range,
+                    Direction::Forward,
+                    1,
+                    Movement::Move,
+                    &TextFormat::default(),
+                    &mut TextAnnotations::default(),
+                )
+                .head
             ),
             (1, 3).into()
         );
@@ -497,7 +591,15 @@ mod test {
         ];
 
         for ((direction, amount), coordinates) in moves_and_expected_coordinates {
-            range = move_horizontally(slice, range, direction, amount, Movement::Move, 0);
+            range = move_horizontally(
+                slice,
+                range,
+                direction,
+                amount,
+                Movement::Move,
+                &TextFormat::default(),
+                &mut TextAnnotations::default(),
+            );
             assert_eq!(coords_at_pos(slice, range.head), coordinates.into())
         }
     }
@@ -523,7 +625,15 @@ mod test {
         ];
 
         for ((direction, amount), coordinates) in moves_and_expected_coordinates {
-            range = move_horizontally(slice, range, direction, amount, Movement::Move, 0);
+            range = move_horizontally(
+                slice,
+                range,
+                direction,
+                amount,
+                Movement::Move,
+                &TextFormat::default(),
+                &mut TextAnnotations::default(),
+            );
             assert_eq!(coords_at_pos(slice, range.head), coordinates.into());
             assert_eq!(range.head, range.anchor);
         }
@@ -545,7 +655,15 @@ mod test {
         ];
 
         for (direction, amount) in moves {
-            range = move_horizontally(slice, range, direction, amount, Movement::Extend, 0);
+            range = move_horizontally(
+                slice,
+                range,
+                direction,
+                amount,
+                Movement::Extend,
+                &TextFormat::default(),
+                &mut TextAnnotations::default(),
+            );
             assert_eq!(range.anchor, original_anchor);
         }
     }
@@ -569,7 +687,15 @@ mod test {
         ];
 
         for ((direction, amount), coordinates) in moves_and_expected_coordinates {
-            range = move_vertically(slice, range, direction, amount, Movement::Move, 4);
+            range = move_vertically_visual(
+                slice,
+                range,
+                direction,
+                amount,
+                Movement::Move,
+                &TextFormat::default(),
+                &mut TextAnnotations::default(),
+            );
             assert_eq!(coords_at_pos(slice, range.head), coordinates.into());
             assert_eq!(range.head, range.anchor);
         }
@@ -603,8 +729,24 @@ mod test {
 
         for ((axis, direction, amount), coordinates) in moves_and_expected_coordinates {
             range = match axis {
-                Axis::H => move_horizontally(slice, range, direction, amount, Movement::Move, 0),
-                Axis::V => move_vertically(slice, range, direction, amount, Movement::Move, 4),
+                Axis::H => move_horizontally(
+                    slice,
+                    range,
+                    direction,
+                    amount,
+                    Movement::Move,
+                    &TextFormat::default(),
+                    &mut TextAnnotations::default(),
+                ),
+                Axis::V => move_vertically_visual(
+                    slice,
+                    range,
+                    direction,
+                    amount,
+                    Movement::Move,
+                    &TextFormat::default(),
+                    &mut TextAnnotations::default(),
+                ),
             };
             assert_eq!(coords_at_pos(slice, range.head), coordinates.into());
             assert_eq!(range.head, range.anchor);
@@ -638,8 +780,24 @@ mod test {
 
         for ((axis, direction, amount), coordinates) in moves_and_expected_coordinates {
             range = match axis {
-                Axis::H => move_horizontally(slice, range, direction, amount, Movement::Move, 0),
-                Axis::V => move_vertically(slice, range, direction, amount, Movement::Move, 4),
+                Axis::H => move_horizontally(
+                    slice,
+                    range,
+                    direction,
+                    amount,
+                    Movement::Move,
+                    &TextFormat::default(),
+                    &mut TextAnnotations::default(),
+                ),
+                Axis::V => move_vertically_visual(
+                    slice,
+                    range,
+                    direction,
+                    amount,
+                    Movement::Move,
+                    &TextFormat::default(),
+                    &mut TextAnnotations::default(),
+                ),
             };
             assert_eq!(coords_at_pos(slice, range.head), coordinates.into());
             assert_eq!(range.head, range.anchor);