lsp: support both utf-8 and utf-16 offsets.

Still need to implement the clangd encoding negotiation, but it's
a start. Should also manually override to utf8 for pyls.

8 files changed, 179 insertions, 52 deletions

diff --git a/helix-lsp/src/client.rs b/helix-lsp/src/client.rs
index 64070be0..c70e6e78 100644
--- a/helix-lsp/src/client.rs
+++ b/helix-lsp/src/client.rs
@@ -1,6 +1,6 @@
 use crate::{
     transport::{Payload, Transport},
-    Call, Error, Result,
+    Call, Error, OffsetEncoding, Result,
 };
 
 use helix_core::{ChangeSet, Rope};
@@ -29,8 +29,7 @@ pub struct Client {
     pub request_counter: AtomicU64,
 
     capabilities: Option<lsp::ServerCapabilities>,
-    // TODO: handle PublishDiagnostics Version
-    // diagnostics: HashMap<lsp::Url, Vec<lsp::Diagnostic>>,
+    offset_encoding: OffsetEncoding,
 }
 
 impl Client {
@@ -70,6 +69,7 @@ impl Client {
 
             capabilities: None,
             // diagnostics: HashMap::new(),
+            offset_encoding: OffsetEncoding::Utf8,
         };
 
         // TODO: async client.initialize()
@@ -100,6 +100,10 @@ impl Client {
             .expect("language server not yet initialized!")
     }
 
+    pub fn offset_encoding(&self) -> OffsetEncoding {
+        self.offset_encoding
+    }
+
     /// Execute a RPC request on the language server.
     pub async fn request<R: lsp::request::Request>(&self, params: R::Params) -> Result<R::Result>
     where
@@ -291,6 +295,7 @@ impl Client {
         old_text: &Rope,
         new_text: &Rope,
         changeset: &ChangeSet,
+        offset_encoding: OffsetEncoding,
     ) -> Vec<lsp::TextDocumentContentChangeEvent> {
         let mut iter = changeset.changes().iter().peekable();
         let mut old_pos = 0;
@@ -340,7 +345,7 @@ impl Client {
                     new_pos += i;
                 }
                 Delete(_) => {
-                    let start = pos_to_lsp_pos(new_text, new_pos);
+                    let start = pos_to_lsp_pos(new_text, new_pos, offset_encoding);
                     let end = traverse(start, old_text.slice(old_pos..old_end));
 
                     // deletion
@@ -351,7 +356,7 @@ impl Client {
                     });
                 }
                 Insert(s) => {
-                    let start = pos_to_lsp_pos(new_text, new_pos);
+                    let start = pos_to_lsp_pos(new_text, new_pos, offset_encoding);
 
                     new_pos += s.chars().count();
 
@@ -413,7 +418,7 @@ impl Client {
                 }]
             }
             lsp::TextDocumentSyncKind::Incremental => {
-                Self::changeset_to_changes(old_text, new_text, changes)
+                Self::changeset_to_changes(old_text, new_text, changes, self.offset_encoding)
             }
             lsp::TextDocumentSyncKind::None => return Ok(()),
         };
diff --git a/helix-lsp/src/lib.rs b/helix-lsp/src/lib.rs
index 40d2e477..dd925c14 100644
--- a/helix-lsp/src/lib.rs
+++ b/helix-lsp/src/lib.rs
@@ -16,6 +16,8 @@ use thiserror::Error;
 
 use std::{collections::HashMap, sync::Arc};
 
+use serde::{Deserialize, Serialize};
+
 #[derive(Error, Debug)]
 pub enum Error {
     #[error("protocol error: {0}")]
@@ -28,31 +30,76 @@ pub enum Error {
     Other(#[from] anyhow::Error),
 }
 
+#[derive(Clone, Copy, Debug, Serialize, Deserialize)]
+pub enum OffsetEncoding {
+    /// UTF-8 code units aka bytes
+    #[serde(rename = "utf-8")]
+    Utf8,
+    /// UTF-16 code units
+    #[serde(rename = "utf-16")]
+    Utf16,
+}
+
 pub mod util {
     use super::*;
     use helix_core::{Range, Rope, Transaction};
 
-    pub fn lsp_pos_to_pos(doc: &Rope, pos: lsp::Position) -> usize {
-        let line = doc.line_to_char(pos.line as usize);
-        let line_start = doc.char_to_utf16_cu(line);
-        doc.utf16_cu_to_char(line_start + pos.character as usize)
+    pub fn lsp_pos_to_pos(
+        doc: &Rope,
+        pos: lsp::Position,
+        offset_encoding: OffsetEncoding,
+    ) -> usize {
+        match offset_encoding {
+            OffsetEncoding::Utf8 => {
+                let line = doc.line_to_char(pos.line as usize);
+                line + pos.character as usize
+            }
+            OffsetEncoding::Utf16 => {
+                let line = doc.line_to_char(pos.line as usize);
+                let line_start = doc.char_to_utf16_cu(line);
+                doc.utf16_cu_to_char(line_start + pos.character as usize)
+            }
+        }
     }
-    pub fn pos_to_lsp_pos(doc: &Rope, pos: usize) -> lsp::Position {
-        let line = doc.char_to_line(pos);
-        let line_start = doc.char_to_utf16_cu(doc.line_to_char(line));
-        let col = doc.char_to_utf16_cu(pos) - line_start;
+    pub fn pos_to_lsp_pos(
+        doc: &Rope,
+        pos: usize,
+        offset_encoding: OffsetEncoding,
+    ) -> lsp::Position {
+        match offset_encoding {
+            OffsetEncoding::Utf8 => {
+                let line = doc.char_to_line(pos);
+                let line_start = doc.line_to_char(line);
+                let col = pos - line_start;
+
+                lsp::Position::new(line as u32, col as u32)
+            }
+            OffsetEncoding::Utf16 => {
+                let line = doc.char_to_line(pos);
+                let line_start = doc.char_to_utf16_cu(doc.line_to_char(line));
+                let col = doc.char_to_utf16_cu(pos) - line_start;
 
-        lsp::Position::new(line as u32, col as u32)
+                lsp::Position::new(line as u32, col as u32)
+            }
+        }
     }
 
-    pub fn range_to_lsp_range(doc: &Rope, range: Range) -> lsp::Range {
-        let start = pos_to_lsp_pos(doc, range.from());
-        let end = pos_to_lsp_pos(doc, range.to());
+    pub fn range_to_lsp_range(
+        doc: &Rope,
+        range: Range,
+        offset_encoding: OffsetEncoding,
+    ) -> lsp::Range {
+        let start = pos_to_lsp_pos(doc, range.from(), offset_encoding);
+        let end = pos_to_lsp_pos(doc, range.to(), offset_encoding);
 
         lsp::Range::new(start, end)
     }
 
-    pub fn generate_transaction_from_edits(doc: &Rope, edits: Vec<lsp::TextEdit>) -> Transaction {
+    pub fn generate_transaction_from_edits(
+        doc: &Rope,
+        edits: Vec<lsp::TextEdit>,
+        offset_encoding: OffsetEncoding,
+    ) -> Transaction {
         Transaction::change(
             doc,
             edits.into_iter().map(|edit| {
@@ -63,8 +110,8 @@ pub mod util {
                     None
                 };
 
-                let start = lsp_pos_to_pos(doc, edit.range.start);
-                let end = lsp_pos_to_pos(doc, edit.range.end);
+                let start = lsp_pos_to_pos(doc, edit.range.start, offset_encoding);
+                let end = lsp_pos_to_pos(doc, edit.range.end, offset_encoding);
                 (start, end, replacement)
             }),
         )
diff --git a/helix-term/src/application.rs b/helix-term/src/application.rs
index 71c6fba1..b3ddbe15 100644
--- a/helix-term/src/application.rs
+++ b/helix-term/src/application.rs
@@ -175,8 +175,20 @@ impl Application {
                                     };
                                     use helix_lsp::{lsp, util::lsp_pos_to_pos};
                                     use lsp::DiagnosticSeverity;
-                                    let start = lsp_pos_to_pos(text, diagnostic.range.start);
-                                    let end = lsp_pos_to_pos(text, diagnostic.range.end);
+
+                                    let language_server = doc.language_server().unwrap();
+
+                                    // TODO: convert inside server
+                                    let start = lsp_pos_to_pos(
+                                        text,
+                                        diagnostic.range.start,
+                                        language_server.offset_encoding(),
+                                    );
+                                    let end = lsp_pos_to_pos(
+                                        text,
+                                        diagnostic.range.end,
+                                        language_server.offset_encoding(),
+                                    );
 
                                     Diagnostic {
                                         range: Range { start, end },
diff --git a/helix-term/src/commands.rs b/helix-term/src/commands.rs
index 3854f416..4c04b0a2 100644
--- a/helix-term/src/commands.rs
+++ b/helix-term/src/commands.rs
@@ -16,6 +16,7 @@ use helix_view::{
 use helix_lsp::{
     lsp,
     util::{lsp_pos_to_pos, pos_to_lsp_pos, range_to_lsp_range},
+    OffsetEncoding,
 };
 
 use crate::{
@@ -1117,18 +1118,24 @@ pub fn exit_select_mode(cx: &mut Context) {
     cx.doc().mode = Mode::Normal;
 }
 
-fn _goto(cx: &mut Context, locations: Vec<lsp::Location>) {
+fn _goto(cx: &mut Context, locations: Vec<lsp::Location>, offset_encoding: OffsetEncoding) {
     use helix_view::editor::Action;
 
     push_jump(cx);
 
-    fn jump_to(editor: &mut Editor, location: &lsp::Location, action: Action) {
+    fn jump_to(
+        editor: &mut Editor,
+        location: &lsp::Location,
+        offset_encoding: OffsetEncoding,
+        action: Action,
+    ) {
         let id = editor
             .open(PathBuf::from(location.uri.path()), action)
             .expect("editor.open failed");
         let (view, doc) = editor.current();
         let definition_pos = location.range.start;
-        let new_pos = lsp_pos_to_pos(doc.text(), definition_pos);
+        // TODO: convert inside server
+        let new_pos = lsp_pos_to_pos(doc.text(), definition_pos, offset_encoding);
         doc.set_selection(view.id, Selection::point(new_pos));
         let line = doc.text().char_to_line(new_pos);
         view.first_line = line.saturating_sub(view.area.height as usize / 2);
@@ -1136,7 +1143,7 @@ fn _goto(cx: &mut Context, locations: Vec<lsp::Location>) {
 
     match locations.as_slice() {
         [location] => {
-            jump_to(cx.editor, location, Action::Replace);
+            jump_to(cx.editor, location, offset_encoding, Action::Replace);
         }
         [] => (), // maybe show user message that no definition was found?
         _locations => {
@@ -1147,7 +1154,9 @@ fn _goto(cx: &mut Context, locations: Vec<lsp::Location>) {
                     let line = location.range.start.line;
                     format!("{}:{}", file, line).into()
                 },
-                move |editor: &mut Editor, location, action| jump_to(editor, location, action),
+                move |editor: &mut Editor, location, action| {
+                    jump_to(editor, location, offset_encoding, action)
+                },
             );
             cx.push_layer(Box::new(picker));
         }
@@ -1161,12 +1170,14 @@ pub fn goto_definition(cx: &mut Context) {
         None => return,
     };
 
-    let pos = pos_to_lsp_pos(doc.text(), doc.selection(view.id).cursor());
+    let offset_encoding = language_server.offset_encoding();
+
+    let pos = pos_to_lsp_pos(doc.text(), doc.selection(view.id).cursor(), offset_encoding);
 
     // TODO: handle fails
     let res =
         smol::block_on(language_server.goto_definition(doc.identifier(), pos)).unwrap_or_default();
-    _goto(cx, res);
+    _goto(cx, res, offset_encoding);
 }
 
 pub fn goto_type_definition(cx: &mut Context) {
@@ -1176,12 +1187,14 @@ pub fn goto_type_definition(cx: &mut Context) {
         None => return,
     };
 
-    let pos = pos_to_lsp_pos(doc.text(), doc.selection(view.id).cursor());
+    let offset_encoding = language_server.offset_encoding();
+
+    let pos = pos_to_lsp_pos(doc.text(), doc.selection(view.id).cursor(), offset_encoding);
 
     // TODO: handle fails
     let res = smol::block_on(language_server.goto_type_definition(doc.identifier(), pos))
         .unwrap_or_default();
-    _goto(cx, res);
+    _goto(cx, res, offset_encoding);
 }
 
 pub fn goto_implementation(cx: &mut Context) {
@@ -1191,12 +1204,14 @@ pub fn goto_implementation(cx: &mut Context) {
         None => return,
     };
 
-    let pos = pos_to_lsp_pos(doc.text(), doc.selection(view.id).cursor());
+    let offset_encoding = language_server.offset_encoding();
+
+    let pos = pos_to_lsp_pos(doc.text(), doc.selection(view.id).cursor(), offset_encoding);
 
     // TODO: handle fails
     let res = smol::block_on(language_server.goto_implementation(doc.identifier(), pos))
         .unwrap_or_default();
-    _goto(cx, res);
+    _goto(cx, res, offset_encoding);
 }
 
 pub fn goto_reference(cx: &mut Context) {
@@ -1206,12 +1221,14 @@ pub fn goto_reference(cx: &mut Context) {
         None => return,
     };
 
-    let pos = pos_to_lsp_pos(doc.text(), doc.selection(view.id).cursor());
+    let offset_encoding = language_server.offset_encoding();
+
+    let pos = pos_to_lsp_pos(doc.text(), doc.selection(view.id).cursor(), offset_encoding);
 
     // TODO: handle fails
     let res =
         smol::block_on(language_server.goto_reference(doc.identifier(), pos)).unwrap_or_default();
-    _goto(cx, res);
+    _goto(cx, res, offset_encoding);
 }
 
 pub fn signature_help(cx: &mut Context) {
@@ -1222,7 +1239,11 @@ pub fn signature_help(cx: &mut Context) {
         None => return,
     };
 
-    let pos = pos_to_lsp_pos(doc.text(), doc.selection(view.id).cursor());
+    let pos = pos_to_lsp_pos(
+        doc.text(),
+        doc.selection(view.id).cursor(),
+        language_server.offset_encoding(),
+    );
 
     // TODO: handle fails
 
@@ -1579,10 +1600,15 @@ pub fn format_selections(cx: &mut Context) {
     // via lsp if available
     // else via tree-sitter indentation calculations
 
+    let language_server = match doc.language_server() {
+        Some(language_server) => language_server,
+        None => return,
+    };
+
     let ranges: Vec<lsp::Range> = doc
         .selection(view.id)
         .iter()
-        .map(|range| range_to_lsp_range(doc.text(), *range))
+        .map(|range| range_to_lsp_range(doc.text(), *range, language_server.offset_encoding()))
         .collect();
 
     for range in ranges {
@@ -1590,7 +1616,6 @@ pub fn format_selections(cx: &mut Context) {
             Some(language_server) => language_server,
             None => return,
         };
-
         // TODO: handle fails
         // TODO: concurrent map
         let edits = smol::block_on(language_server.text_document_range_formatting(
@@ -1600,7 +1625,11 @@ pub fn format_selections(cx: &mut Context) {
         ))
         .unwrap_or_default();
 
-        let transaction = helix_lsp::util::generate_transaction_from_edits(doc.text(), edits);
+        let transaction = helix_lsp::util::generate_transaction_from_edits(
+            doc.text(),
+            edits,
+            language_server.offset_encoding(),
+        );
 
         doc.apply(&transaction, view.id);
     }
@@ -1726,7 +1755,13 @@ pub fn completion(cx: &mut Context) {
         None => return,
     };
 
-    let pos = pos_to_lsp_pos(doc.text(), doc.selection(view.id).cursor());
+    let offset_encoding = language_server.offset_encoding();
+
+    let pos = pos_to_lsp_pos(
+        doc.text(),
+        doc.selection(view.id).cursor(),
+        language_server.offset_encoding(),
+    );
 
     // TODO: handle fails
     let res = smol::block_on(language_server.completion(doc.identifier(), pos)).unwrap();
@@ -1754,7 +1789,7 @@ pub fn completion(cx: &mut Context) {
                 let size = compositor.size();
                 let ui = compositor.find("hx::ui::editor::EditorView").unwrap();
                 if let Some(ui) = ui.as_any_mut().downcast_mut::<ui::EditorView>() {
-                    ui.set_completion(items, trigger_offset, size);
+                    ui.set_completion(items, offset_encoding, trigger_offset, size);
                 };
             }
         },
@@ -1779,7 +1814,11 @@ pub fn hover(cx: &mut Context) {
 
     // TODO: blocking here is not ideal, make commands async fn?
     // not like we can process additional input meanwhile though
-    let pos = pos_to_lsp_pos(doc.text(), doc.selection(view.id).cursor());
+    let pos = pos_to_lsp_pos(
+        doc.text(),
+        doc.selection(view.id).cursor(),
+        language_server.offset_encoding(),
+    );
 
     // TODO: handle fails
     let res = smol::block_on(language_server.text_document_hover(doc.identifier(), pos))
diff --git a/helix-term/src/ui/completion.rs b/helix-term/src/ui/completion.rs
index a804b5f2..7a8413f8 100644
--- a/helix-term/src/ui/completion.rs
+++ b/helix-term/src/ui/completion.rs
@@ -26,7 +26,11 @@ pub struct Completion {
 }
 
 impl Completion {
-    pub fn new(items: Vec<CompletionItem>, trigger_offset: usize) -> Self {
+    pub fn new(
+        items: Vec<CompletionItem>,
+        offset_encoding: helix_lsp::OffsetEncoding,
+        trigger_offset: usize,
+    ) -> Self {
         // let items: Vec<CompletionItem> = Vec::new();
         let mut menu = Menu::new(
             items,
@@ -99,8 +103,12 @@ impl Completion {
                             doc.apply(&remove, view.id);
                         }
 
-                        let transaction =
-                            util::generate_transaction_from_edits(doc.text(), vec![edit]);
+                        use helix_lsp::OffsetEncoding;
+                        let transaction = util::generate_transaction_from_edits(
+                            doc.text(),
+                            vec![edit],
+                            offset_encoding, // TODO: should probably transcode in Client
+                        );
                         doc.apply(&transaction, view.id);
                     }
                     _ => (),
diff --git a/helix-term/src/ui/editor.rs b/helix-term/src/ui/editor.rs
index 8879612c..227ccdaa 100644
--- a/helix-term/src/ui/editor.rs
+++ b/helix-term/src/ui/editor.rs
@@ -517,10 +517,11 @@ impl EditorView {
     pub fn set_completion(
         &mut self,
         items: Vec<helix_lsp::lsp::CompletionItem>,
+        offset_encoding: helix_lsp::OffsetEncoding,
         trigger_offset: usize,
         size: Rect,
     ) {
-        let mut completion = Completion::new(items, trigger_offset);
+        let mut completion = Completion::new(items, offset_encoding, trigger_offset);
         // TODO : propagate required size on resize to completion too
         completion.required_size((size.width, size.height));
         self.completion = Some(completion);
diff --git a/helix-term/src/ui/mod.rs b/helix-term/src/ui/mod.rs
index d9a05a9c..105ba7ad 100644
--- a/helix-term/src/ui/mod.rs
+++ b/helix-term/src/ui/mod.rs
@@ -20,7 +20,7 @@ pub use tui::layout::Rect;
 pub use tui::style::{Color, Modifier, Style};
 
 use helix_core::regex::Regex;
-use helix_view::{View, Document, Editor};
+use helix_view::{Document, Editor, View};
 
 use std::path::{Path, PathBuf};
 
diff --git a/helix-view/src/document.rs b/helix-view/src/document.rs
index 8e836f03..a04600b6 100644
--- a/helix-view/src/document.rs
+++ b/helix-view/src/document.rs
@@ -406,7 +406,7 @@ mod test {
 
     #[test]
     fn changeset_to_changes() {
-        use helix_lsp::{lsp, Client};
+        use helix_lsp::{lsp, Client, OffsetEncoding};
         let text = Rope::from("hello");
         let mut doc = Document::new(text);
         let view = ViewId::default();
@@ -417,7 +417,12 @@ mod test {
         let transaction = Transaction::insert(doc.text(), doc.selection(view), " world".into());
         let old_doc = doc.text().clone();
         doc.apply(&transaction, view);
-        let changes = Client::changeset_to_changes(&old_doc, doc.text(), transaction.changes());
+        let changes = Client::changeset_to_changes(
+            &old_doc,
+            doc.text(),
+            transaction.changes(),
+            OffsetEncoding::Utf8,
+        );
 
         assert_eq!(
             changes,
@@ -436,7 +441,12 @@ mod test {
         let transaction = transaction.invert(&old_doc);
         let old_doc = doc.text().clone();
         doc.apply(&transaction, view);
-        let changes = Client::changeset_to_changes(&old_doc, doc.text(), transaction.changes());
+        let changes = Client::changeset_to_changes(
+            &old_doc,
+            doc.text(),
+            transaction.changes(),
+            OffsetEncoding::Utf8,
+        );
 
         // line: 0-based.
         // col: 0-based, gaps between chars.
@@ -468,7 +478,12 @@ mod test {
         // aeilou
         let old_doc = doc.text().clone();
         doc.apply(&transaction, view);
-        let changes = Client::changeset_to_changes(&old_doc, doc.text(), transaction.changes());
+        let changes = Client::changeset_to_changes(
+            &old_doc,
+            doc.text(),
+            transaction.changes(),
+            OffsetEncoding::Utf8,
+        );
 
         assert_eq!(
             changes,