From a4f5a0134e308ffad318ab6920fe1ed0264a93cb Mon Sep 17 00:00:00 2001
From: Jan Hrastnik
Date: Mon, 14 Jun 2021 15:09:54 +0200
Subject: trying out line ending helper functions in commands.rs

---
 helix-term/src/commands.rs | 44 ++++++++++++++++++++++++++++++++++----------
 1 file changed, 34 insertions(+), 10 deletions(-)

(limited to 'helix-term/src/commands.rs')

diff --git a/helix-term/src/commands.rs b/helix-term/src/commands.rs
index c80716d4..144e1f3c 100644
--- a/helix-term/src/commands.rs
+++ b/helix-term/src/commands.rs
@@ -4,9 +4,14 @@ use helix_core::{
     movement::{self, Direction},
     object, pos_at_coords,
     regex::{self, Regex},
+<<<<<<< HEAD
     register::{self, Register, Registers},
     search, selection, Change, ChangeSet, Position, Range, Rope, RopeSlice, Selection, SmallVec,
     Tendril, Transaction,
+=======
+    register, search, selection, Change, ChangeSet, LineEnding, Position, Range, Rope, RopeSlice,
+    Selection, SmallVec, Tendril, Transaction,
+>>>>>>> 856fd95 (trying out line ending helper functions in commands.rs)
 };
 
 use helix_view::{
@@ -184,8 +189,14 @@ pub fn move_line_end(cx: &mut Context) {
         let line = text.char_to_line(range.head);
 
         // Line end is pos at the start of next line - 1
-        // subtract another 1 because the line ends with \n
-        let pos = text.line_to_char(line + 1).saturating_sub(2);
+        // subtract 3 if the line ending is \r\n, otherwise subtract 2 as I assume all others are just 1 char length
+        let pos =
+            text.line_to_char(line + 1)
+                .saturating_sub(if doc.line_ending == LineEnding::Crlf {
+                    3
+                } else {
+                    2
+                });
         Range::new(pos, pos)
     });
 
@@ -337,7 +348,7 @@ where
             KeyEvent {
                 code: KeyCode::Enter,
                 ..
-            } => '\n',
+            } => '\n', // TODO: we should be calling doc.line_ending() here
             KeyEvent {
                 code: KeyCode::Char(ch),
                 ..
@@ -465,7 +476,7 @@ pub fn replace(cx: &mut Context) {
             KeyEvent {
                 code: KeyCode::Enter,
                 ..
-            } => Some('\n'),
+            } => Some('\n'), // TODO: we should be calling doc.line_ending() here
             _ => None,
         };
 
@@ -606,8 +617,14 @@ pub fn extend_line_end(cx: &mut Context) {
         let line = text.char_to_line(range.head);
 
         // Line end is pos at the start of next line - 1
-        // subtract another 1 because the line ends with \n
-        let pos = text.line_to_char(line + 1).saturating_sub(2);
+        // subtract 3 if the line ending is \r\n, otherwise subtract 2 as I assume all others are just 1 char length
+        let pos =
+            text.line_to_char(line + 1)
+                .saturating_sub(if doc.line_ending == LineEnding::Crlf {
+                    3
+                } else {
+                    2
+                });
         Range::new(range.anchor, pos)
     });
 
@@ -896,7 +913,7 @@ pub fn append_mode(cx: &mut Context) {
     if selection.iter().any(|range| range.head == end) {
         let transaction = Transaction::change(
             doc.text(),
-            std::array::IntoIter::new([(end, end, Some(Tendril::from_char('\n')))]),
+            std::array::IntoIter::new([(end, end, Some(Tendril::from_char('\n')))]), // TODO: change \n to doc.line_ending()
         );
         doc.apply(&transaction, view.id);
     }
@@ -1523,7 +1540,7 @@ fn open(cx: &mut Context, open: Open) {
             );
             let indent = doc.indent_unit().repeat(indent_level);
             let mut text = String::with_capacity(1 + indent.len());
-            text.push('\n');
+            text.push_str(doc.line_ending());
             text.push_str(&indent);
             let text = text.repeat(count);
 
@@ -2131,7 +2148,7 @@ pub mod insert {
             );
             let indent = doc.indent_unit().repeat(indent_level);
             let mut text = String::with_capacity(1 + indent.len());
-            text.push('\n');
+            text.push_str(doc.line_ending());
             text.push_str(&indent);
 
             let head = pos + offs + text.chars().count();
@@ -2152,7 +2169,7 @@ pub mod insert {
             if helix_core::auto_pairs::PAIRS.contains(&(prev, curr)) {
                 // another newline, indent the end bracket one level less
                 let indent = doc.indent_unit().repeat(indent_level.saturating_sub(1));
-                text.push('\n');
+                text.push_str(doc.line_ending());
                 text.push_str(&indent);
             }
 
@@ -2268,8 +2285,15 @@ fn paste_impl(
             .unwrap(),
     );
 
+<<<<<<< HEAD
     // if any of values ends \n it's linewise paste
     let linewise = values.iter().any(|value| value.ends_with('\n'));
+=======
+        // if any of values ends \n it's linewise paste
+        let linewise = values
+            .iter()
+            .any(|value| value.ends_with(doc.line_ending()));
+>>>>>>> 856fd95 (trying out line ending helper functions in commands.rs)
 
     let mut values = values.iter().cloned().map(Tendril::from).chain(repeat);
 
-- 
cgit v1.2.3-70-g09d2


From 7cf0fa05a4ca6bf62d38836f3aa99a1ac585a261 Mon Sep 17 00:00:00 2001
From: Jan Hrastnik
Date: Wed, 16 Jun 2021 16:57:47 +0200
Subject: doc.line_ending() now returns &'static str

---
 helix-term/src/commands.rs | 4 ++--
 helix-view/src/document.rs | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'helix-term/src/commands.rs')

diff --git a/helix-term/src/commands.rs b/helix-term/src/commands.rs
index 144e1f3c..31e80345 100644
--- a/helix-term/src/commands.rs
+++ b/helix-term/src/commands.rs
@@ -348,7 +348,7 @@ where
             KeyEvent {
                 code: KeyCode::Enter,
                 ..
-            } => '\n', // TODO: we should be calling doc.line_ending() here
+            } => '\n', 
             KeyEvent {
                 code: KeyCode::Char(ch),
                 ..
@@ -476,7 +476,7 @@ pub fn replace(cx: &mut Context) {
             KeyEvent {
                 code: KeyCode::Enter,
                 ..
-            } => Some('\n'), // TODO: we should be calling doc.line_ending() here
+            } => Some('\n'),
             _ => None,
         };
 
diff --git a/helix-view/src/document.rs b/helix-view/src/document.rs
index 5f61f05a..c15a42ab 100644
--- a/helix-view/src/document.rs
+++ b/helix-view/src/document.rs
@@ -741,7 +741,7 @@ impl Document {
         self.diagnostics = diagnostics;
     }
 
-    pub fn line_ending(&self) -> &str {
+    pub fn line_ending(&self) -> &'static str {
         match self.line_ending {
             LineEnding::Crlf => "\u{000D}\u{000A}",
             LineEnding::LF => "\u{000A}",
-- 
cgit v1.2.3-70-g09d2


From 9c3eadb2e4fd297abcc8ceb02b3088ab3b9b1ceb Mon Sep 17 00:00:00 2001
From: Jan Hrastnik
Date: Wed, 16 Jun 2021 17:22:55 +0200
Subject: fixed some problems from rebasing

---
 helix-term/src/commands.rs | 22 ++++++----------------
 helix-view/src/document.rs | 17 ++---------------
 2 files changed, 8 insertions(+), 31 deletions(-)

(limited to 'helix-term/src/commands.rs')

diff --git a/helix-term/src/commands.rs b/helix-term/src/commands.rs
index 31e80345..644c2e23 100644
--- a/helix-term/src/commands.rs
+++ b/helix-term/src/commands.rs
@@ -4,14 +4,9 @@ use helix_core::{
     movement::{self, Direction},
     object, pos_at_coords,
     regex::{self, Regex},
-<<<<<<< HEAD
     register::{self, Register, Registers},
-    search, selection, Change, ChangeSet, Position, Range, Rope, RopeSlice, Selection, SmallVec,
-    Tendril, Transaction,
-=======
-    register, search, selection, Change, ChangeSet, LineEnding, Position, Range, Rope, RopeSlice,
-    Selection, SmallVec, Tendril, Transaction,
->>>>>>> 856fd95 (trying out line ending helper functions in commands.rs)
+    search, selection, Change, ChangeSet, LineEnding, Position, Range, Rope, RopeSlice, Selection,
+    SmallVec, Tendril, Transaction,
 };
 
 use helix_view::{
@@ -348,7 +343,7 @@ where
             KeyEvent {
                 code: KeyCode::Enter,
                 ..
-            } => '\n', 
+            } => '\n',
             KeyEvent {
                 code: KeyCode::Char(ch),
                 ..
@@ -2285,15 +2280,10 @@ fn paste_impl(
             .unwrap(),
     );
 
-<<<<<<< HEAD
     // if any of values ends \n it's linewise paste
-    let linewise = values.iter().any(|value| value.ends_with('\n'));
-=======
-        // if any of values ends \n it's linewise paste
-        let linewise = values
-            .iter()
-            .any(|value| value.ends_with(doc.line_ending()));
->>>>>>> 856fd95 (trying out line ending helper functions in commands.rs)
+    let linewise = values
+        .iter()
+        .any(|value| value.ends_with(doc.line_ending()));
 
     let mut values = values.iter().cloned().map(Tendril::from).chain(repeat);
 
diff --git a/helix-view/src/document.rs b/helix-view/src/document.rs
index c15a42ab..bf26d8a0 100644
--- a/helix-view/src/document.rs
+++ b/helix-view/src/document.rs
@@ -5,7 +5,9 @@ use std::path::{Component, Path, PathBuf};
 use std::sync::Arc;
 
 use helix_core::{
+    auto_detect_line_ending,
     chars::{char_is_linebreak, char_is_whitespace},
+    history::History,
     syntax::{LanguageConfiguration, LOADER},
     ChangeSet, Diagnostic, LineEnding, Rope, Selection, State, Syntax, Transaction,
     DEFAULT_LINE_ENDING,
@@ -28,21 +30,6 @@ pub enum IndentStyle {
     Spaces(u8),
 }
 
-/// Represents one of the valid Unicode line endings.
-/// Also acts as an index into `LINE_ENDINGS`.
-#[derive(PartialEq, Copy, Clone, Debug)]
-pub enum LineEnding {
-    None = 0, // No line ending
-    Crlf = 1, // CarriageReturn followed by LineFeed
-    LF = 2,   // U+000A -- LineFeed
-    VT = 3,   // U+000B -- VerticalTab
-    FF = 4,   // U+000C -- FormFeed
-    CR = 5,   // U+000D -- CarriageReturn
-    Nel = 6,  // U+0085 -- NextLine
-    LS = 7,   // U+2028 -- Line Separator
-    PS = 8,   // U+2029 -- ParagraphSeparator
-}
-
 pub struct Document {
     // rope + selection
     pub(crate) id: DocumentId,
-- 
cgit v1.2.3-70-g09d2


From 8bccd6df3054143baf128157d8dcecb10a911956 Mon Sep 17 00:00:00 2001
From: Jan Hrastnik
Date: Thu, 17 Jun 2021 13:49:50 +0200
Subject: applied changes from pr review

---
 helix-core/src/line_ending.rs | 23 +++++++++++++++++++++++
 helix-term/src/commands.rs    | 34 +++++++++++++---------------------
 helix-view/src/document.rs    | 16 +++-------------
 3 files changed, 39 insertions(+), 34 deletions(-)

(limited to 'helix-term/src/commands.rs')

diff --git a/helix-core/src/line_ending.rs b/helix-core/src/line_ending.rs
index 4f5708ec..f9d67b57 100644
--- a/helix-core/src/line_ending.rs
+++ b/helix-core/src/line_ending.rs
@@ -13,6 +13,29 @@ pub enum LineEnding {
     PS,   // U+2029 -- ParagraphSeparator
 }
 
+impl LineEnding {
+    pub fn len(&self) -> usize {
+        match self {
+            Self::Crlf => 2,
+            _ => 1,
+        }
+    }
+
+    pub fn as_str(&self) -> &str {
+        match self {
+            Self::Crlf => "\u{000D}\u{000A}",
+            Self::LF => "\u{000A}",
+            Self::Nel => "\u{0085}",
+            Self::LS => "\u{2028}",
+            Self::CR => "\u{000D}",
+            _ => panic!(
+                "Unexpected line ending: {:?}, expected Crlf, LF, CR, Nel, or LS.",
+                self
+            ),
+        }
+    }
+}
+
 pub fn rope_slice_to_line_ending(g: &RopeSlice) -> Option<LineEnding> {
     if let Some(text) = g.as_str() {
         str_to_line_ending(text)
diff --git a/helix-term/src/commands.rs b/helix-term/src/commands.rs
index 644c2e23..62faadf9 100644
--- a/helix-term/src/commands.rs
+++ b/helix-term/src/commands.rs
@@ -184,14 +184,10 @@ pub fn move_line_end(cx: &mut Context) {
         let line = text.char_to_line(range.head);
 
         // Line end is pos at the start of next line - 1
-        // subtract 3 if the line ending is \r\n, otherwise subtract 2 as I assume all others are just 1 char length
-        let pos =
-            text.line_to_char(line + 1)
-                .saturating_sub(if doc.line_ending == LineEnding::Crlf {
-                    3
-                } else {
-                    2
-                });
+        // subtract another 1 because the line ends with \n
+        let pos = text
+            .line_to_char(line + 1)
+            .saturating_sub(doc.line_ending().len() + 1);
         Range::new(pos, pos)
     });
 
@@ -612,14 +608,10 @@ pub fn extend_line_end(cx: &mut Context) {
         let line = text.char_to_line(range.head);
 
         // Line end is pos at the start of next line - 1
-        // subtract 3 if the line ending is \r\n, otherwise subtract 2 as I assume all others are just 1 char length
-        let pos =
-            text.line_to_char(line + 1)
-                .saturating_sub(if doc.line_ending == LineEnding::Crlf {
-                    3
-                } else {
-                    2
-                });
+        // subtract another 1 because the line ends with \n
+        let pos = text
+            .line_to_char(line + 1)
+            .saturating_sub(doc.line_ending().len() + 1);
         Range::new(range.anchor, pos)
     });
 
@@ -908,7 +900,7 @@ pub fn append_mode(cx: &mut Context) {
     if selection.iter().any(|range| range.head == end) {
         let transaction = Transaction::change(
             doc.text(),
-            std::array::IntoIter::new([(end, end, Some(Tendril::from_char('\n')))]), // TODO: change \n to doc.line_ending()
+            std::array::IntoIter::new([(end, end, Some(doc.line_ending().as_str().into()))]),
         );
         doc.apply(&transaction, view.id);
     }
@@ -1535,7 +1527,7 @@ fn open(cx: &mut Context, open: Open) {
             );
             let indent = doc.indent_unit().repeat(indent_level);
             let mut text = String::with_capacity(1 + indent.len());
-            text.push_str(doc.line_ending());
+            text.push_str(doc.line_ending().as_str());
             text.push_str(&indent);
             let text = text.repeat(count);
 
@@ -2143,7 +2135,7 @@ pub mod insert {
             );
             let indent = doc.indent_unit().repeat(indent_level);
             let mut text = String::with_capacity(1 + indent.len());
-            text.push_str(doc.line_ending());
+            text.push_str(doc.line_ending().as_str());
             text.push_str(&indent);
 
             let head = pos + offs + text.chars().count();
@@ -2164,7 +2156,7 @@ pub mod insert {
             if helix_core::auto_pairs::PAIRS.contains(&(prev, curr)) {
                 // another newline, indent the end bracket one level less
                 let indent = doc.indent_unit().repeat(indent_level.saturating_sub(1));
-                text.push_str(doc.line_ending());
+                text.push_str(doc.line_ending().as_str());
                 text.push_str(&indent);
             }
 
@@ -2283,7 +2275,7 @@ fn paste_impl(
     // if any of values ends \n it's linewise paste
     let linewise = values
         .iter()
-        .any(|value| value.ends_with(doc.line_ending()));
+        .any(|value| value.ends_with(doc.line_ending().as_str()));
 
     let mut values = values.iter().cloned().map(Tendril::from).chain(repeat);
 
diff --git a/helix-view/src/document.rs b/helix-view/src/document.rs
index bf26d8a0..fe06d09d 100644
--- a/helix-view/src/document.rs
+++ b/helix-view/src/document.rs
@@ -63,7 +63,7 @@ pub struct Document {
 
     diagnostics: Vec<Diagnostic>,
     language_server: Option<Arc<helix_lsp::Client>>,
-    pub line_ending: LineEnding,
+    line_ending: LineEnding,
 }
 
 use std::fmt;
@@ -728,18 +728,8 @@ impl Document {
         self.diagnostics = diagnostics;
     }
 
-    pub fn line_ending(&self) -> &'static str {
-        match self.line_ending {
-            LineEnding::Crlf => "\u{000D}\u{000A}",
-            LineEnding::LF => "\u{000A}",
-            LineEnding::Nel => "\u{0085}",
-            LineEnding::LS => "\u{2028}",
-            LineEnding::CR => "\u{000D}",
-            _ => panic!(
-                "Unexpected line ending: {:?}, expected Crlf, LF, CR, Nel, or LS.",
-                self.line_ending
-            ),
-        }
+    pub fn line_ending(&self) -> LineEnding {
+        self.line_ending
     }
 }
 
-- 
cgit v1.2.3-70-g09d2


From ecb884db98fbe6ed70743d1080ce7f78e121ba50 Mon Sep 17 00:00:00 2001
From: Jan Hrastnik
Date: Sat, 19 Jun 2021 14:03:14 +0200
Subject: added get_line_ending from pr comment

---
 helix-core/src/lib.rs         |  2 +-
 helix-core/src/line_ending.rs | 23 ++++++++++++++++++-----
 helix-term/src/commands.rs    | 12 +++++-------
 3 files changed, 24 insertions(+), 13 deletions(-)

(limited to 'helix-term/src/commands.rs')

diff --git a/helix-core/src/lib.rs b/helix-core/src/lib.rs
index 55365500..3f6bea5a 100644
--- a/helix-core/src/lib.rs
+++ b/helix-core/src/lib.rs
@@ -111,6 +111,6 @@ pub use diagnostic::Diagnostic;
 pub use state::State;
 
 pub use line_ending::{
-    auto_detect_line_ending, rope_slice_to_line_ending, LineEnding, DEFAULT_LINE_ENDING,
+    auto_detect_line_ending, rope_slice_to_line_ending, LineEnding, DEFAULT_LINE_ENDING, get_line_ending
 };
 pub use transaction::{Assoc, Change, ChangeSet, Operation, Transaction};
diff --git a/helix-core/src/line_ending.rs b/helix-core/src/line_ending.rs
index f9d67b57..423f4b92 100644
--- a/helix-core/src/line_ending.rs
+++ b/helix-core/src/line_ending.rs
@@ -14,7 +14,7 @@ pub enum LineEnding {
 }
 
 impl LineEnding {
-    pub fn len(&self) -> usize {
+    pub fn len_chars(&self) -> usize {
         match self {
             Self::Crlf => 2,
             _ => 1,
@@ -28,10 +28,9 @@ impl LineEnding {
             Self::Nel => "\u{0085}",
             Self::LS => "\u{2028}",
             Self::CR => "\u{000D}",
-            _ => panic!(
-                "Unexpected line ending: {:?}, expected Crlf, LF, CR, Nel, or LS.",
-                self
-            ),
+            Self::VT => "\u{000B}",
+            Self::FF => "\u{000C}",
+            Self::PS => "\u{2029}",
         }
     }
 }
@@ -93,6 +92,20 @@ pub fn auto_detect_line_ending(doc: &Rope) -> Option<LineEnding> {
     ending
 }
 
+/// Returns the passed line's line ending, if any.
+pub fn get_line_ending(line: &RopeSlice) -> Option<LineEnding> {
+    // Last character as str.
+    let g1 = line.slice(line.len_chars().saturating_sub(1)..).as_str().unwrap();
+
+    // Last two characters as str, or empty str if they're not contiguous.
+    // It's fine to punt on the non-contiguous case, because Ropey guarantees
+    // that CRLF is always contiguous.
+    let g2 = line.slice(line.len_chars().saturating_sub(2)..).as_str().unwrap_or("");
+
+    // First check the two-character case for CRLF, then check the single-character case.
+    str_to_line_ending(g2).or_else(|| str_to_line_ending(g1))
+}
+
 #[cfg(target_os = "windows")]
 pub const DEFAULT_LINE_ENDING: LineEnding = LineEnding::Crlf;
 #[cfg(not(target_os = "windows"))]
diff --git a/helix-term/src/commands.rs b/helix-term/src/commands.rs
index 62faadf9..d894a646 100644
--- a/helix-term/src/commands.rs
+++ b/helix-term/src/commands.rs
@@ -5,7 +5,7 @@ use helix_core::{
     object, pos_at_coords,
     regex::{self, Regex},
     register::{self, Register, Registers},
-    search, selection, Change, ChangeSet, LineEnding, Position, Range, Rope, RopeSlice, Selection,
+    search, selection, Change, ChangeSet, LineEnding, Position, Range, Rope, RopeSlice, Selection, get_line_ending,
     SmallVec, Tendril, Transaction,
 };
 
@@ -183,11 +183,10 @@ pub fn move_line_end(cx: &mut Context) {
         let text = doc.text();
         let line = text.char_to_line(range.head);
 
-        // Line end is pos at the start of next line - 1
-        // subtract another 1 because the line ends with \n
         let pos = text
             .line_to_char(line + 1)
-            .saturating_sub(doc.line_ending().len() + 1);
+            .saturating_sub(get_line_ending(&text.line(line)).map(|le| le.len_chars()).unwrap_or(0));
+
         Range::new(pos, pos)
     });
 
@@ -607,11 +606,10 @@ pub fn extend_line_end(cx: &mut Context) {
         let text = doc.text();
         let line = text.char_to_line(range.head);
 
-        // Line end is pos at the start of next line - 1
-        // subtract another 1 because the line ends with \n
         let pos = text
             .line_to_char(line + 1)
-            .saturating_sub(doc.line_ending().len() + 1);
+            .saturating_sub(get_line_ending(&text.line(line)).map(|le| le.len_chars()).unwrap_or(0));
+
         Range::new(range.anchor, pos)
     });
 
-- 
cgit v1.2.3-70-g09d2


From 97323dc2f90f81afc82bd929d111abda540bebe5 Mon Sep 17 00:00:00 2001
From: Jan Hrastnik
Date: Sat, 19 Jun 2021 14:05:11 +0200
Subject: ran cargo fmt

---
 helix-core/src/lib.rs         |  3 ++-
 helix-core/src/line_ending.rs | 10 ++++++++--
 helix-term/src/commands.rs    | 22 +++++++++++++---------
 3 files changed, 23 insertions(+), 12 deletions(-)

(limited to 'helix-term/src/commands.rs')

diff --git a/helix-core/src/lib.rs b/helix-core/src/lib.rs
index 3f6bea5a..c02d41f8 100644
--- a/helix-core/src/lib.rs
+++ b/helix-core/src/lib.rs
@@ -111,6 +111,7 @@ pub use diagnostic::Diagnostic;
 pub use state::State;
 
 pub use line_ending::{
-    auto_detect_line_ending, rope_slice_to_line_ending, LineEnding, DEFAULT_LINE_ENDING, get_line_ending
+    auto_detect_line_ending, get_line_ending, rope_slice_to_line_ending, LineEnding,
+    DEFAULT_LINE_ENDING,
 };
 pub use transaction::{Assoc, Change, ChangeSet, Operation, Transaction};
diff --git a/helix-core/src/line_ending.rs b/helix-core/src/line_ending.rs
index 423f4b92..2cc5b5d8 100644
--- a/helix-core/src/line_ending.rs
+++ b/helix-core/src/line_ending.rs
@@ -95,12 +95,18 @@ pub fn auto_detect_line_ending(doc: &Rope) -> Option<LineEnding> {
 /// Returns the passed line's line ending, if any.
 pub fn get_line_ending(line: &RopeSlice) -> Option<LineEnding> {
     // Last character as str.
-    let g1 = line.slice(line.len_chars().saturating_sub(1)..).as_str().unwrap();
+    let g1 = line
+        .slice(line.len_chars().saturating_sub(1)..)
+        .as_str()
+        .unwrap();
 
     // Last two characters as str, or empty str if they're not contiguous.
     // It's fine to punt on the non-contiguous case, because Ropey guarantees
     // that CRLF is always contiguous.
-    let g2 = line.slice(line.len_chars().saturating_sub(2)..).as_str().unwrap_or("");
+    let g2 = line
+        .slice(line.len_chars().saturating_sub(2)..)
+        .as_str()
+        .unwrap_or("");
 
     // First check the two-character case for CRLF, then check the single-character case.
     str_to_line_ending(g2).or_else(|| str_to_line_ending(g1))
diff --git a/helix-term/src/commands.rs b/helix-term/src/commands.rs
index d894a646..463869f6 100644
--- a/helix-term/src/commands.rs
+++ b/helix-term/src/commands.rs
@@ -1,11 +1,11 @@
 use helix_core::{
-    comment, coords_at_pos, find_first_non_whitespace_char, find_root, graphemes, indent,
-    match_brackets,
+    comment, coords_at_pos, find_first_non_whitespace_char, find_root, get_line_ending, graphemes,
+    indent, match_brackets,
     movement::{self, Direction},
     object, pos_at_coords,
     regex::{self, Regex},
     register::{self, Register, Registers},
-    search, selection, Change, ChangeSet, LineEnding, Position, Range, Rope, RopeSlice, Selection, get_line_ending,
+    search, selection, Change, ChangeSet, LineEnding, Position, Range, Rope, RopeSlice, Selection,
     SmallVec, Tendril, Transaction,
 };
 
@@ -183,9 +183,11 @@ pub fn move_line_end(cx: &mut Context) {
         let text = doc.text();
         let line = text.char_to_line(range.head);
 
-        let pos = text
-            .line_to_char(line + 1)
-            .saturating_sub(get_line_ending(&text.line(line)).map(|le| le.len_chars()).unwrap_or(0));
+        let pos = text.line_to_char(line + 1).saturating_sub(
+            get_line_ending(&text.line(line))
+                .map(|le| le.len_chars())
+                .unwrap_or(0),
+        );
 
         Range::new(pos, pos)
     });
@@ -606,9 +608,11 @@ pub fn extend_line_end(cx: &mut Context) {
         let text = doc.text();
         let line = text.char_to_line(range.head);
 
-        let pos = text
-            .line_to_char(line + 1)
-            .saturating_sub(get_line_ending(&text.line(line)).map(|le| le.len_chars()).unwrap_or(0));
+        let pos = text.line_to_char(line + 1).saturating_sub(
+            get_line_ending(&text.line(line))
+                .map(|le| le.len_chars())
+                .unwrap_or(0),
+        );
 
         Range::new(range.anchor, pos)
     });
-- 
cgit v1.2.3-70-g09d2


From 1e80fbb6020c75bdc5d7e3edc9513b4ed52c4e6b Mon Sep 17 00:00:00 2001
From: Jan Hrastnik
Date: Sat, 19 Jun 2021 14:58:49 +0200
Subject: fix merge issue

---
 helix-term/src/commands.rs | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

(limited to 'helix-term/src/commands.rs')

diff --git a/helix-term/src/commands.rs b/helix-term/src/commands.rs
index 07d2999b..af2e8eee 100644
--- a/helix-term/src/commands.rs
+++ b/helix-term/src/commands.rs
@@ -1668,16 +1668,16 @@ fn open(cx: &mut Context, open: Open) {
     let mut ranges = SmallVec::with_capacity(selection.len());
     let mut offs = 0;
 
-    let line = match open {
-        // adjust position to the end of the line (next line - 1)
-        Open::Below => line + 1,
-        // adjust position to the end of the previous line (current line - 1)
-        Open::Above => line,
-    };
-
     let mut transaction = Transaction::change_by_selection(contents, selection, |range| {
         let line = text.char_to_line(range.head);
 
+        let line = match open {
+            // adjust position to the end of the line (next line - 1)
+            Open::Below => line + 1,
+            // adjust position to the end of the previous line (current line - 1)
+            Open::Above => line,
+        };
+
         // insert newlines after this index for both Above and Below variants
         let linend_index = doc.text().line_to_char(line).saturating_sub(1);
 
-- 
cgit v1.2.3-70-g09d2


From 701eb0dd6800e75116f36e503787dd0f50df709e Mon Sep 17 00:00:00 2001
From: Jan Hrastnik
Date: Sun, 20 Jun 2021 01:24:36 +0200
Subject: changed some hardcoded newlines, removed a else if in line_ending.rs

---
 helix-core/src/line_ending.rs |  5 ++---
 helix-core/src/movement.rs    |  6 ++++--
 helix-term/src/commands.rs    | 20 +++++++++++++++-----
 helix-view/src/document.rs    |  2 +-
 4 files changed, 22 insertions(+), 11 deletions(-)

(limited to 'helix-term/src/commands.rs')

diff --git a/helix-core/src/line_ending.rs b/helix-core/src/line_ending.rs
index 2cc5b5d8..33f8d078 100644
--- a/helix-core/src/line_ending.rs
+++ b/helix-core/src/line_ending.rs
@@ -38,9 +38,8 @@ impl LineEnding {
 pub fn rope_slice_to_line_ending(g: &RopeSlice) -> Option<LineEnding> {
     if let Some(text) = g.as_str() {
         str_to_line_ending(text)
-    } else if g == "\u{000D}\u{000A}" {
-        Some(LineEnding::Crlf)
-    } else {
+    }
+    else {
         // Not a line ending
         None
     }
diff --git a/helix-core/src/movement.rs b/helix-core/src/movement.rs
index 8b1e802f..cf7ea854 100644
--- a/helix-core/src/movement.rs
+++ b/helix-core/src/movement.rs
@@ -5,7 +5,7 @@ use ropey::iter::Chars;
 use crate::{
     coords_at_pos,
     graphemes::{nth_next_grapheme_boundary, nth_prev_grapheme_boundary},
-    pos_at_coords, Position, Range, RopeSlice,
+    pos_at_coords, Position, Range, RopeSlice, get_line_ending
 };
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
@@ -38,7 +38,9 @@ pub fn move_horizontally(
         }
         Direction::Forward => {
             // Line end is pos at the start of next line - 1
-            let end = slice.line_to_char(line + 1).saturating_sub(1);
+            let end = slice.line_to_char(line + 1).saturating_sub(get_line_ending(&slice.line(line))
+                .map(|le| le.len_chars())
+                .unwrap_or(0));
             nth_next_grapheme_boundary(slice, pos, count).min(end)
         }
     };
diff --git a/helix-term/src/commands.rs b/helix-term/src/commands.rs
index af2e8eee..cee0e752 100644
--- a/helix-term/src/commands.rs
+++ b/helix-term/src/commands.rs
@@ -6,7 +6,7 @@ use helix_core::{
     regex::{self, Regex},
     register::{self, Register, Registers},
     search, selection, Change, ChangeSet, LineEnding, Position, Range, Rope, RopeSlice, Selection,
-    SmallVec, Tendril, Transaction,
+    SmallVec, Tendril, Transaction, DEFAULT_LINE_ENDING,
 };
 
 use helix_view::{
@@ -627,7 +627,7 @@ fn replace(cx: &mut Context) {
             KeyEvent {
                 code: KeyCode::Enter,
                 ..
-            } => Some('\n'),
+            } => Some('\n'), // TODO: replace this with DEFAULT_LINE_ENDING
             _ => None,
         };
 
@@ -982,7 +982,13 @@ fn delete_selection_impl(reg: &mut Register, doc: &mut Document, view_id: ViewId
     // then delete
     let transaction =
         Transaction::change_by_selection(doc.text(), doc.selection(view_id), |range| {
-            let max_to = doc.text().len_chars().saturating_sub(1);
+            let alltext = doc.text();
+            let line = alltext.char_to_line(range.head);
+            let max_to = doc.text().len_chars().saturating_sub(
+                get_line_ending(&alltext.line(line))
+                    .map(|le| le.len_chars())
+                    .unwrap_or(0),
+            );
             let to = std::cmp::min(max_to, range.to() + 1);
             (range.from(), to, None)
         });
@@ -1644,8 +1650,12 @@ fn append_to_line(cx: &mut Context) {
     let selection = doc.selection(view.id).transform(|range| {
         let text = doc.text();
         let line = text.char_to_line(range.head);
-        // we can't use line_to_char(line + 1) - 2 because the last line might not contain \n
-        let pos = (text.line_to_char(line) + text.line(line).len_chars()).saturating_sub(1);
+        // we can't use line_to_char(line + 1) - 2 because the last line might not contain a newline
+        let pos = (text.line_to_char(line) + text.line(line).len_chars()).saturating_sub(
+            get_line_ending(&text.line(line))
+                .map(|le| le.len_chars())
+                .unwrap_or(0),
+        );
         Range::new(pos, pos)
     });
     doc.set_selection(view.id, selection);
diff --git a/helix-view/src/document.rs b/helix-view/src/document.rs
index 49d270e4..44d50583 100644
--- a/helix-view/src/document.rs
+++ b/helix-view/src/document.rs
@@ -255,7 +255,7 @@ impl Document {
         use std::{fs::File, io::BufReader};
 
         let doc = if !path.exists() {
-            Rope::from("\n")
+            Rope::from(DEFAULT_LINE_ENDING.as_str()) 
         } else {
             let file = File::open(&path).context(format!("unable to open {:?}", path))?;
             let mut doc = Rope::from_reader(BufReader::new(file))?;
-- 
cgit v1.2.3-70-g09d2


From 8634e04a31f8f761b3d0505528295d31d63c7918 Mon Sep 17 00:00:00 2001
From: Jan Hrastnik
Date: Sun, 20 Jun 2021 02:22:10 +0200
Subject: added the line_end helper function

---
 helix-core/src/lib.rs         |  2 +-
 helix-core/src/line_ending.rs |  6 ++++++
 helix-core/src/movement.rs    |  6 ++----
 helix-term/src/commands.rs    | 21 ++++-----------------
 4 files changed, 13 insertions(+), 22 deletions(-)

(limited to 'helix-term/src/commands.rs')

diff --git a/helix-core/src/lib.rs b/helix-core/src/lib.rs
index 9ac506a6..d99bb66d 100644
--- a/helix-core/src/lib.rs
+++ b/helix-core/src/lib.rs
@@ -114,6 +114,6 @@ pub use state::State;
 
 pub use line_ending::{
     auto_detect_line_ending, get_line_ending, rope_slice_to_line_ending, LineEnding,
-    DEFAULT_LINE_ENDING,
+    DEFAULT_LINE_ENDING, line_end
 };
 pub use transaction::{Assoc, Change, ChangeSet, Operation, Transaction};
diff --git a/helix-core/src/line_ending.rs b/helix-core/src/line_ending.rs
index 33f8d078..f6118493 100644
--- a/helix-core/src/line_ending.rs
+++ b/helix-core/src/line_ending.rs
@@ -111,6 +111,12 @@ pub fn get_line_ending(line: &RopeSlice) -> Option<LineEnding> {
     str_to_line_ending(g2).or_else(|| str_to_line_ending(g1))
 }
 
+pub fn line_end(slice: &RopeSlice, line: usize) -> usize {
+    slice.line_to_char(line + 1).saturating_sub(get_line_ending(&slice.line(line))
+        .map(|le| le.len_chars())
+        .unwrap_or(0))
+}
+
 #[cfg(target_os = "windows")]
 pub const DEFAULT_LINE_ENDING: LineEnding = LineEnding::Crlf;
 #[cfg(not(target_os = "windows"))]
diff --git a/helix-core/src/movement.rs b/helix-core/src/movement.rs
index cf7ea854..a3cd9b96 100644
--- a/helix-core/src/movement.rs
+++ b/helix-core/src/movement.rs
@@ -5,7 +5,7 @@ use ropey::iter::Chars;
 use crate::{
     coords_at_pos,
     graphemes::{nth_next_grapheme_boundary, nth_prev_grapheme_boundary},
-    pos_at_coords, Position, Range, RopeSlice, get_line_ending
+    pos_at_coords, Position, Range, RopeSlice, get_line_ending, line_end
 };
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
@@ -38,9 +38,7 @@ pub fn move_horizontally(
         }
         Direction::Forward => {
             // Line end is pos at the start of next line - 1
-            let end = slice.line_to_char(line + 1).saturating_sub(get_line_ending(&slice.line(line))
-                .map(|le| le.len_chars())
-                .unwrap_or(0));
+            let end = line_end(&slice, line);
             nth_next_grapheme_boundary(slice, pos, count).min(end)
         }
     };
diff --git a/helix-term/src/commands.rs b/helix-term/src/commands.rs
index cee0e752..8124c17a 100644
--- a/helix-term/src/commands.rs
+++ b/helix-term/src/commands.rs
@@ -1,6 +1,6 @@
 use helix_core::{
     comment, coords_at_pos, find_first_non_whitespace_char, find_root, get_line_ending, graphemes,
-    indent, match_brackets,
+    indent, line_end, match_brackets,
     movement::{self, Direction},
     object, pos_at_coords,
     regex::{self, Regex},
@@ -342,11 +342,7 @@ fn move_line_end(cx: &mut Context) {
         let text = doc.text();
         let line = text.char_to_line(range.head);
 
-        let pos = text.line_to_char(line + 1).saturating_sub(
-            get_line_ending(&text.line(line))
-                .map(|le| le.len_chars())
-                .unwrap_or(0),
-        );
+        let pos = line_end(&text.slice(..), line);
 
         Range::new(pos, pos)
     });
@@ -767,11 +763,7 @@ fn extend_line_end(cx: &mut Context) {
         let text = doc.text();
         let line = text.char_to_line(range.head);
 
-        let pos = text.line_to_char(line + 1).saturating_sub(
-            get_line_ending(&text.line(line))
-                .map(|le| le.len_chars())
-                .unwrap_or(0),
-        );
+        let pos = line_end(&text.slice(..), line);
 
         Range::new(range.anchor, pos)
     });
@@ -1650,12 +1642,7 @@ fn append_to_line(cx: &mut Context) {
     let selection = doc.selection(view.id).transform(|range| {
         let text = doc.text();
         let line = text.char_to_line(range.head);
-        // we can't use line_to_char(line + 1) - 2 because the last line might not contain a newline
-        let pos = (text.line_to_char(line) + text.line(line).len_chars()).saturating_sub(
-            get_line_ending(&text.line(line))
-                .map(|le| le.len_chars())
-                .unwrap_or(0),
-        );
+        let pos = line_end(&text.slice(..), line);
         Range::new(pos, pos)
     });
     doc.set_selection(view.id, selection);
-- 
cgit v1.2.3-70-g09d2


From 4efd6713c5b30b33c497a1f85b77a7b0a7fd17e0 Mon Sep 17 00:00:00 2001
From: Nathan Vegdahl
Date: Sun, 20 Jun 2021 15:09:10 -0700
Subject: Work on moving code over to LineEnding instead of assuming '\n'.

Also some general cleanup and some minor fixes along the way.
---
 Cargo.lock                      |   1 +
 helix-core/src/auto_pairs.rs    |   2 +-
 helix-core/src/chars.rs         | 122 +++++++++++++++++++++++++----
 helix-core/src/lib.rs           |   2 +-
 helix-core/src/line_ending.rs   |  47 ++++++++---
 helix-core/src/movement.rs      | 167 +++++++++-------------------------------
 helix-core/src/position.rs      |   6 +-
 helix-core/src/syntax.rs        |   9 ++-
 helix-lsp/src/client.rs         |   7 +-
 helix-term/src/commands.rs      |  12 +--
 helix-term/src/ui/markdown.rs   |   2 +
 helix-tui/Cargo.toml            |   1 +
 helix-tui/src/text.rs           |   3 +-
 helix-tui/src/widgets/reflow.rs |   9 ++-
 helix-view/src/document.rs      |  24 +++---
 helix-view/src/editor.rs        |   4 +-
 16 files changed, 228 insertions(+), 190 deletions(-)

(limited to 'helix-term/src/commands.rs')

diff --git a/Cargo.lock b/Cargo.lock
index 24c277e1..a1de7138 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -331,6 +331,7 @@ dependencies = [
  "bitflags",
  "cassowary",
  "crossterm",
+ "helix-core",
  "serde",
  "unicode-segmentation",
  "unicode-width",
diff --git a/helix-core/src/auto_pairs.rs b/helix-core/src/auto_pairs.rs
index 74e25ac9..746f201a 100644
--- a/helix-core/src/auto_pairs.rs
+++ b/helix-core/src/auto_pairs.rs
@@ -12,7 +12,7 @@ pub const PAIRS: &[(char, char)] = &[
     ('`', '`'),
 ];
 
-const CLOSE_BEFORE: &str = ")]}'\":;> \n"; // includes space and newline
+const CLOSE_BEFORE: &str = ")]}'\":;> \n\r\u{000B}\u{000C}\u{0085}\u{2028}\u{2029}"; // includes space and newlines
 
 // insert hook:
 // Fn(doc, selection, char) => Option<Transaction>
diff --git a/helix-core/src/chars.rs b/helix-core/src/chars.rs
index 243a1374..24133dd3 100644
--- a/helix-core/src/chars.rs
+++ b/helix-core/src/chars.rs
@@ -1,25 +1,44 @@
-/// Determine whether a character is a line break.
-pub fn char_is_linebreak(c: char) -> bool {
-    matches!(
-        c,
-        '\u{000A}' | // LineFeed
-        '\u{000B}' | // VerticalTab
-        '\u{000C}' | // FormFeed
-        '\u{000D}' | // CarriageReturn
-        '\u{0085}' | // NextLine
-        '\u{2028}' | // Line Separator
-        '\u{2029}' // ParagraphSeparator
-    )
+use crate::LineEnding;
+
+#[derive(Debug, Eq, PartialEq)]
+pub enum CharCategory {
+    Whitespace,
+    Eol,
+    Word,
+    Punctuation,
+    Unknown,
+}
+
+#[inline]
+pub fn categorize_char(ch: char) -> CharCategory {
+    if char_is_line_ending(ch) {
+        CharCategory::Eol
+    } else if ch.is_whitespace() {
+        CharCategory::Whitespace
+    } else if char_is_word(ch) {
+        CharCategory::Word
+    } else if char_is_punctuation(ch) {
+        CharCategory::Punctuation
+    } else {
+        CharCategory::Unknown
+    }
+}
+
+/// Determine whether a character is a line ending.
+#[inline]
+pub fn char_is_line_ending(ch: char) -> bool {
+    LineEnding::from_char(ch).is_some()
 }
 
 /// Determine whether a character qualifies as (non-line-break)
 /// whitespace.
-pub fn char_is_whitespace(c: char) -> bool {
+#[inline]
+pub fn char_is_whitespace(ch: char) -> bool {
     // TODO: this is a naive binary categorization of whitespace
     // characters.  For display, word wrapping, etc. we'll need a better
     // categorization based on e.g. breaking vs non-breaking spaces
     // and whether they're zero-width or not.
-    match c {
+    match ch {
         //'\u{1680}' | // Ogham Space Mark (here for completeness, but usually displayed as a dash, not as whitespace)
         '\u{0009}' | // Character Tabulation
         '\u{0020}' | // Space
@@ -34,8 +53,81 @@ pub fn char_is_whitespace(c: char) -> bool {
         // En Quad, Em Quad, En Space, Em Space, Three-per-em Space,
         // Four-per-em Space, Six-per-em Space, Figure Space,
         // Punctuation Space, Thin Space, Hair Space, Zero Width Space.
-        c if ('\u{2000}' ..= '\u{200B}').contains(&c) => true,
+        ch if ('\u{2000}' ..= '\u{200B}').contains(&ch) => true,
 
         _ => false,
     }
 }
+
+#[inline]
+pub fn char_is_punctuation(ch: char) -> bool {
+    use unicode_general_category::{get_general_category, GeneralCategory};
+
+    matches!(
+        get_general_category(ch),
+        GeneralCategory::OtherPunctuation
+            | GeneralCategory::OpenPunctuation
+            | GeneralCategory::ClosePunctuation
+            | GeneralCategory::InitialPunctuation
+            | GeneralCategory::FinalPunctuation
+            | GeneralCategory::ConnectorPunctuation
+            | GeneralCategory::DashPunctuation
+            | GeneralCategory::MathSymbol
+            | GeneralCategory::CurrencySymbol
+            | GeneralCategory::ModifierSymbol
+    )
+}
+
+#[inline]
+pub fn char_is_word(ch: char) -> bool {
+    ch.is_alphanumeric() || ch == '_'
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_categorize() {
+        const EOL_TEST_CASE: &'static str = "\n\r\u{000B}\u{000C}\u{0085}\u{2028}\u{2029}";
+        const WORD_TEST_CASE: &'static str =
+            "_hello_world_あいうえおー1234567890１２３４５６７８９０";
+        const PUNCTUATION_TEST_CASE: &'static str =
+            "!\"#$%&\'()*+,-./:;<=>?@[\\]^`{|}~！”＃＄％＆’（）＊＋、。：；＜＝＞？＠「」＾｀｛｜｝～";
+        const WHITESPACE_TEST_CASE: &'static str = "  　   ";
+
+        for ch in EOL_TEST_CASE.chars() {
+            assert_eq!(CharCategory::Eol, categorize_char(ch));
+        }
+
+        for ch in WHITESPACE_TEST_CASE.chars() {
+            assert_eq!(
+                CharCategory::Whitespace,
+                categorize_char(ch),
+                "Testing '{}', but got `{:?}` instead of `Category::Whitespace`",
+                ch,
+                categorize_char(ch)
+            );
+        }
+
+        for ch in WORD_TEST_CASE.chars() {
+            assert_eq!(
+                CharCategory::Word,
+                categorize_char(ch),
+                "Testing '{}', but got `{:?}` instead of `Category::Word`",
+                ch,
+                categorize_char(ch)
+            );
+        }
+
+        for ch in PUNCTUATION_TEST_CASE.chars() {
+            assert_eq!(
+                CharCategory::Punctuation,
+                categorize_char(ch),
+                "Testing '{}', but got `{:?}` instead of `Category::Punctuation`",
+                ch,
+                categorize_char(ch)
+            );
+        }
+    }
+}
diff --git a/helix-core/src/lib.rs b/helix-core/src/lib.rs
index e00e56be..183b9f0a 100644
--- a/helix-core/src/lib.rs
+++ b/helix-core/src/lib.rs
@@ -113,6 +113,6 @@ pub use diagnostic::Diagnostic;
 pub use state::State;
 
 pub use line_ending::{
-    auto_detect_line_ending, get_line_ending, line_end, LineEnding, DEFAULT_LINE_ENDING,
+    auto_detect_line_ending, get_line_ending, line_end_char_index, LineEnding, DEFAULT_LINE_ENDING,
 };
 pub use transaction::{Assoc, Change, ChangeSet, Operation, Transaction};
diff --git a/helix-core/src/line_ending.rs b/helix-core/src/line_ending.rs
index 45e20c88..c4636c63 100644
--- a/helix-core/src/line_ending.rs
+++ b/helix-core/src/line_ending.rs
@@ -1,5 +1,10 @@
 use crate::{Rope, RopeGraphemes, RopeSlice};
 
+#[cfg(target_os = "windows")]
+pub const DEFAULT_LINE_ENDING: LineEnding = LineEnding::Crlf;
+#[cfg(not(target_os = "windows"))]
+pub const DEFAULT_LINE_ENDING: LineEnding = LineEnding::LF;
+
 /// Represents one of the valid Unicode line endings.
 #[derive(PartialEq, Copy, Clone, Debug)]
 pub enum LineEnding {
@@ -14,6 +19,7 @@ pub enum LineEnding {
 }
 
 impl LineEnding {
+    #[inline]
     pub fn len_chars(&self) -> usize {
         match self {
             Self::Crlf => 2,
@@ -21,6 +27,7 @@ impl LineEnding {
         }
     }
 
+    #[inline]
     pub fn as_str(&self) -> &'static str {
         match self {
             Self::Crlf => "\u{000D}\u{000A}",
@@ -34,6 +41,22 @@ impl LineEnding {
         }
     }
 
+    #[inline]
+    pub fn from_char(ch: char) -> Option<LineEnding> {
+        match ch {
+            '\u{000A}' => Some(LineEnding::LF),
+            '\u{000B}' => Some(LineEnding::VT),
+            '\u{000C}' => Some(LineEnding::FF),
+            '\u{000D}' => Some(LineEnding::CR),
+            '\u{0085}' => Some(LineEnding::Nel),
+            '\u{2028}' => Some(LineEnding::LS),
+            '\u{2029}' => Some(LineEnding::PS),
+            // Not a line ending
+            _ => None,
+        }
+    }
+
+    #[inline]
     pub fn from_str(g: &str) -> Option<LineEnding> {
         match g {
             "\u{000D}\u{000A}" => Some(LineEnding::Crlf),
@@ -49,6 +72,7 @@ impl LineEnding {
         }
     }
 
+    #[inline]
     pub fn from_rope_slice(g: &RopeSlice) -> Option<LineEnding> {
         if let Some(text) = g.as_str() {
             LineEnding::from_str(text)
@@ -62,6 +86,11 @@ impl LineEnding {
     }
 }
 
+#[inline]
+pub fn str_is_line_ending(s: &str) -> bool {
+    LineEnding::from_str(s).is_some()
+}
+
 /// Attempts to detect what line ending the passed document uses.
 pub fn auto_detect_line_ending(doc: &Rope) -> Option<LineEnding> {
     // Return first matched line ending. Not all possible line endings
@@ -96,19 +125,13 @@ pub fn get_line_ending(line: &RopeSlice) -> Option<LineEnding> {
 }
 
 /// Returns the char index of the end of the given line, not including its line ending.
-pub fn line_end(slice: &RopeSlice, line: usize) -> usize {
-    slice.line_to_char(line + 1).saturating_sub(
-        get_line_ending(&slice.line(line))
+pub fn line_end_char_index(slice: &RopeSlice, line: usize) -> usize {
+    slice.line_to_char(line + 1)
+        - get_line_ending(&slice.line(line))
             .map(|le| le.len_chars())
-            .unwrap_or(0),
-    )
+            .unwrap_or(0)
 }
 
-#[cfg(target_os = "windows")]
-pub const DEFAULT_LINE_ENDING: LineEnding = LineEnding::Crlf;
-#[cfg(not(target_os = "windows"))]
-pub const DEFAULT_LINE_ENDING: LineEnding = LineEnding::LF;
-
 #[cfg(test)]
 mod line_ending_tests {
     use super::*;
@@ -150,11 +173,11 @@ mod line_ending_tests {
     fn test_rope_slice_to_line_ending() {
         let r = Rope::from_str("\r\n");
         assert_eq!(
-            rope_slice_to_line_ending(&r.slice(1..2)),
+            LineEnding::from_rope_slice(&r.slice(1..2)),
             Some(LineEnding::LF)
         );
         assert_eq!(
-            rope_slice_to_line_ending(&r.slice(0..2)),
+            LineEnding::from_rope_slice(&r.slice(0..2)),
             Some(LineEnding::Crlf)
         );
     }
diff --git a/helix-core/src/movement.rs b/helix-core/src/movement.rs
index 7f47e662..d0023e9f 100644
--- a/helix-core/src/movement.rs
+++ b/helix-core/src/movement.rs
@@ -3,9 +3,13 @@ use std::iter::{self, from_fn, Peekable, SkipWhile};
 use ropey::iter::Chars;
 
 use crate::{
+    chars::{
+        categorize_char, char_is_line_ending, char_is_punctuation, char_is_whitespace,
+        char_is_word, CharCategory,
+    },
     coords_at_pos, get_line_ending,
     graphemes::{nth_next_grapheme_boundary, nth_prev_grapheme_boundary},
-    line_end, pos_at_coords, Position, Range, RopeSlice,
+    line_end_char_index, pos_at_coords, Position, Range, RopeSlice,
 };
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
@@ -37,9 +41,8 @@ pub fn move_horizontally(
             nth_prev_grapheme_boundary(slice, pos, count).max(start)
         }
         Direction::Forward => {
-            // Line end is pos at the start of next line - 1
-            let end = line_end(&slice, line);
-            nth_next_grapheme_boundary(slice, pos, count).min(end)
+            let end_char_idx = line_end_char_index(&slice, line);
+            nth_next_grapheme_boundary(slice, pos, count).min(end_char_idx)
         }
     };
     let anchor = match behaviour {
@@ -68,8 +71,11 @@ pub fn move_vertically(
         ),
     };
 
-    // convert to 0-indexed, subtract another 1 because len_chars() counts \n
-    let new_line_len = slice.line(new_line).len_chars().saturating_sub(2);
+    // Length of the line sans line-ending.
+    let new_line_len = {
+        let line = slice.line(new_line);
+        line.len_chars() - get_line_ending(&line).map(|le| le.len_chars()).unwrap_or(0)
+    };
 
     let new_col = std::cmp::min(horiz as usize, new_line_len);
 
@@ -104,64 +110,6 @@ fn word_move(slice: RopeSlice, mut range: Range, count: usize, target: WordMotio
 }
 
 // ---- util ------------
-#[inline]
-pub(crate) fn is_word(ch: char) -> bool {
-    ch.is_alphanumeric() || ch == '_'
-}
-
-#[inline]
-pub(crate) fn is_end_of_line(ch: char) -> bool {
-    ch == '\n'
-}
-
-#[inline]
-// Whitespace, but not end of line
-pub(crate) fn is_strict_whitespace(ch: char) -> bool {
-    ch.is_whitespace() && !is_end_of_line(ch)
-}
-
-#[inline]
-pub(crate) fn is_punctuation(ch: char) -> bool {
-    use unicode_general_category::{get_general_category, GeneralCategory};
-
-    matches!(
-        get_general_category(ch),
-        GeneralCategory::OtherPunctuation
-            | GeneralCategory::OpenPunctuation
-            | GeneralCategory::ClosePunctuation
-            | GeneralCategory::InitialPunctuation
-            | GeneralCategory::FinalPunctuation
-            | GeneralCategory::ConnectorPunctuation
-            | GeneralCategory::DashPunctuation
-            | GeneralCategory::MathSymbol
-            | GeneralCategory::CurrencySymbol
-            | GeneralCategory::ModifierSymbol
-    )
-}
-
-#[derive(Debug, Eq, PartialEq)]
-pub enum Category {
-    Whitespace,
-    Eol,
-    Word,
-    Punctuation,
-    Unknown,
-}
-
-#[inline]
-pub(crate) fn categorize(ch: char) -> Category {
-    if is_end_of_line(ch) {
-        Category::Eol
-    } else if ch.is_whitespace() {
-        Category::Whitespace
-    } else if is_word(ch) {
-        Category::Word
-    } else if is_punctuation(ch) {
-        Category::Punctuation
-    } else {
-        Category::Unknown
-    }
-}
 
 #[inline]
 /// Returns first index that doesn't satisfy a given predicate when
@@ -235,7 +183,8 @@ impl CharHelpers for Chars<'_> {
         let mut phase = WordMotionPhase::Start;
         let mut head = origin.head;
         let mut anchor: Option<usize> = None;
-        let is_boundary = |a: char, b: Option<char>| categorize(a) != categorize(b.unwrap_or(a));
+        let is_boundary =
+            |a: char, b: Option<char>| categorize_char(a) != categorize_char(b.unwrap_or(a));
         while let Some(peek) = characters.peek().copied() {
             phase = match phase {
                 WordMotionPhase::Start => {
@@ -244,7 +193,8 @@ impl CharHelpers for Chars<'_> {
                         break; // We're at the end, so there's nothing to do.
                     }
                     // Anchor may remain here if the head wasn't at a boundary
-                    if !is_boundary(peek, characters.peek().copied()) && !is_end_of_line(peek) {
+                    if !is_boundary(peek, characters.peek().copied()) && !char_is_line_ending(peek)
+                    {
                         anchor = Some(head);
                     }
                     // First character is always skipped by the head
@@ -252,7 +202,7 @@ impl CharHelpers for Chars<'_> {
                     WordMotionPhase::SkipNewlines
                 }
                 WordMotionPhase::SkipNewlines => {
-                    if is_end_of_line(peek) {
+                    if char_is_line_ending(peek) {
                         characters.next();
                         if characters.peek().is_some() {
                             advance(&mut head);
@@ -286,12 +236,12 @@ fn reached_target(target: WordMotionTarget, peek: char, next_peek: Option<&char>
 
     match target {
         WordMotionTarget::NextWordStart => {
-            ((categorize(peek) != categorize(*next_peek))
-                && (is_end_of_line(*next_peek) || !next_peek.is_whitespace()))
+            ((categorize_char(peek) != categorize_char(*next_peek))
+                && (char_is_line_ending(*next_peek) || !next_peek.is_whitespace()))
         }
         WordMotionTarget::NextWordEnd | WordMotionTarget::PrevWordStart => {
-            ((categorize(peek) != categorize(*next_peek))
-                && (!peek.is_whitespace() || is_end_of_line(*next_peek)))
+            ((categorize_char(peek) != categorize_char(*next_peek))
+                && (!peek.is_whitespace() || char_is_line_ending(*next_peek)))
         }
     }
 }
@@ -330,7 +280,7 @@ mod test {
                 slice,
                 move_vertically(slice, range, Direction::Forward, 1, Movement::Move).head
             ),
-            (1, 2).into()
+            (1, 3).into()
         );
     }
 
@@ -343,12 +293,12 @@ mod test {
         let mut range = Range::point(position);
 
         let moves_and_expected_coordinates = [
-            ((Direction::Forward, 1usize), (0, 1)),
-            ((Direction::Forward, 2usize), (0, 3)),
-            ((Direction::Forward, 0usize), (0, 3)),
-            ((Direction::Forward, 999usize), (0, 31)),
-            ((Direction::Forward, 999usize), (0, 31)),
-            ((Direction::Backward, 999usize), (0, 0)),
+            ((Direction::Forward, 1usize), (0, 1)), // T|his is a simple alphabetic line
+            ((Direction::Forward, 2usize), (0, 3)), // Thi|s is a simple alphabetic line
+            ((Direction::Forward, 0usize), (0, 3)), // Thi|s is a simple alphabetic line
+            ((Direction::Forward, 999usize), (0, 32)), // This is a simple alphabetic line|
+            ((Direction::Forward, 999usize), (0, 32)), // This is a simple alphabetic line|
+            ((Direction::Backward, 999usize), (0, 0)), // |This is a simple alphabetic line
         ];
 
         for ((direction, amount), coordinates) in IntoIter::new(moves_and_expected_coordinates) {
@@ -366,15 +316,15 @@ mod test {
         let mut range = Range::point(position);
 
         let moves_and_expected_coordinates = IntoIter::new([
-            ((Direction::Forward, 1usize), (0, 1)),    // M_ltiline
-            ((Direction::Forward, 2usize), (0, 3)),    // Mul_iline
-            ((Direction::Backward, 6usize), (0, 0)),   // _ultiline
-            ((Direction::Backward, 999usize), (0, 0)), // _ultiline
-            ((Direction::Forward, 3usize), (0, 3)),    // Mul_iline
-            ((Direction::Forward, 0usize), (0, 3)),    // Mul_iline
-            ((Direction::Backward, 0usize), (0, 3)),   // Mul_iline
-            ((Direction::Forward, 999usize), (0, 9)),  // Multilin_
-            ((Direction::Forward, 999usize), (0, 9)),  // Multilin_
+            ((Direction::Forward, 1usize), (0, 1)),    // M|ultiline\n
+            ((Direction::Forward, 2usize), (0, 3)),    // Mul|tiline\n
+            ((Direction::Backward, 6usize), (0, 0)),   // |Multiline\n
+            ((Direction::Backward, 999usize), (0, 0)), // |Multiline\n
+            ((Direction::Forward, 3usize), (0, 3)),    // Mul|tiline\n
+            ((Direction::Forward, 0usize), (0, 3)),    // Mul|tiline\n
+            ((Direction::Backward, 0usize), (0, 3)),   // Mul|tiline\n
+            ((Direction::Forward, 999usize), (0, 9)),  // Multiline|\n
+            ((Direction::Forward, 999usize), (0, 9)),  // Multiline|\n
         ]);
 
         for ((direction, amount), coordinates) in moves_and_expected_coordinates {
@@ -446,7 +396,7 @@ mod test {
             // First descent preserves column as the target line is wider
             ((Axis::V, Direction::Forward, 1usize), (1, 8)),
             // Second descent clamps column as the target line is shorter
-            ((Axis::V, Direction::Forward, 1usize), (2, 4)),
+            ((Axis::V, Direction::Forward, 1usize), (2, 5)),
             // Third descent restores the original column
             ((Axis::V, Direction::Forward, 1usize), (3, 8)),
             // Behaviour is preserved even through long jumps
@@ -760,45 +710,4 @@ mod test {
             }
         }
     }
-
-    #[test]
-    fn test_categorize() {
-        const WORD_TEST_CASE: &'static str =
-            "_hello_world_あいうえおー1234567890１２３４５６７８９０";
-        const PUNCTUATION_TEST_CASE: &'static str =
-            "!\"#$%&\'()*+,-./:;<=>?@[\\]^`{|}~！”＃＄％＆’（）＊＋、。：；＜＝＞？＠「」＾｀｛｜｝～";
-        const WHITESPACE_TEST_CASE: &'static str = "  　   ";
-
-        assert_eq!(Category::Eol, categorize('\n'));
-
-        for ch in WHITESPACE_TEST_CASE.chars() {
-            assert_eq!(
-                Category::Whitespace,
-                categorize(ch),
-                "Testing '{}', but got `{:?}` instead of `Category::Whitespace`",
-                ch,
-                categorize(ch)
-            );
-        }
-
-        for ch in WORD_TEST_CASE.chars() {
-            assert_eq!(
-                Category::Word,
-                categorize(ch),
-                "Testing '{}', but got `{:?}` instead of `Category::Word`",
-                ch,
-                categorize(ch)
-            );
-        }
-
-        for ch in PUNCTUATION_TEST_CASE.chars() {
-            assert_eq!(
-                Category::Punctuation,
-                categorize(ch),
-                "Testing '{}', but got `{:?}` instead of `Category::Punctuation`",
-                ch,
-                categorize(ch)
-            );
-        }
-    }
 }
diff --git a/helix-core/src/position.rs b/helix-core/src/position.rs
index 3d85ff2f..392eee9c 100644
--- a/helix-core/src/position.rs
+++ b/helix-core/src/position.rs
@@ -1,4 +1,5 @@
 use crate::{
+    chars::char_is_line_ending,
     graphemes::{nth_next_grapheme_boundary, RopeGraphemes},
     Rope, RopeSlice,
 };
@@ -23,8 +24,9 @@ impl Position {
     pub fn traverse(self, text: &crate::Tendril) -> Self {
         let Self { mut row, mut col } = self;
         // TODO: there should be a better way here
-        for ch in text.chars() {
-            if ch == '\n' {
+        let mut chars = text.chars().peekable();
+        while let Some(ch) = chars.next() {
+            if char_is_line_ending(ch) && !(ch == '\r' && chars.peek() == Some(&'\n')) {
                 row += 1;
                 col = 0;
             } else {
diff --git a/helix-core/src/syntax.rs b/helix-core/src/syntax.rs
index ae058eb1..92e52d73 100644
--- a/helix-core/src/syntax.rs
+++ b/helix-core/src/syntax.rs
@@ -1,4 +1,4 @@
-use crate::{regex::Regex, Change, Rope, RopeSlice, Transaction};
+use crate::{chars::char_is_line_ending, regex::Regex, Change, Rope, RopeSlice, Transaction};
 pub use helix_syntax::{get_language, get_language_name, Lang};
 
 use std::{
@@ -579,9 +579,10 @@ impl LanguageLayer {
                 mut column,
             } = point;
 
-            // TODO: there should be a better way here
-            for ch in text.bytes() {
-                if ch == b'\n' {
+            // TODO: there should be a better way here.
+            let mut chars = text.chars().peekable();
+            while let Some(ch) = chars.next() {
+                if char_is_line_ending(ch) && !(ch == '\r' && chars.peek() == Some(&'\n')) {
                     row += 1;
                     column = 0;
                 } else {
diff --git a/helix-lsp/src/client.rs b/helix-lsp/src/client.rs
index 101d2f9b..7f136fe8 100644
--- a/helix-lsp/src/client.rs
+++ b/helix-lsp/src/client.rs
@@ -3,7 +3,7 @@ use crate::{
     Call, Error, OffsetEncoding, Result,
 };
 
-use helix_core::{find_root, ChangeSet, Rope};
+use helix_core::{chars::char_is_line_ending, find_root, ChangeSet, Rope};
 use jsonrpc_core as jsonrpc;
 use lsp_types as lsp;
 use serde_json::Value;
@@ -337,8 +337,9 @@ impl Client {
                 mut character,
             } = pos;
 
-            for ch in text.chars() {
-                if ch == '\n' {
+            let mut chars = text.chars().peekable();
+            while let Some(ch) = chars.next() {
+                if char_is_line_ending(ch) && !(ch == '\r' && chars.peek() == Some(&'\n')) {
                     line += 1;
                     character = 0;
                 } else {
diff --git a/helix-term/src/commands.rs b/helix-term/src/commands.rs
index 8124c17a..b006504b 100644
--- a/helix-term/src/commands.rs
+++ b/helix-term/src/commands.rs
@@ -1,6 +1,6 @@
 use helix_core::{
     comment, coords_at_pos, find_first_non_whitespace_char, find_root, get_line_ending, graphemes,
-    indent, line_end, match_brackets,
+    indent, line_end_char_index, match_brackets,
     movement::{self, Direction},
     object, pos_at_coords,
     regex::{self, Regex},
@@ -342,7 +342,7 @@ fn move_line_end(cx: &mut Context) {
         let text = doc.text();
         let line = text.char_to_line(range.head);
 
-        let pos = line_end(&text.slice(..), line);
+        let pos = line_end_char_index(&text.slice(..), line);
 
         Range::new(pos, pos)
     });
@@ -490,6 +490,8 @@ where
     let count = cx.count();
 
     // need to wait for next key
+    // TODO: should this be done by grapheme rather than char?  For example,
+    // we can't properly handle the line-ending case here in terms of char.
     cx.on_next_key(move |cx, event| {
         let ch = match event {
             KeyEvent {
@@ -623,7 +625,7 @@ fn replace(cx: &mut Context) {
             KeyEvent {
                 code: KeyCode::Enter,
                 ..
-            } => Some('\n'), // TODO: replace this with DEFAULT_LINE_ENDING
+            } => Some('\n'), // TODO: use the document's default line ending.
             _ => None,
         };
 
@@ -763,7 +765,7 @@ fn extend_line_end(cx: &mut Context) {
         let text = doc.text();
         let line = text.char_to_line(range.head);
 
-        let pos = line_end(&text.slice(..), line);
+        let pos = line_end_char_index(&text.slice(..), line);
 
         Range::new(range.anchor, pos)
     });
@@ -1642,7 +1644,7 @@ fn append_to_line(cx: &mut Context) {
     let selection = doc.selection(view.id).transform(|range| {
         let text = doc.text();
         let line = text.char_to_line(range.head);
-        let pos = line_end(&text.slice(..), line);
+        let pos = line_end_char_index(&text.slice(..), line);
         Range::new(pos, pos)
     });
     doc.set_selection(view.id, selection);
diff --git a/helix-term/src/ui/markdown.rs b/helix-term/src/ui/markdown.rs
index be113747..3ce3a5b8 100644
--- a/helix-term/src/ui/markdown.rs
+++ b/helix-term/src/ui/markdown.rs
@@ -110,6 +110,8 @@ fn parse<'a>(contents: &'a str, theme: Option<&Theme>) -> tui::text::Text<'a> {
                                         // TODO: replace tabs with indentation
 
                                         let mut slice = &text[start..end];
+                                        // TODO: do we need to handle all unicode line endings
+                                        // here, or is just '\n' okay?
                                         while let Some(end) = slice.find('\n') {
                                             // emit span up to newline
                                             let text = &slice[..end];
diff --git a/helix-tui/Cargo.toml b/helix-tui/Cargo.toml
index 89fa755d..30e2374d 100644
--- a/helix-tui/Cargo.toml
+++ b/helix-tui/Cargo.toml
@@ -22,3 +22,4 @@ unicode-segmentation = "1.2"
 unicode-width = "0.1"
 crossterm = { version = "0.20", optional = true }
 serde = { version = "1", "optional" = true, features = ["derive"]}
+helix-core = { version = "0.2", path = "../helix-core" }
diff --git a/helix-tui/src/text.rs b/helix-tui/src/text.rs
index c671e918..b23bfd81 100644
--- a/helix-tui/src/text.rs
+++ b/helix-tui/src/text.rs
@@ -47,6 +47,7 @@
 //! ]);
 //! ```
 use crate::style::Style;
+use helix_core::line_ending::str_is_line_ending;
 use std::borrow::Cow;
 use unicode_segmentation::UnicodeSegmentation;
 use unicode_width::UnicodeWidthStr;
@@ -177,7 +178,7 @@ impl<'a> Span<'a> {
                 symbol: g,
                 style: base_style.patch(self.style),
             })
-            .filter(|s| s.symbol != "\n")
+            .filter(|s| !str_is_line_ending(s.symbol))
     }
 }
 
diff --git a/helix-tui/src/widgets/reflow.rs b/helix-tui/src/widgets/reflow.rs
index 94ff7330..ae561a4f 100644
--- a/helix-tui/src/widgets/reflow.rs
+++ b/helix-tui/src/widgets/reflow.rs
@@ -1,4 +1,5 @@
 use crate::text::StyledGrapheme;
+use helix_core::line_ending::str_is_line_ending;
 use unicode_segmentation::UnicodeSegmentation;
 use unicode_width::UnicodeWidthStr;
 
@@ -62,13 +63,13 @@ impl<'a, 'b> LineComposer<'a> for WordWrapper<'a, 'b> {
             // Ignore characters wider that the total max width.
             if symbol.width() as u16 > self.max_line_width
                 // Skip leading whitespace when trim is enabled.
-                || self.trim && symbol_whitespace && symbol != "\n" && current_line_width == 0
+                || self.trim && symbol_whitespace && !str_is_line_ending(symbol) && current_line_width == 0
             {
                 continue;
             }
 
             // Break on newline and discard it.
-            if symbol == "\n" {
+            if str_is_line_ending(symbol) {
                 if prev_whitespace {
                     current_line_width = width_to_last_word_end;
                     self.current_line.truncate(symbols_to_last_word_end);
@@ -170,7 +171,7 @@ impl<'a, 'b> LineComposer<'a> for LineTruncator<'a, 'b> {
             }
 
             // Break on newline and discard it.
-            if symbol == "\n" {
+            if str_is_line_ending(symbol) {
                 break;
             }
 
@@ -199,7 +200,7 @@ impl<'a, 'b> LineComposer<'a> for LineTruncator<'a, 'b> {
 
         if skip_rest {
             for StyledGrapheme { symbol, .. } in &mut self.symbols {
-                if symbol == "\n" {
+                if str_is_line_ending(symbol) {
                     break;
                 }
             }
diff --git a/helix-view/src/document.rs b/helix-view/src/document.rs
index 80be1ed2..3e38c24d 100644
--- a/helix-view/src/document.rs
+++ b/helix-view/src/document.rs
@@ -10,7 +10,7 @@ use std::sync::Arc;
 
 use helix_core::{
     auto_detect_line_ending,
-    chars::{char_is_linebreak, char_is_whitespace},
+    chars::{char_is_line_ending, char_is_whitespace},
     history::History,
     syntax::{LanguageConfiguration, LOADER},
     ChangeSet, Diagnostic, LineEnding, Rope, Selection, State, Syntax, Transaction,
@@ -81,6 +81,9 @@ pub struct Document {
     /// Current indent style.
     pub indent_style: IndentStyle,
 
+    /// The document's default line ending.
+    pub line_ending: LineEnding,
+
     syntax: Option<Syntax>,
     // /// Corresponding language scope name. Usually `source.<lang>`.
     pub(crate) language: Option<Arc<LanguageConfiguration>>,
@@ -99,7 +102,6 @@ pub struct Document {
 
     diagnostics: Vec<Diagnostic>,
     language_server: Option<Arc<helix_lsp::Client>>,
-    line_ending: LineEnding,
 }
 
 use std::fmt;
@@ -254,21 +256,21 @@ impl Document {
     pub fn load(path: PathBuf) -> Result<Self, Error> {
         use std::{fs::File, io::BufReader};
 
-        let doc = if !path.exists() {
+        let mut doc = if !path.exists() {
             Rope::from(DEFAULT_LINE_ENDING.as_str())
         } else {
             let file = File::open(&path).context(format!("unable to open {:?}", path))?;
-            let mut doc = Rope::from_reader(BufReader::new(file))?;
-            // add missing newline at the end of file
-            if doc.len_bytes() == 0 || doc.byte(doc.len_bytes() - 1) != b'\n' {
-                doc.insert_char(doc.len_chars(), '\n');
-            }
-            doc
+            Rope::from_reader(BufReader::new(file))?
         };
 
         // search for line endings
         let line_ending = auto_detect_line_ending(&doc).unwrap_or(DEFAULT_LINE_ENDING);
 
+        // add missing newline at the end of file
+        if doc.len_bytes() == 0 || char_is_line_ending(doc.char(doc.len_chars() - 1)) {
+            doc.insert(doc.len_chars(), line_ending.as_str());
+        }
+
         let mut doc = Self::new(doc);
         // set the path and try detecting the language
         doc.set_path(&path)?;
@@ -379,7 +381,7 @@ impl Document {
                     Some(' ') => false,
 
                     // Ignore blank lines.
-                    Some(c) if char_is_linebreak(c) => continue,
+                    Some(c) if char_is_line_ending(c) => continue,
 
                     _ => {
                         prev_line_is_tabs = false;
@@ -403,7 +405,7 @@ impl Document {
                         c if char_is_whitespace(c) => count_is_done = true,
 
                         // Ignore blank lines.
-                        c if char_is_linebreak(c) => continue 'outer,
+                        c if char_is_line_ending(c) => continue 'outer,
 
                         _ => break,
                     }
diff --git a/helix-view/src/editor.rs b/helix-view/src/editor.rs
index db8ae87a..fb2eb36d 100644
--- a/helix-view/src/editor.rs
+++ b/helix-view/src/editor.rs
@@ -12,7 +12,7 @@ use anyhow::Error;
 
 pub use helix_core::diagnostic::Severity;
 pub use helix_core::register::Registers;
-use helix_core::Position;
+use helix_core::{Position, DEFAULT_LINE_ENDING};
 
 #[derive(Debug)]
 pub struct Editor {
@@ -150,7 +150,7 @@ impl Editor {
 
     pub fn new_file(&mut self, action: Action) -> DocumentId {
         use helix_core::Rope;
-        let doc = Document::new(Rope::from("\n"));
+        let doc = Document::new(Rope::from(DEFAULT_LINE_ENDING.as_str()));
         let id = self.documents.insert(doc);
         self.documents[id].id = id;
         self.switch(id, action);
-- 
cgit v1.2.3-70-g09d2


From 714002048cc9601bf0981435c6d3ad43d1c765e8 Mon Sep 17 00:00:00 2001
From: Nathan Vegdahl
Date: Mon, 21 Jun 2021 09:52:21 -0700
Subject: Don't need getters/setters for line_ending property.

It's plain-old-data.  If we want to do fancier things later, it's
easy to switch back.
---
 helix-term/src/commands.rs |  8 ++++----
 helix-view/src/document.rs | 10 +---------
 2 files changed, 5 insertions(+), 13 deletions(-)

(limited to 'helix-term/src/commands.rs')

diff --git a/helix-term/src/commands.rs b/helix-term/src/commands.rs
index 28c4fe3a..70441fcf 100644
--- a/helix-term/src/commands.rs
+++ b/helix-term/src/commands.rs
@@ -1009,7 +1009,7 @@ fn append_mode(cx: &mut Context) {
     if selection.iter().any(|range| range.head == end) {
         let transaction = Transaction::change(
             doc.text(),
-            std::array::IntoIter::new([(end, end, Some(doc.line_ending().as_str().into()))]),
+            std::array::IntoIter::new([(end, end, Some(doc.line_ending.as_str().into()))]),
         );
         doc.apply(&transaction, view.id);
     }
@@ -2349,7 +2349,7 @@ pub mod insert {
             );
             let indent = doc.indent_unit().repeat(indent_level);
             let mut text = String::with_capacity(1 + indent.len());
-            text.push_str(doc.line_ending().as_str());
+            text.push_str(doc.line_ending.as_str());
             text.push_str(&indent);
 
             let head = pos + offs + text.chars().count();
@@ -2370,7 +2370,7 @@ pub mod insert {
             if helix_core::auto_pairs::PAIRS.contains(&(prev, curr)) {
                 // another newline, indent the end bracket one level less
                 let indent = doc.indent_unit().repeat(indent_level.saturating_sub(1));
-                text.push_str(doc.line_ending().as_str());
+                text.push_str(doc.line_ending.as_str());
                 text.push_str(&indent);
             }
 
@@ -2537,7 +2537,7 @@ fn paste_impl(
     // if any of values ends \n it's linewise paste
     let linewise = values
         .iter()
-        .any(|value| value.ends_with(doc.line_ending().as_str()));
+        .any(|value| value.ends_with(doc.line_ending.as_str()));
 
     let mut values = values.iter().cloned().map(Tendril::from).chain(repeat);
 
diff --git a/helix-view/src/document.rs b/helix-view/src/document.rs
index 9326fb79..945271ea 100644
--- a/helix-view/src/document.rs
+++ b/helix-view/src/document.rs
@@ -268,7 +268,7 @@ impl Document {
         // set the path and try detecting the language
         doc.set_path(&path)?;
         doc.detect_indent_style();
-        doc.set_line_ending(line_ending);
+        doc.line_ending = line_ending;
 
         if let Some(loader) = config_loader {
             doc.detect_language(theme, loader);
@@ -531,10 +531,6 @@ impl Document {
         self.selections.insert(view_id, selection);
     }
 
-    pub fn set_line_ending(&mut self, line_ending: LineEnding) {
-        self.line_ending = line_ending;
-    }
-
     fn _apply(&mut self, transaction: &Transaction, view_id: ViewId) -> bool {
         let old_doc = self.text().clone();
 
@@ -805,10 +801,6 @@ impl Document {
     pub fn set_diagnostics(&mut self, diagnostics: Vec<Diagnostic>) {
         self.diagnostics = diagnostics;
     }
-
-    pub fn line_ending(&self) -> LineEnding {
-        self.line_ending
-    }
 }
 
 #[cfg(test)]
-- 
cgit v1.2.3-70-g09d2


From 07e28802f6b61a17e839d05b2a031575f323b9c9 Mon Sep 17 00:00:00 2001
From: Nathan Vegdahl
Date: Mon, 21 Jun 2021 10:29:29 -0700
Subject: Add function to get the line ending of a str slice.

This is needed in some places.
---
 helix-core/src/lib.rs         |  4 +---
 helix-core/src/line_ending.rs | 23 +++++++++++++++++++++++
 helix-core/src/movement.rs    |  5 +++--
 helix-term/src/commands.rs    |  9 +++++----
 helix-view/src/document.rs    |  2 +-
 5 files changed, 33 insertions(+), 10 deletions(-)

(limited to 'helix-term/src/commands.rs')

diff --git a/helix-core/src/lib.rs b/helix-core/src/lib.rs
index 69294688..f697bc7f 100644
--- a/helix-core/src/lib.rs
+++ b/helix-core/src/lib.rs
@@ -116,7 +116,5 @@ pub use syntax::Syntax;
 pub use diagnostic::Diagnostic;
 pub use state::State;
 
-pub use line_ending::{
-    auto_detect_line_ending, get_line_ending, line_end_char_index, LineEnding, DEFAULT_LINE_ENDING,
-};
+pub use line_ending::{LineEnding, DEFAULT_LINE_ENDING};
 pub use transaction::{Assoc, Change, ChangeSet, Operation, Transaction};
diff --git a/helix-core/src/line_ending.rs b/helix-core/src/line_ending.rs
index 3055c96e..19de2231 100644
--- a/helix-core/src/line_ending.rs
+++ b/helix-core/src/line_ending.rs
@@ -128,6 +128,29 @@ pub fn get_line_ending(line: &RopeSlice) -> Option<LineEnding> {
     LineEnding::from_str(g2).or_else(|| LineEnding::from_str(g1))
 }
 
+/// Returns the passed line's line ending, if any.
+pub fn get_line_ending_of_str(line: &str) -> Option<LineEnding> {
+    if line.ends_with("\u{000D}\u{000A}") {
+        Some(LineEnding::Crlf)
+    } else if line.ends_with("\u{000A}") {
+        Some(LineEnding::LF)
+    } else if line.ends_with("\u{000B}") {
+        Some(LineEnding::VT)
+    } else if line.ends_with("\u{000C}") {
+        Some(LineEnding::FF)
+    } else if line.ends_with("\u{000D}") {
+        Some(LineEnding::CR)
+    } else if line.ends_with("\u{0085}") {
+        Some(LineEnding::Nel)
+    } else if line.ends_with("\u{2028}") {
+        Some(LineEnding::LS)
+    } else if line.ends_with("\u{2029}") {
+        Some(LineEnding::PS)
+    } else {
+        None
+    }
+}
+
 /// Returns the char index of the end of the given line, not including its line ending.
 pub fn line_end_char_index(slice: &RopeSlice, line: usize) -> usize {
     slice.line_to_char(line + 1)
diff --git a/helix-core/src/movement.rs b/helix-core/src/movement.rs
index d0023e9f..bfceb4ef 100644
--- a/helix-core/src/movement.rs
+++ b/helix-core/src/movement.rs
@@ -7,9 +7,10 @@ use crate::{
         categorize_char, char_is_line_ending, char_is_punctuation, char_is_whitespace,
         char_is_word, CharCategory,
     },
-    coords_at_pos, get_line_ending,
+    coords_at_pos,
     graphemes::{nth_next_grapheme_boundary, nth_prev_grapheme_boundary},
-    line_end_char_index, pos_at_coords, Position, Range, RopeSlice,
+    line_ending::{get_line_ending, line_end_char_index},
+    pos_at_coords, Position, Range, RopeSlice,
 };
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
diff --git a/helix-term/src/commands.rs b/helix-term/src/commands.rs
index 70441fcf..3d2a9028 100644
--- a/helix-term/src/commands.rs
+++ b/helix-term/src/commands.rs
@@ -1,6 +1,7 @@
 use helix_core::{
-    comment, coords_at_pos, find_first_non_whitespace_char, find_root, get_line_ending, graphemes,
-    indent, line_end_char_index, match_brackets,
+    comment, coords_at_pos, find_first_non_whitespace_char, find_root, graphemes, indent,
+    line_ending::{get_line_ending, get_line_ending_of_str, line_end_char_index},
+    match_brackets,
     movement::{self, Direction},
     object, pos_at_coords,
     regex::{self, Regex},
@@ -2534,10 +2535,10 @@ fn paste_impl(
             .unwrap(),
     );
 
-    // if any of values ends \n it's linewise paste
+    // if any of values ends with a line ending, it's linewise paste
     let linewise = values
         .iter()
-        .any(|value| value.ends_with(doc.line_ending.as_str()));
+        .any(|value| get_line_ending_of_str(value).is_some());
 
     let mut values = values.iter().cloned().map(Tendril::from).chain(repeat);
 
diff --git a/helix-view/src/document.rs b/helix-view/src/document.rs
index 945271ea..bd45db5a 100644
--- a/helix-view/src/document.rs
+++ b/helix-view/src/document.rs
@@ -7,9 +7,9 @@ use std::str::FromStr;
 use std::sync::Arc;
 
 use helix_core::{
-    auto_detect_line_ending,
     chars::{char_is_line_ending, char_is_whitespace},
     history::History,
+    line_ending::auto_detect_line_ending,
     syntax::{self, LanguageConfiguration},
     ChangeSet, Diagnostic, LineEnding, Rope, Selection, State, Syntax, Transaction,
     DEFAULT_LINE_ENDING,
-- 
cgit v1.2.3-70-g09d2


From 23d618853553afbac78c8d2a4ad048195f3484b8 Mon Sep 17 00:00:00 2001
From: Nathan Vegdahl
Date: Mon, 21 Jun 2021 11:08:05 -0700
Subject: Update `replace` command to use document line ending setting.

---
 helix-term/src/commands.rs | 31 +++++++++++++++++++------------
 1 file changed, 19 insertions(+), 12 deletions(-)

(limited to 'helix-term/src/commands.rs')

diff --git a/helix-term/src/commands.rs b/helix-term/src/commands.rs
index 3d2a9028..7f60f2f9 100644
--- a/helix-term/src/commands.rs
+++ b/helix-term/src/commands.rs
@@ -1,13 +1,15 @@
 use helix_core::{
     comment, coords_at_pos, find_first_non_whitespace_char, find_root, graphemes, indent,
-    line_ending::{get_line_ending, get_line_ending_of_str, line_end_char_index},
+    line_ending::{
+        get_line_ending, get_line_ending_of_str, line_end_char_index, str_is_line_ending,
+    },
     match_brackets,
     movement::{self, Direction},
     object, pos_at_coords,
     regex::{self, Regex},
     register::{self, Register, Registers},
-    search, selection, Change, ChangeSet, LineEnding, Position, Range, Rope, RopeSlice, Selection,
-    SmallVec, Tendril, Transaction, DEFAULT_LINE_ENDING,
+    search, selection, Change, ChangeSet, LineEnding, Position, Range, Rope, RopeGraphemes,
+    RopeSlice, Selection, SmallVec, Tendril, Transaction, DEFAULT_LINE_ENDING,
 };
 
 use helix_view::{
@@ -577,32 +579,37 @@ fn extend_first_nonwhitespace(cx: &mut Context) {
 }
 
 fn replace(cx: &mut Context) {
+    let mut buf = [0u8; 4]; // To hold utf8 encoded char.
+
     // need to wait for next key
     cx.on_next_key(move |cx, event| {
+        let (view, doc) = current!(cx.editor);
         let ch = match event {
             KeyEvent {
                 code: KeyCode::Char(ch),
                 ..
-            } => Some(ch),
+            } => Some(&ch.encode_utf8(&mut buf[..])[..]),
             KeyEvent {
                 code: KeyCode::Enter,
                 ..
-            } => Some('\n'), // TODO: use the document's default line ending.
+            } => Some(doc.line_ending.as_str()),
             _ => None,
         };
 
         if let Some(ch) = ch {
-            let (view, doc) = current!(cx.editor);
-
             let transaction =
                 Transaction::change_by_selection(doc.text(), doc.selection(view.id), |range| {
                     let max_to = doc.text().len_chars().saturating_sub(1);
                     let to = std::cmp::min(max_to, range.to() + 1);
-                    let text: String = doc
-                        .text()
-                        .slice(range.from()..to)
-                        .chars()
-                        .map(|c| if c == '\n' { '\n' } else { ch })
+                    let text: String = RopeGraphemes::new(doc.text().slice(range.from()..to))
+                        .map(|g| {
+                            let cow: Cow<str> = g.into();
+                            if str_is_line_ending(&cow) {
+                                cow
+                            } else {
+                                ch.into()
+                            }
+                        })
                         .collect();
 
                     (range.from(), to, Some(text.into()))
-- 
cgit v1.2.3-70-g09d2


From e436c30ed7df303a32455c4cf9e0574ab87c0683 Mon Sep 17 00:00:00 2001
From: Nathan Vegdahl
Date: Mon, 21 Jun 2021 11:22:07 -0700
Subject: Make split_selection_on_newline command handle all line endings.

---
 helix-term/src/commands.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'helix-term/src/commands.rs')

diff --git a/helix-term/src/commands.rs b/helix-term/src/commands.rs
index 7f60f2f9..dc7f8913 100644
--- a/helix-term/src/commands.rs
+++ b/helix-term/src/commands.rs
@@ -791,7 +791,8 @@ fn split_selection_on_newline(cx: &mut Context) {
     let text = doc.text().slice(..);
     // only compile the regex once
     #[allow(clippy::trivial_regex)]
-    static REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"\n").unwrap());
+    static REGEX: Lazy<Regex> =
+        Lazy::new(|| Regex::new(r"\r\n|[\n\r\u{000B}\u{000C}\u{0085}\u{2028}\u{2029}]").unwrap());
     let selection = selection::split_on_matches(text, doc.selection(view.id), &REGEX);
     doc.set_selection(view.id, selection);
 }
-- 
cgit v1.2.3-70-g09d2


From d33355650fd53c05b4e3d4e0f421eaf013b5ea1e Mon Sep 17 00:00:00 2001
From: Nathan Vegdahl
Date: Mon, 21 Jun 2021 11:59:03 -0700
Subject: Convert remaining commands to use the document's line ending setting.

---
 helix-term/src/commands.rs | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

(limited to 'helix-term/src/commands.rs')

diff --git a/helix-term/src/commands.rs b/helix-term/src/commands.rs
index dc7f8913..6ccbaaff 100644
--- a/helix-term/src/commands.rs
+++ b/helix-term/src/commands.rs
@@ -455,13 +455,27 @@ where
 
     // need to wait for next key
     // TODO: should this be done by grapheme rather than char?  For example,
-    // we can't properly handle the line-ending case here in terms of char.
+    // we can't properly handle the line-ending CRLF case here in terms of char.
     cx.on_next_key(move |cx, event| {
         let ch = match event {
             KeyEvent {
                 code: KeyCode::Enter,
                 ..
-            } => '\n',
+            } =>
+            // TODO: this isn't quite correct when CRLF is involved.
+            // This hack will work in most cases, since documents don't
+            // usually mix line endings.  But we should fix it eventually
+            // anyway.
+            {
+                current!(cx.editor)
+                    .1
+                    .line_ending
+                    .as_str()
+                    .chars()
+                    .next()
+                    .unwrap()
+            }
+
             KeyEvent {
                 code: KeyCode::Char(ch),
                 ..
@@ -1289,7 +1303,8 @@ mod cmd {
     }
 
     fn yank_joined_to_clipboard(editor: &mut Editor, args: &[&str], _: PromptEvent) {
-        let separator = args.first().copied().unwrap_or("\n");
+        let (_, doc) = current!(editor);
+        let separator = args.first().copied().unwrap_or(doc.line_ending.as_str());
         yank_joined_to_clipboard_impl(editor, separator);
     }
 
@@ -1745,7 +1760,7 @@ fn open(cx: &mut Context, open: Open) {
         let indent = doc.indent_unit().repeat(indent_level);
         let indent_len = indent.len();
         let mut text = String::with_capacity(1 + indent_len);
-        text.push('\n');
+        text.push_str(doc.line_ending.as_str());
         text.push_str(&indent);
         let text = text.repeat(count);
 
@@ -2502,7 +2517,8 @@ fn yank_joined_to_clipboard_impl(editor: &mut Editor, separator: &str) {
 }
 
 fn yank_joined_to_clipboard(cx: &mut Context) {
-    yank_joined_to_clipboard_impl(&mut cx.editor, "\n");
+    let line_ending = current!(cx.editor).1.line_ending;
+    yank_joined_to_clipboard_impl(&mut cx.editor, line_ending.as_str());
 }
 
 fn yank_main_selection_to_clipboard_impl(editor: &mut Editor) {
-- 
cgit v1.2.3-70-g09d2


From 7c4fa18764ed6a6c30125ed6bfc9b025216e9668 Mon Sep 17 00:00:00 2001
From: Nathan Vegdahl
Date: Mon, 21 Jun 2021 12:02:44 -0700
Subject: Fix clippy warnings.

---
 helix-core/src/line_ending.rs | 14 +++++++-------
 helix-term/src/commands.rs    |  5 ++++-
 2 files changed, 11 insertions(+), 8 deletions(-)

(limited to 'helix-term/src/commands.rs')

diff --git a/helix-core/src/line_ending.rs b/helix-core/src/line_ending.rs
index 19de2231..dfc74551 100644
--- a/helix-core/src/line_ending.rs
+++ b/helix-core/src/line_ending.rs
@@ -132,19 +132,19 @@ pub fn get_line_ending(line: &RopeSlice) -> Option<LineEnding> {
 pub fn get_line_ending_of_str(line: &str) -> Option<LineEnding> {
     if line.ends_with("\u{000D}\u{000A}") {
         Some(LineEnding::Crlf)
-    } else if line.ends_with("\u{000A}") {
+    } else if line.ends_with('\u{000A}') {
         Some(LineEnding::LF)
-    } else if line.ends_with("\u{000B}") {
+    } else if line.ends_with('\u{000B}') {
         Some(LineEnding::VT)
-    } else if line.ends_with("\u{000C}") {
+    } else if line.ends_with('\u{000C}') {
         Some(LineEnding::FF)
-    } else if line.ends_with("\u{000D}") {
+    } else if line.ends_with('\u{000D}') {
         Some(LineEnding::CR)
-    } else if line.ends_with("\u{0085}") {
+    } else if line.ends_with('\u{0085}') {
         Some(LineEnding::Nel)
-    } else if line.ends_with("\u{2028}") {
+    } else if line.ends_with('\u{2028}') {
         Some(LineEnding::LS)
-    } else if line.ends_with("\u{2029}") {
+    } else if line.ends_with('\u{2029}') {
         Some(LineEnding::PS)
     } else {
         None
diff --git a/helix-term/src/commands.rs b/helix-term/src/commands.rs
index 6ccbaaff..5cb30da6 100644
--- a/helix-term/src/commands.rs
+++ b/helix-term/src/commands.rs
@@ -1304,7 +1304,10 @@ mod cmd {
 
     fn yank_joined_to_clipboard(editor: &mut Editor, args: &[&str], _: PromptEvent) {
         let (_, doc) = current!(editor);
-        let separator = args.first().copied().unwrap_or(doc.line_ending.as_str());
+        let separator = args
+            .first()
+            .copied()
+            .unwrap_or_else(|| doc.line_ending.as_str());
         yank_joined_to_clipboard_impl(editor, separator);
     }
 
-- 
cgit v1.2.3-70-g09d2


From a18d50b77759d6305b99ff59baa1ccaf9d644a88 Mon Sep 17 00:00:00 2001
From: Nathan Vegdahl
Date: Mon, 21 Jun 2021 12:36:01 -0700
Subject: Add command to set the document's default line ending.

---
 helix-term/src/commands.rs | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

(limited to 'helix-term/src/commands.rs')

diff --git a/helix-term/src/commands.rs b/helix-term/src/commands.rs
index 5cb30da6..63a4d901 100644
--- a/helix-term/src/commands.rs
+++ b/helix-term/src/commands.rs
@@ -1160,6 +1160,45 @@ mod cmd {
         }
     }
 
+    /// Sets or reports the current document's line ending setting.
+    fn set_line_ending(editor: &mut Editor, args: &[&str], event: PromptEvent) {
+        use LineEnding::*;
+
+        // If no argument, report current line ending setting.
+        if args.is_empty() {
+            let line_ending = current!(editor).1.line_ending;
+            editor.set_status(match line_ending {
+                Crlf => "crlf".into(),
+                LF => "line feed".into(),
+                FF => "form feed".into(),
+                CR => "carriage return".into(),
+                Nel => "next line".into(),
+
+                // These should never be a document's default line ending.
+                VT | LS | PS => "error".into(),
+            });
+            return;
+        }
+
+        // Attempt to parse argument as a line ending.
+        let line_ending = match args.get(0) {
+            // We check for CR first because it shares a common prefix with CRLF.
+            Some(arg) if "cr".starts_with(&arg.to_lowercase()) => Some(CR),
+            Some(arg) if "crlf".starts_with(&arg.to_lowercase()) => Some(Crlf),
+            Some(arg) if "lf".starts_with(&arg.to_lowercase()) => Some(LF),
+            Some(arg) if "ff".starts_with(&arg.to_lowercase()) => Some(FF),
+            Some(arg) if "nel".starts_with(&arg.to_lowercase()) => Some(Nel),
+            _ => None,
+        };
+
+        if let Some(le) = line_ending {
+            doc_mut!(editor).line_ending = le;
+        } else {
+            // Invalid argument.
+            editor.set_error(format!("invalid line ending '{}'", args[0],));
+        }
+    }
+
     fn earlier(editor: &mut Editor, args: &[&str], event: PromptEvent) {
         let uk = match args.join(" ").parse::<helix_core::history::UndoKind>() {
             Ok(uk) => uk,
@@ -1392,6 +1431,13 @@ mod cmd {
             fun: set_indent_style,
             completer: None,
         },
+        TypableCommand {
+            name: "line-ending",
+            alias: None,
+            doc: "Set the document's default line ending. Options: crlf, lf, cr, ff, nel.",
+            fun: set_line_ending,
+            completer: None,
+        },
         TypableCommand {
             name: "earlier",
             alias: Some("ear"),
-- 
cgit v1.2.3-70-g09d2