From 0fa32e833237affd401ec644704cda4f84d07374 Mon Sep 17 00:00:00 2001
From: JJ
Date: Thu, 26 Oct 2023 14:24:33 -0700
Subject: compiler: fix bugs in lexer. we're lexing!!

---
 src/lex.rs | 42 ++++++++++++++++++++++++++++++++----------
 1 file changed, 32 insertions(+), 10 deletions(-)

(limited to 'src')

diff --git a/src/lex.rs b/src/lex.rs
index 49be668..01b24c5 100644
--- a/src/lex.rs
+++ b/src/lex.rs
@@ -200,24 +200,26 @@ pub fn tokenize(input: &str) -> Result<TokenStream> {
                     },
                     Some(&'#') => { // documentation comment
                         input.next();
-                        while let Some(x) = input.next() {
+                        while let Some(x) = input.peek() {
                             match x {
                                 '\n' => break,
                                 _ => {
-                                    buf.push(x);
+                                    buf.push(*x);
                                 }
                             }
+                            input.next();
                         }
                         res.push(Lit(DocComment(String::from(&buf))));
                     },
                     _ => { // standard comment, runs til EOL
-                        while let Some(x) = input.next() {
+                        while let Some(x) = input.peek() {
                             match x {
                                 '\n' => break,
                                 _ => {
-                                    buf.push(x);
+                                    buf.push(*x);
                                 }
                             }
+                            input.next();
                         }
                         res.push(Lit(Comment(String::from(&buf))));
                     }
@@ -225,19 +227,24 @@ pub fn tokenize(input: &str) -> Result<TokenStream> {
             },
             c if c.is_alphabetic() || c == '_' => { // valid identifiers!
                 buf.push(c);
-                while let Some(x) = input.next() {
+                while let Some(x) = input.peek() {
                     match x {
-                        x if x.is_alphanumeric() || x == '_' => buf.push(x),
+                        x if x.is_alphanumeric() || x == &'_' => {
+                            buf.push(*x);
+                            input.next();
+                        },
                         _ => {
                             res.push(Word(String::from(&buf)));
                             match x { // () and [] denote both parameters/generics and tuples/arrays
                                 '(' => { // we must disambiguate by treating those *directly* after words as such
                                     res.push(Sep(FuncLeftParen));
                                     state.paren_stack.push(Paren::Func);
+                                    input.next();
                                 },
                                 '[' => {
                                     res.push(Sep(GenericLeftBracket));
                                     state.bracket_stack.push(Bracket::Generic);
+                                    input.next();
                                 },
                                 _ => {},
                             }
@@ -248,10 +255,10 @@ pub fn tokenize(input: &str) -> Result<TokenStream> {
             },
             '0'..='9' => { // numeric literals!
                 buf.push(c);
-                while let Some(x) = input.next() {
+                while let Some(x) = input.peek() {
                     match x {
                         'a'..='z' | 'A'..='Z' | '0'..='9' | '_' => {
-                            buf.push(x);
+                            buf.push(*x);
                             input.next();
                         },
                         _ => break
@@ -286,17 +293,32 @@ pub fn tokenize(input: &str) -> Result<TokenStream> {
                     Some(Bracket::Array) => res.push(Sep(ArrayRightBracket)),
                     None => return Err(MismatchedBrackets.into()),
                 }
-                if input.peek() == Some(&'[') { // parameters following generics
+                if input.peek() == Some(&'(') { // parameters following generics
                     res.push(Sep(FuncLeftParen));
                     state.paren_stack.push(Paren::Func);
                     input.next();
                 }
             },
+            '`' => {
+                res.push(Sep(BackTick));
+                match input.peek() {
+                    Some('(') => {
+                        res.push(Sep(FuncLeftParen));
+                        state.paren_stack.push(Paren::Func);
+                        input.next();
+                    },
+                    Some('[') => {
+                        res.push(Sep(GenericLeftBracket));
+                        state.bracket_stack.push(Bracket::Generic);
+                        input.next();
+                    },
+                    _ => {}
+                }
+            },
             ',' => res.push(Sep(Comma)),
             '.' => res.push(Sep(Period)),
             ';' => res.push(Sep(Semicolon)),
             ':' => res.push(Sep(Colon)),
-            '`' => res.push(Sep(BackTick)),
             '{' => res.push(Sep(StructLeftBrace)),
             '}' => res.push(Sep(StructRightBrace)),
             '=' => res.push(Sep(Equals)),
-- 
cgit v1.2.3-70-g09d2