diff options
| -rw-r--r-- | src/lex.rs | 42 |
1 files changed, 32 insertions, 10 deletions
@@ -200,24 +200,26 @@ pub fn tokenize(input: &str) -> Result<TokenStream> { }, Some(&'#') => { // documentation comment input.next(); - while let Some(x) = input.next() { + while let Some(x) = input.peek() { match x { '\n' => break, _ => { - buf.push(x); + buf.push(*x); } } + input.next(); } res.push(Lit(DocComment(String::from(&buf)))); }, _ => { // standard comment, runs til EOL - while let Some(x) = input.next() { + while let Some(x) = input.peek() { match x { '\n' => break, _ => { - buf.push(x); + buf.push(*x); } } + input.next(); } res.push(Lit(Comment(String::from(&buf)))); } @@ -225,19 +227,24 @@ pub fn tokenize(input: &str) -> Result<TokenStream> { }, c if c.is_alphabetic() || c == '_' => { // valid identifiers! buf.push(c); - while let Some(x) = input.next() { + while let Some(x) = input.peek() { match x { - x if x.is_alphanumeric() || x == '_' => buf.push(x), + x if x.is_alphanumeric() || x == &'_' => { + buf.push(*x); + input.next(); + }, _ => { res.push(Word(String::from(&buf))); match x { // () and [] denote both parameters/generics and tuples/arrays '(' => { // we must disambiguate by treating those *directly* after words as such res.push(Sep(FuncLeftParen)); state.paren_stack.push(Paren::Func); + input.next(); }, '[' => { res.push(Sep(GenericLeftBracket)); state.bracket_stack.push(Bracket::Generic); + input.next(); }, _ => {}, } @@ -248,10 +255,10 @@ pub fn tokenize(input: &str) -> Result<TokenStream> { }, '0'..='9' => { // numeric literals! buf.push(c); - while let Some(x) = input.next() { + while let Some(x) = input.peek() { match x { 'a'..='z' | 'A'..='Z' | '0'..='9' | '_' => { - buf.push(x); + buf.push(*x); input.next(); }, _ => break @@ -286,17 +293,32 @@ pub fn tokenize(input: &str) -> Result<TokenStream> { Some(Bracket::Array) => res.push(Sep(ArrayRightBracket)), None => return Err(MismatchedBrackets.into()), } - if input.peek() == Some(&'[') { // parameters following generics + if input.peek() == Some(&'(') { // parameters following generics res.push(Sep(FuncLeftParen)); state.paren_stack.push(Paren::Func); input.next(); } }, + '`' => { + res.push(Sep(BackTick)); + match input.peek() { + Some('(') => { + res.push(Sep(FuncLeftParen)); + state.paren_stack.push(Paren::Func); + input.next(); + }, + Some('[') => { + res.push(Sep(GenericLeftBracket)); + state.bracket_stack.push(Bracket::Generic); + input.next(); + }, + _ => {} + } + }, ',' => res.push(Sep(Comma)), '.' => res.push(Sep(Period)), ';' => res.push(Sep(Semicolon)), ':' => res.push(Sep(Colon)), - '`' => res.push(Sep(BackTick)), '{' => res.push(Sep(StructLeftBrace)), '}' => res.push(Sep(StructRightBrace)), '=' => res.push(Sep(Equals)), |