aboutsummaryrefslogtreecommitdiff
path: root/src/lex.rs
diff options
context:
space:
mode:
Diffstat (limited to 'src/lex.rs')
-rw-r--r--src/lex.rs42
1 files changed, 32 insertions, 10 deletions
diff --git a/src/lex.rs b/src/lex.rs
index 49be668..01b24c5 100644
--- a/src/lex.rs
+++ b/src/lex.rs
@@ -200,24 +200,26 @@ pub fn tokenize(input: &str) -> Result<TokenStream> {
},
Some(&'#') => { // documentation comment
input.next();
- while let Some(x) = input.next() {
+ while let Some(x) = input.peek() {
match x {
'\n' => break,
_ => {
- buf.push(x);
+ buf.push(*x);
}
}
+ input.next();
}
res.push(Lit(DocComment(String::from(&buf))));
},
_ => { // standard comment, runs til EOL
- while let Some(x) = input.next() {
+ while let Some(x) = input.peek() {
match x {
'\n' => break,
_ => {
- buf.push(x);
+ buf.push(*x);
}
}
+ input.next();
}
res.push(Lit(Comment(String::from(&buf))));
}
@@ -225,19 +227,24 @@ pub fn tokenize(input: &str) -> Result<TokenStream> {
},
c if c.is_alphabetic() || c == '_' => { // valid identifiers!
buf.push(c);
- while let Some(x) = input.next() {
+ while let Some(x) = input.peek() {
match x {
- x if x.is_alphanumeric() || x == '_' => buf.push(x),
+ x if x.is_alphanumeric() || x == &'_' => {
+ buf.push(*x);
+ input.next();
+ },
_ => {
res.push(Word(String::from(&buf)));
match x { // () and [] denote both parameters/generics and tuples/arrays
'(' => { // we must disambiguate by treating those *directly* after words as such
res.push(Sep(FuncLeftParen));
state.paren_stack.push(Paren::Func);
+ input.next();
},
'[' => {
res.push(Sep(GenericLeftBracket));
state.bracket_stack.push(Bracket::Generic);
+ input.next();
},
_ => {},
}
@@ -248,10 +255,10 @@ pub fn tokenize(input: &str) -> Result<TokenStream> {
},
'0'..='9' => { // numeric literals!
buf.push(c);
- while let Some(x) = input.next() {
+ while let Some(x) = input.peek() {
match x {
'a'..='z' | 'A'..='Z' | '0'..='9' | '_' => {
- buf.push(x);
+ buf.push(*x);
input.next();
},
_ => break
@@ -286,17 +293,32 @@ pub fn tokenize(input: &str) -> Result<TokenStream> {
Some(Bracket::Array) => res.push(Sep(ArrayRightBracket)),
None => return Err(MismatchedBrackets.into()),
}
- if input.peek() == Some(&'[') { // parameters following generics
+ if input.peek() == Some(&'(') { // parameters following generics
res.push(Sep(FuncLeftParen));
state.paren_stack.push(Paren::Func);
input.next();
}
},
+ '`' => {
+ res.push(Sep(BackTick));
+ match input.peek() {
+ Some('(') => {
+ res.push(Sep(FuncLeftParen));
+ state.paren_stack.push(Paren::Func);
+ input.next();
+ },
+ Some('[') => {
+ res.push(Sep(GenericLeftBracket));
+ state.bracket_stack.push(Bracket::Generic);
+ input.next();
+ },
+ _ => {}
+ }
+ },
',' => res.push(Sep(Comma)),
'.' => res.push(Sep(Period)),
';' => res.push(Sep(Semicolon)),
':' => res.push(Sep(Colon)),
- '`' => res.push(Sep(BackTick)),
'{' => res.push(Sep(StructLeftBrace)),
'}' => res.push(Sep(StructRightBrace)),
'=' => res.push(Sep(Equals)),