diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/frontend/lex.rs | 13 |
1 files changed, 7 insertions, 6 deletions
diff --git a/src/frontend/lex.rs b/src/frontend/lex.rs index f751c63..e0aa76d 100644 --- a/src/frontend/lex.rs +++ b/src/frontend/lex.rs @@ -31,12 +31,13 @@ impl std::error::Error for LexicalError {} /// **Basic** syntax tokens. Form an unambiguous TokenStream. #[derive(Clone, PartialEq)] pub enum Token { - Key(Keyword), // keyword identifiers. - Word(String), // non-keyword identifiers. - Num(String), // numeric value, ex. 413, 0b101011, 0xabcd - Lit(Literal), // literal value, ex. for strings/comments. - Sep(Punctuation), // punctuation. non-word tokens. operators are lexed as this and later transformed to words. - Indent(usize), // indentation. denotes line breaks and scope at which a line starts. + Key(Keyword), // keyword identifiers. + Word(String), // non-keyword identifiers. + Op(String), // operators. separate tokens lexographically so the parser knows to parse infix notation. + Num(String), // numeric value, ex. 413, 0b101011, 0xabcd + Lit(Literal), // literal value, ex. for strings/comments. + Sep(Punctuation), // punctuation. non-word tokens. ~~operators are lexed as this and later transformed to words.~~ + Indent(usize), // indentation. denotes line breaks and scope at which a line starts. } #[derive(Clone, PartialEq)] |