added separators
This commit is contained in:
parent
d294a06820
commit
95e427f8d1
1 changed files with 2 additions and 0 deletions
|
@ -210,6 +210,8 @@ pub fn lex(input: String) -> Vec<Token> {
|
|||
read_comment(&splitted, &mut pos, true);
|
||||
} else if is_operator(char) {
|
||||
out.push(Token { typ: TokenType::OPERATOR, value: String::from(char), pos: pos-1 });
|
||||
} else if is_sep(char) {
|
||||
out.push(Token { typ: TokenType::SEPARATOR, value: String::from(char), pos: pos-1 });
|
||||
} else {
|
||||
read_identifier(&splitted, &mut pos, &mut out);
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue