begin parser

src/lexer.rs

@@ -5,7 +5,8 @@ pub enum TokenType {
     IDENTIFIER,
     OPERATOR,
     KEYWORD,
-    SEPARATOR
+    SEPARATOR,
+    OPEND
 }
 #[derive(Debug)]
 pub struct Token {
@@ -14,6 +15,10 @@ pub struct Token {
     pub pos: usize,
 }
 
+fn is_opend(char: &str) -> bool {
+    let chars = vec![";","\n"];
+    return chars.contains(&char);
+}
 fn is_ignored(char: &str) -> bool {
     let chars = vec![""," ","\n","\r","\t"];
     return chars.contains(&char);
@@ -194,6 +199,10 @@ pub fn lex(input: String) -> Vec<Token> {
     while pos < splitted.len() {
         let char = splitted[pos];
         pos += 1;
+        if is_opend(char) {
+            out.push(Token { typ: TokenType::OPEND, value: String::from(char), pos: pos-1 });
+            continue;
+        }
         if is_ignored(char) {
             continue;
         }
@@ -224,5 +233,6 @@ pub fn lex(input: String) -> Vec<Token> {
             read_identifier(&splitted, &mut pos, &mut out);
         }
     }
+    out.push(Token { typ: TokenType::OPEND, value: String::from("EOF"), pos: pos-1 });
     return out;
 }