From f441022881945db08846e09a8f3bfee2f8cfa95d Mon Sep 17 00:00:00 2001
From: Jan Tuomi <jans.tuomi@gmail.com>
Date: Wed, 15 Jun 2016 21:56:03 +0300
Subject: Add interpreter and AST

---
 main/core/tokenizer/Tokenizer.java   | 15 ++++++++++-----
 main/core/tokenizer/token/Token.java |  4 ++++
 2 files changed, 14 insertions(+), 5 deletions(-)

(limited to 'main/core/tokenizer')

diff --git a/main/core/tokenizer/Tokenizer.java b/main/core/tokenizer/Tokenizer.java
index 4a12014..c04db22 100644
--- a/main/core/tokenizer/Tokenizer.java
+++ b/main/core/tokenizer/Tokenizer.java
@@ -20,7 +20,8 @@ public class Tokenizer {
     private List<Token> tokens = new ArrayList<>();
 
     private static SortedMap<Token.Type, String> tokenRegexes = new TreeMap<>();
-    private static final List<Token.Type>illegalTokenTypes = new ArrayList<>();
+    private static final List<Token.Type> discardedTokenTypes = new ArrayList<>();
+    private static final List<Token.Type> erroneousTokenTypes = new ArrayList<>();
 
     private static final Tokenizer instance = new Tokenizer();
 
@@ -51,7 +52,9 @@ public class Tokenizer {
         tokenRegexes.put(Token.Type.IntegerLiteralToken, "^(\\d+)");
         tokenRegexes.put(Token.Type.SymbolToken, "^([a-zA-Z]+\\w*)");
 
-        illegalTokenTypes.add(Token.Type.NotAToken);
+        discardedTokenTypes.add(Token.Type.WhitespaceToken);
+
+        erroneousTokenTypes.add(Token.Type.NotAToken);
     }
 
     public List<Token> tokenize(String string) {
@@ -71,10 +74,12 @@ public class Tokenizer {
                 line++;
             }
 
-            if (!illegalTokenTypes.contains(token.getTokenType())) {
+            if (!discardedTokenTypes.contains(token.getTokenType())) {
                 tokens.add(token);
-            } else {
-                ExceptionManager.raise(InterpreterException.Exception.IllegalTokenException, line, Arrays.asList(token.getText()));
+            }
+
+            if (erroneousTokenTypes.contains(token.getTokenType())) {
+                ExceptionManager.raise(InterpreterException.Exception.IllegalTokenError, line, Arrays.asList(token.getText()));
             }
 
             String tokenRawText = token.getRawText();
diff --git a/main/core/tokenizer/token/Token.java b/main/core/tokenizer/token/Token.java
index 6bb5f3b..99ae913 100644
--- a/main/core/tokenizer/token/Token.java
+++ b/main/core/tokenizer/token/Token.java
@@ -148,6 +148,10 @@ public class Token {
 
     @Override
     public String toString() {
+        return String.format(String.format("%s %s", getTokenType(), getText()));
+    }
+
+    public String toFormattedString() {
         String textRepr = text;
         return String.format("%-40s %s", getTokenType(), textRepr);
     }
-- 
cgit v1.3