From f441022881945db08846e09a8f3bfee2f8cfa95d Mon Sep 17 00:00:00 2001 From: Jan Tuomi Date: Wed, 15 Jun 2016 21:56:03 +0300 Subject: Add interpreter and AST --- main/core/tokenizer/Tokenizer.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'main/core/tokenizer/Tokenizer.java') diff --git a/main/core/tokenizer/Tokenizer.java b/main/core/tokenizer/Tokenizer.java index 4a12014..c04db22 100644 --- a/main/core/tokenizer/Tokenizer.java +++ b/main/core/tokenizer/Tokenizer.java @@ -20,7 +20,8 @@ public class Tokenizer { private List tokens = new ArrayList<>(); private static SortedMap tokenRegexes = new TreeMap<>(); - private static final ListillegalTokenTypes = new ArrayList<>(); + private static final List discardedTokenTypes = new ArrayList<>(); + private static final List erroneousTokenTypes = new ArrayList<>(); private static final Tokenizer instance = new Tokenizer(); @@ -51,7 +52,9 @@ public class Tokenizer { tokenRegexes.put(Token.Type.IntegerLiteralToken, "^(\\d+)"); tokenRegexes.put(Token.Type.SymbolToken, "^([a-zA-Z]+\\w*)"); - illegalTokenTypes.add(Token.Type.NotAToken); + discardedTokenTypes.add(Token.Type.WhitespaceToken); + + erroneousTokenTypes.add(Token.Type.NotAToken); } public List tokenize(String string) { @@ -71,10 +74,12 @@ public class Tokenizer { line++; } - if (!illegalTokenTypes.contains(token.getTokenType())) { + if (!discardedTokenTypes.contains(token.getTokenType())) { tokens.add(token); - } else { - ExceptionManager.raise(InterpreterException.Exception.IllegalTokenException, line, Arrays.asList(token.getText())); + } + + if (erroneousTokenTypes.contains(token.getTokenType())) { + ExceptionManager.raise(InterpreterException.Exception.IllegalTokenError, line, Arrays.asList(token.getText())); } String tokenRawText = token.getRawText(); -- cgit v1.3