Add tokenizer, parser and support for certain operations

1 files changed, 100 insertions, 0 deletions

diff --git a/main/core/tokenizer/Tokenizer.java b/main/core/tokenizer/Tokenizer.java
new file mode 100644
index 0000000..4a12014
--- /dev/null
+++ b/main/core/tokenizer/Tokenizer.java
@@ -0,0 +1,100 @@
+package com.jantuomi.interpreter.main.core.tokenizer;
+
+
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+import com.jantuomi.interpreter.main.exception.ExceptionManager;
+import com.jantuomi.interpreter.main.exception.InterpreterException;
+
+import java.util.*;
+
+/**
+ * Created by jan on 10.6.2016.
+ */
+public class Tokenizer {
+    private String sourceString;
+
+    public List<Token> getTokens() {
+        return tokens;
+    }
+
+    private List<Token> tokens = new ArrayList<>();
+
+    private static SortedMap<Token.Type, String> tokenRegexes = new TreeMap<>();
+    private static final List<Token.Type>illegalTokenTypes = new ArrayList<>();
+
+    private static final Tokenizer instance = new Tokenizer();
+
+    public static final Tokenizer getInstance() {
+        return instance;
+    }
+
+    private Tokenizer() {
+        tokenRegexes.put(Token.Type.CommentToken, "^\\/\\*(.*)\\*\\/");
+        tokenRegexes.put(Token.Type.StringLiteralToken, "^\"(.*)\"");
+        tokenRegexes.put(Token.Type.WhitespaceToken, "^( |\t)");
+        tokenRegexes.put(Token.Type.NewlineToken, "^(\n|\r\n)");
+        tokenRegexes.put(Token.Type.AdditionToken, "^(\\+)");
+        tokenRegexes.put(Token.Type.SubtractionToken, "^(\\-)");
+        tokenRegexes.put(Token.Type.DivisionToken, "^(\\/)");
+        tokenRegexes.put(Token.Type.MultiplicationToken, "^(\\*)");
+        tokenRegexes.put(Token.Type.AssignmentToken, "^(\\<\\-)");
+        tokenRegexes.put(Token.Type.LessThanToken, "^(\\<)");
+        tokenRegexes.put(Token.Type.GreaterThanToken, "^(\\>)");
+        tokenRegexes.put(Token.Type.LessOrEqualThanToken, "^(\\<\\=)");
+        tokenRegexes.put(Token.Type.GreaterOrEqualThanToken, "^(\\>\\=)");
+        tokenRegexes.put(Token.Type.EqualsToken, "^(\\=\\=)");
+        tokenRegexes.put(Token.Type.NotEqualsToken, "^(\\!\\=)");
+        tokenRegexes.put(Token.Type.OpenParenToken, "^(\\()");
+        tokenRegexes.put(Token.Type.ClosedParenToken, "^(\\))");
+        tokenRegexes.put(Token.Type.FunctionDefineToken, "^(func)");
+        tokenRegexes.put(Token.Type.DeclarationToken, "^(decl)");
+        tokenRegexes.put(Token.Type.IntegerLiteralToken, "^(\\d+)");
+        tokenRegexes.put(Token.Type.SymbolToken, "^([a-zA-Z]+\\w*)");
+
+        illegalTokenTypes.add(Token.Type.NotAToken);
+    }
+
+    public List<Token> tokenize(String string) {
+        sourceString = string;
+        tokens.clear();
+        int line = 1;
+        for (int i = 0; i < string.length();) {
+
+            Token token = Token.makeToken(string.substring(i), tokenRegexes, line);
+
+            if (token == null) {
+                i++;
+                continue;
+            }
+
+            if (token.getTokenType() == Token.Type.NewlineToken) {
+                line++;
+            }
+
+            if (!illegalTokenTypes.contains(token.getTokenType())) {
+                tokens.add(token);
+            } else {
+                ExceptionManager.raise(InterpreterException.Exception.IllegalTokenException, line, Arrays.asList(token.getText()));
+            }
+
+            String tokenRawText = token.getRawText();
+            if (tokenRawText == null) {
+                tokenRawText = token.getText();
+            }
+            if (tokenRawText != null) {
+                i += tokenRawText.length();
+            } else {
+                i++;
+            }
+        }
+        return tokens;
+    }
+
+    public static void printTokens(List<Token> tokens) {
+        System.out.println("### Tokens: ###");
+        for (Token token : tokens) {
+            System.out.println(String.format("%-40s %s", token.getTokenType(), token.getText()));
+        }
+        System.out.println("### End ###");
+    }
+}