Add tokenizer, parser and support for certain operations

13 files changed, 727 insertions, 0 deletions

diff --git a/main/core/CommandLineArgumentContainer.java b/main/core/CommandLineArgumentContainer.java
new file mode 100644
index 0000000..9ca0b5e
--- /dev/null
+++ b/main/core/CommandLineArgumentContainer.java
@@ -0,0 +1,56 @@
+package com.jantuomi.interpreter.main.core;
+
+import org.kohsuke.args4j.Option;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+
+/**
+ * Created by jan on 10.6.2016.
+ */
+
+public class CommandLineArgumentContainer {
+
+    private CommandLineArgumentContainer() {}
+
+    private static final CommandLineArgumentContainer instance = new CommandLineArgumentContainer();
+    private File srcFile;
+
+    @Option(name="-f", usage="Execute script in file FILE.")
+    public void setFile(File file) {
+        this.srcFile = file;
+    }
+
+    public static CommandLineArgumentContainer getInstance() {
+        return instance;
+    }
+
+    public String getSourceFileContents() {
+        if (srcFile == null) {
+            return null;
+        }
+
+        BufferedReader br;
+        String contents = null;
+        try {
+            br = new BufferedReader(new FileReader(srcFile));
+            StringBuilder sb = new StringBuilder();
+            String line = br.readLine();
+
+            while (line != null) {
+                sb.append(line);
+                sb.append(System.lineSeparator());
+                line = br.readLine();
+            }
+            contents = sb.toString();
+        } catch (IOException e) {
+            e.printStackTrace();
+            return null;
+        } finally {
+            // Add a newline at the end for comment rows to terminate nicely
+            return contents + "\n";
+        }
+    }
+}
diff --git a/main/core/parser/Parser.java b/main/core/parser/Parser.java
new file mode 100644
index 0000000..af62f7e
--- /dev/null
+++ b/main/core/parser/Parser.java
@@ -0,0 +1,171 @@
+package com.jantuomi.interpreter.main.core.parser;
+
+import com.jantuomi.interpreter.main.core.parser.ast.*;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+import com.jantuomi.interpreter.main.utils.Counter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Created by jan on 11.6.2016.
+ */
+public class Parser {
+    private static final Parser instance = new Parser();
+
+    private List<Token> tokens;
+    private List<ASTNode> statementSequence;
+
+    public static Parser getInstance() {
+        return instance;
+    }
+
+    private Parser() {
+
+    }
+
+    public List<ASTNode> parse(List<Token> tokens) {
+        this.tokens = tokens;
+        this.statementSequence = new ArrayList<>();
+
+        Counter c = new Counter();
+        while (c.getValue() < tokens.size()) {
+            AssignmentNode asn = expectAssignmentRoutine(c);
+            if (asn != null) {
+                statementSequence.add(asn);
+                continue;
+            }
+            AdditionNode adn = expectAdditionRoutine(c);
+            if (adn != null) {
+                statementSequence.add(adn);
+                continue;
+            }
+        }
+
+        return statementSequence;
+    }
+
+    public AssignmentNode expectAssignmentRoutine(Counter c) {
+        Counter d = c.clone();
+
+        SymbolNode lhs = expectLHS(d);
+        if (lhs == null) {
+            return null;
+        }
+        boolean isAssign = expectAssignmentAndAdvance(d);
+        if (!isAssign) {
+            return null;
+        }
+        ExpressionNode rhs = expectRHS(d);
+        if (rhs == null) {
+            return null;
+        }
+
+        AssignmentNode an = new AssignmentNode(lhs, rhs);
+        c.setValue(d.getValue());
+        return an;
+    }
+
+    public SymbolNode expectLHS(Counter c) {
+        SymbolNode sn = parseSymbolAndAdvance(c);
+        if (sn != null) {
+            return sn;
+        }
+        return null;
+    }
+
+    public ExpressionNode expectRHS(Counter c) {
+        ExpressionNode en = expectExpression(c);
+        if (en != null) {
+            return en;
+        }
+        return null;
+    }
+
+    private ExpressionNode expectExpression(Counter c) {
+        Counter d = c.clone();
+
+        AdditionNode an = expectAdditionRoutine(d);
+        if (an != null) {
+            c.setValue(d.getValue());
+            return an;
+        }
+        IntegerLiteralNode in = parseIntegerLiteralAndAdvance(d);
+        if (in != null) {
+            c.setValue(d.getValue());
+            return in;
+        }
+        SymbolNode sn = parseSymbolAndAdvance(d);
+        if (sn != null) {
+            c.setValue(d.getValue());
+            return sn;
+        }
+
+        return null;
+    }
+
+    private AdditionNode expectAdditionRoutine(Counter c) {
+        Counter d = c.clone();
+
+        ExpressionNode lhs = expectLHS(d);
+        if (lhs == null) {
+            return null;
+        }
+        boolean isAddition = expectAdditionAndAdvance(d);
+        if (!isAddition) {
+            return null;
+        }
+        ExpressionNode rhs = expectRHS(d);
+        if (rhs == null) {
+            return null;
+        }
+
+        AdditionNode an = new AdditionNode(lhs, rhs);
+        c.setValue(d.getValue());
+        return an;
+    }
+
+    private boolean expectAdditionAndAdvance(Counter c) {
+        if (tokens.get(c.getValue()).is(Token.Type.AdditionToken)) {
+            c.advance();
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    private IntegerLiteralNode parseIntegerLiteralAndAdvance(Counter c) {
+        if (tokens.get(c.getValue()).is(Token.Type.IntegerLiteralToken)) {
+            IntegerLiteralNode in = new IntegerLiteralNode(tokens.get(c.getValue()));
+            c.advance();
+            return in;
+        } else {
+            return null;
+        }
+    }
+
+    public boolean expectAssignmentAndAdvance(Counter c) {
+        if (tokens.get(c.getValue()).is(Token.Type.AssignmentToken)) {
+            c.advance();
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    public SymbolNode parseSymbolAndAdvance(Counter c) {
+        if (tokens.get(c.getValue()).is(Token.Type.SymbolToken)) {
+            SymbolNode sn = new SymbolNode(tokens.get(c.getValue()));
+            c.advance();
+            return sn;
+        } else {
+            return null;
+        }
+    }
+
+    public void printTree(ASTNode e) {
+        System.out.println("### Tree begin ###");
+        e.print(0);
+        System.out.println("### Tree end ###");
+    }
+}
diff --git a/main/core/parser/ast/ASTNode.java b/main/core/parser/ast/ASTNode.java
new file mode 100644
index 0000000..106ff75
--- /dev/null
+++ b/main/core/parser/ast/ASTNode.java
@@ -0,0 +1,33 @@
+package com.jantuomi.interpreter.main.core.parser.ast;
+
+import com.jantuomi.interpreter.main.core.parser.datatype.DataContainer;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.List;
+
+/**
+ * Created by jan on 11.6.2016.
+ */
+abstract public class ASTNode {
+
+    public abstract DataContainer evaluate();
+
+    protected Token source;
+
+    public ASTNode(Token token) {
+        this.source = token;
+    }
+
+    abstract List<ASTNode> getChildren();
+
+    public void print(int indent) {
+        for (int i = 0; i < indent; i++) {
+            System.out.print("\t");
+        }
+
+        System.out.println(source.toString());
+        for (ASTNode node : getChildren()) {
+            node.print(indent + 1);
+        }
+    }
+}
diff --git a/main/core/parser/ast/AdditionNode.java b/main/core/parser/ast/AdditionNode.java
new file mode 100644
index 0000000..3117e41
--- /dev/null
+++ b/main/core/parser/ast/AdditionNode.java
@@ -0,0 +1,34 @@
+package com.jantuomi.interpreter.main.core.parser.ast;
+
+import com.jantuomi.interpreter.main.core.parser.datatype.DataContainer;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.Arrays;
+import java.util.List;
+
+
+/**
+ * Created by jan on 13.6.2016.
+ */
+public class AdditionNode extends ExpressionNode {
+
+    private ExpressionNode lhs;
+    private ExpressionNode rhs;
+
+    public AdditionNode(ExpressionNode lhs, ExpressionNode rhs) {
+        super(new Token(Token.Type.AdditionToken));
+
+        this.lhs = lhs;
+        this.rhs = rhs;
+    }
+
+    @Override
+    public List<ASTNode> getChildren() {
+        return Arrays.asList(lhs, rhs);
+    }
+
+    @Override
+    public DataContainer evaluate() {
+        return null;
+    }
+}
diff --git a/main/core/parser/ast/AssignmentNode.java b/main/core/parser/ast/AssignmentNode.java
new file mode 100644
index 0000000..8d8f8bc
--- /dev/null
+++ b/main/core/parser/ast/AssignmentNode.java
@@ -0,0 +1,36 @@
+package com.jantuomi.interpreter.main.core.parser.ast;
+
+import com.jantuomi.interpreter.main.core.parser.datatype.DataContainer;
+import com.jantuomi.interpreter.main.core.runtime.State;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Created by jan on 11.6.2016.
+ */
+public class AssignmentNode extends ASTNode {
+
+    private SymbolNode lhs;
+    private ExpressionNode rhs;
+
+    public AssignmentNode(SymbolNode lhs, ExpressionNode rhs) {
+        super(new Token(Token.Type.AssignmentToken));
+
+        this.lhs = lhs;
+        this.rhs = rhs;
+    }
+
+    @Override
+    public DataContainer evaluate() {
+        DataContainer value = rhs.evaluate();
+        State.getInstance().setSymbolValue(lhs.getSymbol(), value);
+        return value;
+    }
+
+    @Override
+    List<ASTNode> getChildren() {
+        return Arrays.asList(lhs, rhs);
+    }
+}
diff --git a/main/core/parser/ast/ExpressionNode.java b/main/core/parser/ast/ExpressionNode.java
new file mode 100644
index 0000000..ac1a190
--- /dev/null
+++ b/main/core/parser/ast/ExpressionNode.java
@@ -0,0 +1,12 @@
+package com.jantuomi.interpreter.main.core.parser.ast;
+
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+/**
+ * Created by jan on 11.6.2016.
+ */
+abstract public class ExpressionNode extends ASTNode {
+    public ExpressionNode(Token token) {
+        super(token);
+    }
+}
diff --git a/main/core/parser/ast/IntegerLiteralNode.java b/main/core/parser/ast/IntegerLiteralNode.java
new file mode 100644
index 0000000..b68adf0
--- /dev/null
+++ b/main/core/parser/ast/IntegerLiteralNode.java
@@ -0,0 +1,30 @@
+package com.jantuomi.interpreter.main.core.parser.ast;
+
+import com.jantuomi.interpreter.main.core.parser.datatype.DataContainer;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Created by jan on 13.6.2016.
+ */
+public class IntegerLiteralNode extends ExpressionNode {
+
+    private int data;
+
+    public IntegerLiteralNode(Token token) {
+        super(token);
+        this.data = Integer.parseInt(token.getText());
+    }
+
+    @Override
+    List<ASTNode> getChildren() {
+        return Arrays.asList();
+    }
+
+    @Override
+    public DataContainer evaluate() {
+        return null;
+    }
+}
diff --git a/main/core/parser/ast/SymbolNode.java b/main/core/parser/ast/SymbolNode.java
new file mode 100644
index 0000000..1457851
--- /dev/null
+++ b/main/core/parser/ast/SymbolNode.java
@@ -0,0 +1,35 @@
+package com.jantuomi.interpreter.main.core.parser.ast;
+
+import com.jantuomi.interpreter.main.core.parser.datatype.DataContainer;
+import com.jantuomi.interpreter.main.core.runtime.State;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Created by jan on 11.6.2016.
+ */
+public class SymbolNode extends ExpressionNode {
+    private String symbol;
+
+    public SymbolNode(Token token) {
+        super(token);
+
+        this.symbol = token.getText();
+    }
+
+    @Override
+    List<ASTNode> getChildren() {
+        return Arrays.asList();
+    }
+
+    public String getSymbol() {
+        return symbol;
+    }
+
+    @Override
+    public DataContainer evaluate() {
+        return State.getInstance().getSymbolValue(symbol);
+    }
+}
diff --git a/main/core/parser/datatype/DataContainer.java b/main/core/parser/datatype/DataContainer.java
new file mode 100644
index 0000000..48e72a3
--- /dev/null
+++ b/main/core/parser/datatype/DataContainer.java
@@ -0,0 +1,17 @@
+package com.jantuomi.interpreter.main.core.parser.datatype;
+
+
+/**
+ * Created by jan on 11.6.2016.
+ */
+abstract public class DataContainer<T> {
+    public T getData() {
+        return data;
+    }
+
+    public void setData(T data) {
+        this.data = data;
+    }
+
+    private T data;
+}
diff --git a/main/core/parser/datatype/IntegerDataContainer.java b/main/core/parser/datatype/IntegerDataContainer.java
new file mode 100644
index 0000000..c5b9fb0
--- /dev/null
+++ b/main/core/parser/datatype/IntegerDataContainer.java
@@ -0,0 +1,10 @@
+package com.jantuomi.interpreter.main.core.parser.datatype;
+
+/**
+ * Created by jan on 11.6.2016.
+ */
+public class IntegerDataContainer extends DataContainer<Integer> {
+    public IntegerDataContainer(Integer data) {
+        this.setData(data);
+    }
+}
diff --git a/main/core/runtime/State.java b/main/core/runtime/State.java
new file mode 100644
index 0000000..7e4fb71
--- /dev/null
+++ b/main/core/runtime/State.java
@@ -0,0 +1,35 @@
+package com.jantuomi.interpreter.main.core.runtime;
+
+import com.jantuomi.interpreter.main.core.parser.datatype.DataContainer;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Created by jan on 11.6.2016.
+ */
+public class State {
+    private static final State instance = new State();
+
+    public static State getInstance() {
+        return instance;
+    }
+
+    private Map<String, DataContainer> variables = new HashMap<>();
+
+    private State() {
+
+    }
+
+    public DataContainer getSymbolValue(String symbol) {
+        if (variables.keySet().contains(symbol)) {
+            return variables.get(symbol);
+        } else {
+            return null;
+        }
+    }
+
+    public void setSymbolValue(String symbol, DataContainer value) {
+        variables.put(symbol, value);
+    }
+}
diff --git a/main/core/tokenizer/Tokenizer.java b/main/core/tokenizer/Tokenizer.java
new file mode 100644
index 0000000..4a12014
--- /dev/null
+++ b/main/core/tokenizer/Tokenizer.java
@@ -0,0 +1,100 @@
+package com.jantuomi.interpreter.main.core.tokenizer;
+
+
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+import com.jantuomi.interpreter.main.exception.ExceptionManager;
+import com.jantuomi.interpreter.main.exception.InterpreterException;
+
+import java.util.*;
+
+/**
+ * Created by jan on 10.6.2016.
+ */
+public class Tokenizer {
+    private String sourceString;
+
+    public List<Token> getTokens() {
+        return tokens;
+    }
+
+    private List<Token> tokens = new ArrayList<>();
+
+    private static SortedMap<Token.Type, String> tokenRegexes = new TreeMap<>();
+    private static final List<Token.Type>illegalTokenTypes = new ArrayList<>();
+
+    private static final Tokenizer instance = new Tokenizer();
+
+    public static final Tokenizer getInstance() {
+        return instance;
+    }
+
+    private Tokenizer() {
+        tokenRegexes.put(Token.Type.CommentToken, "^\\/\\*(.*)\\*\\/");
+        tokenRegexes.put(Token.Type.StringLiteralToken, "^\"(.*)\"");
+        tokenRegexes.put(Token.Type.WhitespaceToken, "^( |\t)");
+        tokenRegexes.put(Token.Type.NewlineToken, "^(\n|\r\n)");
+        tokenRegexes.put(Token.Type.AdditionToken, "^(\\+)");
+        tokenRegexes.put(Token.Type.SubtractionToken, "^(\\-)");
+        tokenRegexes.put(Token.Type.DivisionToken, "^(\\/)");
+        tokenRegexes.put(Token.Type.MultiplicationToken, "^(\\*)");
+        tokenRegexes.put(Token.Type.AssignmentToken, "^(\\<\\-)");
+        tokenRegexes.put(Token.Type.LessThanToken, "^(\\<)");
+        tokenRegexes.put(Token.Type.GreaterThanToken, "^(\\>)");
+        tokenRegexes.put(Token.Type.LessOrEqualThanToken, "^(\\<\\=)");
+        tokenRegexes.put(Token.Type.GreaterOrEqualThanToken, "^(\\>\\=)");
+        tokenRegexes.put(Token.Type.EqualsToken, "^(\\=\\=)");
+        tokenRegexes.put(Token.Type.NotEqualsToken, "^(\\!\\=)");
+        tokenRegexes.put(Token.Type.OpenParenToken, "^(\\()");
+        tokenRegexes.put(Token.Type.ClosedParenToken, "^(\\))");
+        tokenRegexes.put(Token.Type.FunctionDefineToken, "^(func)");
+        tokenRegexes.put(Token.Type.DeclarationToken, "^(decl)");
+        tokenRegexes.put(Token.Type.IntegerLiteralToken, "^(\\d+)");
+        tokenRegexes.put(Token.Type.SymbolToken, "^([a-zA-Z]+\\w*)");
+
+        illegalTokenTypes.add(Token.Type.NotAToken);
+    }
+
+    public List<Token> tokenize(String string) {
+        sourceString = string;
+        tokens.clear();
+        int line = 1;
+        for (int i = 0; i < string.length();) {
+
+            Token token = Token.makeToken(string.substring(i), tokenRegexes, line);
+
+            if (token == null) {
+                i++;
+                continue;
+            }
+
+            if (token.getTokenType() == Token.Type.NewlineToken) {
+                line++;
+            }
+
+            if (!illegalTokenTypes.contains(token.getTokenType())) {
+                tokens.add(token);
+            } else {
+                ExceptionManager.raise(InterpreterException.Exception.IllegalTokenException, line, Arrays.asList(token.getText()));
+            }
+
+            String tokenRawText = token.getRawText();
+            if (tokenRawText == null) {
+                tokenRawText = token.getText();
+            }
+            if (tokenRawText != null) {
+                i += tokenRawText.length();
+            } else {
+                i++;
+            }
+        }
+        return tokens;
+    }
+
+    public static void printTokens(List<Token> tokens) {
+        System.out.println("### Tokens: ###");
+        for (Token token : tokens) {
+            System.out.println(String.format("%-40s %s", token.getTokenType(), token.getText()));
+        }
+        System.out.println("### End ###");
+    }
+}
diff --git a/main/core/tokenizer/token/Token.java b/main/core/tokenizer/token/Token.java
new file mode 100644
index 0000000..6bb5f3b
--- /dev/null
+++ b/main/core/tokenizer/token/Token.java
@@ -0,0 +1,158 @@
+package com.jantuomi.interpreter.main.core.tokenizer.token;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Created by jan on 10.6.2016.
+ */
+public class Token {
+
+    /* Token types, ordered by precedence */
+    public enum Type {
+        CommentToken,
+        AdditionToken,
+        SubtractionToken,
+        DivisionToken,
+        MultiplicationToken,
+        AssignmentToken,
+        LessThanToken,
+        GreaterThanToken,
+        LessOrEqualThanToken,
+        GreaterOrEqualThanToken,
+        EqualsToken,
+        NotEqualsToken,
+        OpenParenToken,
+        ClosedParenToken,
+        FunctionDefineToken,
+        DeclarationToken,
+        StringLiteralToken,
+        IntegerLiteralToken,
+        WhitespaceToken,
+        NewlineToken,
+        EndStatementToken,
+        SymbolToken,
+        EndFunctionDefineToken,
+        NotAToken
+    }
+
+    public boolean is(Type type) {
+        return getTokenType() == type;
+    }
+
+    public int getLine() {
+        return line;
+    }
+
+    public void setLine(int line) {
+        this.line = line;
+    }
+
+    private int line;
+    private String rawText;
+    private String text;
+    private Type type;
+
+    public String getText() {
+        return text;
+    }
+
+    public Type getTokenType() {
+        return this.type;
+    }
+
+    public String getRawText() {
+        return rawText;
+    }
+
+    public boolean isHigherPrecedenceThan(Token other) {
+        return getTokenType().ordinal() <= other.getTokenType().ordinal();
+    }
+
+    public Token(Type type, String text, String rawText) {
+        initialize(type, text, rawText);
+    }
+
+    public Token(Type type, String text) {
+        initialize(type, text);
+    }
+
+    public Token(Type type) {
+        initialize(type, null);
+    }
+
+    private void initialize(Type type, String text, String rawText) {
+        this.type = type;
+        this.text = text;
+        this.rawText = rawText;
+    }
+
+    private void initialize(Type type, String text) {
+        initialize(type, text, text);
+    }
+
+    public static Token makeToken(String string, Map<Type, String> regexes, int line) {
+        Type type = Type.NotAToken;
+        String text = Character.toString(string.charAt(0));
+
+        for (Type t : regexes.keySet()) {
+            String regex = regexes.get(t);
+            Token found = matchToken(string, t, regex);
+            if (found != null) {
+                found.setLine(line);
+                return found;
+            }
+        }
+
+        // TODO raise error
+        return null;
+    }
+
+    public static Token matchToken(String string, Type type, String regex) {
+        Pattern pattern = Pattern.compile(regex, Pattern.DOTALL);
+        Matcher matcher = pattern.matcher(string);
+
+        if (matcher.find()) {
+            String text = matcher.group(1);
+            String rawText = matcher.group(0);
+
+            return new Token(type, text, rawText);
+        }
+
+        return null;
+    }
+
+    @Override
+    public int hashCode() {
+        return new HashCodeBuilder(17, 37)
+                .append(text)
+                .append(getTokenType())
+                .toHashCode();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (!(o instanceof Token)) return false;
+        if (o == this) return true;
+
+        Token rhs = (Token) o;
+        return new EqualsBuilder()
+                .append(getTokenType(), rhs.getTokenType())
+                .append(text, rhs.text)
+                .isEquals();
+    }
+
+    @Override
+    public String toString() {
+        String textRepr = text;
+        return String.format("%-40s %s", getTokenType(), textRepr);
+    }
+
+    public boolean isSameType(Token other) {
+        return type == other.type;
+    }
+}