Implemented general enhancements

13 files changed, 410 insertions, 15 deletions

diff --git a/main/core/tokenizer/Tokenizer.java b/main/core/tokenizer/Tokenizer.java
index 7e88a43..9aed09c 100644
--- a/main/core/tokenizer/Tokenizer.java
+++ b/main/core/tokenizer/Tokenizer.java
@@ -5,7 +5,10 @@ import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
 import com.jantuomi.interpreter.main.exception.ExceptionManager;
 import com.jantuomi.interpreter.main.exception.InterpreterException;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
 
 /**
  * Created by jan on 10.6.2016.
@@ -19,7 +22,7 @@ public class Tokenizer {
 
     private List<Token> tokens = new ArrayList<>();
 
-    private static SortedMap<Token.Type, String> tokenRegexes = new TreeMap<>();
+    private static Map<Token.Type, String> tokenRegexes = new LinkedHashMap<>();
     private static final List<Token.Type> discardedTokenTypes = new ArrayList<>();
     private static final List<Token.Type> erroneousTokenTypes = new ArrayList<>();
 
@@ -47,12 +50,15 @@ public class Tokenizer {
         tokenRegexes.put(Token.Type.NotEqualsToken, "^(\\!\\=)");
         tokenRegexes.put(Token.Type.OpenParenToken, "^(\\()");
         tokenRegexes.put(Token.Type.ClosedParenToken, "^(\\))");
-        tokenRegexes.put(Token.Type.FunctionDefineToken, "^(func)");
-        tokenRegexes.put(Token.Type.DeclarationToken, "^(decl)");
+        tokenRegexes.put(Token.Type.FunctionDefineToken, "^(func)\\b");
+        tokenRegexes.put(Token.Type.FunctionBodyToken, "^(as)\\b");
+        tokenRegexes.put(Token.Type.EndFunctionDefineToken, "^(end)\\b");
+        tokenRegexes.put(Token.Type.DeclarationToken, "^(decl)\\b");
         tokenRegexes.put(Token.Type.IntegerLiteralToken, "^(\\d+)");
         tokenRegexes.put(Token.Type.SymbolToken, "^([a-zA-Z]+\\w*)");
 
         discardedTokenTypes.add(Token.Type.WhitespaceToken);
+        discardedTokenTypes.add(Token.Type.CommentToken);
 
         erroneousTokenTypes.add(Token.Type.NotAToken);
     }
@@ -79,7 +85,7 @@ public class Tokenizer {
             }
 
             if (erroneousTokenTypes.contains(token.getTokenType())) {
-                ExceptionManager.raise(InterpreterException.ExceptionType.IllegalTokenError, line, Arrays.asList(token.getText()));
+                ExceptionManager.raise(InterpreterException.ExceptionType.IllegalTokenError, line, token.getText());
             }
 
             String tokenRawText = token.getRawText();
diff --git a/main/core/tokenizer/token/ArgumentInfo.java b/main/core/tokenizer/token/ArgumentInfo.java
index a9c3d90..b062a04 100644
--- a/main/core/tokenizer/token/ArgumentInfo.java
+++ b/main/core/tokenizer/token/ArgumentInfo.java
@@ -6,6 +6,27 @@ package com.jantuomi.interpreter.main.core.tokenizer.token;
 public class ArgumentInfo {
     private int count = 0;
     private boolean isVarargs = false;
+
+    public Token.Type getOptionalArgument() {
+        return optionalArgument;
+    }
+
+    public void setOptionalArgument(Token.Type optionalArgument) {
+        this.optionalArgument = optionalArgument;
+    }
+
+    private Token.Type optionalArgument = null;
+
+    public Token.Type getTerminator() {
+        return terminator;
+    }
+
+    public void setTerminator(Token.Type terminator) {
+        this.terminator = terminator;
+    }
+
+    private Token.Type terminator;
+
     public ArgumentInfo(int count) {
         this.count = count;
     }
diff --git a/main/core/tokenizer/token/Token.java b/main/core/tokenizer/token/Token.java
index 0d4395e..5248799 100644
--- a/main/core/tokenizer/token/Token.java
+++ b/main/core/tokenizer/token/Token.java
@@ -1,8 +1,9 @@
 package com.jantuomi.interpreter.main.core.tokenizer.token;
 
 import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
-import com.jantuomi.interpreter.main.core.tokenizer.token.types.AdditionToken;
-import com.jantuomi.interpreter.main.core.tokenizer.token.types.IntegerLiteralToken;
+import com.jantuomi.interpreter.main.core.tokenizer.token.types.*;
+import com.jantuomi.interpreter.main.exception.ExceptionManager;
+import com.jantuomi.interpreter.main.exception.InterpreterException;
 import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
 
@@ -11,6 +12,7 @@ import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+
 /**
  * Created by jan on 10.6.2016.
  */
@@ -53,6 +55,7 @@ abstract public class Token {
         EndStatementToken,
         SymbolToken,
         EndFunctionDefineToken,
+        FunctionBodyToken,
         NotAToken
     }
 
@@ -115,13 +118,13 @@ abstract public class Token {
         initialize(type, text, text);
     }
 
-    public static Token makeToken(String string, Map<Type, String> regexes, int line) {
+    public static Token makeToken(String string, Map<Type, String> regexes, int line) throws InterpreterException {
         Type type = Type.NotAToken;
         String text = Character.toString(string.charAt(0));
 
         for (Type t : regexes.keySet()) {
             String regex = regexes.get(t);
-            Token found = matchToken(string, t, regex);
+            Token found = matchToken(string, t, regex, line);
             if (found != null) {
                 found.setLine(line);
                 return found;
@@ -132,7 +135,7 @@ abstract public class Token {
         return null;
     }
 
-    public static Token matchToken(String string, Type type, String regex) {
+    public static Token matchToken(String string, Type type, String regex, int line) throws InterpreterException {
         Pattern pattern = Pattern.compile(regex, Pattern.DOTALL);
         Matcher matcher = pattern.matcher(string);
 
@@ -141,12 +144,37 @@ abstract public class Token {
             String rawText = matcher.group(0);
 
             switch (type) {
+                case WhitespaceToken:
+                    return new WhitespaceToken();
+                case CommentToken:
+                    return new CommentToken(text, rawText);
+
+                // TODO placeholder whitespace instead of newline
+                case NewlineToken:
+                    return new WhitespaceToken();
+
                 case IntegerLiteralToken:
                     return new IntegerLiteralToken(text);
                 case AdditionToken:
                     return new AdditionToken();
+                case SubtractionToken:
+                    return new SubtractionToken();
+                case FunctionDefineToken:
+                    return new FunctionDefineToken();
+                case EndFunctionDefineToken:
+                    return new EndFunctionDefineToken();
+                case FunctionBodyToken:
+                    return new FunctionBodyToken();
+                case SymbolToken:
+                    return new SymbolToken(text);
+                case OpenParenToken:
+                    return new OpenParenToken();
+                case ClosedParenToken:
+                    return new ClosedParenToken();
                 default:
-                    return null;
+                    ExceptionManager.raise(InterpreterException.ExceptionType.IllegalTokenError, line,
+                            type.toString()
+                    );
             }
         }
 
@@ -157,9 +185,24 @@ abstract public class Token {
         switch (getTokenType()) {
 
             case FunctionDefineToken:
-                ArgumentInfo ai = new ArgumentInfo();
-                ai.setVariable(true);
-                return ai;
+                ArgumentInfo fdai = new ArgumentInfo();
+                fdai.setVariable(true);
+                fdai.setTerminator(Type.FunctionBodyToken);
+                return fdai;
+            case FunctionBodyToken:
+                ArgumentInfo fbai = new ArgumentInfo();
+                fbai.setVariable(true);
+                fbai.setTerminator(Type.EndFunctionDefineToken);
+                return fbai;
+            case OpenParenToken:
+                ArgumentInfo opai = new ArgumentInfo();
+                opai.setVariable(true);
+                opai.setTerminator(Type.ClosedParenToken);
+                return opai;
+            case SymbolToken:
+                ArgumentInfo sai = new ArgumentInfo();
+                sai.setOptionalArgument(Type.OpenParenToken);
+                return sai;
             case AdditionToken:
             case SubtractionToken:
             case DivisionToken:
@@ -201,7 +244,11 @@ abstract public class Token {
 
     @Override
     public String toString() {
-        return String.format(String.format("%s %s", getTokenType(), getText()));
+        return String.format(String.format(
+                "%s%s",
+                getTokenType().toString().replace("Token", ""),
+                getText() == null ? "" : " " + getText()
+        ));
     }
 
     public String toFormattedString() {
diff --git a/main/core/tokenizer/token/types/ClosedParenToken.java b/main/core/tokenizer/token/types/ClosedParenToken.java
new file mode 100644
index 0000000..17e5c67
--- /dev/null
+++ b/main/core/tokenizer/token/types/ClosedParenToken.java
@@ -0,0 +1,31 @@
+package com.jantuomi.interpreter.main.core.tokenizer.token.types;
+
+import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Created by jan on 19.6.2016.
+ */
+public class ClosedParenToken extends Token {
+    public ClosedParenToken() {
+        super(Type.ClosedParenToken);
+    }
+
+    @Override
+    public ASTNode generateNode() {
+        return null;
+    }
+
+    @Override
+    public Token setArguments(List<Token> args) {
+        return this;
+    }
+
+    @Override
+    public List<Token> getChildren() {
+        return Arrays.asList();
+    }
+}
diff --git a/main/core/tokenizer/token/types/CommentToken.java b/main/core/tokenizer/token/types/CommentToken.java
new file mode 100644
index 0000000..45ca995
--- /dev/null
+++ b/main/core/tokenizer/token/types/CommentToken.java
@@ -0,0 +1,32 @@
+package com.jantuomi.interpreter.main.core.tokenizer.token.types;
+
+import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
+import com.jantuomi.interpreter.main.core.parser.ast.EmptyNode;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Created by jan on 19.6.2016.
+ */
+public class CommentToken extends Token {
+    public CommentToken(String text, String rawText) {
+        super(Type.CommentToken, text, rawText);
+    }
+
+    @Override
+    public ASTNode generateNode() {
+        return new EmptyNode(this);
+    }
+
+    @Override
+    public Token setArguments(List<Token> args) {
+        return this;
+    }
+
+    @Override
+    public List<Token> getChildren() {
+        return Arrays.asList();
+    }
+}
diff --git a/main/core/tokenizer/token/types/EndFunctionDefineToken.java b/main/core/tokenizer/token/types/EndFunctionDefineToken.java
new file mode 100644
index 0000000..ffd33c5
--- /dev/null
+++ b/main/core/tokenizer/token/types/EndFunctionDefineToken.java
@@ -0,0 +1,31 @@
+package com.jantuomi.interpreter.main.core.tokenizer.token.types;
+
+import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Created by jan on 16.6.2016.
+ */
+public class EndFunctionDefineToken extends Token {
+    public EndFunctionDefineToken() {
+        super(Type.EndFunctionDefineToken, "end");
+    }
+
+    @Override
+    public ASTNode generateNode() {
+        return null;
+    }
+
+    @Override
+    public Token setArguments(List<Token> args) {
+        return this;
+    }
+
+    @Override
+    public List<Token> getChildren() {
+        return Arrays.asList();
+    }
+}
diff --git a/main/core/tokenizer/token/types/FunctionBodyToken.java b/main/core/tokenizer/token/types/FunctionBodyToken.java
new file mode 100644
index 0000000..647ed55
--- /dev/null
+++ b/main/core/tokenizer/token/types/FunctionBodyToken.java
@@ -0,0 +1,29 @@
+package com.jantuomi.interpreter.main.core.tokenizer.token.types;
+
+import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
+import com.jantuomi.interpreter.main.core.parser.ast.FunctionBodyNode;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Created by jan on 16.6.2016.
+ */
+public class FunctionBodyToken extends VarargOperatorToken {
+    public FunctionBodyToken() {
+        super(Type.FunctionBodyToken, "as");
+    }
+
+    @Override
+    public ASTNode generateNode() {
+        FunctionBodyNode node = new FunctionBodyNode(this);
+        List<ASTNode> statements = new ArrayList<>();
+        for (int i = 0; i < args.size() - 1; i++) {
+            Token arg = args.get(i);
+            statements.add(arg.generateNode());
+        }
+        node.setArgs(statements);
+        return node;
+    }
+}
diff --git a/main/core/tokenizer/token/types/FunctionDefineToken.java b/main/core/tokenizer/token/types/FunctionDefineToken.java
new file mode 100644
index 0000000..6bbfd84
--- /dev/null
+++ b/main/core/tokenizer/token/types/FunctionDefineToken.java
@@ -0,0 +1,33 @@
+package com.jantuomi.interpreter.main.core.tokenizer.token.types;
+
+import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
+import com.jantuomi.interpreter.main.core.parser.ast.FunctionBodyNode;
+import com.jantuomi.interpreter.main.core.parser.ast.FunctionDefineNode;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Created by jan on 16.6.2016.
+ */
+public class FunctionDefineToken extends VarargOperatorToken {
+    public FunctionDefineToken() {
+        super(Type.FunctionDefineToken, "func");
+    }
+
+    @Override
+    public ASTNode generateNode() {
+        FunctionDefineNode node = new FunctionDefineNode(this);
+        node.setName(args.get(0).getText());
+
+        List<ASTNode> funcArgs = new ArrayList<>();
+        for (int i = 1; i < args.size() - 1; i++) {
+            funcArgs.add(args.get(i).generateNode());
+        }
+        node.setArgs(funcArgs);
+
+        FunctionBodyNode bodyNode = (FunctionBodyNode) args.get(args.size() - 1).generateNode();
+        node.setBody(bodyNode);
+        return node;
+    }
+}
diff --git a/main/core/tokenizer/token/types/OpenParenToken.java b/main/core/tokenizer/token/types/OpenParenToken.java
new file mode 100644
index 0000000..07385c9
--- /dev/null
+++ b/main/core/tokenizer/token/types/OpenParenToken.java
@@ -0,0 +1,28 @@
+package com.jantuomi.interpreter.main.core.tokenizer.token.types;
+
+import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
+import com.jantuomi.interpreter.main.core.parser.ast.ParameterListNode;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+/**
+ * Created by jan on 19.6.2016.
+ */
+public class OpenParenToken extends VarargOperatorToken {
+    public OpenParenToken() {
+        super(Type.OpenParenToken);
+    }
+
+    @Override
+    public ASTNode generateNode() {
+        ParameterListNode node = new ParameterListNode(this);
+
+        for (Token token : args) {
+            if (token.getTokenType() == Type.ClosedParenToken) {
+                continue;
+            }
+
+            node.addParameter(token.generateNode());
+        }
+        return node;
+    }
+}
diff --git a/main/core/tokenizer/token/types/SubtractionToken.java b/main/core/tokenizer/token/types/SubtractionToken.java
new file mode 100644
index 0000000..a1b9a83
--- /dev/null
+++ b/main/core/tokenizer/token/types/SubtractionToken.java
@@ -0,0 +1,21 @@
+package com.jantuomi.interpreter.main.core.tokenizer.token.types;
+
+import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
+import com.jantuomi.interpreter.main.core.parser.ast.SubtractionNode;
+
+/**
+ * Created by jan on 16.6.2016.
+ */
+public class SubtractionToken extends BinaryOperatorToken {
+    public SubtractionToken() {
+        super(Type.SubtractionToken);
+    }
+
+    @Override
+    public ASTNode generateNode() {
+        return new SubtractionNode(this,
+                lhs.generateNode(),
+                rhs.generateNode()
+        );
+    }
+}
diff --git a/main/core/tokenizer/token/types/SymbolToken.java b/main/core/tokenizer/token/types/SymbolToken.java
new file mode 100644
index 0000000..b091ac9
--- /dev/null
+++ b/main/core/tokenizer/token/types/SymbolToken.java
@@ -0,0 +1,52 @@
+package com.jantuomi.interpreter.main.core.tokenizer.token.types;
+
+import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
+import com.jantuomi.interpreter.main.core.parser.ast.ParameterListNode;
+import com.jantuomi.interpreter.main.core.parser.ast.SymbolNode;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Created by jan on 16.6.2016.
+ */
+public class SymbolToken extends Token {
+
+    public void setCallArgumentList(OpenParenToken callArgumentList) {
+        this.callArgumentList = callArgumentList;
+    }
+
+    private OpenParenToken callArgumentList = null;
+
+    public SymbolToken(String name) {
+        super(Type.SymbolToken, name);
+    }
+
+    @Override
+    public ASTNode generateNode() {
+        SymbolNode node = new SymbolNode(this);
+        node.setName(this.getText());
+        if (callArgumentList != null) {
+            node.setParameterListNode((ParameterListNode) callArgumentList.generateNode());
+        }
+        return node;
+    }
+
+    @Override
+    public Token setArguments(List<Token> args) {
+        if (args.size() > 0) {
+            setCallArgumentList((OpenParenToken) args.get(0));
+        }
+        return this;
+    }
+
+    @Override
+    public List<Token> getChildren() {
+        if (callArgumentList == null) {
+            return Arrays.asList();
+        } else {
+            return Arrays.asList(callArgumentList);
+        }
+    }
+}
diff --git a/main/core/tokenizer/token/types/VarargOperatorToken.java b/main/core/tokenizer/token/types/VarargOperatorToken.java
new file mode 100644
index 0000000..f8901ee
--- /dev/null
+++ b/main/core/tokenizer/token/types/VarargOperatorToken.java
@@ -0,0 +1,33 @@
+package com.jantuomi.interpreter.main.core.tokenizer.token.types;
+
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.List;
+
+/**
+ * Created by jan on 16.6.2016.
+ */
+abstract public class VarargOperatorToken extends Token {
+
+    protected List<Token> args;
+
+    public VarargOperatorToken(Type type, String text) {
+        super(type, text);
+    }
+
+    public VarargOperatorToken(Type type) {
+        super(type);
+    }
+
+    @Override
+    public Token setArguments(List<Token> args) {
+        this.args = args;
+        return this;
+    }
+
+    @Override
+    public List<Token> getChildren() {
+        return args;
+    }
+
+}
diff --git a/main/core/tokenizer/token/types/WhitespaceToken.java b/main/core/tokenizer/token/types/WhitespaceToken.java
new file mode 100644
index 0000000..fb4949c
--- /dev/null
+++ b/main/core/tokenizer/token/types/WhitespaceToken.java
@@ -0,0 +1,31 @@
+package com.jantuomi.interpreter.main.core.tokenizer.token.types;
+
+import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Created by jan on 16.6.2016.
+ */
+public class WhitespaceToken extends Token {
+    public WhitespaceToken() {
+        super(Type.WhitespaceToken);
+    }
+
+    @Override
+    public ASTNode generateNode() {
+        return null;
+    }
+
+    @Override
+    public Token setArguments(List<Token> args) {
+        return this;
+    }
+
+    @Override
+    public List<Token> getChildren() {
+        return Arrays.asList();
+    }
+}