Implemented general enhancements

37 files changed, 857 insertions, 51 deletions

diff --git a/main/Main.java b/main/Main.java
index 31bbd7b..77c2f51 100644
--- a/main/Main.java
+++ b/main/Main.java
@@ -53,9 +53,20 @@ public class Main {
         Collections.reverse(sequence);
 
         List<Token> trees = Parser.getInstance().parse(sequence);
+        if (CommandLineArgumentContainer.getInstance().isDebugModeActive()) {
+            Parser.getInstance().printAllTrees(trees);
+        }
+
         List<ASTNode> nodes = ASTGenerator.getInstance().generate(trees);
+        if (CommandLineArgumentContainer.getInstance().isDebugModeActive()) {
+            ASTGenerator.getInstance().printAllTrees(nodes);
+        }
+
         String output = Interpreter.execute(nodes);
-        System.out.println(output);
+
+        if (output.trim().length() > 0) {
+            System.out.println(output);
+        }
     }
 
     public static boolean parseArguments(String[] args) {
diff --git a/main/core/CommandLineArgumentContainer.java b/main/core/CommandLineArgumentContainer.java
index 8984616..2bcb160 100644
--- a/main/core/CommandLineArgumentContainer.java
+++ b/main/core/CommandLineArgumentContainer.java
@@ -27,6 +27,10 @@ public class CommandLineArgumentContainer {
     @Option(name="-i", usage="Run in interactive mode.")
     public boolean interactiveModeActive = false;
 
+
+    @Option(name="-d", usage="Display verbose debug output.")
+    public boolean debugModeActive = false;
+
     public static CommandLineArgumentContainer getInstance() {
         return instance;
     }
@@ -35,6 +39,10 @@ public class CommandLineArgumentContainer {
         return interactiveModeActive;
     }
 
+    public boolean isDebugModeActive() {
+        return debugModeActive;
+    }
+
     public String getSourceFileContents() {
         if (srcFile == null) {
             return null;
diff --git a/main/core/parser/ASTGenerator.java b/main/core/parser/ASTGenerator.java
index e0a3146..548735b 100644
--- a/main/core/parser/ASTGenerator.java
+++ b/main/core/parser/ASTGenerator.java
@@ -27,4 +27,16 @@ public class ASTGenerator {
 
         return list;
     }
+
+    public void printTree(ASTNode root) {
+        System.out.println("### AST Tree begin ###");
+        root.print(0);
+        System.out.println("### AST Tree end ###");
+    }
+
+    public void printAllTrees(List<ASTNode> trees) {
+        for (ASTNode tree : trees) {
+            printTree(tree);
+        }
+    }
 }
diff --git a/main/core/parser/Parser.java b/main/core/parser/Parser.java
index e8b8813..f6e1208 100644
--- a/main/core/parser/Parser.java
+++ b/main/core/parser/Parser.java
@@ -3,9 +3,10 @@ package com.jantuomi.interpreter.main.core.parser;
 import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
 import com.jantuomi.interpreter.main.core.tokenizer.token.ArgumentInfo;
 import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+import com.jantuomi.interpreter.main.exception.ExceptionManager;
+import com.jantuomi.interpreter.main.exception.InterpreterException;
 
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.Stack;
 
@@ -28,11 +29,11 @@ public class Parser {
 
     private Stack<Token> stack;
 
-    public List<Token> parse(List<Token> tokens) {
+    public List<Token> parse(List<Token> tokens) throws InterpreterException {
         this.tokens = tokens;
         this.stack = new Stack<>();
 
-        List<Token> args = new ArrayList<>();
+        List<Token> args;
         List<Token> output = new ArrayList<>();
         for (Token t : tokens) {
             if (t.getTokenType() == Token.Type.NewlineToken) {
@@ -42,16 +43,48 @@ public class Parser {
                 continue;
             }
 
-            args.clear();
+            args = new ArrayList<>();
             ArgumentInfo argumentInfo = t.getArgumentInfo();
-            if (!argumentInfo.getVarargs()) {
-                for (int i = 0; i < argumentInfo.getCount(); i++) {
+
+            /*
+            This branch is executed if the token has a variable list of arguments
+             */
+            if (argumentInfo.getVarargs()) {
+                while (true) {
+                    if (stack.size() > 0) {
+                        Token arg = stack.pop();
+                        if (arg.getTokenType() != argumentInfo.getTerminator()) {
+                            args.add(arg);
+                        }
+                        else {
+                            args.add(arg);
+                            break;
+                        }
+                    }
+                }
+            }
+            /*
+            If the token has an optional argument type (it is only an argument if
+            it is of certain type)
+             */
+            else if (argumentInfo.getOptionalArgument() != null) {
+                if (stack.size() > 0 && stack.peek().getTokenType() == argumentInfo.getOptionalArgument()) {
                     args.add(stack.pop());
                 }
-            } else {
-                System.err.println("no vararg support yet");
             }
-            Collections.reverse(args);
+            /* Normal cases with a fixed list of arguments */
+            else {
+                for (int i = 0; i < argumentInfo.getCount(); i++) {
+                    if (stack.size() > 0) {
+                        args.add(stack.pop());
+                    } else {
+                        ExceptionManager.raise(InterpreterException.ExceptionType.ArgumentError, t.getLine(),
+                                t.toString(), Integer.toString(argumentInfo.getCount()));
+                        return null;
+                    }
+                }
+            }
+
             Token result = t.setArguments(args);
             stack.push(result);
         }
@@ -62,8 +95,14 @@ public class Parser {
     }
 
     public void printTree(Token e) {
-        System.out.println("### Tree begin ###");
+        System.out.println("### Token Tree begin ###");
         e.print(0);
-        System.out.println("### Tree end ###");
+        System.out.println("### Token Tree end ###");
+    }
+
+    public void printAllTrees(List<Token> trees) {
+        for (Token tree : trees) {
+            printTree(tree);
+        }
     }
 }
diff --git a/main/core/parser/ast/ASTNode.java b/main/core/parser/ast/ASTNode.java
index 4f840e3..99bdc06 100644
--- a/main/core/parser/ast/ASTNode.java
+++ b/main/core/parser/ast/ASTNode.java
@@ -31,7 +31,9 @@ abstract public class ASTNode {
 
         System.out.println(source.toString());
         for (ASTNode node : getChildren()) {
-            node.print(indent + 1);
+            if (node != null) {
+                node.print(indent + 1);
+            }
         }
     }
 }
diff --git a/main/core/parser/ast/AdditionNode.java b/main/core/parser/ast/AdditionNode.java
index 46f67a4..a6406ae 100644
--- a/main/core/parser/ast/AdditionNode.java
+++ b/main/core/parser/ast/AdditionNode.java
@@ -9,15 +9,10 @@ import java.util.List;
 /**
  * Created by jan on 16.6.2016.
  */
-public class AdditionNode extends ASTNode {
-
-    private ASTNode lhs;
-    private ASTNode rhs;
+public class AdditionNode extends BinaryOperatorNode {
 
     public AdditionNode(Token token, ASTNode lhs, ASTNode rhs) {
-        super(token);
-        this.lhs = lhs;
-        this.rhs = rhs;
+        super(token, lhs, rhs);
     }
 
     @Override
diff --git a/main/core/parser/ast/BinaryOperatorNode.java b/main/core/parser/ast/BinaryOperatorNode.java
new file mode 100644
index 0000000..9ec21a4
--- /dev/null
+++ b/main/core/parser/ast/BinaryOperatorNode.java
@@ -0,0 +1,17 @@
+package com.jantuomi.interpreter.main.core.parser.ast;
+
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+/**
+ * Created by jan on 16.6.2016.
+ */
+abstract public class BinaryOperatorNode extends ASTNode {
+    protected ASTNode lhs;
+    protected ASTNode rhs;
+
+    public BinaryOperatorNode(Token token, ASTNode lhs, ASTNode rhs) {
+        super(token);
+        this.lhs = lhs;
+        this.rhs = rhs;
+    }
+}
diff --git a/main/core/parser/ast/EmptyNode.java b/main/core/parser/ast/EmptyNode.java
new file mode 100644
index 0000000..44bfcc9
--- /dev/null
+++ b/main/core/parser/ast/EmptyNode.java
@@ -0,0 +1,27 @@
+package com.jantuomi.interpreter.main.core.parser.ast;
+
+import com.jantuomi.interpreter.main.core.parser.datatype.DataContainer;
+import com.jantuomi.interpreter.main.core.parser.datatype.VoidDataContainer;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Created by jan on 17.6.2016.
+ */
+public class EmptyNode extends ASTNode {
+    public EmptyNode(Token source) {
+        super(source);
+    }
+
+    @Override
+    public DataContainer evaluate() {
+        return new VoidDataContainer();
+    }
+
+    @Override
+    List<ASTNode> getChildren() {
+        return Arrays.asList();
+    }
+}
diff --git a/main/core/parser/ast/FunctionBodyNode.java b/main/core/parser/ast/FunctionBodyNode.java
new file mode 100644
index 0000000..394710c
--- /dev/null
+++ b/main/core/parser/ast/FunctionBodyNode.java
@@ -0,0 +1,25 @@
+package com.jantuomi.interpreter.main.core.parser.ast;
+
+import com.jantuomi.interpreter.main.core.parser.datatype.DataContainer;
+import com.jantuomi.interpreter.main.core.parser.datatype.VoidDataContainer;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+/**
+ * Created by jan on 17.6.2016.
+ */
+public class FunctionBodyNode extends VarargOperatorNode {
+    public FunctionBodyNode(Token token) {
+        super(token);
+    }
+
+    @Override
+    public DataContainer evaluate() {
+        DataContainer returnValue = new VoidDataContainer();
+
+        for (ASTNode node : args) {
+            returnValue = node.evaluate();
+        }
+
+        return returnValue;
+    }
+}
diff --git a/main/core/parser/ast/FunctionDefineNode.java b/main/core/parser/ast/FunctionDefineNode.java
new file mode 100644
index 0000000..e867f78
--- /dev/null
+++ b/main/core/parser/ast/FunctionDefineNode.java
@@ -0,0 +1,59 @@
+package com.jantuomi.interpreter.main.core.parser.ast;
+
+import com.jantuomi.interpreter.main.core.parser.datatype.DataContainer;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+/**
+ * Created by jan on 17.6.2016.
+ */
+public class FunctionDefineNode extends VarargOperatorNode {
+
+    private String name;
+
+    public void setBody(ASTNode body) {
+        this.body = body;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    private ASTNode body;
+
+    public FunctionDefineNode(Token token) {
+        super(token);
+    }
+
+    @Override
+    public DataContainer evaluate() {
+        // TODO add function in State
+        return null;
+    }
+
+    @Override
+    public void print(int indent) {
+
+        for (int i = 0; i < indent; i++) {
+            System.out.print("\t");
+        }
+
+        System.out.println(String.format("Function %s", name));
+
+        for (int i = 0; i < indent + 1; i++) {
+            System.out.print("\t");
+        }
+
+        System.out.println(String.format("Arguments: "));
+
+        for (ASTNode node : args) {
+            node.print(indent + 2);
+        }
+
+        for (int i = 0; i < indent + 1; i++) {
+            System.out.print("\t");
+        }
+
+        System.out.println(String.format("Body statements: "));
+        body.print(indent + 2);
+    }
+}
diff --git a/main/core/parser/ast/ParameterListNode.java b/main/core/parser/ast/ParameterListNode.java
new file mode 100644
index 0000000..50f919a
--- /dev/null
+++ b/main/core/parser/ast/ParameterListNode.java
@@ -0,0 +1,38 @@
+package com.jantuomi.interpreter.main.core.parser.ast;
+
+import com.jantuomi.interpreter.main.core.parser.datatype.DataContainer;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Created by jan on 19.6.2016.
+ */
+public class ParameterListNode extends ASTNode {
+
+    private List<ASTNode> parameters = new ArrayList<>();
+
+    public void addParameter(ASTNode param) {
+        parameters.add(param);
+    }
+
+    public List<ASTNode> getParameters() {
+        return parameters;
+    }
+
+    public ParameterListNode(Token token) {
+        super(token);
+    }
+
+    @Override
+    public DataContainer evaluate() {
+        // TODO use State to evaluate function
+        return null;
+    }
+
+    @Override
+    List<ASTNode> getChildren() {
+        return parameters;
+    }
+}
diff --git a/main/core/parser/ast/SubtractionNode.java b/main/core/parser/ast/SubtractionNode.java
new file mode 100644
index 0000000..0865ab1
--- /dev/null
+++ b/main/core/parser/ast/SubtractionNode.java
@@ -0,0 +1,30 @@
+package com.jantuomi.interpreter.main.core.parser.ast;
+
+import com.jantuomi.interpreter.main.core.parser.datatype.DataContainer;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Created by jan on 16.6.2016.
+ */
+public class SubtractionNode extends BinaryOperatorNode {
+    public SubtractionNode(Token token, ASTNode lhs, ASTNode rhs) {
+        super(token, lhs, rhs);
+    }
+
+    @Override
+    public DataContainer evaluate() {
+        DataContainer operand1 = lhs.evaluate();
+        DataContainer operand2 = rhs.evaluate();
+
+        DataContainer result = operand1.subtract(operand2);
+        return result;
+
+    }
+    @Override
+    List<ASTNode> getChildren() {
+        return Arrays.asList(lhs, rhs);
+    }
+}
diff --git a/main/core/parser/ast/SymbolNode.java b/main/core/parser/ast/SymbolNode.java
new file mode 100644
index 0000000..7ea8d96
--- /dev/null
+++ b/main/core/parser/ast/SymbolNode.java
@@ -0,0 +1,48 @@
+package com.jantuomi.interpreter.main.core.parser.ast;
+
+import com.jantuomi.interpreter.main.core.parser.datatype.DataContainer;
+import com.jantuomi.interpreter.main.core.runtime.State;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Created by jan on 17.6.2016.
+ */
+public class SymbolNode extends ASTNode {
+
+    public ParameterListNode getParameterListNode() {
+        return parameterListNode;
+    }
+
+    public void setParameterListNode(ParameterListNode parameterListNode) {
+        this.parameterListNode = parameterListNode;
+    }
+
+    private ParameterListNode parameterListNode;
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    private String name;
+
+    public SymbolNode(Token token) {
+        super(token);
+    }
+
+    @Override
+    public DataContainer evaluate() {
+        return State.getInstance().getSymbolValue(name, parameterListNode.getParameters());
+    }
+
+    @Override
+    List<ASTNode> getChildren() {
+        if (parameterListNode == null) {
+            return Arrays.asList();
+        } else {
+            return Arrays.asList(parameterListNode);
+        }
+    }
+}
diff --git a/main/core/parser/ast/VarargOperatorNode.java b/main/core/parser/ast/VarargOperatorNode.java
new file mode 100644
index 0000000..96a7320
--- /dev/null
+++ b/main/core/parser/ast/VarargOperatorNode.java
@@ -0,0 +1,26 @@
+package com.jantuomi.interpreter.main.core.parser.ast;
+
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.List;
+
+/**
+ * Created by jan on 17.6.2016.
+ */
+public abstract class VarargOperatorNode extends ASTNode {
+
+    protected List<ASTNode> args;
+
+    public VarargOperatorNode(Token token) {
+        super(token);
+    }
+
+    @Override
+    public List<ASTNode> getChildren() {
+        return args;
+    }
+
+    public void setArgs(List<ASTNode> args) {
+        this.args = args;
+    }
+}
diff --git a/main/core/parser/datatype/DataContainer.java b/main/core/parser/datatype/DataContainer.java
index 593855b..b504bd6 100644
--- a/main/core/parser/datatype/DataContainer.java
+++ b/main/core/parser/datatype/DataContainer.java
@@ -25,4 +25,5 @@ abstract public class DataContainer<T> {
     abstract public String toString();
 
     abstract public DataContainer<T> add(DataContainer<T> other);
+    public abstract DataContainer<T> subtract(DataContainer<T> other);
 }
diff --git a/main/core/parser/datatype/IntegerDataContainer.java b/main/core/parser/datatype/IntegerDataContainer.java
index b5fdac3..650db5c 100644
--- a/main/core/parser/datatype/IntegerDataContainer.java
+++ b/main/core/parser/datatype/IntegerDataContainer.java
@@ -22,4 +22,11 @@ public class IntegerDataContainer extends DataContainer<Integer> {
                 this.value + ((IntegerDataContainer) other).value
         );
     }
+
+    @Override
+    public DataContainer<Integer> subtract(DataContainer<Integer> other) {
+        return new IntegerDataContainer(
+                this.value - ((IntegerDataContainer) other).value
+        );
+    }
 }
diff --git a/main/core/parser/datatype/StringDataContainer.java b/main/core/parser/datatype/StringDataContainer.java
new file mode 100644
index 0000000..9916f77
--- /dev/null
+++ b/main/core/parser/datatype/StringDataContainer.java
@@ -0,0 +1,29 @@
+package com.jantuomi.interpreter.main.core.parser.datatype;
+
+/**
+ * Created by jan on 16.6.2016.
+ */
+public class StringDataContainer extends DataContainer<String> {
+    private String value;
+
+    public StringDataContainer(String text) {
+        this.value = text;
+    }
+
+    @Override
+    public String toString() {
+        return value;
+    }
+
+    @Override
+    public DataContainer<String> add(DataContainer<String> other) {
+        return new StringDataContainer(
+                value + ((StringDataContainer) other).value
+        );
+    }
+
+    @Override
+    public DataContainer<String> subtract(DataContainer<String> other) {
+        return null;
+    }
+}
diff --git a/main/core/parser/datatype/VoidDataContainer.java b/main/core/parser/datatype/VoidDataContainer.java
new file mode 100644
index 0000000..8e078d9
--- /dev/null
+++ b/main/core/parser/datatype/VoidDataContainer.java
@@ -0,0 +1,22 @@
+package com.jantuomi.interpreter.main.core.parser.datatype;
+
+/**
+ * Created by jan on 17.6.2016.
+ */
+public class VoidDataContainer extends DataContainer<Integer> {
+
+    @Override
+    public String toString() {
+        return "";
+    }
+
+    @Override
+    public DataContainer<Integer> add(DataContainer<Integer> other) {
+        return null;
+    }
+
+    @Override
+    public DataContainer<Integer> subtract(DataContainer<Integer> other) {
+        return null;
+    }
+}
diff --git a/main/core/runtime/Interpreter.java b/main/core/runtime/Interpreter.java
index 00294e2..0789822 100644
--- a/main/core/runtime/Interpreter.java
+++ b/main/core/runtime/Interpreter.java
@@ -2,7 +2,9 @@ package com.jantuomi.interpreter.main.core.runtime;
 
 import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
 import com.jantuomi.interpreter.main.core.parser.datatype.DataContainer;
+import com.sun.deploy.util.StringUtils;
 
+import java.util.ArrayList;
 import java.util.List;
 
 /**
@@ -19,17 +21,17 @@ public class Interpreter {
 
     }
     public static String execute(List<ASTNode> sequence) {
-        StringBuilder output = new StringBuilder();
+        List<String> output = new ArrayList<>();
         for (ASTNode node : sequence) {
             DataContainer data = node.evaluate();
 
             if (data != null) {
                 String out = data.toString();
-                output.append(out);
+                output.add(out);
             } else {
                 break;
             }
         }
-        return output.toString();
+        return StringUtils.join(output, " ");
     }
 }
diff --git a/main/core/runtime/State.java b/main/core/runtime/State.java
index 7e4fb71..7d1a2fa 100644
--- a/main/core/runtime/State.java
+++ b/main/core/runtime/State.java
@@ -1,8 +1,10 @@
 package com.jantuomi.interpreter.main.core.runtime;
 
+import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
 import com.jantuomi.interpreter.main.core.parser.datatype.DataContainer;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -29,6 +31,11 @@ public class State {
         }
     }
 
+    public DataContainer getSymbolValue(String symbol, List<ASTNode> parameters) {
+        // TODO
+        return null;
+    }
+
     public void setSymbolValue(String symbol, DataContainer value) {
         variables.put(symbol, value);
     }
diff --git a/main/core/tokenizer/Tokenizer.java b/main/core/tokenizer/Tokenizer.java
index 7e88a43..9aed09c 100644
--- a/main/core/tokenizer/Tokenizer.java
+++ b/main/core/tokenizer/Tokenizer.java
@@ -5,7 +5,10 @@ import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
 import com.jantuomi.interpreter.main.exception.ExceptionManager;
 import com.jantuomi.interpreter.main.exception.InterpreterException;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
 
 /**
  * Created by jan on 10.6.2016.
@@ -19,7 +22,7 @@ public class Tokenizer {
 
     private List<Token> tokens = new ArrayList<>();
 
-    private static SortedMap<Token.Type, String> tokenRegexes = new TreeMap<>();
+    private static Map<Token.Type, String> tokenRegexes = new LinkedHashMap<>();
     private static final List<Token.Type> discardedTokenTypes = new ArrayList<>();
     private static final List<Token.Type> erroneousTokenTypes = new ArrayList<>();
 
@@ -47,12 +50,15 @@ public class Tokenizer {
         tokenRegexes.put(Token.Type.NotEqualsToken, "^(\\!\\=)");
         tokenRegexes.put(Token.Type.OpenParenToken, "^(\\()");
         tokenRegexes.put(Token.Type.ClosedParenToken, "^(\\))");
-        tokenRegexes.put(Token.Type.FunctionDefineToken, "^(func)");
-        tokenRegexes.put(Token.Type.DeclarationToken, "^(decl)");
+        tokenRegexes.put(Token.Type.FunctionDefineToken, "^(func)\\b");
+        tokenRegexes.put(Token.Type.FunctionBodyToken, "^(as)\\b");
+        tokenRegexes.put(Token.Type.EndFunctionDefineToken, "^(end)\\b");
+        tokenRegexes.put(Token.Type.DeclarationToken, "^(decl)\\b");
         tokenRegexes.put(Token.Type.IntegerLiteralToken, "^(\\d+)");
         tokenRegexes.put(Token.Type.SymbolToken, "^([a-zA-Z]+\\w*)");
 
         discardedTokenTypes.add(Token.Type.WhitespaceToken);
+        discardedTokenTypes.add(Token.Type.CommentToken);
 
         erroneousTokenTypes.add(Token.Type.NotAToken);
     }
@@ -79,7 +85,7 @@ public class Tokenizer {
             }
 
             if (erroneousTokenTypes.contains(token.getTokenType())) {
-                ExceptionManager.raise(InterpreterException.ExceptionType.IllegalTokenError, line, Arrays.asList(token.getText()));
+                ExceptionManager.raise(InterpreterException.ExceptionType.IllegalTokenError, line, token.getText());
             }
 
             String tokenRawText = token.getRawText();
diff --git a/main/core/tokenizer/token/ArgumentInfo.java b/main/core/tokenizer/token/ArgumentInfo.java
index a9c3d90..b062a04 100644
--- a/main/core/tokenizer/token/ArgumentInfo.java
+++ b/main/core/tokenizer/token/ArgumentInfo.java
@@ -6,6 +6,27 @@ package com.jantuomi.interpreter.main.core.tokenizer.token;
 public class ArgumentInfo {
     private int count = 0;
     private boolean isVarargs = false;
+
+    public Token.Type getOptionalArgument() {
+        return optionalArgument;
+    }
+
+    public void setOptionalArgument(Token.Type optionalArgument) {
+        this.optionalArgument = optionalArgument;
+    }
+
+    private Token.Type optionalArgument = null;
+
+    public Token.Type getTerminator() {
+        return terminator;
+    }
+
+    public void setTerminator(Token.Type terminator) {
+        this.terminator = terminator;
+    }
+
+    private Token.Type terminator;
+
     public ArgumentInfo(int count) {
         this.count = count;
     }
diff --git a/main/core/tokenizer/token/Token.java b/main/core/tokenizer/token/Token.java
index 0d4395e..5248799 100644
--- a/main/core/tokenizer/token/Token.java
+++ b/main/core/tokenizer/token/Token.java
@@ -1,8 +1,9 @@
 package com.jantuomi.interpreter.main.core.tokenizer.token;
 
 import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
-import com.jantuomi.interpreter.main.core.tokenizer.token.types.AdditionToken;
-import com.jantuomi.interpreter.main.core.tokenizer.token.types.IntegerLiteralToken;
+import com.jantuomi.interpreter.main.core.tokenizer.token.types.*;
+import com.jantuomi.interpreter.main.exception.ExceptionManager;
+import com.jantuomi.interpreter.main.exception.InterpreterException;
 import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
 
@@ -11,6 +12,7 @@ import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+
 /**
  * Created by jan on 10.6.2016.
  */
@@ -53,6 +55,7 @@ abstract public class Token {
         EndStatementToken,
         SymbolToken,
         EndFunctionDefineToken,
+        FunctionBodyToken,
         NotAToken
     }
 
@@ -115,13 +118,13 @@ abstract public class Token {
         initialize(type, text, text);
     }
 
-    public static Token makeToken(String string, Map<Type, String> regexes, int line) {
+    public static Token makeToken(String string, Map<Type, String> regexes, int line) throws InterpreterException {
         Type type = Type.NotAToken;
         String text = Character.toString(string.charAt(0));
 
         for (Type t : regexes.keySet()) {
             String regex = regexes.get(t);
-            Token found = matchToken(string, t, regex);
+            Token found = matchToken(string, t, regex, line);
             if (found != null) {
                 found.setLine(line);
                 return found;
@@ -132,7 +135,7 @@ abstract public class Token {
         return null;
     }
 
-    public static Token matchToken(String string, Type type, String regex) {
+    public static Token matchToken(String string, Type type, String regex, int line) throws InterpreterException {
         Pattern pattern = Pattern.compile(regex, Pattern.DOTALL);
         Matcher matcher = pattern.matcher(string);
 
@@ -141,12 +144,37 @@ abstract public class Token {
             String rawText = matcher.group(0);
 
             switch (type) {
+                case WhitespaceToken:
+                    return new WhitespaceToken();
+                case CommentToken:
+                    return new CommentToken(text, rawText);
+
+                // TODO placeholder whitespace instead of newline
+                case NewlineToken:
+                    return new WhitespaceToken();
+
                 case IntegerLiteralToken:
                     return new IntegerLiteralToken(text);
                 case AdditionToken:
                     return new AdditionToken();
+                case SubtractionToken:
+                    return new SubtractionToken();
+                case FunctionDefineToken:
+                    return new FunctionDefineToken();
+                case EndFunctionDefineToken:
+                    return new EndFunctionDefineToken();
+                case FunctionBodyToken:
+                    return new FunctionBodyToken();
+                case SymbolToken:
+                    return new SymbolToken(text);
+                case OpenParenToken:
+                    return new OpenParenToken();
+                case ClosedParenToken:
+                    return new ClosedParenToken();
                 default:
-                    return null;
+                    ExceptionManager.raise(InterpreterException.ExceptionType.IllegalTokenError, line,
+                            type.toString()
+                    );
             }
         }
 
@@ -157,9 +185,24 @@ abstract public class Token {
         switch (getTokenType()) {
 
             case FunctionDefineToken:
-                ArgumentInfo ai = new ArgumentInfo();
-                ai.setVariable(true);
-                return ai;
+                ArgumentInfo fdai = new ArgumentInfo();
+                fdai.setVariable(true);
+                fdai.setTerminator(Type.FunctionBodyToken);
+                return fdai;
+            case FunctionBodyToken:
+                ArgumentInfo fbai = new ArgumentInfo();
+                fbai.setVariable(true);
+                fbai.setTerminator(Type.EndFunctionDefineToken);
+                return fbai;
+            case OpenParenToken:
+                ArgumentInfo opai = new ArgumentInfo();
+                opai.setVariable(true);
+                opai.setTerminator(Type.ClosedParenToken);
+                return opai;
+            case SymbolToken:
+                ArgumentInfo sai = new ArgumentInfo();
+                sai.setOptionalArgument(Type.OpenParenToken);
+                return sai;
             case AdditionToken:
             case SubtractionToken:
             case DivisionToken:
@@ -201,7 +244,11 @@ abstract public class Token {
 
     @Override
     public String toString() {
-        return String.format(String.format("%s %s", getTokenType(), getText()));
+        return String.format(String.format(
+                "%s%s",
+                getTokenType().toString().replace("Token", ""),
+                getText() == null ? "" : " " + getText()
+        ));
     }
 
     public String toFormattedString() {
diff --git a/main/core/tokenizer/token/types/ClosedParenToken.java b/main/core/tokenizer/token/types/ClosedParenToken.java
new file mode 100644
index 0000000..17e5c67
--- /dev/null
+++ b/main/core/tokenizer/token/types/ClosedParenToken.java
@@ -0,0 +1,31 @@
+package com.jantuomi.interpreter.main.core.tokenizer.token.types;
+
+import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Created by jan on 19.6.2016.
+ */
+public class ClosedParenToken extends Token {
+    public ClosedParenToken() {
+        super(Type.ClosedParenToken);
+    }
+
+    @Override
+    public ASTNode generateNode() {
+        return null;
+    }
+
+    @Override
+    public Token setArguments(List<Token> args) {
+        return this;
+    }
+
+    @Override
+    public List<Token> getChildren() {
+        return Arrays.asList();
+    }
+}
diff --git a/main/core/tokenizer/token/types/CommentToken.java b/main/core/tokenizer/token/types/CommentToken.java
new file mode 100644
index 0000000..45ca995
--- /dev/null
+++ b/main/core/tokenizer/token/types/CommentToken.java
@@ -0,0 +1,32 @@
+package com.jantuomi.interpreter.main.core.tokenizer.token.types;
+
+import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
+import com.jantuomi.interpreter.main.core.parser.ast.EmptyNode;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Created by jan on 19.6.2016.
+ */
+public class CommentToken extends Token {
+    public CommentToken(String text, String rawText) {
+        super(Type.CommentToken, text, rawText);
+    }
+
+    @Override
+    public ASTNode generateNode() {
+        return new EmptyNode(this);
+    }
+
+    @Override
+    public Token setArguments(List<Token> args) {
+        return this;
+    }
+
+    @Override
+    public List<Token> getChildren() {
+        return Arrays.asList();
+    }
+}
diff --git a/main/core/tokenizer/token/types/EndFunctionDefineToken.java b/main/core/tokenizer/token/types/EndFunctionDefineToken.java
new file mode 100644
index 0000000..ffd33c5
--- /dev/null
+++ b/main/core/tokenizer/token/types/EndFunctionDefineToken.java
@@ -0,0 +1,31 @@
+package com.jantuomi.interpreter.main.core.tokenizer.token.types;
+
+import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Created by jan on 16.6.2016.
+ */
+public class EndFunctionDefineToken extends Token {
+    public EndFunctionDefineToken() {
+        super(Type.EndFunctionDefineToken, "end");
+    }
+
+    @Override
+    public ASTNode generateNode() {
+        return null;
+    }
+
+    @Override
+    public Token setArguments(List<Token> args) {
+        return this;
+    }
+
+    @Override
+    public List<Token> getChildren() {
+        return Arrays.asList();
+    }
+}
diff --git a/main/core/tokenizer/token/types/FunctionBodyToken.java b/main/core/tokenizer/token/types/FunctionBodyToken.java
new file mode 100644
index 0000000..647ed55
--- /dev/null
+++ b/main/core/tokenizer/token/types/FunctionBodyToken.java
@@ -0,0 +1,29 @@
+package com.jantuomi.interpreter.main.core.tokenizer.token.types;
+
+import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
+import com.jantuomi.interpreter.main.core.parser.ast.FunctionBodyNode;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Created by jan on 16.6.2016.
+ */
+public class FunctionBodyToken extends VarargOperatorToken {
+    public FunctionBodyToken() {
+        super(Type.FunctionBodyToken, "as");
+    }
+
+    @Override
+    public ASTNode generateNode() {
+        FunctionBodyNode node = new FunctionBodyNode(this);
+        List<ASTNode> statements = new ArrayList<>();
+        for (int i = 0; i < args.size() - 1; i++) {
+            Token arg = args.get(i);
+            statements.add(arg.generateNode());
+        }
+        node.setArgs(statements);
+        return node;
+    }
+}
diff --git a/main/core/tokenizer/token/types/FunctionDefineToken.java b/main/core/tokenizer/token/types/FunctionDefineToken.java
new file mode 100644
index 0000000..6bbfd84
--- /dev/null
+++ b/main/core/tokenizer/token/types/FunctionDefineToken.java
@@ -0,0 +1,33 @@
+package com.jantuomi.interpreter.main.core.tokenizer.token.types;
+
+import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
+import com.jantuomi.interpreter.main.core.parser.ast.FunctionBodyNode;
+import com.jantuomi.interpreter.main.core.parser.ast.FunctionDefineNode;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Created by jan on 16.6.2016.
+ */
+public class FunctionDefineToken extends VarargOperatorToken {
+    public FunctionDefineToken() {
+        super(Type.FunctionDefineToken, "func");
+    }
+
+    @Override
+    public ASTNode generateNode() {
+        FunctionDefineNode node = new FunctionDefineNode(this);
+        node.setName(args.get(0).getText());
+
+        List<ASTNode> funcArgs = new ArrayList<>();
+        for (int i = 1; i < args.size() - 1; i++) {
+            funcArgs.add(args.get(i).generateNode());
+        }
+        node.setArgs(funcArgs);
+
+        FunctionBodyNode bodyNode = (FunctionBodyNode) args.get(args.size() - 1).generateNode();
+        node.setBody(bodyNode);
+        return node;
+    }
+}
diff --git a/main/core/tokenizer/token/types/OpenParenToken.java b/main/core/tokenizer/token/types/OpenParenToken.java
new file mode 100644
index 0000000..07385c9
--- /dev/null
+++ b/main/core/tokenizer/token/types/OpenParenToken.java
@@ -0,0 +1,28 @@
+package com.jantuomi.interpreter.main.core.tokenizer.token.types;
+
+import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
+import com.jantuomi.interpreter.main.core.parser.ast.ParameterListNode;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+/**
+ * Created by jan on 19.6.2016.
+ */
+public class OpenParenToken extends VarargOperatorToken {
+    public OpenParenToken() {
+        super(Type.OpenParenToken);
+    }
+
+    @Override
+    public ASTNode generateNode() {
+        ParameterListNode node = new ParameterListNode(this);
+
+        for (Token token : args) {
+            if (token.getTokenType() == Type.ClosedParenToken) {
+                continue;
+            }
+
+            node.addParameter(token.generateNode());
+        }
+        return node;
+    }
+}
diff --git a/main/core/tokenizer/token/types/SubtractionToken.java b/main/core/tokenizer/token/types/SubtractionToken.java
new file mode 100644
index 0000000..a1b9a83
--- /dev/null
+++ b/main/core/tokenizer/token/types/SubtractionToken.java
@@ -0,0 +1,21 @@
+package com.jantuomi.interpreter.main.core.tokenizer.token.types;
+
+import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
+import com.jantuomi.interpreter.main.core.parser.ast.SubtractionNode;
+
+/**
+ * Created by jan on 16.6.2016.
+ */
+public class SubtractionToken extends BinaryOperatorToken {
+    public SubtractionToken() {
+        super(Type.SubtractionToken);
+    }
+
+    @Override
+    public ASTNode generateNode() {
+        return new SubtractionNode(this,
+                lhs.generateNode(),
+                rhs.generateNode()
+        );
+    }
+}
diff --git a/main/core/tokenizer/token/types/SymbolToken.java b/main/core/tokenizer/token/types/SymbolToken.java
new file mode 100644
index 0000000..b091ac9
--- /dev/null
+++ b/main/core/tokenizer/token/types/SymbolToken.java
@@ -0,0 +1,52 @@
+package com.jantuomi.interpreter.main.core.tokenizer.token.types;
+
+import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
+import com.jantuomi.interpreter.main.core.parser.ast.ParameterListNode;
+import com.jantuomi.interpreter.main.core.parser.ast.SymbolNode;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Created by jan on 16.6.2016.
+ */
+public class SymbolToken extends Token {
+
+    public void setCallArgumentList(OpenParenToken callArgumentList) {
+        this.callArgumentList = callArgumentList;
+    }
+
+    private OpenParenToken callArgumentList = null;
+
+    public SymbolToken(String name) {
+        super(Type.SymbolToken, name);
+    }
+
+    @Override
+    public ASTNode generateNode() {
+        SymbolNode node = new SymbolNode(this);
+        node.setName(this.getText());
+        if (callArgumentList != null) {
+            node.setParameterListNode((ParameterListNode) callArgumentList.generateNode());
+        }
+        return node;
+    }
+
+    @Override
+    public Token setArguments(List<Token> args) {
+        if (args.size() > 0) {
+            setCallArgumentList((OpenParenToken) args.get(0));
+        }
+        return this;
+    }
+
+    @Override
+    public List<Token> getChildren() {
+        if (callArgumentList == null) {
+            return Arrays.asList();
+        } else {
+            return Arrays.asList(callArgumentList);
+        }
+    }
+}
diff --git a/main/core/tokenizer/token/types/VarargOperatorToken.java b/main/core/tokenizer/token/types/VarargOperatorToken.java
new file mode 100644
index 0000000..f8901ee
--- /dev/null
+++ b/main/core/tokenizer/token/types/VarargOperatorToken.java
@@ -0,0 +1,33 @@
+package com.jantuomi.interpreter.main.core.tokenizer.token.types;
+
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.List;
+
+/**
+ * Created by jan on 16.6.2016.
+ */
+abstract public class VarargOperatorToken extends Token {
+
+    protected List<Token> args;
+
+    public VarargOperatorToken(Type type, String text) {
+        super(type, text);
+    }
+
+    public VarargOperatorToken(Type type) {
+        super(type);
+    }
+
+    @Override
+    public Token setArguments(List<Token> args) {
+        this.args = args;
+        return this;
+    }
+
+    @Override
+    public List<Token> getChildren() {
+        return args;
+    }
+
+}
diff --git a/main/core/tokenizer/token/types/WhitespaceToken.java b/main/core/tokenizer/token/types/WhitespaceToken.java
new file mode 100644
index 0000000..fb4949c
--- /dev/null
+++ b/main/core/tokenizer/token/types/WhitespaceToken.java
@@ -0,0 +1,31 @@
+package com.jantuomi.interpreter.main.core.tokenizer.token.types;
+
+import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Created by jan on 16.6.2016.
+ */
+public class WhitespaceToken extends Token {
+    public WhitespaceToken() {
+        super(Type.WhitespaceToken);
+    }
+
+    @Override
+    public ASTNode generateNode() {
+        return null;
+    }
+
+    @Override
+    public Token setArguments(List<Token> args) {
+        return this;
+    }
+
+    @Override
+    public List<Token> getChildren() {
+        return Arrays.asList();
+    }
+}
diff --git a/main/exception/ExceptionManager.java b/main/exception/ExceptionManager.java
index dbb59d3..2dd46f5 100644
--- a/main/exception/ExceptionManager.java
+++ b/main/exception/ExceptionManager.java
@@ -1,7 +1,5 @@
 package com.jantuomi.interpreter.main.exception;
 
-import java.util.List;
-
 /**
  * Created by jan on 10.6.2016.
  */
@@ -16,7 +14,7 @@ public class ExceptionManager {
         return instance;
     }
 
-    public static void raise(InterpreterException.ExceptionType ex, int line, List<String> args) throws InterpreterException {
+    public static void raise(InterpreterException.ExceptionType ex, int line, String... args) throws InterpreterException {
         InterpreterException e = new InterpreterException(ex, line, args);
         throw e;
     }
diff --git a/main/exception/InterpreterException.java b/main/exception/InterpreterException.java
index 26e39fd..1565df9 100644
--- a/main/exception/InterpreterException.java
+++ b/main/exception/InterpreterException.java
@@ -2,7 +2,6 @@ package com.jantuomi.interpreter.main.exception;
 
 
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 /**
@@ -14,7 +13,8 @@ public class InterpreterException extends Exception {
         IllegalTokenError,
         UnknownOperatorError,
         SyntaxError,
-        TypeError
+        TypeError,
+        ArgumentError
     }
 
     public static Map<ExceptionType, String> errorTexts = new HashMap<>();
@@ -24,20 +24,20 @@ public class InterpreterException extends Exception {
         errorTexts.put(ExceptionType.UnknownOperatorError, "Unexpected operator %s found.");
         errorTexts.put(ExceptionType.SyntaxError, "Unexpected %s.");
         errorTexts.put(ExceptionType.TypeError, "Incompatible types %s and %s.");
+        errorTexts.put(ExceptionType.ArgumentError, "Function %s requires %s arguments.");
     }
 
     private ExceptionType exceptionType;
 
-    public InterpreterException(ExceptionType exceptionType, int line, List<String> args) {
-        super(formatMessage(exceptionType.toString() + what(exceptionType), line, args));
+    public InterpreterException(ExceptionType exceptionType, int line, String... args) {
+        super(formatMessage(exceptionType.toString() + ": " + what(exceptionType), line, args), null, false, false);
         this.exceptionType = exceptionType;
     }
 
-    private static String formatMessage(String message, int line, List<String> args) {
-        for (String arg : args) {
-            message = String.format(message, arg);
-        }
-        return String.format("line %d: ", line) + message;
+    private static String formatMessage(String message, int line, String... args) {
+
+        message = String.format(message, args);
+        return String.format("\nline %d: ", line) + message;
     }
 
     public static String what(ExceptionType type) {
diff --git a/test/MainTest.java b/test/MainTest.java
index d6171a5..1730a6b 100644
--- a/test/MainTest.java
+++ b/test/MainTest.java
@@ -25,7 +25,7 @@ public class MainTest {
 
     @Test
     public void testWholeProcedure() throws InterpreterException {
-        String input = "x <- 1 + 2";
+        String input = "- + 2 5 1";
         Main.run(input);
     }
 }
\ No newline at end of file
diff --git a/test/resources/funcdef.bork b/test/resources/funcdef.bork
new file mode 100644
index 0000000..8412129
--- /dev/null
+++ b/test/resources/funcdef.bork
@@ -0,0 +1,8 @@
+/* kommentti juu jee */
+
+func my_func x y as
+    + x 1
+    - y x
+end
+
+my_func(1 2)
\ No newline at end of file