6 files changed, 170 insertions, 4 deletions
diff --git a/main/core/tokenizer/Tokenizer.java b/main/core/tokenizer/Tokenizer.java
index c04db22..7e88a43 100644
--- a/main/core/tokenizer/Tokenizer.java
+++ b/main/core/tokenizer/Tokenizer.java
@@ -57,7 +57,7 @@ public class Tokenizer {
         erroneousTokenTypes.add(Token.Type.NotAToken);
     }
 
-    public List<Token> tokenize(String string) {
+    public List<Token> tokenize(String string) throws InterpreterException {
         sourceString = string;
         tokens.clear();
         int line = 1;
@@ -79,7 +79,7 @@ public class Tokenizer {
             }
 
             if (erroneousTokenTypes.contains(token.getTokenType())) {
-                ExceptionManager.raise(InterpreterException.Exception.IllegalTokenError, line, Arrays.asList(token.getText()));
+                ExceptionManager.raise(InterpreterException.ExceptionType.IllegalTokenError, line, Arrays.asList(token.getText()));
             }
 
             String tokenRawText = token.getRawText();
diff --git a/main/core/tokenizer/token/ArgumentInfo.java b/main/core/tokenizer/token/ArgumentInfo.java
new file mode 100644
index 0000000..a9c3d90
--- /dev/null
+++ b/main/core/tokenizer/token/ArgumentInfo.java
@@ -0,0 +1,28 @@
+package com.jantuomi.interpreter.main.core.tokenizer.token;
+
+/**
+ * Created by jan on 15.6.2016.
+ */
+public class ArgumentInfo {
+    private int count = 0;
+    private boolean isVarargs = false;
+    public ArgumentInfo(int count) {
+        this.count = count;
+    }
+
+    public ArgumentInfo() {
+
+    }
+
+    public void setVariable(boolean value) {
+        this.isVarargs = value;
+    }
+
+    public boolean getVarargs() {
+        return isVarargs;
+    }
+
+    public int getCount() {
+        return count;
+    }
+}
diff --git a/main/core/tokenizer/token/Token.java b/main/core/tokenizer/token/Token.java
index 99ae913..0d4395e 100644
--- a/main/core/tokenizer/token/Token.java
+++ b/main/core/tokenizer/token/Token.java
@@ -1,8 +1,12 @@
 package com.jantuomi.interpreter.main.core.tokenizer.token;
 
+import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
+import com.jantuomi.interpreter.main.core.tokenizer.token.types.AdditionToken;
+import com.jantuomi.interpreter.main.core.tokenizer.token.types.IntegerLiteralToken;
 import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
 
+import java.util.List;
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -10,7 +14,19 @@ import java.util.regex.Pattern;
 /**
  * Created by jan on 10.6.2016.
  */
-public class Token {
+abstract public class Token {
+
+    public void print(int indent) {
+        for (int i = 0; i < indent; i++) {
+            System.out.print("\t");
+        }
+        System.out.println(toString());
+        for (Token child : getChildren()) {
+            child.print(indent + 1);
+        }
+    }
+
+    abstract public ASTNode generateNode();
 
     /* Token types, ordered by precedence */
     public enum Type {
@@ -40,6 +56,10 @@ public class Token {
         NotAToken
     }
 
+    abstract public Token setArguments(List<Token> args);
+
+    abstract public List<Token> getChildren();
+
     public boolean is(Type type) {
         return getTokenType() == type;
     }
@@ -120,12 +140,45 @@ public class Token {
             String text = matcher.group(1);
             String rawText = matcher.group(0);
 
-            return new Token(type, text, rawText);
+            switch (type) {
+                case IntegerLiteralToken:
+                    return new IntegerLiteralToken(text);
+                case AdditionToken:
+                    return new AdditionToken();
+                default:
+                    return null;
+            }
         }
 
         return null;
     }
 
+    public ArgumentInfo getArgumentInfo() {
+        switch (getTokenType()) {
+
+            case FunctionDefineToken:
+                ArgumentInfo ai = new ArgumentInfo();
+                ai.setVariable(true);
+                return ai;
+            case AdditionToken:
+            case SubtractionToken:
+            case DivisionToken:
+            case MultiplicationToken:
+            case AssignmentToken:
+            case LessThanToken:
+            case GreaterThanToken:
+            case LessOrEqualThanToken:
+            case GreaterOrEqualThanToken:
+            case EqualsToken:
+            case NotEqualsToken:
+                return new ArgumentInfo(2);
+            case DeclarationToken:
+                return new ArgumentInfo(1);
+            default:
+                return new ArgumentInfo(0);
+        }
+    }
+
     @Override
     public int hashCode() {
         return new HashCodeBuilder(17, 37)
diff --git a/main/core/tokenizer/token/types/AdditionToken.java b/main/core/tokenizer/token/types/AdditionToken.java
new file mode 100644
index 0000000..2d77b46
--- /dev/null
+++ b/main/core/tokenizer/token/types/AdditionToken.java
@@ -0,0 +1,22 @@
+package com.jantuomi.interpreter.main.core.tokenizer.token.types;
+
+import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
+import com.jantuomi.interpreter.main.core.parser.ast.AdditionNode;
+
+/**
+ * Created by jan on 15.6.2016.
+ */
+public class AdditionToken extends BinaryOperatorToken {
+    public AdditionToken() {
+        super(Type.AdditionToken);
+    }
+
+    @Override
+    public ASTNode generateNode() {
+        return new AdditionNode(this,
+                lhs.generateNode(),
+                rhs.generateNode()
+        );
+
+    }
+}
diff --git a/main/core/tokenizer/token/types/BinaryOperatorToken.java b/main/core/tokenizer/token/types/BinaryOperatorToken.java
new file mode 100644
index 0000000..3998bb9
--- /dev/null
+++ b/main/core/tokenizer/token/types/BinaryOperatorToken.java
@@ -0,0 +1,31 @@
+package com.jantuomi.interpreter.main.core.tokenizer.token.types;
+
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Created by jan on 15.6.2016.
+ */
+abstract public class BinaryOperatorToken extends Token {
+    protected Token lhs;
+    protected Token rhs;
+
+    public BinaryOperatorToken(Type type) {
+        super(type);
+    }
+
+    @Override
+    public Token setArguments(List<Token> args) {
+        lhs = args.get(0);
+        rhs = args.get(1);
+        return this;
+    }
+
+    @Override
+    public List<Token> getChildren() {
+        return Arrays.asList(lhs, rhs);
+
+    }
+}
diff --git a/main/core/tokenizer/token/types/IntegerLiteralToken.java b/main/core/tokenizer/token/types/IntegerLiteralToken.java
new file mode 100644
index 0000000..cdb4fde
--- /dev/null
+++ b/main/core/tokenizer/token/types/IntegerLiteralToken.java
@@ -0,0 +1,32 @@
+package com.jantuomi.interpreter.main.core.tokenizer.token.types;
+
+import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
+import com.jantuomi.interpreter.main.core.parser.ast.IntegerLiteralNode;
+import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Created by jan on 15.6.2016.
+ */
+public class IntegerLiteralToken extends Token{
+    public IntegerLiteralToken(String text) {
+        super(Type.IntegerLiteralToken, text);
+    }
+
+    @Override
+    public ASTNode generateNode() {
+        return new IntegerLiteralNode(this);
+    }
+
+    @Override
+    public Token setArguments(List<Token> args) {
+        return this;
+    }
+
+    @Override
+    public List<Token> getChildren() {
+        return Arrays.asList();
+    }
+}