1 files changed, 57 insertions, 10 deletions

diff --git a/main/core/tokenizer/token/Token.java b/main/core/tokenizer/token/Token.java
index 0d4395e..5248799 100644
--- a/main/core/tokenizer/token/Token.java
+++ b/main/core/tokenizer/token/Token.java
@@ -1,8 +1,9 @@
 package com.jantuomi.interpreter.main.core.tokenizer.token;
 
 import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
-import com.jantuomi.interpreter.main.core.tokenizer.token.types.AdditionToken;
-import com.jantuomi.interpreter.main.core.tokenizer.token.types.IntegerLiteralToken;
+import com.jantuomi.interpreter.main.core.tokenizer.token.types.*;
+import com.jantuomi.interpreter.main.exception.ExceptionManager;
+import com.jantuomi.interpreter.main.exception.InterpreterException;
 import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
 
@@ -11,6 +12,7 @@ import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+
 /**
  * Created by jan on 10.6.2016.
  */
@@ -53,6 +55,7 @@ abstract public class Token {
         EndStatementToken,
         SymbolToken,
         EndFunctionDefineToken,
+        FunctionBodyToken,
         NotAToken
     }
 
@@ -115,13 +118,13 @@ abstract public class Token {
         initialize(type, text, text);
     }
 
-    public static Token makeToken(String string, Map<Type, String> regexes, int line) {
+    public static Token makeToken(String string, Map<Type, String> regexes, int line) throws InterpreterException {
         Type type = Type.NotAToken;
         String text = Character.toString(string.charAt(0));
 
         for (Type t : regexes.keySet()) {
             String regex = regexes.get(t);
-            Token found = matchToken(string, t, regex);
+            Token found = matchToken(string, t, regex, line);
             if (found != null) {
                 found.setLine(line);
                 return found;
@@ -132,7 +135,7 @@ abstract public class Token {
         return null;
     }
 
-    public static Token matchToken(String string, Type type, String regex) {
+    public static Token matchToken(String string, Type type, String regex, int line) throws InterpreterException {
         Pattern pattern = Pattern.compile(regex, Pattern.DOTALL);
         Matcher matcher = pattern.matcher(string);
 
@@ -141,12 +144,37 @@ abstract public class Token {
             String rawText = matcher.group(0);
 
             switch (type) {
+                case WhitespaceToken:
+                    return new WhitespaceToken();
+                case CommentToken:
+                    return new CommentToken(text, rawText);
+
+                // TODO placeholder whitespace instead of newline
+                case NewlineToken:
+                    return new WhitespaceToken();
+
                 case IntegerLiteralToken:
                     return new IntegerLiteralToken(text);
                 case AdditionToken:
                     return new AdditionToken();
+                case SubtractionToken:
+                    return new SubtractionToken();
+                case FunctionDefineToken:
+                    return new FunctionDefineToken();
+                case EndFunctionDefineToken:
+                    return new EndFunctionDefineToken();
+                case FunctionBodyToken:
+                    return new FunctionBodyToken();
+                case SymbolToken:
+                    return new SymbolToken(text);
+                case OpenParenToken:
+                    return new OpenParenToken();
+                case ClosedParenToken:
+                    return new ClosedParenToken();
                 default:
-                    return null;
+                    ExceptionManager.raise(InterpreterException.ExceptionType.IllegalTokenError, line,
+                            type.toString()
+                    );
             }
         }
 
@@ -157,9 +185,24 @@ abstract public class Token {
         switch (getTokenType()) {
 
             case FunctionDefineToken:
-                ArgumentInfo ai = new ArgumentInfo();
-                ai.setVariable(true);
-                return ai;
+                ArgumentInfo fdai = new ArgumentInfo();
+                fdai.setVariable(true);
+                fdai.setTerminator(Type.FunctionBodyToken);
+                return fdai;
+            case FunctionBodyToken:
+                ArgumentInfo fbai = new ArgumentInfo();
+                fbai.setVariable(true);
+                fbai.setTerminator(Type.EndFunctionDefineToken);
+                return fbai;
+            case OpenParenToken:
+                ArgumentInfo opai = new ArgumentInfo();
+                opai.setVariable(true);
+                opai.setTerminator(Type.ClosedParenToken);
+                return opai;
+            case SymbolToken:
+                ArgumentInfo sai = new ArgumentInfo();
+                sai.setOptionalArgument(Type.OpenParenToken);
+                return sai;
             case AdditionToken:
             case SubtractionToken:
             case DivisionToken:
@@ -201,7 +244,11 @@ abstract public class Token {
 
     @Override
     public String toString() {
-        return String.format(String.format("%s %s", getTokenType(), getText()));
+        return String.format(String.format(
+                "%s%s",
+                getTokenType().toString().replace("Token", ""),
+                getText() == null ? "" : " " + getText()
+        ));
     }
 
     public String toFormattedString() {