Add ObjectMember token type

Code cleanup in Token and Tokenizer

2 files changed, 3 insertions, 21 deletions

diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/Tokenizer.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/Tokenizer.java
index 66b8251..2571b61 100644
--- a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/Tokenizer.java
+++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/Tokenizer.java
@@ -67,6 +67,7 @@ public class Tokenizer {
 
         tokenRegexes.put(Token.Type.CallToken, "^([a-öA-Ö]+\\w*)\\?[\\s|\\!]");
         tokenRegexes.put(Token.Type.SymbolToken, "^([a-öA-Ö]+\\w*)\\b");
+        tokenRegexes.put(Token.Type.ObjectMemberToken, "^([a-öA-Ö]+\\w*)\\:\\b");
 
         discardedTokenTypes.add(Token.Type.WhitespaceToken);
         discardedTokenTypes.add(Token.Type.CommentToken);
diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/Token.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/Token.java
index 720efcd..97ac488 100644
--- a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/Token.java
+++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/Token.java
@@ -30,7 +30,7 @@ abstract public class Token {
 
     abstract public ASTNode generateNode() throws TunkkiError;
 
-    /* Token types, ordered by precedence */
+    /* Token types */
     public enum Type {
         CommentToken,
 
@@ -72,6 +72,7 @@ abstract public class Token {
         NegationToken,
 
         DeclareAssignToken,
+        ObjectMemberToken,
         NotAToken
     }
 
@@ -79,10 +80,6 @@ abstract public class Token {
 
     abstract public List<Token> getChildren();
 
-    public boolean is(Type type) {
-        return getTokenType() == type;
-    }
-
     public int getLine() {
         return line;
     }
@@ -108,10 +105,6 @@ abstract public class Token {
         return rawText;
     }
 
-    public boolean isHigherPrecedenceThan(Token other) {
-        return getTokenType().ordinal() <= other.getTokenType().ordinal();
-    }
-
     public Token(Type type, String text, String rawText) {
         initialize(type, text, rawText);
     }
@@ -135,9 +128,6 @@ abstract public class Token {
     }
 
     public static Token makeToken(String string, Map<Type, String> regexes, int line) throws TunkkiError {
-        Type type = Type.NotAToken;
-        String text = Character.toString(string.charAt(0));
-
         for (Type t : regexes.keySet()) {
             String regex = regexes.get(t);
             Token found = matchToken(string, t, regex, line);
@@ -202,13 +192,4 @@ abstract public class Token {
                 getText() == null ? "" : " " + getText()
         ));
     }
-
-    public String toFormattedString() {
-        String textRepr = text;
-        return String.format("%-40s %s", getTokenType(), textRepr);
-    }
-
-    public boolean isSameType(Token other) {
-        return type == other.type;
-    }
 }