package com.jantuomi.interpreter.main.core.tokenizer.token; import com.jantuomi.interpreter.main.core.parser.ast.ASTNode; import com.jantuomi.interpreter.main.core.tokenizer.token.types.AdditionToken; import com.jantuomi.interpreter.main.core.tokenizer.token.types.IntegerLiteralToken; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; /** * Created by jan on 10.6.2016. */ abstract public class Token { public void print(int indent) { for (int i = 0; i < indent; i++) { System.out.print("\t"); } System.out.println(toString()); for (Token child : getChildren()) { child.print(indent + 1); } } abstract public ASTNode generateNode(); /* Token types, ordered by precedence */ public enum Type { CommentToken, AdditionToken, SubtractionToken, DivisionToken, MultiplicationToken, AssignmentToken, LessThanToken, GreaterThanToken, LessOrEqualThanToken, GreaterOrEqualThanToken, EqualsToken, NotEqualsToken, OpenParenToken, ClosedParenToken, FunctionDefineToken, DeclarationToken, StringLiteralToken, IntegerLiteralToken, WhitespaceToken, NewlineToken, EndStatementToken, SymbolToken, EndFunctionDefineToken, NotAToken } abstract public Token setArguments(List args); abstract public List getChildren(); public boolean is(Type type) { return getTokenType() == type; } public int getLine() { return line; } public void setLine(int line) { this.line = line; } private int line; private String rawText; private String text; private Type type; public String getText() { return text; } public Type getTokenType() { return this.type; } public String getRawText() { return rawText; } public boolean isHigherPrecedenceThan(Token other) { return getTokenType().ordinal() <= other.getTokenType().ordinal(); } public Token(Type type, String text, String rawText) { initialize(type, text, rawText); } public Token(Type type, String text) { initialize(type, text); } public Token(Type type) { initialize(type, null); } private void initialize(Type type, String text, String rawText) { this.type = type; this.text = text; this.rawText = rawText; } private void initialize(Type type, String text) { initialize(type, text, text); } public static Token makeToken(String string, Map regexes, int line) { Type type = Type.NotAToken; String text = Character.toString(string.charAt(0)); for (Type t : regexes.keySet()) { String regex = regexes.get(t); Token found = matchToken(string, t, regex); if (found != null) { found.setLine(line); return found; } } // TODO raise error return null; } public static Token matchToken(String string, Type type, String regex) { Pattern pattern = Pattern.compile(regex, Pattern.DOTALL); Matcher matcher = pattern.matcher(string); if (matcher.find()) { String text = matcher.group(1); String rawText = matcher.group(0); switch (type) { case IntegerLiteralToken: return new IntegerLiteralToken(text); case AdditionToken: return new AdditionToken(); default: return null; } } return null; } public ArgumentInfo getArgumentInfo() { switch (getTokenType()) { case FunctionDefineToken: ArgumentInfo ai = new ArgumentInfo(); ai.setVariable(true); return ai; case AdditionToken: case SubtractionToken: case DivisionToken: case MultiplicationToken: case AssignmentToken: case LessThanToken: case GreaterThanToken: case LessOrEqualThanToken: case GreaterOrEqualThanToken: case EqualsToken: case NotEqualsToken: return new ArgumentInfo(2); case DeclarationToken: return new ArgumentInfo(1); default: return new ArgumentInfo(0); } } @Override public int hashCode() { return new HashCodeBuilder(17, 37) .append(text) .append(getTokenType()) .toHashCode(); } @Override public boolean equals(Object o) { if (!(o instanceof Token)) return false; if (o == this) return true; Token rhs = (Token) o; return new EqualsBuilder() .append(getTokenType(), rhs.getTokenType()) .append(text, rhs.text) .isEquals(); } @Override public String toString() { return String.format(String.format("%s %s", getTokenType(), getText())); } public String toFormattedString() { String textRepr = text; return String.format("%-40s %s", getTokenType(), textRepr); } public boolean isSameType(Token other) { return type == other.type; } }