diff options
Diffstat (limited to 'main/core/tokenizer')
25 files changed, 0 insertions, 1198 deletions
diff --git a/main/core/tokenizer/Tokenizer.java b/main/core/tokenizer/Tokenizer.java deleted file mode 100644 index 9ff1e4c..0000000 --- a/main/core/tokenizer/Tokenizer.java +++ /dev/null @@ -1,119 +0,0 @@ -package com.jantuomi.interpreter.main.core.tokenizer; - - -import com.jantuomi.interpreter.main.core.tokenizer.token.Token; -import com.jantuomi.interpreter.main.exception.ExceptionManager; -import com.jantuomi.interpreter.main.exception.InterpreterException; - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -/** - * Created by jan on 10.6.2016. - */ -public class Tokenizer { - private String sourceString; - - public List<Token> getTokens() { - return tokens; - } - - private List<Token> tokens = new ArrayList<>(); - - private static Map<Token.Type, String> tokenRegexes = new LinkedHashMap<>(); - private static final List<Token.Type> discardedTokenTypes = new ArrayList<>(); - private static final List<Token.Type> erroneousTokenTypes = new ArrayList<>(); - - private static final Tokenizer instance = new Tokenizer(); - - public static final Tokenizer getInstance() { - return instance; - } - - private Tokenizer() { - tokenRegexes.put(Token.Type.CommentToken, "^\\/\\*(.*?)\\*\\/"); - - tokenRegexes.put(Token.Type.DoubleLiteralToken, "^([-+]?[0-9]*\\.?[0-9]+)f\\b"); - tokenRegexes.put(Token.Type.IntegerLiteralToken, "^([-+]?\\d+)\\b"); - tokenRegexes.put(Token.Type.BooleanLiteralToken, "^(true|false)\\b"); - tokenRegexes.put(Token.Type.StringLiteralToken, "^\"(.*?)\""); - - tokenRegexes.put(Token.Type.WhitespaceToken, "^( |\t)"); - tokenRegexes.put(Token.Type.NewlineToken, "^(\n|\r\n)"); - tokenRegexes.put(Token.Type.AdditionToken, "^(\\+)"); - tokenRegexes.put(Token.Type.SubtractionToken, "^(\\-)"); - tokenRegexes.put(Token.Type.DivisionToken, "^(\\/)"); - tokenRegexes.put(Token.Type.MultiplicationToken, "^(\\*)"); - tokenRegexes.put(Token.Type.AssignmentToken, "^(set)\\b"); - tokenRegexes.put(Token.Type.LessThanToken, "^(\\<)"); - tokenRegexes.put(Token.Type.GreaterThanToken, "^(\\>)"); - tokenRegexes.put(Token.Type.LessOrEqualThanToken, "^(\\<\\=)"); - tokenRegexes.put(Token.Type.GreaterOrEqualThanToken, "^(\\>\\=)"); - tokenRegexes.put(Token.Type.EqualsToken, "^(\\=\\=)"); - tokenRegexes.put(Token.Type.NotEqualsToken, "^(\\!\\=)"); - tokenRegexes.put(Token.Type.OpenParenToken, "^(\\()"); - tokenRegexes.put(Token.Type.ClosedParenToken, "^(\\))"); - tokenRegexes.put(Token.Type.FunctionDefineToken, "^(func)\\b"); - tokenRegexes.put(Token.Type.FunctionBodyToken, "^(as)\\b"); - tokenRegexes.put(Token.Type.BranchBodyToken, "^(then)\\b"); - tokenRegexes.put(Token.Type.EndBlockToken, "^(end)\\b"); - tokenRegexes.put(Token.Type.NegationToken, "^(not)\\b"); - tokenRegexes.put(Token.Type.DeclarationToken, "^(decl)\\b"); - tokenRegexes.put(Token.Type.BranchToken, "^(if)\\b"); - - tokenRegexes.put(Token.Type.SymbolToken, "^([a-zA-Z]+\\w*)"); - - discardedTokenTypes.add(Token.Type.WhitespaceToken); - discardedTokenTypes.add(Token.Type.CommentToken); - - erroneousTokenTypes.add(Token.Type.NotAToken); - } - - public List<Token> tokenize(String string) throws InterpreterException { - sourceString = string; - tokens.clear(); - int line = 1; - for (int i = 0; i < string.length();) { - - Token token = Token.makeToken(string.substring(i), tokenRegexes, line); - - if (token == null) { - i++; - continue; - } - - if (token.getTokenType() == Token.Type.NewlineToken) { - line++; - } - - if (!discardedTokenTypes.contains(token.getTokenType())) { - tokens.add(token); - } - - if (erroneousTokenTypes.contains(token.getTokenType())) { - ExceptionManager.raise(InterpreterException.ExceptionType.IllegalTokenError, line, token.getText()); - } - - String tokenRawText = token.getRawText(); - if (tokenRawText == null) { - tokenRawText = token.getText(); - } - if (tokenRawText != null) { - i += tokenRawText.length(); - } else { - i++; - } - } - return tokens; - } - - public static void printTokens(List<Token> tokens) { - System.out.println("### Tokens: ###"); - for (Token token : tokens) { - System.out.println(String.format("%-40s %s", token.getTokenType(), token.getText())); - } - System.out.println("### End ###"); - } -} diff --git a/main/core/tokenizer/token/ArgumentInfo.java b/main/core/tokenizer/token/ArgumentInfo.java deleted file mode 100644 index b062a04..0000000 --- a/main/core/tokenizer/token/ArgumentInfo.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.jantuomi.interpreter.main.core.tokenizer.token; - -/** - * Created by jan on 15.6.2016. - */ -public class ArgumentInfo { - private int count = 0; - private boolean isVarargs = false; - - public Token.Type getOptionalArgument() { - return optionalArgument; - } - - public void setOptionalArgument(Token.Type optionalArgument) { - this.optionalArgument = optionalArgument; - } - - private Token.Type optionalArgument = null; - - public Token.Type getTerminator() { - return terminator; - } - - public void setTerminator(Token.Type terminator) { - this.terminator = terminator; - } - - private Token.Type terminator; - - public ArgumentInfo(int count) { - this.count = count; - } - - public ArgumentInfo() { - - } - - public void setVariable(boolean value) { - this.isVarargs = value; - } - - public boolean getVarargs() { - return isVarargs; - } - - public int getCount() { - return count; - } -} diff --git a/main/core/tokenizer/token/Token.java b/main/core/tokenizer/token/Token.java deleted file mode 100644 index d4058cb..0000000 --- a/main/core/tokenizer/token/Token.java +++ /dev/null @@ -1,298 +0,0 @@ -package com.jantuomi.interpreter.main.core.tokenizer.token; - -import com.jantuomi.interpreter.main.core.parser.ast.ASTNode; -import com.jantuomi.interpreter.main.core.tokenizer.token.types.*; -import com.jantuomi.interpreter.main.exception.ExceptionManager; -import com.jantuomi.interpreter.main.exception.InterpreterException; -import org.apache.commons.lang3.builder.EqualsBuilder; -import org.apache.commons.lang3.builder.HashCodeBuilder; - -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - - -/** - * Created by jan on 10.6.2016. - */ -abstract public class Token { - - public void print(int indent) { - for (int i = 0; i < indent; i++) { - System.out.print("\t"); - } - System.out.println(toString()); - for (Token child : getChildren()) { - child.print(indent + 1); - } - } - - abstract public ASTNode generateNode() throws InterpreterException; - - /* Token types, ordered by precedence */ - public enum Type { - CommentToken, - - AdditionToken, - SubtractionToken, - DivisionToken, - MultiplicationToken, - - AssignmentToken, - - LessThanToken, - GreaterThanToken, - LessOrEqualThanToken, - GreaterOrEqualThanToken, - EqualsToken, - NotEqualsToken, - - OpenParenToken, - ClosedParenToken, - - FunctionDefineToken, - DeclarationToken, - - StringLiteralToken, - IntegerLiteralToken, - BooleanLiteralToken, - DoubleLiteralToken, - - WhitespaceToken, - NewlineToken, - - BranchToken, - SymbolToken, - EndBlockToken, - FunctionBodyToken, - BranchBodyToken, - NegationToken, - - NotAToken - } - - abstract public Token setArguments(List<Token> args); - - abstract public List<Token> getChildren(); - - public boolean is(Type type) { - return getTokenType() == type; - } - - public int getLine() { - return line; - } - - public void setLine(int line) { - this.line = line; - } - - private int line; - private String rawText; - private String text; - private Type type; - - public String getText() { - return text; - } - - public Type getTokenType() { - return this.type; - } - - public String getRawText() { - return rawText; - } - - public boolean isHigherPrecedenceThan(Token other) { - return getTokenType().ordinal() <= other.getTokenType().ordinal(); - } - - public Token(Type type, String text, String rawText) { - initialize(type, text, rawText); - } - - public Token(Type type, String text) { - initialize(type, text); - } - - public Token(Type type) { - initialize(type, null); - } - - private void initialize(Type type, String text, String rawText) { - this.type = type; - this.text = text; - this.rawText = rawText; - } - - private void initialize(Type type, String text) { - initialize(type, text, text); - } - - public static Token makeToken(String string, Map<Type, String> regexes, int line) throws InterpreterException { - Type type = Type.NotAToken; - String text = Character.toString(string.charAt(0)); - - for (Type t : regexes.keySet()) { - String regex = regexes.get(t); - Token found = matchToken(string, t, regex, line); - if (found != null) { - found.setLine(line); - return found; - } - } - - // TODO raise error - return null; - } - - public static Token matchToken(String string, Type type, String regex, int line) throws InterpreterException { - Pattern pattern = Pattern.compile(regex, Pattern.DOTALL); - Matcher matcher = pattern.matcher(string); - - if (matcher.find()) { - String text = matcher.group(1); - String rawText = matcher.group(0); - - switch (type) { - case WhitespaceToken: - return new WhitespaceToken(); - case CommentToken: - return new CommentToken(text, rawText); - - // TODO placeholder whitespace instead of newline - case NewlineToken: - return new WhitespaceToken(); - - case IntegerLiteralToken: - return new IntegerLiteralToken(text); - case StringLiteralToken: - return new StringLiteralToken(text, rawText); - case BooleanLiteralToken: - return new BooleanLiteralToken(text); - case DoubleLiteralToken: - return new DoubleLiteralToken(text, rawText); - case AdditionToken: - return new AdditionToken(); - case SubtractionToken: - return new SubtractionToken(); - case MultiplicationToken: - return new MultiplicationToken(); - case DivisionToken: - return new DivisionToken(); - case FunctionDefineToken: - return new FunctionDefineToken(); - case EndBlockToken: - return new EndFunctionDefineToken(); - case FunctionBodyToken: - return new BlockBodyToken("as"); - case BranchBodyToken: - return new BlockBodyToken("then"); - case DeclarationToken: - return new VariableDeclareToken(); - case AssignmentToken: - return new AssignmentToken(); - case SymbolToken: - return new SymbolToken(text); - case OpenParenToken: - return new OpenParenToken(); - case ClosedParenToken: - return new ClosedParenToken(); - case BranchToken: - return new BranchToken(); - case NegationToken: - return new NegationToken(); - default: - ExceptionManager.raise(InterpreterException.ExceptionType.IllegalTokenError, line, - type.toString() - ); - } - } - - return null; - } - - public ArgumentInfo getArgumentInfo() { - switch (getTokenType()) { - - case FunctionDefineToken: - ArgumentInfo fdai = new ArgumentInfo(); - fdai.setVariable(true); - fdai.setTerminator(Type.FunctionBodyToken); - return fdai; - case BranchBodyToken: - case FunctionBodyToken: - ArgumentInfo fbai = new ArgumentInfo(); - fbai.setVariable(true); - fbai.setTerminator(Type.EndBlockToken); - return fbai; - case OpenParenToken: - ArgumentInfo opai = new ArgumentInfo(); - opai.setVariable(true); - opai.setTerminator(Type.ClosedParenToken); - return opai; - case SymbolToken: - ArgumentInfo sai = new ArgumentInfo(); - sai.setOptionalArgument(Type.OpenParenToken); - return sai; - case AdditionToken: - case SubtractionToken: - case DivisionToken: - case MultiplicationToken: - case AssignmentToken: - case LessThanToken: - case GreaterThanToken: - case LessOrEqualThanToken: - case GreaterOrEqualThanToken: - case EqualsToken: - case NotEqualsToken: - case BranchToken: - return new ArgumentInfo(2); - case DeclarationToken: - case NegationToken: - return new ArgumentInfo(1); - default: - return new ArgumentInfo(0); - } - } - - @Override - public int hashCode() { - return new HashCodeBuilder(17, 37) - .append(text) - .append(getTokenType()) - .toHashCode(); - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof Token)) return false; - if (o == this) return true; - - Token rhs = (Token) o; - return new EqualsBuilder() - .append(getTokenType(), rhs.getTokenType()) - .append(text, rhs.text) - .isEquals(); - } - - @Override - public String toString() { - return String.format(String.format( - "%s%s", - getTokenType().toString().replace("Token", ""), - getText() == null ? "" : " " + getText() - )); - } - - public String toFormattedString() { - String textRepr = text; - return String.format("%-40s %s", getTokenType(), textRepr); - } - - public boolean isSameType(Token other) { - return type == other.type; - } -} diff --git a/main/core/tokenizer/token/types/AdditionToken.java b/main/core/tokenizer/token/types/AdditionToken.java deleted file mode 100644 index 4623d73..0000000 --- a/main/core/tokenizer/token/types/AdditionToken.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.jantuomi.interpreter.main.core.tokenizer.token.types; - -import com.jantuomi.interpreter.main.core.parser.ast.ASTNode; -import com.jantuomi.interpreter.main.core.parser.ast.AdditionNode; -import com.jantuomi.interpreter.main.exception.InterpreterException; - -/** - * Created by jan on 15.6.2016. - */ -public class AdditionToken extends BinaryOperatorToken { - public AdditionToken() { - super(Type.AdditionToken); - } - - @Override - public ASTNode generateNode() throws InterpreterException { - return new AdditionNode(this, - lhs.generateNode(), - rhs.generateNode() - ); - - } -} diff --git a/main/core/tokenizer/token/types/AssignmentToken.java b/main/core/tokenizer/token/types/AssignmentToken.java deleted file mode 100644 index 57f404c..0000000 --- a/main/core/tokenizer/token/types/AssignmentToken.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.jantuomi.interpreter.main.core.tokenizer.token.types; - -import com.jantuomi.interpreter.main.core.parser.ast.ASTNode; -import com.jantuomi.interpreter.main.core.parser.ast.AssignmentNode; -import com.jantuomi.interpreter.main.core.parser.ast.SymbolNode; -import com.jantuomi.interpreter.main.core.tokenizer.token.Token; -import com.jantuomi.interpreter.main.exception.ExceptionManager; -import com.jantuomi.interpreter.main.exception.InterpreterException; - -import java.util.Arrays; -import java.util.List; - -/** - * Created by jan on 20.6.2016. - */ -public class AssignmentToken extends Token { - - private Token lhs; - private Token rhs; - - public AssignmentToken() { - super(Type.AssignmentToken, "set"); - } - - // TODO this and the node - - @Override - public ASTNode generateNode() throws InterpreterException { - ASTNode lhsNode = lhs.generateNode(); - - if (!(lhsNode instanceof SymbolNode)) { - ExceptionManager.raise(InterpreterException.ExceptionType.SyntaxError, getLine(), "non-symbol " + toString()); - return null; - } - - AssignmentNode node = new AssignmentNode(this, - (SymbolNode) lhsNode, - rhs.generateNode() - ); - - return node; - } - - @Override - public Token setArguments(List<Token> args) { - this.lhs = args.get(0); - this.rhs = args.get(1); - return this; - } - - @Override - public List<Token> getChildren() { - return Arrays.asList(lhs, rhs); - } -} diff --git a/main/core/tokenizer/token/types/BinaryOperatorToken.java b/main/core/tokenizer/token/types/BinaryOperatorToken.java deleted file mode 100644 index 3998bb9..0000000 --- a/main/core/tokenizer/token/types/BinaryOperatorToken.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.jantuomi.interpreter.main.core.tokenizer.token.types; - -import com.jantuomi.interpreter.main.core.tokenizer.token.Token; - -import java.util.Arrays; -import java.util.List; - -/** - * Created by jan on 15.6.2016. - */ -abstract public class BinaryOperatorToken extends Token { - protected Token lhs; - protected Token rhs; - - public BinaryOperatorToken(Type type) { - super(type); - } - - @Override - public Token setArguments(List<Token> args) { - lhs = args.get(0); - rhs = args.get(1); - return this; - } - - @Override - public List<Token> getChildren() { - return Arrays.asList(lhs, rhs); - - } -} diff --git a/main/core/tokenizer/token/types/BlockBodyToken.java b/main/core/tokenizer/token/types/BlockBodyToken.java deleted file mode 100644 index 444ea1f..0000000 --- a/main/core/tokenizer/token/types/BlockBodyToken.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.jantuomi.interpreter.main.core.tokenizer.token.types; - -import com.jantuomi.interpreter.main.core.parser.ast.ASTNode; -import com.jantuomi.interpreter.main.core.parser.ast.BlockBodyNode; -import com.jantuomi.interpreter.main.core.tokenizer.token.Token; -import com.jantuomi.interpreter.main.exception.InterpreterException; - -import java.util.ArrayList; -import java.util.List; - -/** - * Created by jan on 16.6.2016. - */ -public class BlockBodyToken extends VarargOperatorToken { - public BlockBodyToken(String keyword) { - super(Type.FunctionBodyToken, keyword); - } - - @Override - public ASTNode generateNode() throws InterpreterException { - BlockBodyNode node = new BlockBodyNode(this); - List<ASTNode> statements = new ArrayList<>(); - for (int i = 0; i < args.size() - 1; i++) { - Token arg = args.get(i); - statements.add(arg.generateNode()); - } - node.setArgs(statements); - return node; - } -} diff --git a/main/core/tokenizer/token/types/BooleanLiteralToken.java b/main/core/tokenizer/token/types/BooleanLiteralToken.java deleted file mode 100644 index 6e28b82..0000000 --- a/main/core/tokenizer/token/types/BooleanLiteralToken.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.jantuomi.interpreter.main.core.tokenizer.token.types; - -import com.jantuomi.interpreter.main.core.parser.ast.ASTNode; -import com.jantuomi.interpreter.main.core.parser.ast.BooleanLiteralNode; -import com.jantuomi.interpreter.main.core.tokenizer.token.Token; - -import java.util.Arrays; -import java.util.List; - -/** - * Created by jan on 15.6.2016. - */ -public class BooleanLiteralToken extends Token{ - public BooleanLiteralToken(String text) { - super(Type.BooleanLiteralToken, text); - } - - @Override - public ASTNode generateNode() { - return new BooleanLiteralNode(this); - } - - @Override - public Token setArguments(List<Token> args) { - return this; - } - - @Override - public List<Token> getChildren() { - return Arrays.asList(); - } -} diff --git a/main/core/tokenizer/token/types/BranchToken.java b/main/core/tokenizer/token/types/BranchToken.java deleted file mode 100644 index 84aeabb..0000000 --- a/main/core/tokenizer/token/types/BranchToken.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.jantuomi.interpreter.main.core.tokenizer.token.types; - -import com.jantuomi.interpreter.main.core.parser.ast.ASTNode; -import com.jantuomi.interpreter.main.core.parser.ast.BlockBodyNode; -import com.jantuomi.interpreter.main.core.parser.ast.BranchNode; -import com.jantuomi.interpreter.main.core.tokenizer.token.Token; -import com.jantuomi.interpreter.main.exception.InterpreterException; - -import java.util.Arrays; -import java.util.List; - -/** - * Created by jan on 21.6.2016. - */ -public class BranchToken extends Token { - - private Token expression; - private BlockBodyToken branch; - - public BranchToken() { - super(Type.BranchToken, "if"); - } - - @Override - public ASTNode generateNode() throws InterpreterException { - return new BranchNode(this, - expression.generateNode(), - (BlockBodyNode) branch.generateNode() - ); - } - - @Override - public Token setArguments(List<Token> args) { - expression = args.get(0); - branch = (BlockBodyToken) args.get(1); - return this; - } - - @Override - public List<Token> getChildren() { - return Arrays.asList(expression, branch); - } -} diff --git a/main/core/tokenizer/token/types/ClosedParenToken.java b/main/core/tokenizer/token/types/ClosedParenToken.java deleted file mode 100644 index 17e5c67..0000000 --- a/main/core/tokenizer/token/types/ClosedParenToken.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.jantuomi.interpreter.main.core.tokenizer.token.types; - -import com.jantuomi.interpreter.main.core.parser.ast.ASTNode; -import com.jantuomi.interpreter.main.core.tokenizer.token.Token; - -import java.util.Arrays; -import java.util.List; - -/** - * Created by jan on 19.6.2016. - */ -public class ClosedParenToken extends Token { - public ClosedParenToken() { - super(Type.ClosedParenToken); - } - - @Override - public ASTNode generateNode() { - return null; - } - - @Override - public Token setArguments(List<Token> args) { - return this; - } - - @Override - public List<Token> getChildren() { - return Arrays.asList(); - } -} diff --git a/main/core/tokenizer/token/types/CommentToken.java b/main/core/tokenizer/token/types/CommentToken.java deleted file mode 100644 index 45ca995..0000000 --- a/main/core/tokenizer/token/types/CommentToken.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.jantuomi.interpreter.main.core.tokenizer.token.types; - -import com.jantuomi.interpreter.main.core.parser.ast.ASTNode; -import com.jantuomi.interpreter.main.core.parser.ast.EmptyNode; -import com.jantuomi.interpreter.main.core.tokenizer.token.Token; - -import java.util.Arrays; -import java.util.List; - -/** - * Created by jan on 19.6.2016. - */ -public class CommentToken extends Token { - public CommentToken(String text, String rawText) { - super(Type.CommentToken, text, rawText); - } - - @Override - public ASTNode generateNode() { - return new EmptyNode(this); - } - - @Override - public Token setArguments(List<Token> args) { - return this; - } - - @Override - public List<Token> getChildren() { - return Arrays.asList(); - } -} diff --git a/main/core/tokenizer/token/types/DivisionToken.java b/main/core/tokenizer/token/types/DivisionToken.java deleted file mode 100644 index 14f4d13..0000000 --- a/main/core/tokenizer/token/types/DivisionToken.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.jantuomi.interpreter.main.core.tokenizer.token.types; - -import com.jantuomi.interpreter.main.core.parser.ast.ASTNode; -import com.jantuomi.interpreter.main.core.parser.ast.DivisionNode; -import com.jantuomi.interpreter.main.exception.InterpreterException; - -/** - * Created by jan on 15.6.2016. - */ -public class DivisionToken extends BinaryOperatorToken { - public DivisionToken() { - super(Type.DivisionToken); - } - - @Override - public ASTNode generateNode() throws InterpreterException { - return new DivisionNode(this, - lhs.generateNode(), - rhs.generateNode() - ); - - } -} diff --git a/main/core/tokenizer/token/types/DoubleLiteralToken.java b/main/core/tokenizer/token/types/DoubleLiteralToken.java deleted file mode 100644 index 1a379eb..0000000 --- a/main/core/tokenizer/token/types/DoubleLiteralToken.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.jantuomi.interpreter.main.core.tokenizer.token.types; - -import com.jantuomi.interpreter.main.core.parser.ast.ASTNode; -import com.jantuomi.interpreter.main.core.parser.ast.DoubleLiteralNode; -import com.jantuomi.interpreter.main.core.tokenizer.token.Token; - -import java.util.Arrays; -import java.util.List; - -/** - * Created by jan on 15.6.2016. - */ -public class DoubleLiteralToken extends Token{ - public DoubleLiteralToken(String text, String rawText) { - super(Type.DoubleLiteralToken, text, rawText); - } - - @Override - public ASTNode generateNode() { - return new DoubleLiteralNode(this); - } - - @Override - public Token setArguments(List<Token> args) { - return this; - } - - @Override - public List<Token> getChildren() { - return Arrays.asList(); - } -} diff --git a/main/core/tokenizer/token/types/EndFunctionDefineToken.java b/main/core/tokenizer/token/types/EndFunctionDefineToken.java deleted file mode 100644 index 14b21f8..0000000 --- a/main/core/tokenizer/token/types/EndFunctionDefineToken.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.jantuomi.interpreter.main.core.tokenizer.token.types; - -import com.jantuomi.interpreter.main.core.parser.ast.ASTNode; -import com.jantuomi.interpreter.main.core.tokenizer.token.Token; - -import java.util.Arrays; -import java.util.List; - -/** - * Created by jan on 16.6.2016. - */ -public class EndFunctionDefineToken extends Token { - public EndFunctionDefineToken() { - super(Type.EndBlockToken, "end"); - } - - @Override - public ASTNode generateNode() { - return null; - } - - @Override - public Token setArguments(List<Token> args) { - return this; - } - - @Override - public List<Token> getChildren() { - return Arrays.asList(); - } -} diff --git a/main/core/tokenizer/token/types/FunctionDefineToken.java b/main/core/tokenizer/token/types/FunctionDefineToken.java deleted file mode 100644 index 7d715a9..0000000 --- a/main/core/tokenizer/token/types/FunctionDefineToken.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.jantuomi.interpreter.main.core.tokenizer.token.types; - -import com.jantuomi.interpreter.main.core.parser.ast.ASTNode; -import com.jantuomi.interpreter.main.core.parser.ast.BlockBodyNode; -import com.jantuomi.interpreter.main.core.parser.ast.FunctionDefineNode; -import com.jantuomi.interpreter.main.exception.InterpreterException; - -import java.util.ArrayList; -import java.util.List; - -/** - * Created by jan on 16.6.2016. - */ -public class FunctionDefineToken extends VarargOperatorToken { - public FunctionDefineToken() { - super(Type.FunctionDefineToken, "func"); - } - - @Override - public ASTNode generateNode() throws InterpreterException { - FunctionDefineNode node = new FunctionDefineNode(this); - node.setName(args.get(0).getText()); - - List<ASTNode> funcArgs = new ArrayList<>(); - for (int i = 1; i < args.size() - 1; i++) { - funcArgs.add(args.get(i).generateNode()); - } - node.setArgs(funcArgs); - - BlockBodyNode bodyNode = (BlockBodyNode) args.get(args.size() - 1).generateNode(); - node.setBody(bodyNode); - return node; - } -} diff --git a/main/core/tokenizer/token/types/IntegerLiteralToken.java b/main/core/tokenizer/token/types/IntegerLiteralToken.java deleted file mode 100644 index cdb4fde..0000000 --- a/main/core/tokenizer/token/types/IntegerLiteralToken.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.jantuomi.interpreter.main.core.tokenizer.token.types; - -import com.jantuomi.interpreter.main.core.parser.ast.ASTNode; -import com.jantuomi.interpreter.main.core.parser.ast.IntegerLiteralNode; -import com.jantuomi.interpreter.main.core.tokenizer.token.Token; - -import java.util.Arrays; -import java.util.List; - -/** - * Created by jan on 15.6.2016. - */ -public class IntegerLiteralToken extends Token{ - public IntegerLiteralToken(String text) { - super(Type.IntegerLiteralToken, text); - } - - @Override - public ASTNode generateNode() { - return new IntegerLiteralNode(this); - } - - @Override - public Token setArguments(List<Token> args) { - return this; - } - - @Override - public List<Token> getChildren() { - return Arrays.asList(); - } -} diff --git a/main/core/tokenizer/token/types/MultiplicationToken.java b/main/core/tokenizer/token/types/MultiplicationToken.java deleted file mode 100644 index 521f01a..0000000 --- a/main/core/tokenizer/token/types/MultiplicationToken.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.jantuomi.interpreter.main.core.tokenizer.token.types; - -import com.jantuomi.interpreter.main.core.parser.ast.ASTNode; -import com.jantuomi.interpreter.main.core.parser.ast.MultiplicationNode; -import com.jantuomi.interpreter.main.exception.InterpreterException; - -/** - * Created by jan on 15.6.2016. - */ -public class MultiplicationToken extends BinaryOperatorToken { - public MultiplicationToken() { - super(Type.MultiplicationToken); - } - - @Override - public ASTNode generateNode() throws InterpreterException { - return new MultiplicationNode(this, - lhs.generateNode(), - rhs.generateNode() - ); - - } -} diff --git a/main/core/tokenizer/token/types/NegationToken.java b/main/core/tokenizer/token/types/NegationToken.java deleted file mode 100644 index 5658f10..0000000 --- a/main/core/tokenizer/token/types/NegationToken.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.jantuomi.interpreter.main.core.tokenizer.token.types; - -import com.jantuomi.interpreter.main.core.parser.ast.ASTNode; -import com.jantuomi.interpreter.main.core.parser.ast.NegationNode; -import com.jantuomi.interpreter.main.core.tokenizer.token.Token; -import com.jantuomi.interpreter.main.exception.InterpreterException; - -import java.util.Arrays; -import java.util.List; - -/** - * Created by jan on 21.6.2016. - */ -public class NegationToken extends Token { - - private Token operand; - - public NegationToken() { - super(Type.NegationToken, "not"); - } - - @Override - public ASTNode generateNode() throws InterpreterException { - return new NegationNode(this, operand.generateNode()); - } - - @Override - public Token setArguments(List<Token> args) { - operand = args.get(0); - return this; - } - - @Override - public List<Token> getChildren() { - return Arrays.asList(operand); - } -} diff --git a/main/core/tokenizer/token/types/OpenParenToken.java b/main/core/tokenizer/token/types/OpenParenToken.java deleted file mode 100644 index 80b507c..0000000 --- a/main/core/tokenizer/token/types/OpenParenToken.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.jantuomi.interpreter.main.core.tokenizer.token.types; - -import com.jantuomi.interpreter.main.core.parser.ast.ASTNode; -import com.jantuomi.interpreter.main.core.parser.ast.ParameterListNode; -import com.jantuomi.interpreter.main.core.tokenizer.token.Token; -import com.jantuomi.interpreter.main.exception.InterpreterException; - -/** - * Created by jan on 19.6.2016. - */ -public class OpenParenToken extends VarargOperatorToken { - public OpenParenToken() { - super(Type.OpenParenToken); - } - - @Override - public ASTNode generateNode() throws InterpreterException { - ParameterListNode node = new ParameterListNode(this); - - for (Token token : args) { - if (token.getTokenType() == Type.ClosedParenToken) { - continue; - } - - node.addParameter(token.generateNode()); - } - return node; - } -} diff --git a/main/core/tokenizer/token/types/StringLiteralToken.java b/main/core/tokenizer/token/types/StringLiteralToken.java deleted file mode 100644 index 8d6cb8a..0000000 --- a/main/core/tokenizer/token/types/StringLiteralToken.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.jantuomi.interpreter.main.core.tokenizer.token.types; - -import com.jantuomi.interpreter.main.core.parser.ast.ASTNode; -import com.jantuomi.interpreter.main.core.parser.ast.StringLiteralNode; -import com.jantuomi.interpreter.main.core.tokenizer.token.Token; -import com.jantuomi.interpreter.main.exception.InterpreterException; - -import java.util.Arrays; -import java.util.List; - -/** - * Created by jan on 21.6.2016. - */ -public class StringLiteralToken extends Token { - public StringLiteralToken(String text, String rawText) { - super(Type.StringLiteralToken, text, rawText); - } - - @Override - public ASTNode generateNode() throws InterpreterException { - return new StringLiteralNode(this); - } - - @Override - public Token setArguments(List<Token> args) { - return this; - } - - @Override - public List<Token> getChildren() { - return Arrays.asList(); - } -} diff --git a/main/core/tokenizer/token/types/SubtractionToken.java b/main/core/tokenizer/token/types/SubtractionToken.java deleted file mode 100644 index 9f7ff10..0000000 --- a/main/core/tokenizer/token/types/SubtractionToken.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.jantuomi.interpreter.main.core.tokenizer.token.types; - -import com.jantuomi.interpreter.main.core.parser.ast.ASTNode; -import com.jantuomi.interpreter.main.core.parser.ast.SubtractionNode; -import com.jantuomi.interpreter.main.exception.InterpreterException; - -/** - * Created by jan on 16.6.2016. - */ -public class SubtractionToken extends BinaryOperatorToken { - public SubtractionToken() { - super(Type.SubtractionToken); - } - - @Override - public ASTNode generateNode() throws InterpreterException { - return new SubtractionNode(this, - lhs.generateNode(), - rhs.generateNode() - ); - } -} diff --git a/main/core/tokenizer/token/types/SymbolToken.java b/main/core/tokenizer/token/types/SymbolToken.java deleted file mode 100644 index d750921..0000000 --- a/main/core/tokenizer/token/types/SymbolToken.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.jantuomi.interpreter.main.core.tokenizer.token.types; - -import com.jantuomi.interpreter.main.core.parser.ast.ASTNode; -import com.jantuomi.interpreter.main.core.parser.ast.ParameterListNode; -import com.jantuomi.interpreter.main.core.parser.ast.SymbolNode; -import com.jantuomi.interpreter.main.core.tokenizer.token.Token; -import com.jantuomi.interpreter.main.exception.InterpreterException; - -import java.util.Arrays; -import java.util.List; - -/** - * Created by jan on 16.6.2016. - */ -public class SymbolToken extends Token { - - public void setCallArgumentList(OpenParenToken callArgumentList) { - this.callArgumentList = callArgumentList; - } - - private OpenParenToken callArgumentList = null; - - public SymbolToken(String name) { - super(Type.SymbolToken, name); - } - - @Override - public ASTNode generateNode() throws InterpreterException { - SymbolNode node = new SymbolNode(this); - node.setName(this.getText()); - if (callArgumentList != null) { - node.setParameterListNode((ParameterListNode) callArgumentList.generateNode()); - } - return node; - } - - @Override - public Token setArguments(List<Token> args) { - if (args.size() > 0) { - setCallArgumentList((OpenParenToken) args.get(0)); - } - return this; - } - - @Override - public List<Token> getChildren() { - if (callArgumentList == null) { - return Arrays.asList(); - } else { - return Arrays.asList(callArgumentList); - } - } -} diff --git a/main/core/tokenizer/token/types/VarargOperatorToken.java b/main/core/tokenizer/token/types/VarargOperatorToken.java deleted file mode 100644 index f8901ee..0000000 --- a/main/core/tokenizer/token/types/VarargOperatorToken.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.jantuomi.interpreter.main.core.tokenizer.token.types; - -import com.jantuomi.interpreter.main.core.tokenizer.token.Token; - -import java.util.List; - -/** - * Created by jan on 16.6.2016. - */ -abstract public class VarargOperatorToken extends Token { - - protected List<Token> args; - - public VarargOperatorToken(Type type, String text) { - super(type, text); - } - - public VarargOperatorToken(Type type) { - super(type); - } - - @Override - public Token setArguments(List<Token> args) { - this.args = args; - return this; - } - - @Override - public List<Token> getChildren() { - return args; - } - -} diff --git a/main/core/tokenizer/token/types/VariableDeclareToken.java b/main/core/tokenizer/token/types/VariableDeclareToken.java deleted file mode 100644 index 1ec0826..0000000 --- a/main/core/tokenizer/token/types/VariableDeclareToken.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.jantuomi.interpreter.main.core.tokenizer.token.types; - -import com.jantuomi.interpreter.main.core.parser.ast.ASTNode; -import com.jantuomi.interpreter.main.core.parser.ast.SymbolNode; -import com.jantuomi.interpreter.main.core.parser.ast.VariableDeclareNode; -import com.jantuomi.interpreter.main.core.tokenizer.token.Token; -import com.jantuomi.interpreter.main.exception.InterpreterException; - -import java.util.Arrays; -import java.util.List; - -/** - * Created by jan on 20.6.2016. - */ -public class VariableDeclareToken extends Token { - - private SymbolToken variable; - - public VariableDeclareToken() { - super(Type.DeclarationToken, "decl"); - } - - @Override - public ASTNode generateNode() throws InterpreterException { - VariableDeclareNode node = new VariableDeclareNode(this, - (SymbolNode) variable.generateNode()); - return node; - } - - @Override - public Token setArguments(List<Token> args) { - if (args.size() > 0) { - this.variable = (SymbolToken) args.get(0); - } - return this; - } - - @Override - public List<Token> getChildren() { - return Arrays.asList(variable); - } -} diff --git a/main/core/tokenizer/token/types/WhitespaceToken.java b/main/core/tokenizer/token/types/WhitespaceToken.java deleted file mode 100644 index fb4949c..0000000 --- a/main/core/tokenizer/token/types/WhitespaceToken.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.jantuomi.interpreter.main.core.tokenizer.token.types; - -import com.jantuomi.interpreter.main.core.parser.ast.ASTNode; -import com.jantuomi.interpreter.main.core.tokenizer.token.Token; - -import java.util.Arrays; -import java.util.List; - -/** - * Created by jan on 16.6.2016. - */ -public class WhitespaceToken extends Token { - public WhitespaceToken() { - super(Type.WhitespaceToken); - } - - @Override - public ASTNode generateNode() { - return null; - } - - @Override - public Token setArguments(List<Token> args) { - return this; - } - - @Override - public List<Token> getChildren() { - return Arrays.asList(); - } -} |