From 12eeefcdba1ee392f033f51e6b9826eaae3c4de6 Mon Sep 17 00:00:00 2001 From: Jan Tuomi Date: Mon, 13 Jun 2016 23:16:54 +0300 Subject: Add tokenizer, parser and support for certain operations --- main/Main.java | 37 +++++ main/core/CommandLineArgumentContainer.java | 56 +++++++ main/core/parser/Parser.java | 171 +++++++++++++++++++++ main/core/parser/ast/ASTNode.java | 33 ++++ main/core/parser/ast/AdditionNode.java | 34 ++++ main/core/parser/ast/AssignmentNode.java | 36 +++++ main/core/parser/ast/ExpressionNode.java | 12 ++ main/core/parser/ast/IntegerLiteralNode.java | 30 ++++ main/core/parser/ast/SymbolNode.java | 35 +++++ main/core/parser/datatype/DataContainer.java | 17 ++ .../core/parser/datatype/IntegerDataContainer.java | 10 ++ main/core/runtime/State.java | 35 +++++ main/core/tokenizer/Tokenizer.java | 100 ++++++++++++ main/core/tokenizer/token/Token.java | 158 +++++++++++++++++++ main/exception/ExceptionManager.java | 29 ++++ main/exception/InterpreterException.java | 33 ++++ main/utils/Counter.java | 27 ++++ main/utils/Utilities.java | 17 ++ test/MainTest.java | 24 +++ test/core/CommandLineArgumentContainerTest.java | 26 ++++ test/core/parser/ParserTest.java | 34 ++++ test/core/tokenizer/TokenizerTest.java | 80 ++++++++++ test/resources/test.file | 4 + 23 files changed, 1038 insertions(+) create mode 100644 main/Main.java create mode 100644 main/core/CommandLineArgumentContainer.java create mode 100644 main/core/parser/Parser.java create mode 100644 main/core/parser/ast/ASTNode.java create mode 100644 main/core/parser/ast/AdditionNode.java create mode 100644 main/core/parser/ast/AssignmentNode.java create mode 100644 main/core/parser/ast/ExpressionNode.java create mode 100644 main/core/parser/ast/IntegerLiteralNode.java create mode 100644 main/core/parser/ast/SymbolNode.java create mode 100644 main/core/parser/datatype/DataContainer.java create mode 100644 main/core/parser/datatype/IntegerDataContainer.java create mode 100644 main/core/runtime/State.java create mode 100644 main/core/tokenizer/Tokenizer.java create mode 100644 main/core/tokenizer/token/Token.java create mode 100644 main/exception/ExceptionManager.java create mode 100644 main/exception/InterpreterException.java create mode 100644 main/utils/Counter.java create mode 100644 main/utils/Utilities.java create mode 100644 test/MainTest.java create mode 100644 test/core/CommandLineArgumentContainerTest.java create mode 100644 test/core/parser/ParserTest.java create mode 100644 test/core/tokenizer/TokenizerTest.java create mode 100644 test/resources/test.file diff --git a/main/Main.java b/main/Main.java new file mode 100644 index 0000000..488b9fd --- /dev/null +++ b/main/Main.java @@ -0,0 +1,37 @@ +package com.jantuomi.interpreter.main; + +import com.jantuomi.interpreter.main.core.CommandLineArgumentContainer; +import com.jantuomi.interpreter.main.core.parser.Parser; +import com.jantuomi.interpreter.main.core.tokenizer.Tokenizer; +import org.kohsuke.args4j.CmdLineException; +import org.kohsuke.args4j.CmdLineParser; + +public class Main { + + public static void main(String[] args) throws Exception { + boolean parseSuccess = parseArguments(args); + if (!parseSuccess) { + throw new Exception("Argument files could not be parsed successfully."); + } + + String sourceFileContents = CommandLineArgumentContainer.getInstance().getSourceFileContents(); + Tokenizer tokenizer = Tokenizer.getInstance(); + tokenizer.tokenize(sourceFileContents); + Parser.getInstance().parse(tokenizer.getTokens()); + } + + public static boolean parseArguments(String[] args) { + CommandLineArgumentContainer container = CommandLineArgumentContainer.getInstance(); + CmdLineParser parser = new CmdLineParser(container); + + try { + parser.parseArgument(args); + } catch (CmdLineException e) { + System.err.println(e.getMessage()); + parser.printUsage(System.err); + return false; + } + + return true; + } +} diff --git a/main/core/CommandLineArgumentContainer.java b/main/core/CommandLineArgumentContainer.java new file mode 100644 index 0000000..9ca0b5e --- /dev/null +++ b/main/core/CommandLineArgumentContainer.java @@ -0,0 +1,56 @@ +package com.jantuomi.interpreter.main.core; + +import org.kohsuke.args4j.Option; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; + +/** + * Created by jan on 10.6.2016. + */ + +public class CommandLineArgumentContainer { + + private CommandLineArgumentContainer() {} + + private static final CommandLineArgumentContainer instance = new CommandLineArgumentContainer(); + private File srcFile; + + @Option(name="-f", usage="Execute script in file FILE.") + public void setFile(File file) { + this.srcFile = file; + } + + public static CommandLineArgumentContainer getInstance() { + return instance; + } + + public String getSourceFileContents() { + if (srcFile == null) { + return null; + } + + BufferedReader br; + String contents = null; + try { + br = new BufferedReader(new FileReader(srcFile)); + StringBuilder sb = new StringBuilder(); + String line = br.readLine(); + + while (line != null) { + sb.append(line); + sb.append(System.lineSeparator()); + line = br.readLine(); + } + contents = sb.toString(); + } catch (IOException e) { + e.printStackTrace(); + return null; + } finally { + // Add a newline at the end for comment rows to terminate nicely + return contents + "\n"; + } + } +} diff --git a/main/core/parser/Parser.java b/main/core/parser/Parser.java new file mode 100644 index 0000000..af62f7e --- /dev/null +++ b/main/core/parser/Parser.java @@ -0,0 +1,171 @@ +package com.jantuomi.interpreter.main.core.parser; + +import com.jantuomi.interpreter.main.core.parser.ast.*; +import com.jantuomi.interpreter.main.core.tokenizer.token.Token; +import com.jantuomi.interpreter.main.utils.Counter; + +import java.util.ArrayList; +import java.util.List; + +/** + * Created by jan on 11.6.2016. + */ +public class Parser { + private static final Parser instance = new Parser(); + + private List tokens; + private List statementSequence; + + public static Parser getInstance() { + return instance; + } + + private Parser() { + + } + + public List parse(List tokens) { + this.tokens = tokens; + this.statementSequence = new ArrayList<>(); + + Counter c = new Counter(); + while (c.getValue() < tokens.size()) { + AssignmentNode asn = expectAssignmentRoutine(c); + if (asn != null) { + statementSequence.add(asn); + continue; + } + AdditionNode adn = expectAdditionRoutine(c); + if (adn != null) { + statementSequence.add(adn); + continue; + } + } + + return statementSequence; + } + + public AssignmentNode expectAssignmentRoutine(Counter c) { + Counter d = c.clone(); + + SymbolNode lhs = expectLHS(d); + if (lhs == null) { + return null; + } + boolean isAssign = expectAssignmentAndAdvance(d); + if (!isAssign) { + return null; + } + ExpressionNode rhs = expectRHS(d); + if (rhs == null) { + return null; + } + + AssignmentNode an = new AssignmentNode(lhs, rhs); + c.setValue(d.getValue()); + return an; + } + + public SymbolNode expectLHS(Counter c) { + SymbolNode sn = parseSymbolAndAdvance(c); + if (sn != null) { + return sn; + } + return null; + } + + public ExpressionNode expectRHS(Counter c) { + ExpressionNode en = expectExpression(c); + if (en != null) { + return en; + } + return null; + } + + private ExpressionNode expectExpression(Counter c) { + Counter d = c.clone(); + + AdditionNode an = expectAdditionRoutine(d); + if (an != null) { + c.setValue(d.getValue()); + return an; + } + IntegerLiteralNode in = parseIntegerLiteralAndAdvance(d); + if (in != null) { + c.setValue(d.getValue()); + return in; + } + SymbolNode sn = parseSymbolAndAdvance(d); + if (sn != null) { + c.setValue(d.getValue()); + return sn; + } + + return null; + } + + private AdditionNode expectAdditionRoutine(Counter c) { + Counter d = c.clone(); + + ExpressionNode lhs = expectLHS(d); + if (lhs == null) { + return null; + } + boolean isAddition = expectAdditionAndAdvance(d); + if (!isAddition) { + return null; + } + ExpressionNode rhs = expectRHS(d); + if (rhs == null) { + return null; + } + + AdditionNode an = new AdditionNode(lhs, rhs); + c.setValue(d.getValue()); + return an; + } + + private boolean expectAdditionAndAdvance(Counter c) { + if (tokens.get(c.getValue()).is(Token.Type.AdditionToken)) { + c.advance(); + return true; + } else { + return false; + } + } + + private IntegerLiteralNode parseIntegerLiteralAndAdvance(Counter c) { + if (tokens.get(c.getValue()).is(Token.Type.IntegerLiteralToken)) { + IntegerLiteralNode in = new IntegerLiteralNode(tokens.get(c.getValue())); + c.advance(); + return in; + } else { + return null; + } + } + + public boolean expectAssignmentAndAdvance(Counter c) { + if (tokens.get(c.getValue()).is(Token.Type.AssignmentToken)) { + c.advance(); + return true; + } else { + return false; + } + } + + public SymbolNode parseSymbolAndAdvance(Counter c) { + if (tokens.get(c.getValue()).is(Token.Type.SymbolToken)) { + SymbolNode sn = new SymbolNode(tokens.get(c.getValue())); + c.advance(); + return sn; + } else { + return null; + } + } + + public void printTree(ASTNode e) { + System.out.println("### Tree begin ###"); + e.print(0); + System.out.println("### Tree end ###"); + } +} diff --git a/main/core/parser/ast/ASTNode.java b/main/core/parser/ast/ASTNode.java new file mode 100644 index 0000000..106ff75 --- /dev/null +++ b/main/core/parser/ast/ASTNode.java @@ -0,0 +1,33 @@ +package com.jantuomi.interpreter.main.core.parser.ast; + +import com.jantuomi.interpreter.main.core.parser.datatype.DataContainer; +import com.jantuomi.interpreter.main.core.tokenizer.token.Token; + +import java.util.List; + +/** + * Created by jan on 11.6.2016. + */ +abstract public class ASTNode { + + public abstract DataContainer evaluate(); + + protected Token source; + + public ASTNode(Token token) { + this.source = token; + } + + abstract List getChildren(); + + public void print(int indent) { + for (int i = 0; i < indent; i++) { + System.out.print("\t"); + } + + System.out.println(source.toString()); + for (ASTNode node : getChildren()) { + node.print(indent + 1); + } + } +} diff --git a/main/core/parser/ast/AdditionNode.java b/main/core/parser/ast/AdditionNode.java new file mode 100644 index 0000000..3117e41 --- /dev/null +++ b/main/core/parser/ast/AdditionNode.java @@ -0,0 +1,34 @@ +package com.jantuomi.interpreter.main.core.parser.ast; + +import com.jantuomi.interpreter.main.core.parser.datatype.DataContainer; +import com.jantuomi.interpreter.main.core.tokenizer.token.Token; + +import java.util.Arrays; +import java.util.List; + + +/** + * Created by jan on 13.6.2016. + */ +public class AdditionNode extends ExpressionNode { + + private ExpressionNode lhs; + private ExpressionNode rhs; + + public AdditionNode(ExpressionNode lhs, ExpressionNode rhs) { + super(new Token(Token.Type.AdditionToken)); + + this.lhs = lhs; + this.rhs = rhs; + } + + @Override + public List getChildren() { + return Arrays.asList(lhs, rhs); + } + + @Override + public DataContainer evaluate() { + return null; + } +} diff --git a/main/core/parser/ast/AssignmentNode.java b/main/core/parser/ast/AssignmentNode.java new file mode 100644 index 0000000..8d8f8bc --- /dev/null +++ b/main/core/parser/ast/AssignmentNode.java @@ -0,0 +1,36 @@ +package com.jantuomi.interpreter.main.core.parser.ast; + +import com.jantuomi.interpreter.main.core.parser.datatype.DataContainer; +import com.jantuomi.interpreter.main.core.runtime.State; +import com.jantuomi.interpreter.main.core.tokenizer.token.Token; + +import java.util.Arrays; +import java.util.List; + +/** + * Created by jan on 11.6.2016. + */ +public class AssignmentNode extends ASTNode { + + private SymbolNode lhs; + private ExpressionNode rhs; + + public AssignmentNode(SymbolNode lhs, ExpressionNode rhs) { + super(new Token(Token.Type.AssignmentToken)); + + this.lhs = lhs; + this.rhs = rhs; + } + + @Override + public DataContainer evaluate() { + DataContainer value = rhs.evaluate(); + State.getInstance().setSymbolValue(lhs.getSymbol(), value); + return value; + } + + @Override + List getChildren() { + return Arrays.asList(lhs, rhs); + } +} diff --git a/main/core/parser/ast/ExpressionNode.java b/main/core/parser/ast/ExpressionNode.java new file mode 100644 index 0000000..ac1a190 --- /dev/null +++ b/main/core/parser/ast/ExpressionNode.java @@ -0,0 +1,12 @@ +package com.jantuomi.interpreter.main.core.parser.ast; + +import com.jantuomi.interpreter.main.core.tokenizer.token.Token; + +/** + * Created by jan on 11.6.2016. + */ +abstract public class ExpressionNode extends ASTNode { + public ExpressionNode(Token token) { + super(token); + } +} diff --git a/main/core/parser/ast/IntegerLiteralNode.java b/main/core/parser/ast/IntegerLiteralNode.java new file mode 100644 index 0000000..b68adf0 --- /dev/null +++ b/main/core/parser/ast/IntegerLiteralNode.java @@ -0,0 +1,30 @@ +package com.jantuomi.interpreter.main.core.parser.ast; + +import com.jantuomi.interpreter.main.core.parser.datatype.DataContainer; +import com.jantuomi.interpreter.main.core.tokenizer.token.Token; + +import java.util.Arrays; +import java.util.List; + +/** + * Created by jan on 13.6.2016. + */ +public class IntegerLiteralNode extends ExpressionNode { + + private int data; + + public IntegerLiteralNode(Token token) { + super(token); + this.data = Integer.parseInt(token.getText()); + } + + @Override + List getChildren() { + return Arrays.asList(); + } + + @Override + public DataContainer evaluate() { + return null; + } +} diff --git a/main/core/parser/ast/SymbolNode.java b/main/core/parser/ast/SymbolNode.java new file mode 100644 index 0000000..1457851 --- /dev/null +++ b/main/core/parser/ast/SymbolNode.java @@ -0,0 +1,35 @@ +package com.jantuomi.interpreter.main.core.parser.ast; + +import com.jantuomi.interpreter.main.core.parser.datatype.DataContainer; +import com.jantuomi.interpreter.main.core.runtime.State; +import com.jantuomi.interpreter.main.core.tokenizer.token.Token; + +import java.util.Arrays; +import java.util.List; + +/** + * Created by jan on 11.6.2016. + */ +public class SymbolNode extends ExpressionNode { + private String symbol; + + public SymbolNode(Token token) { + super(token); + + this.symbol = token.getText(); + } + + @Override + List getChildren() { + return Arrays.asList(); + } + + public String getSymbol() { + return symbol; + } + + @Override + public DataContainer evaluate() { + return State.getInstance().getSymbolValue(symbol); + } +} diff --git a/main/core/parser/datatype/DataContainer.java b/main/core/parser/datatype/DataContainer.java new file mode 100644 index 0000000..48e72a3 --- /dev/null +++ b/main/core/parser/datatype/DataContainer.java @@ -0,0 +1,17 @@ +package com.jantuomi.interpreter.main.core.parser.datatype; + + +/** + * Created by jan on 11.6.2016. + */ +abstract public class DataContainer { + public T getData() { + return data; + } + + public void setData(T data) { + this.data = data; + } + + private T data; +} diff --git a/main/core/parser/datatype/IntegerDataContainer.java b/main/core/parser/datatype/IntegerDataContainer.java new file mode 100644 index 0000000..c5b9fb0 --- /dev/null +++ b/main/core/parser/datatype/IntegerDataContainer.java @@ -0,0 +1,10 @@ +package com.jantuomi.interpreter.main.core.parser.datatype; + +/** + * Created by jan on 11.6.2016. + */ +public class IntegerDataContainer extends DataContainer { + public IntegerDataContainer(Integer data) { + this.setData(data); + } +} diff --git a/main/core/runtime/State.java b/main/core/runtime/State.java new file mode 100644 index 0000000..7e4fb71 --- /dev/null +++ b/main/core/runtime/State.java @@ -0,0 +1,35 @@ +package com.jantuomi.interpreter.main.core.runtime; + +import com.jantuomi.interpreter.main.core.parser.datatype.DataContainer; + +import java.util.HashMap; +import java.util.Map; + +/** + * Created by jan on 11.6.2016. + */ +public class State { + private static final State instance = new State(); + + public static State getInstance() { + return instance; + } + + private Map variables = new HashMap<>(); + + private State() { + + } + + public DataContainer getSymbolValue(String symbol) { + if (variables.keySet().contains(symbol)) { + return variables.get(symbol); + } else { + return null; + } + } + + public void setSymbolValue(String symbol, DataContainer value) { + variables.put(symbol, value); + } +} diff --git a/main/core/tokenizer/Tokenizer.java b/main/core/tokenizer/Tokenizer.java new file mode 100644 index 0000000..4a12014 --- /dev/null +++ b/main/core/tokenizer/Tokenizer.java @@ -0,0 +1,100 @@ +package com.jantuomi.interpreter.main.core.tokenizer; + + +import com.jantuomi.interpreter.main.core.tokenizer.token.Token; +import com.jantuomi.interpreter.main.exception.ExceptionManager; +import com.jantuomi.interpreter.main.exception.InterpreterException; + +import java.util.*; + +/** + * Created by jan on 10.6.2016. + */ +public class Tokenizer { + private String sourceString; + + public List getTokens() { + return tokens; + } + + private List tokens = new ArrayList<>(); + + private static SortedMap tokenRegexes = new TreeMap<>(); + private static final ListillegalTokenTypes = new ArrayList<>(); + + private static final Tokenizer instance = new Tokenizer(); + + public static final Tokenizer getInstance() { + return instance; + } + + private Tokenizer() { + tokenRegexes.put(Token.Type.CommentToken, "^\\/\\*(.*)\\*\\/"); + tokenRegexes.put(Token.Type.StringLiteralToken, "^\"(.*)\""); + tokenRegexes.put(Token.Type.WhitespaceToken, "^( |\t)"); + tokenRegexes.put(Token.Type.NewlineToken, "^(\n|\r\n)"); + tokenRegexes.put(Token.Type.AdditionToken, "^(\\+)"); + tokenRegexes.put(Token.Type.SubtractionToken, "^(\\-)"); + tokenRegexes.put(Token.Type.DivisionToken, "^(\\/)"); + tokenRegexes.put(Token.Type.MultiplicationToken, "^(\\*)"); + tokenRegexes.put(Token.Type.AssignmentToken, "^(\\<\\-)"); + tokenRegexes.put(Token.Type.LessThanToken, "^(\\<)"); + tokenRegexes.put(Token.Type.GreaterThanToken, "^(\\>)"); + tokenRegexes.put(Token.Type.LessOrEqualThanToken, "^(\\<\\=)"); + tokenRegexes.put(Token.Type.GreaterOrEqualThanToken, "^(\\>\\=)"); + tokenRegexes.put(Token.Type.EqualsToken, "^(\\=\\=)"); + tokenRegexes.put(Token.Type.NotEqualsToken, "^(\\!\\=)"); + tokenRegexes.put(Token.Type.OpenParenToken, "^(\\()"); + tokenRegexes.put(Token.Type.ClosedParenToken, "^(\\))"); + tokenRegexes.put(Token.Type.FunctionDefineToken, "^(func)"); + tokenRegexes.put(Token.Type.DeclarationToken, "^(decl)"); + tokenRegexes.put(Token.Type.IntegerLiteralToken, "^(\\d+)"); + tokenRegexes.put(Token.Type.SymbolToken, "^([a-zA-Z]+\\w*)"); + + illegalTokenTypes.add(Token.Type.NotAToken); + } + + public List tokenize(String string) { + sourceString = string; + tokens.clear(); + int line = 1; + for (int i = 0; i < string.length();) { + + Token token = Token.makeToken(string.substring(i), tokenRegexes, line); + + if (token == null) { + i++; + continue; + } + + if (token.getTokenType() == Token.Type.NewlineToken) { + line++; + } + + if (!illegalTokenTypes.contains(token.getTokenType())) { + tokens.add(token); + } else { + ExceptionManager.raise(InterpreterException.Exception.IllegalTokenException, line, Arrays.asList(token.getText())); + } + + String tokenRawText = token.getRawText(); + if (tokenRawText == null) { + tokenRawText = token.getText(); + } + if (tokenRawText != null) { + i += tokenRawText.length(); + } else { + i++; + } + } + return tokens; + } + + public static void printTokens(List tokens) { + System.out.println("### Tokens: ###"); + for (Token token : tokens) { + System.out.println(String.format("%-40s %s", token.getTokenType(), token.getText())); + } + System.out.println("### End ###"); + } +} diff --git a/main/core/tokenizer/token/Token.java b/main/core/tokenizer/token/Token.java new file mode 100644 index 0000000..6bb5f3b --- /dev/null +++ b/main/core/tokenizer/token/Token.java @@ -0,0 +1,158 @@ +package com.jantuomi.interpreter.main.core.tokenizer.token; + +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; + +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Created by jan on 10.6.2016. + */ +public class Token { + + /* Token types, ordered by precedence */ + public enum Type { + CommentToken, + AdditionToken, + SubtractionToken, + DivisionToken, + MultiplicationToken, + AssignmentToken, + LessThanToken, + GreaterThanToken, + LessOrEqualThanToken, + GreaterOrEqualThanToken, + EqualsToken, + NotEqualsToken, + OpenParenToken, + ClosedParenToken, + FunctionDefineToken, + DeclarationToken, + StringLiteralToken, + IntegerLiteralToken, + WhitespaceToken, + NewlineToken, + EndStatementToken, + SymbolToken, + EndFunctionDefineToken, + NotAToken + } + + public boolean is(Type type) { + return getTokenType() == type; + } + + public int getLine() { + return line; + } + + public void setLine(int line) { + this.line = line; + } + + private int line; + private String rawText; + private String text; + private Type type; + + public String getText() { + return text; + } + + public Type getTokenType() { + return this.type; + } + + public String getRawText() { + return rawText; + } + + public boolean isHigherPrecedenceThan(Token other) { + return getTokenType().ordinal() <= other.getTokenType().ordinal(); + } + + public Token(Type type, String text, String rawText) { + initialize(type, text, rawText); + } + + public Token(Type type, String text) { + initialize(type, text); + } + + public Token(Type type) { + initialize(type, null); + } + + private void initialize(Type type, String text, String rawText) { + this.type = type; + this.text = text; + this.rawText = rawText; + } + + private void initialize(Type type, String text) { + initialize(type, text, text); + } + + public static Token makeToken(String string, Map regexes, int line) { + Type type = Type.NotAToken; + String text = Character.toString(string.charAt(0)); + + for (Type t : regexes.keySet()) { + String regex = regexes.get(t); + Token found = matchToken(string, t, regex); + if (found != null) { + found.setLine(line); + return found; + } + } + + // TODO raise error + return null; + } + + public static Token matchToken(String string, Type type, String regex) { + Pattern pattern = Pattern.compile(regex, Pattern.DOTALL); + Matcher matcher = pattern.matcher(string); + + if (matcher.find()) { + String text = matcher.group(1); + String rawText = matcher.group(0); + + return new Token(type, text, rawText); + } + + return null; + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37) + .append(text) + .append(getTokenType()) + .toHashCode(); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof Token)) return false; + if (o == this) return true; + + Token rhs = (Token) o; + return new EqualsBuilder() + .append(getTokenType(), rhs.getTokenType()) + .append(text, rhs.text) + .isEquals(); + } + + @Override + public String toString() { + String textRepr = text; + return String.format("%-40s %s", getTokenType(), textRepr); + } + + public boolean isSameType(Token other) { + return type == other.type; + } +} diff --git a/main/exception/ExceptionManager.java b/main/exception/ExceptionManager.java new file mode 100644 index 0000000..0ceb25b --- /dev/null +++ b/main/exception/ExceptionManager.java @@ -0,0 +1,29 @@ +package com.jantuomi.interpreter.main.exception; + +import java.util.List; + +/** + * Created by jan on 10.6.2016. + */ +public class ExceptionManager { + + private static final ExceptionManager instance = new ExceptionManager(); + + private ExceptionManager() { + } + + public static ExceptionManager getInstance() { + return instance; + } + + public static void raise(InterpreterException.Exception ex, int line, List args) { + InterpreterException e = new InterpreterException(ex); + String output = e.what(); + for (String arg : args) { + output = String.format(output, arg); + } + System.err.println(String.format("[%s] line: %d %s", ex.toString(), line, output)); + } + + +} diff --git a/main/exception/InterpreterException.java b/main/exception/InterpreterException.java new file mode 100644 index 0000000..8129012 --- /dev/null +++ b/main/exception/InterpreterException.java @@ -0,0 +1,33 @@ +package com.jantuomi.interpreter.main.exception; + + +import java.util.HashMap; +import java.util.Map; + +/** + * Created by jan on 12.6.2016. + */ +public class InterpreterException { + + public enum Exception { + IllegalTokenException, + UnknownOperatorException + } + + public static Map errorTexts = new HashMap<>(); + + static { + errorTexts.put(Exception.IllegalTokenException, "Illegal token %s found."); + errorTexts.put(Exception.UnknownOperatorException, "Unknown operator %s found."); + } + + private Exception exception; + + public InterpreterException(Exception e) { + exception = e; + } + + public String what() { + return errorTexts.get(exception); + } +} diff --git a/main/utils/Counter.java b/main/utils/Counter.java new file mode 100644 index 0000000..a0bf0fd --- /dev/null +++ b/main/utils/Counter.java @@ -0,0 +1,27 @@ +package com.jantuomi.interpreter.main.utils; + +/** + * Created by jan on 13.6.2016. + */ +public class Counter { + private int value = 0; + + public int advance() { + value = value + 1; + return value; + } + + public int getValue() { + return value; + } + + public void setValue(int value) { + this.value = value; + } + + public Counter clone() { + Counter c = new Counter(); + c.value = value; + return c; + } +} diff --git a/main/utils/Utilities.java b/main/utils/Utilities.java new file mode 100644 index 0000000..396248f --- /dev/null +++ b/main/utils/Utilities.java @@ -0,0 +1,17 @@ +package com.jantuomi.interpreter.main.utils; + +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Created by jan on 10.6.2016. + */ +public class Utilities { + private Utilities() {} + + public static String getBasePath() { + Path currentRelativePath = Paths.get(""); + String s = currentRelativePath.toAbsolutePath().toString(); + return s; + } +} diff --git a/test/MainTest.java b/test/MainTest.java new file mode 100644 index 0000000..fec91a7 --- /dev/null +++ b/test/MainTest.java @@ -0,0 +1,24 @@ +package com.jantuomi.interpreter.test; + +import com.jantuomi.interpreter.main.Main; +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Created by jan on 10.6.2016. + */ +public class MainTest { + @Test + public void testParseArguments1() { + String[] notAnArgument = {"--not-an-argument"}; + assertFalse(Main.parseArguments(notAnArgument)); + } + + @Test + public void testParseArguments2() { + String[] fileTestArgument = {"-f", "test.file"}; + assertTrue(Main.parseArguments(fileTestArgument)); + } +} \ No newline at end of file diff --git a/test/core/CommandLineArgumentContainerTest.java b/test/core/CommandLineArgumentContainerTest.java new file mode 100644 index 0000000..f6a6ead --- /dev/null +++ b/test/core/CommandLineArgumentContainerTest.java @@ -0,0 +1,26 @@ +package com.jantuomi.interpreter.test.core; + +import com.jantuomi.interpreter.main.core.CommandLineArgumentContainer; +import com.jantuomi.interpreter.main.utils.Utilities; +import org.junit.Test; + +import java.io.File; + +import static org.junit.Assert.assertTrue; + +/** + * Created by jan on 10.6.2016. + */ +public class CommandLineArgumentContainerTest { + @Test + public void getSourceFileContents() throws Exception { + CommandLineArgumentContainer c = CommandLineArgumentContainer.getInstance(); + + File basePath = new File(Utilities.getBasePath()); + c.setFile(new File(basePath, "src/com/jantuomi/interpreter/test/resources/test.file")); + String contents = c.getSourceFileContents(); + System.out.println(contents); + assertTrue(contents.contains("###")); + } + +} \ No newline at end of file diff --git a/test/core/parser/ParserTest.java b/test/core/parser/ParserTest.java new file mode 100644 index 0000000..913896e --- /dev/null +++ b/test/core/parser/ParserTest.java @@ -0,0 +1,34 @@ +package com.jantuomi.interpreter.test.core.parser; + +import com.jantuomi.interpreter.main.core.parser.Parser; +import com.jantuomi.interpreter.main.core.parser.ast.ASTNode; +import com.jantuomi.interpreter.main.core.tokenizer.token.Token; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +/** + * Created by jan on 12.6.2016. + */ +public class ParserTest { + + @Test + public void parseAssignment() throws Exception { + List tokens = Arrays.asList( + new Token(Token.Type.SymbolToken, "x"), + new Token(Token.Type.AssignmentToken), + new Token(Token.Type.SymbolToken, "y"), + new Token(Token.Type.AdditionToken), + new Token(Token.Type.SymbolToken, "x"), + new Token(Token.Type.AdditionToken), + new Token(Token.Type.IntegerLiteralToken, "1") + ); + + Parser parser = Parser.getInstance(); + List sequence = parser.parse(tokens); + for (ASTNode e : sequence) { + parser.printTree(e); + } + } +} \ No newline at end of file diff --git a/test/core/tokenizer/TokenizerTest.java b/test/core/tokenizer/TokenizerTest.java new file mode 100644 index 0000000..7b29020 --- /dev/null +++ b/test/core/tokenizer/TokenizerTest.java @@ -0,0 +1,80 @@ +package com.jantuomi.interpreter.test.core.tokenizer; + +import com.jantuomi.interpreter.main.core.tokenizer.Tokenizer; +import com.jantuomi.interpreter.main.core.tokenizer.token.Token; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertTrue; + +/** + * Created by jan on 10.6.2016. + */ +public class TokenizerTest { + + @Test + public void testTokenizeCommentAndSymbol() { + String testString = "\"string literal\"\n/*comment*/\n1 2\nsymbol_Test3"; + List list = Tokenizer.getInstance().tokenize(testString); + + System.out.println(String.format("test string:\n%s", testString)); + Tokenizer.printTokens(list); + assertTrue(list.contains(new Token(Token.Type.CommentToken, "comment"))); + } + + @Test + public void testTokenizeMultilineString() { + String testString = "\"this is a\nmultiline\nstring\""; + List list = Tokenizer.getInstance().tokenize(testString); + + System.out.println(String.format("test string:\n%s", testString)); + Tokenizer.printTokens(list); + assertTrue(list.contains(new Token(Token.Type.StringLiteralToken, "this is a\nmultiline\nstring"))); + } + + @Test + public void testTokenizeMath() { + String testString = "x <- (1 + 2) / 3 * 4"; + List list = Tokenizer.getInstance().tokenize(testString); + + System.out.println(String.format("test string:\n%s", testString)); + Tokenizer.printTokens(list); + for (Token t : list) { + if (t.isSameType(new Token(Token.Type.AssignmentToken))) { + return; + } + } + assertTrue("No assignment token found!" == null); + } + + @Test + public void testTokenizeDeclAndAssign() { + String testString = "decl x\nx <- 1 + 2"; + List list = Tokenizer.getInstance().tokenize(testString); + + System.out.println(String.format("test string:\n%s", testString)); + Tokenizer.printTokens(list); + for (Token t : list) { + if (t.isSameType(new Token(Token.Type.DeclarationToken))) { + return; + } + } + assertTrue("No declaration token found!" == null); + } + + @Test + public void testTokenizeFunction() { + String testString = "func name(arg)\nx <- 1\nend"; + List list = Tokenizer.getInstance().tokenize(testString); + + System.out.println(String.format("test string:\n%s", testString)); + Tokenizer.printTokens(list); + for (Token t : list) { + if (t.isSameType(new Token(Token.Type.FunctionDefineToken))) { + return; + } + } + assertTrue("No function declaration token found!" == null); + } +} \ No newline at end of file diff --git a/test/resources/test.file b/test/resources/test.file new file mode 100644 index 0000000..3b723c0 --- /dev/null +++ b/test/resources/test.file @@ -0,0 +1,4 @@ +### +This test file starts and +ends in three # symbols +### \ No newline at end of file -- cgit v1.3