From baceabde7bc971c35ea25016acf60b90d003ac29 Mon Sep 17 00:00:00 2001 From: jantuomi Date: Mon, 1 Aug 2016 14:47:22 +0300 Subject: Move tokenizer package under parser package --- src/main/com/jantuomi/tunkki/Main.java | 14 -- src/main/com/jantuomi/tunkki/Tunkki.java | 4 +- .../jantuomi/tunkki/core/parser/ASTGenerator.java | 2 +- .../com/jantuomi/tunkki/core/parser/Parser.java | 4 +- .../jantuomi/tunkki/core/parser/ast/ASTNode.java | 2 +- .../tunkki/core/parser/ast/AdditionNode.java | 2 +- .../tunkki/core/parser/ast/AssignmentNode.java | 2 +- .../tunkki/core/parser/ast/BinaryOperatorNode.java | 2 +- .../tunkki/core/parser/ast/BlockBodyNode.java | 2 +- .../tunkki/core/parser/ast/BooleanLiteralNode.java | 2 +- .../tunkki/core/parser/ast/BranchNode.java | 2 +- .../jantuomi/tunkki/core/parser/ast/CallNode.java | 2 +- .../tunkki/core/parser/ast/DeclareAssignNode.java | 2 +- .../tunkki/core/parser/ast/DivisionNode.java | 2 +- .../tunkki/core/parser/ast/DoubleLiteralNode.java | 2 +- .../jantuomi/tunkki/core/parser/ast/EmptyNode.java | 2 +- .../tunkki/core/parser/ast/FunctionDefineNode.java | 2 +- .../tunkki/core/parser/ast/IntegerLiteralNode.java | 2 +- .../tunkki/core/parser/ast/MultiplicationNode.java | 2 +- .../tunkki/core/parser/ast/NegationNode.java | 2 +- .../tunkki/core/parser/ast/ParameterListNode.java | 2 +- .../tunkki/core/parser/ast/StringLiteralNode.java | 2 +- .../tunkki/core/parser/ast/SubtractionNode.java | 2 +- .../tunkki/core/parser/ast/SymbolNode.java | 2 +- .../tunkki/core/parser/ast/UnaryOperatorNode.java | 2 +- .../tunkki/core/parser/ast/VarargOperatorNode.java | 2 +- .../core/parser/ast/VariableDeclareNode.java | 2 +- .../tunkki/core/parser/tokenizer/Tokenizer.java | 121 ++++++++++++ .../core/parser/tokenizer/token/ArgumentInfo.java | 49 +++++ .../tunkki/core/parser/tokenizer/token/Token.java | 215 +++++++++++++++++++++ .../core/parser/tokenizer/token/TokenFactory.java | 73 +++++++ .../tokenizer/token/types/AdditionToken.java | 22 +++ .../tokenizer/token/types/AssignmentToken.java | 36 ++++ .../tokenizer/token/types/BinaryOperatorToken.java | 40 ++++ .../tokenizer/token/types/BlockBodyToken.java | 35 ++++ .../tokenizer/token/types/BooleanLiteralToken.java | 18 ++ .../parser/tokenizer/token/types/BranchToken.java | 48 +++++ .../parser/tokenizer/token/types/CallEndToken.java | 18 ++ .../parser/tokenizer/token/types/CallToken.java | 40 ++++ .../parser/tokenizer/token/types/CommentToken.java | 18 ++ .../tokenizer/token/types/DeclareAssignToken.java | 33 ++++ .../tokenizer/token/types/DivisionToken.java | 23 +++ .../tokenizer/token/types/DoubleLiteralToken.java | 18 ++ .../tokenizer/token/types/EndBlockToken.java | 17 ++ .../tokenizer/token/types/FunctionDefineToken.java | 50 +++++ .../tokenizer/token/types/IntegerLiteralToken.java | 18 ++ .../tokenizer/token/types/MultiplicationToken.java | 23 +++ .../tokenizer/token/types/NegationToken.java | 20 ++ .../token/types/OptionalArgumentToken.java | 48 +++++ .../tokenizer/token/types/StringLiteralToken.java | 19 ++ .../tokenizer/token/types/SubtractionToken.java | 22 +++ .../parser/tokenizer/token/types/SymbolToken.java | 23 +++ .../tokenizer/token/types/TerminalToken.java | 39 ++++ .../tokenizer/token/types/UnaryOperatorToken.java | 39 ++++ .../tokenizer/token/types/VarargOperatorToken.java | 49 +++++ .../token/types/VariableDeclareToken.java | 25 +++ .../tokenizer/token/types/WhitespaceToken.java | 37 ++++ .../jantuomi/tunkki/core/tokenizer/Tokenizer.java | 121 ------------ .../tunkki/core/tokenizer/token/ArgumentInfo.java | 49 ----- .../tunkki/core/tokenizer/token/Token.java | 215 --------------------- .../tunkki/core/tokenizer/token/TokenFactory.java | 73 ------- .../core/tokenizer/token/types/AdditionToken.java | 22 --- .../tokenizer/token/types/AssignmentToken.java | 36 ---- .../tokenizer/token/types/BinaryOperatorToken.java | 40 ---- .../core/tokenizer/token/types/BlockBodyToken.java | 35 ---- .../tokenizer/token/types/BooleanLiteralToken.java | 18 -- .../core/tokenizer/token/types/BranchToken.java | 48 ----- .../core/tokenizer/token/types/CallEndToken.java | 18 -- .../core/tokenizer/token/types/CallToken.java | 40 ---- .../core/tokenizer/token/types/CommentToken.java | 18 -- .../tokenizer/token/types/DeclareAssignToken.java | 34 ---- .../core/tokenizer/token/types/DivisionToken.java | 23 --- .../tokenizer/token/types/DoubleLiteralToken.java | 18 -- .../core/tokenizer/token/types/EndBlockToken.java | 17 -- .../tokenizer/token/types/FunctionDefineToken.java | 50 ----- .../tokenizer/token/types/IntegerLiteralToken.java | 18 -- .../tokenizer/token/types/MultiplicationToken.java | 23 --- .../core/tokenizer/token/types/NegationToken.java | 20 -- .../token/types/OptionalArgumentToken.java | 48 ----- .../tokenizer/token/types/StringLiteralToken.java | 19 -- .../tokenizer/token/types/SubtractionToken.java | 22 --- .../core/tokenizer/token/types/SymbolToken.java | 23 --- .../core/tokenizer/token/types/TerminalToken.java | 39 ---- .../tokenizer/token/types/UnaryOperatorToken.java | 39 ---- .../tokenizer/token/types/VarargOperatorToken.java | 49 ----- .../token/types/VariableDeclareToken.java | 25 --- .../tokenizer/token/types/WhitespaceToken.java | 37 ---- .../jantuomi/tunkki/core/parser/ParserTest.java | 6 +- .../core/parser/tokenizer/TokenizerTest.java | 26 +++ .../tunkki/core/tokenizer/TokenizerTest.java | 26 --- 90 files changed, 1293 insertions(+), 1308 deletions(-) create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/Tokenizer.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/ArgumentInfo.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/Token.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/TokenFactory.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/AdditionToken.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/AssignmentToken.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/BinaryOperatorToken.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/BlockBodyToken.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/BooleanLiteralToken.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/BranchToken.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/CallEndToken.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/CallToken.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/CommentToken.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/DeclareAssignToken.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/DivisionToken.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/DoubleLiteralToken.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/EndBlockToken.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/FunctionDefineToken.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/IntegerLiteralToken.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/MultiplicationToken.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/NegationToken.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/OptionalArgumentToken.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/StringLiteralToken.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/SubtractionToken.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/SymbolToken.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/TerminalToken.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/UnaryOperatorToken.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/VarargOperatorToken.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/VariableDeclareToken.java create mode 100644 src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/WhitespaceToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/Tokenizer.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/ArgumentInfo.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/Token.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/TokenFactory.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/AdditionToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/AssignmentToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/BinaryOperatorToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/BlockBodyToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/BooleanLiteralToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/BranchToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/CallEndToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/CallToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/CommentToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/DeclareAssignToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/DivisionToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/DoubleLiteralToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/EndBlockToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/FunctionDefineToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/IntegerLiteralToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/MultiplicationToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/NegationToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/OptionalArgumentToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/StringLiteralToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/SubtractionToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/SymbolToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/TerminalToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/UnaryOperatorToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/VarargOperatorToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/VariableDeclareToken.java delete mode 100644 src/main/com/jantuomi/tunkki/core/tokenizer/token/types/WhitespaceToken.java create mode 100644 src/test/com/jantuomi/tunkki/core/parser/tokenizer/TokenizerTest.java delete mode 100644 src/test/com/jantuomi/tunkki/core/tokenizer/TokenizerTest.java diff --git a/src/main/com/jantuomi/tunkki/Main.java b/src/main/com/jantuomi/tunkki/Main.java index 37979ab..098f0ff 100644 --- a/src/main/com/jantuomi/tunkki/Main.java +++ b/src/main/com/jantuomi/tunkki/Main.java @@ -1,20 +1,6 @@ package com.jantuomi.tunkki; import com.jantuomi.tunkki.core.CommandLineArgumentContainer; -import com.jantuomi.tunkki.core.parser.ASTGenerator; -import com.jantuomi.tunkki.core.parser.Parser; -import com.jantuomi.tunkki.core.parser.ast.ASTNode; -import com.jantuomi.tunkki.core.runtime.Interpreter; -import com.jantuomi.tunkki.core.tokenizer.Tokenizer; -import com.jantuomi.tunkki.core.tokenizer.token.Token; -import com.jantuomi.tunkki.exception.TunkkiError; -import com.jantuomi.tunkki.repl.Repl; -import org.kohsuke.args4j.CmdLineException; -import org.kohsuke.args4j.CmdLineParser; - -import java.util.Collections; -import java.util.List; -import java.util.Scanner; public class Main { diff --git a/src/main/com/jantuomi/tunkki/Tunkki.java b/src/main/com/jantuomi/tunkki/Tunkki.java index 30a728b..cae5644 100644 --- a/src/main/com/jantuomi/tunkki/Tunkki.java +++ b/src/main/com/jantuomi/tunkki/Tunkki.java @@ -5,8 +5,8 @@ import com.jantuomi.tunkki.core.parser.ASTGenerator; import com.jantuomi.tunkki.core.parser.Parser; import com.jantuomi.tunkki.core.parser.ast.ASTNode; import com.jantuomi.tunkki.core.runtime.Interpreter; -import com.jantuomi.tunkki.core.tokenizer.Tokenizer; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.Tokenizer; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; import com.jantuomi.tunkki.exception.TunkkiError; import com.jantuomi.tunkki.repl.Repl; import com.jantuomi.tunkki.utils.Utilities; diff --git a/src/main/com/jantuomi/tunkki/core/parser/ASTGenerator.java b/src/main/com/jantuomi/tunkki/core/parser/ASTGenerator.java index 41ea0ec..4aee626 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/ASTGenerator.java +++ b/src/main/com/jantuomi/tunkki/core/parser/ASTGenerator.java @@ -1,7 +1,7 @@ package com.jantuomi.tunkki.core.parser; import com.jantuomi.tunkki.core.parser.ast.ASTNode; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; import com.jantuomi.tunkki.exception.TunkkiError; import java.util.ArrayList; diff --git a/src/main/com/jantuomi/tunkki/core/parser/Parser.java b/src/main/com/jantuomi/tunkki/core/parser/Parser.java index 08b02bb..6698bba 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/Parser.java +++ b/src/main/com/jantuomi/tunkki/core/parser/Parser.java @@ -1,8 +1,8 @@ package com.jantuomi.tunkki.core.parser; import com.jantuomi.tunkki.core.parser.ast.ASTNode; -import com.jantuomi.tunkki.core.tokenizer.token.ArgumentInfo; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.ArgumentInfo; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; import com.jantuomi.tunkki.exception.ExceptionManager; import com.jantuomi.tunkki.exception.TunkkiError; diff --git a/src/main/com/jantuomi/tunkki/core/parser/ast/ASTNode.java b/src/main/com/jantuomi/tunkki/core/parser/ast/ASTNode.java index a3b0f47..99b68d6 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/ast/ASTNode.java +++ b/src/main/com/jantuomi/tunkki/core/parser/ast/ASTNode.java @@ -1,7 +1,7 @@ package com.jantuomi.tunkki.core.parser.ast; import com.jantuomi.tunkki.core.parser.datatype.Datatype; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; import com.jantuomi.tunkki.exception.TunkkiError; import java.util.List; diff --git a/src/main/com/jantuomi/tunkki/core/parser/ast/AdditionNode.java b/src/main/com/jantuomi/tunkki/core/parser/ast/AdditionNode.java index 6d4e7ca..27b8662 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/ast/AdditionNode.java +++ b/src/main/com/jantuomi/tunkki/core/parser/ast/AdditionNode.java @@ -1,7 +1,7 @@ package com.jantuomi.tunkki.core.parser.ast; import com.jantuomi.tunkki.core.parser.datatype.Datatype; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; import com.jantuomi.tunkki.exception.ExceptionManager; import com.jantuomi.tunkki.exception.TunkkiError; diff --git a/src/main/com/jantuomi/tunkki/core/parser/ast/AssignmentNode.java b/src/main/com/jantuomi/tunkki/core/parser/ast/AssignmentNode.java index 0500ad3..cf705c1 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/ast/AssignmentNode.java +++ b/src/main/com/jantuomi/tunkki/core/parser/ast/AssignmentNode.java @@ -2,7 +2,7 @@ package com.jantuomi.tunkki.core.parser.ast; import com.jantuomi.tunkki.core.parser.datatype.Datatype; import com.jantuomi.tunkki.core.runtime.State; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; import com.jantuomi.tunkki.exception.TunkkiError; import java.util.Arrays; diff --git a/src/main/com/jantuomi/tunkki/core/parser/ast/BinaryOperatorNode.java b/src/main/com/jantuomi/tunkki/core/parser/ast/BinaryOperatorNode.java index 437366d..a83b136 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/ast/BinaryOperatorNode.java +++ b/src/main/com/jantuomi/tunkki/core/parser/ast/BinaryOperatorNode.java @@ -1,6 +1,6 @@ package com.jantuomi.tunkki.core.parser.ast; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; /** * Created by jan on 16.6.2016. diff --git a/src/main/com/jantuomi/tunkki/core/parser/ast/BlockBodyNode.java b/src/main/com/jantuomi/tunkki/core/parser/ast/BlockBodyNode.java index 4899b8f..b394223 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/ast/BlockBodyNode.java +++ b/src/main/com/jantuomi/tunkki/core/parser/ast/BlockBodyNode.java @@ -2,7 +2,7 @@ package com.jantuomi.tunkki.core.parser.ast; import com.jantuomi.tunkki.core.parser.datatype.Datatype; import com.jantuomi.tunkki.core.parser.datatype.VoidDatatype; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; import com.jantuomi.tunkki.exception.TunkkiError; /** diff --git a/src/main/com/jantuomi/tunkki/core/parser/ast/BooleanLiteralNode.java b/src/main/com/jantuomi/tunkki/core/parser/ast/BooleanLiteralNode.java index d9d2c6a..79d959d 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/ast/BooleanLiteralNode.java +++ b/src/main/com/jantuomi/tunkki/core/parser/ast/BooleanLiteralNode.java @@ -2,7 +2,7 @@ package com.jantuomi.tunkki.core.parser.ast; import com.jantuomi.tunkki.core.parser.datatype.BooleanDatatype; import com.jantuomi.tunkki.core.parser.datatype.Datatype; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; import java.util.Arrays; import java.util.List; diff --git a/src/main/com/jantuomi/tunkki/core/parser/ast/BranchNode.java b/src/main/com/jantuomi/tunkki/core/parser/ast/BranchNode.java index 51a5f45..adcd098 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/ast/BranchNode.java +++ b/src/main/com/jantuomi/tunkki/core/parser/ast/BranchNode.java @@ -2,7 +2,7 @@ package com.jantuomi.tunkki.core.parser.ast; import com.jantuomi.tunkki.core.parser.datatype.BooleanDatatype; import com.jantuomi.tunkki.core.parser.datatype.Datatype; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; import com.jantuomi.tunkki.exception.TunkkiError; import java.util.Arrays; diff --git a/src/main/com/jantuomi/tunkki/core/parser/ast/CallNode.java b/src/main/com/jantuomi/tunkki/core/parser/ast/CallNode.java index 6f4e240..5e9cd01 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/ast/CallNode.java +++ b/src/main/com/jantuomi/tunkki/core/parser/ast/CallNode.java @@ -2,7 +2,7 @@ package com.jantuomi.tunkki.core.parser.ast; import com.jantuomi.tunkki.core.parser.datatype.CallableDatatype; import com.jantuomi.tunkki.core.parser.datatype.Datatype; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; import com.jantuomi.tunkki.exception.TunkkiError; /** diff --git a/src/main/com/jantuomi/tunkki/core/parser/ast/DeclareAssignNode.java b/src/main/com/jantuomi/tunkki/core/parser/ast/DeclareAssignNode.java index ea882cf..ed841aa 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/ast/DeclareAssignNode.java +++ b/src/main/com/jantuomi/tunkki/core/parser/ast/DeclareAssignNode.java @@ -2,7 +2,7 @@ package com.jantuomi.tunkki.core.parser.ast; import com.jantuomi.tunkki.core.parser.datatype.Datatype; import com.jantuomi.tunkki.core.runtime.State; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; import com.jantuomi.tunkki.exception.TunkkiError; import java.util.Arrays; diff --git a/src/main/com/jantuomi/tunkki/core/parser/ast/DivisionNode.java b/src/main/com/jantuomi/tunkki/core/parser/ast/DivisionNode.java index c1311c0..aed971a 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/ast/DivisionNode.java +++ b/src/main/com/jantuomi/tunkki/core/parser/ast/DivisionNode.java @@ -1,7 +1,7 @@ package com.jantuomi.tunkki.core.parser.ast; import com.jantuomi.tunkki.core.parser.datatype.Datatype; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; import com.jantuomi.tunkki.exception.ExceptionManager; import com.jantuomi.tunkki.exception.TunkkiError; diff --git a/src/main/com/jantuomi/tunkki/core/parser/ast/DoubleLiteralNode.java b/src/main/com/jantuomi/tunkki/core/parser/ast/DoubleLiteralNode.java index 1272519..3c41775 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/ast/DoubleLiteralNode.java +++ b/src/main/com/jantuomi/tunkki/core/parser/ast/DoubleLiteralNode.java @@ -2,7 +2,7 @@ package com.jantuomi.tunkki.core.parser.ast; import com.jantuomi.tunkki.core.parser.datatype.Datatype; import com.jantuomi.tunkki.core.parser.datatype.DoubleDatatype; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; import java.util.Arrays; import java.util.List; diff --git a/src/main/com/jantuomi/tunkki/core/parser/ast/EmptyNode.java b/src/main/com/jantuomi/tunkki/core/parser/ast/EmptyNode.java index 136a0c3..85ca506 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/ast/EmptyNode.java +++ b/src/main/com/jantuomi/tunkki/core/parser/ast/EmptyNode.java @@ -2,7 +2,7 @@ package com.jantuomi.tunkki.core.parser.ast; import com.jantuomi.tunkki.core.parser.datatype.Datatype; import com.jantuomi.tunkki.core.parser.datatype.VoidDatatype; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; import java.util.Arrays; import java.util.List; diff --git a/src/main/com/jantuomi/tunkki/core/parser/ast/FunctionDefineNode.java b/src/main/com/jantuomi/tunkki/core/parser/ast/FunctionDefineNode.java index a9b0a7e..f73fbf4 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/ast/FunctionDefineNode.java +++ b/src/main/com/jantuomi/tunkki/core/parser/ast/FunctionDefineNode.java @@ -4,7 +4,7 @@ import com.jantuomi.tunkki.core.parser.datatype.CallableDatatype; import com.jantuomi.tunkki.core.parser.datatype.Datatype; import com.jantuomi.tunkki.core.runtime.Function; import com.jantuomi.tunkki.core.runtime.State; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; import com.jantuomi.tunkki.exception.TunkkiError; import java.util.ArrayList; diff --git a/src/main/com/jantuomi/tunkki/core/parser/ast/IntegerLiteralNode.java b/src/main/com/jantuomi/tunkki/core/parser/ast/IntegerLiteralNode.java index f0a1a6d..b6756c3 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/ast/IntegerLiteralNode.java +++ b/src/main/com/jantuomi/tunkki/core/parser/ast/IntegerLiteralNode.java @@ -2,7 +2,7 @@ package com.jantuomi.tunkki.core.parser.ast; import com.jantuomi.tunkki.core.parser.datatype.Datatype; import com.jantuomi.tunkki.core.parser.datatype.IntegerDatatype; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; import java.util.Arrays; import java.util.List; diff --git a/src/main/com/jantuomi/tunkki/core/parser/ast/MultiplicationNode.java b/src/main/com/jantuomi/tunkki/core/parser/ast/MultiplicationNode.java index a1e55d6..67bdf18 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/ast/MultiplicationNode.java +++ b/src/main/com/jantuomi/tunkki/core/parser/ast/MultiplicationNode.java @@ -1,7 +1,7 @@ package com.jantuomi.tunkki.core.parser.ast; import com.jantuomi.tunkki.core.parser.datatype.Datatype; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; import com.jantuomi.tunkki.exception.ExceptionManager; import com.jantuomi.tunkki.exception.TunkkiError; diff --git a/src/main/com/jantuomi/tunkki/core/parser/ast/NegationNode.java b/src/main/com/jantuomi/tunkki/core/parser/ast/NegationNode.java index 7006975..e23b25b 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/ast/NegationNode.java +++ b/src/main/com/jantuomi/tunkki/core/parser/ast/NegationNode.java @@ -3,7 +3,7 @@ package com.jantuomi.tunkki.core.parser.ast; import com.jantuomi.tunkki.core.parser.datatype.BooleanDatatype; import com.jantuomi.tunkki.core.parser.datatype.Datatype; import com.jantuomi.tunkki.core.parser.datatype.IntegerDatatype; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; import com.jantuomi.tunkki.exception.ExceptionManager; import com.jantuomi.tunkki.exception.TunkkiError; diff --git a/src/main/com/jantuomi/tunkki/core/parser/ast/ParameterListNode.java b/src/main/com/jantuomi/tunkki/core/parser/ast/ParameterListNode.java index fbc0992..a05eaee 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/ast/ParameterListNode.java +++ b/src/main/com/jantuomi/tunkki/core/parser/ast/ParameterListNode.java @@ -1,7 +1,7 @@ package com.jantuomi.tunkki.core.parser.ast; import com.jantuomi.tunkki.core.parser.datatype.Datatype; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; import java.util.ArrayList; import java.util.List; diff --git a/src/main/com/jantuomi/tunkki/core/parser/ast/StringLiteralNode.java b/src/main/com/jantuomi/tunkki/core/parser/ast/StringLiteralNode.java index 0976d93..4191917 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/ast/StringLiteralNode.java +++ b/src/main/com/jantuomi/tunkki/core/parser/ast/StringLiteralNode.java @@ -2,7 +2,7 @@ package com.jantuomi.tunkki.core.parser.ast; import com.jantuomi.tunkki.core.parser.datatype.Datatype; import com.jantuomi.tunkki.core.parser.datatype.StringDatatype; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; import java.util.Arrays; import java.util.List; diff --git a/src/main/com/jantuomi/tunkki/core/parser/ast/SubtractionNode.java b/src/main/com/jantuomi/tunkki/core/parser/ast/SubtractionNode.java index 56fd833..1362801 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/ast/SubtractionNode.java +++ b/src/main/com/jantuomi/tunkki/core/parser/ast/SubtractionNode.java @@ -1,7 +1,7 @@ package com.jantuomi.tunkki.core.parser.ast; import com.jantuomi.tunkki.core.parser.datatype.Datatype; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; import com.jantuomi.tunkki.exception.ExceptionManager; import com.jantuomi.tunkki.exception.TunkkiError; diff --git a/src/main/com/jantuomi/tunkki/core/parser/ast/SymbolNode.java b/src/main/com/jantuomi/tunkki/core/parser/ast/SymbolNode.java index 4cfa1df..a768d8d 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/ast/SymbolNode.java +++ b/src/main/com/jantuomi/tunkki/core/parser/ast/SymbolNode.java @@ -2,7 +2,7 @@ package com.jantuomi.tunkki.core.parser.ast; import com.jantuomi.tunkki.core.parser.datatype.Datatype; import com.jantuomi.tunkki.core.runtime.State; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; import com.jantuomi.tunkki.exception.ExceptionManager; import com.jantuomi.tunkki.exception.TunkkiError; diff --git a/src/main/com/jantuomi/tunkki/core/parser/ast/UnaryOperatorNode.java b/src/main/com/jantuomi/tunkki/core/parser/ast/UnaryOperatorNode.java index c047dd4..8cd0f47 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/ast/UnaryOperatorNode.java +++ b/src/main/com/jantuomi/tunkki/core/parser/ast/UnaryOperatorNode.java @@ -1,6 +1,6 @@ package com.jantuomi.tunkki.core.parser.ast; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; import java.util.Arrays; import java.util.List; diff --git a/src/main/com/jantuomi/tunkki/core/parser/ast/VarargOperatorNode.java b/src/main/com/jantuomi/tunkki/core/parser/ast/VarargOperatorNode.java index e935f72..86373c1 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/ast/VarargOperatorNode.java +++ b/src/main/com/jantuomi/tunkki/core/parser/ast/VarargOperatorNode.java @@ -1,6 +1,6 @@ package com.jantuomi.tunkki.core.parser.ast; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; import java.util.List; diff --git a/src/main/com/jantuomi/tunkki/core/parser/ast/VariableDeclareNode.java b/src/main/com/jantuomi/tunkki/core/parser/ast/VariableDeclareNode.java index 07aa8e2..b53438a 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/ast/VariableDeclareNode.java +++ b/src/main/com/jantuomi/tunkki/core/parser/ast/VariableDeclareNode.java @@ -3,7 +3,7 @@ package com.jantuomi.tunkki.core.parser.ast; import com.jantuomi.tunkki.core.parser.datatype.Datatype; import com.jantuomi.tunkki.core.parser.datatype.StringDatatype; import com.jantuomi.tunkki.core.runtime.State; -import com.jantuomi.tunkki.core.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; import java.util.Arrays; import java.util.List; diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/Tokenizer.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/Tokenizer.java new file mode 100644 index 0000000..3b5eaf2 --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/Tokenizer.java @@ -0,0 +1,121 @@ +package com.jantuomi.tunkki.core.parser.tokenizer; + + +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; +import com.jantuomi.tunkki.exception.ExceptionManager; +import com.jantuomi.tunkki.exception.TunkkiError; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Created by jan on 10.6.2016. + */ +public class Tokenizer { + private String sourceString; + + public List getTokens() { + return tokens; + } + + private List tokens = new ArrayList<>(); + + private static Map tokenRegexes = new LinkedHashMap<>(); + private static final List discardedTokenTypes = new ArrayList<>(); + private static final List erroneousTokenTypes = new ArrayList<>(); + + private static final Tokenizer instance = new Tokenizer(); + + public static final Tokenizer getInstance() { + return instance; + } + + private Tokenizer() { + tokenRegexes.put(Token.Type.CommentToken, "^\\/\\*(.*?)\\*\\/"); + + tokenRegexes.put(Token.Type.DoubleLiteralToken, "^([-+]?[0-9]*\\.?[0-9]+)f\\b"); + tokenRegexes.put(Token.Type.IntegerLiteralToken, "^([-+]?\\d+)\\b"); + tokenRegexes.put(Token.Type.BooleanLiteralToken, "^(true|false)\\b"); + tokenRegexes.put(Token.Type.StringLiteralToken, "^\"(.*?)\""); + + tokenRegexes.put(Token.Type.WhitespaceToken, "^( |\t)"); + tokenRegexes.put(Token.Type.NewlineToken, "^(\n|\r\n)"); + tokenRegexes.put(Token.Type.AdditionToken, "^(\\+)"); + tokenRegexes.put(Token.Type.SubtractionToken, "^(\\-)"); + tokenRegexes.put(Token.Type.DivisionToken, "^(\\/)"); + tokenRegexes.put(Token.Type.MultiplicationToken, "^(\\*)"); + tokenRegexes.put(Token.Type.DeclareAssignToken, "^(dset)\\b"); + tokenRegexes.put(Token.Type.AssignmentToken, "^(?)"); + tokenRegexes.put(Token.Type.LessOrEqualThanToken, "^(\\<\\=)"); + tokenRegexes.put(Token.Type.GreaterOrEqualThanToken, "^(\\>\\=)"); + tokenRegexes.put(Token.Type.EqualsToken, "^(\\=\\=)"); + tokenRegexes.put(Token.Type.NotEqualsToken, "^(\\!\\=)"); + tokenRegexes.put(Token.Type.OpenParenToken, "^(\\()"); + tokenRegexes.put(Token.Type.CallEndToken, "^(\\!)"); + tokenRegexes.put(Token.Type.FunctionDefineToken, "^(func)\\b"); + tokenRegexes.put(Token.Type.FunctionBodyToken, "^(as)\\b"); + tokenRegexes.put(Token.Type.BranchBodyToken, "^(then)\\b"); + tokenRegexes.put(Token.Type.EndBlockToken, "^(end)\\b"); + tokenRegexes.put(Token.Type.NegationToken, "^(not)\\b"); + tokenRegexes.put(Token.Type.DeclarationToken, "^(decl)\\b"); + tokenRegexes.put(Token.Type.BranchToken, "^(if)\\b"); + + tokenRegexes.put(Token.Type.CallToken, "^([a-zA-Z]+\\w*)\\?[\\s|\\!]"); + tokenRegexes.put(Token.Type.SymbolToken, "^([a-zA-Z]+\\w*)\\b"); + + discardedTokenTypes.add(Token.Type.WhitespaceToken); + discardedTokenTypes.add(Token.Type.CommentToken); + + erroneousTokenTypes.add(Token.Type.NotAToken); + } + + public List tokenize(String string) throws TunkkiError { + sourceString = string; + tokens.clear(); + int line = 1; + for (int i = 0; i < string.length();) { + + Token token = Token.makeToken(string.substring(i), tokenRegexes, line); + + if (token == null) { + i++; + continue; + } + + if (token.getTokenType() == Token.Type.NewlineToken) { + line++; + } + + if (!discardedTokenTypes.contains(token.getTokenType())) { + tokens.add(token); + } + + if (erroneousTokenTypes.contains(token.getTokenType())) { + ExceptionManager.raise(TunkkiError.ExceptionType.IllegalTokenError, line, token.getText()); + } + + String tokenRawText = token.getRawText(); + if (tokenRawText == null) { + tokenRawText = token.getText(); + } + if (tokenRawText != null) { + i += tokenRawText.length(); + } else { + i++; + } + } + return tokens; + } + + public static void printTokens(List tokens) { + System.out.println("### Tokens: ###"); + for (Token token : tokens) { + System.out.println(String.format("%-40s %s", token.getTokenType(), token.getText())); + } + System.out.println("### End ###"); + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/ArgumentInfo.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/ArgumentInfo.java new file mode 100644 index 0000000..e194b17 --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/ArgumentInfo.java @@ -0,0 +1,49 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token; + +/** + * Created by jan on 15.6.2016. + */ +public class ArgumentInfo { + private int count = 0; + private boolean isVarargs = false; + + public Token.Type getOptionalArgument() { + return optionalArgument; + } + + public void setOptionalArgument(Token.Type optionalArgument) { + this.optionalArgument = optionalArgument; + } + + private Token.Type optionalArgument = null; + + public Token.Type getTerminator() { + return terminator; + } + + public void setTerminator(Token.Type terminator) { + this.terminator = terminator; + } + + private Token.Type terminator; + + public ArgumentInfo(int count) { + this.count = count; + } + + public ArgumentInfo() { + + } + + public void setVariable(boolean value) { + this.isVarargs = value; + } + + public boolean getVarargs() { + return isVarargs; + } + + public int getCount() { + return count; + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/Token.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/Token.java new file mode 100644 index 0000000..17d3cf1 --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/Token.java @@ -0,0 +1,215 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token; + +import com.jantuomi.tunkki.core.parser.ast.ASTNode; +import com.jantuomi.tunkki.exception.ExceptionManager; +import com.jantuomi.tunkki.exception.TunkkiError; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; + +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +/** + * Created by jan on 10.6.2016. + */ +abstract public class Token { + + public void print(int indent) { + for (int i = 0; i < indent; i++) { + System.out.print("\t"); + } + System.out.println(toString()); + for (Token child : getChildren()) { + child.print(indent + 1); + } + } + + abstract public ASTNode generateNode() throws TunkkiError; + + /* Token types, ordered by precedence */ + public enum Type { + CommentToken, + + AdditionToken, + SubtractionToken, + DivisionToken, + MultiplicationToken, + + AssignmentToken, + + LessThanToken, + GreaterThanToken, + LessOrEqualThanToken, + GreaterOrEqualThanToken, + EqualsToken, + NotEqualsToken, + + OpenParenToken, + CallEndToken, + + FunctionDefineToken, + DeclarationToken, + + StringLiteralToken, + IntegerLiteralToken, + BooleanLiteralToken, + DoubleLiteralToken, + + WhitespaceToken, + NewlineToken, + + BranchToken, + SymbolToken, + CallToken, + + EndBlockToken, + FunctionBodyToken, + BranchBodyToken, + NegationToken, + + DeclareAssignToken, + NotAToken + } + + abstract public Token setArguments(List args); + + abstract public List getChildren(); + + public boolean is(Type type) { + return getTokenType() == type; + } + + public int getLine() { + return line; + } + + public void setLine(int line) { + this.line = line; + } + + private int line; + private String rawText; + private String text; + private Type type; + + public String getText() { + return text; + } + + public Type getTokenType() { + return this.type; + } + + public String getRawText() { + return rawText; + } + + public boolean isHigherPrecedenceThan(Token other) { + return getTokenType().ordinal() <= other.getTokenType().ordinal(); + } + + public Token(Type type, String text, String rawText) { + initialize(type, text, rawText); + } + + public Token(Type type, String text) { + initialize(type, text); + } + + public Token(Type type) { + initialize(type, null); + } + + private void initialize(Type type, String text, String rawText) { + this.type = type; + this.text = text; + this.rawText = rawText; + } + + private void initialize(Type type, String text) { + initialize(type, text, text); + } + + public static Token makeToken(String string, Map regexes, int line) throws TunkkiError { + Type type = Type.NotAToken; + String text = Character.toString(string.charAt(0)); + + for (Type t : regexes.keySet()) { + String regex = regexes.get(t); + Token found = matchToken(string, t, regex, line); + if (found != null) { + found.setLine(line); + return found; + } + } + + // TODO raise error + return null; + } + + public static Token matchToken(String string, Type type, String regex, int line) throws TunkkiError { + Pattern pattern = Pattern.compile(regex, Pattern.DOTALL); + Matcher matcher = pattern.matcher(string); + + if (matcher.find()) { + String lexeme = matcher.group(1); + String rawText = matcher.group(0); + + Token token = TokenFactory.create(type, lexeme, rawText); + if (token != null) { + token.setLine(line); + } + else { + ExceptionManager.raise(TunkkiError.ExceptionType.IllegalTokenError, line, + type.toString() + ); + } + + return token; + } + return null; + } + + abstract public ArgumentInfo getArgumentInfo(); + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37) + .append(text) + .append(getTokenType()) + .toHashCode(); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof Token)) return false; + if (o == this) return true; + + Token rhs = (Token) o; + return new EqualsBuilder() + .append(getTokenType(), rhs.getTokenType()) + .append(text, rhs.text) + .isEquals(); + } + + @Override + public String toString() { + return String.format(String.format( + "%s%s", + getTokenType().toString().replace("Token", ""), + getText() == null ? "" : " " + getText() + )); + } + + public String toFormattedString() { + String textRepr = text; + return String.format("%-40s %s", getTokenType(), textRepr); + } + + public boolean isSameType(Token other) { + return type == other.type; + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/TokenFactory.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/TokenFactory.java new file mode 100644 index 0000000..21cd13b --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/TokenFactory.java @@ -0,0 +1,73 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token; + +import com.jantuomi.tunkki.core.parser.tokenizer.token.types.*; + +/** + * Created by jan on 28.6.2016. + */ +public class TokenFactory { + private static TokenFactory instance = new TokenFactory(); + + public static TokenFactory getInstance() { + return instance; + } + + private TokenFactory() { + } + + public static Token create(Token.Type type, String text, String rawText) { + switch (type) { + case WhitespaceToken: + return new WhitespaceToken(); + case CommentToken: + return new CommentToken(text, rawText); + + // TODO placeholder whitespace instead of newline + case NewlineToken: + return new WhitespaceToken(); + + case IntegerLiteralToken: + return new IntegerLiteralToken(text); + case StringLiteralToken: + return new StringLiteralToken(text, rawText); + case BooleanLiteralToken: + return new BooleanLiteralToken(text); + case DoubleLiteralToken: + return new DoubleLiteralToken(text, rawText); + case AdditionToken: + return new AdditionToken(); + case SubtractionToken: + return new SubtractionToken(); + case MultiplicationToken: + return new MultiplicationToken(); + case DivisionToken: + return new DivisionToken(); + case FunctionDefineToken: + return new FunctionDefineToken(); + case EndBlockToken: + return new EndBlockToken(); + case FunctionBodyToken: + return new BlockBodyToken("as"); + case BranchBodyToken: + return new BlockBodyToken("then"); + case DeclarationToken: + return new VariableDeclareToken(); + case AssignmentToken: + return new AssignmentToken(); + case DeclareAssignToken: + return new DeclareAssignToken("dset"); + case SymbolToken: + return new SymbolToken(text); + case CallToken: + return new CallToken(text, rawText); + case CallEndToken: + return new CallEndToken(); + case BranchToken: + return new BranchToken(); + case NegationToken: + return new NegationToken(); + default: + return null; + } + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/AdditionToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/AdditionToken.java new file mode 100644 index 0000000..5379ffa --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/AdditionToken.java @@ -0,0 +1,22 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.ast.ASTNode; +import com.jantuomi.tunkki.core.parser.ast.AdditionNode; +import com.jantuomi.tunkki.exception.TunkkiError; + +/** + * Created by jan on 15.6.2016. + */ +public class AdditionToken extends BinaryOperatorToken { + public AdditionToken() { + super(Type.AdditionToken); + } + + @Override + public ASTNode generateNode() throws TunkkiError { + return new AdditionNode(this, + lhs.generateNode(), + rhs.generateNode() + ); + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/AssignmentToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/AssignmentToken.java new file mode 100644 index 0000000..e70e258 --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/AssignmentToken.java @@ -0,0 +1,36 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.ast.ASTNode; +import com.jantuomi.tunkki.core.parser.ast.AssignmentNode; +import com.jantuomi.tunkki.core.parser.ast.SymbolNode; +import com.jantuomi.tunkki.exception.ExceptionManager; +import com.jantuomi.tunkki.exception.TunkkiError; + +/** + * Created by jan on 20.6.2016. + */ +public class AssignmentToken extends BinaryOperatorToken { + + public AssignmentToken() { + super(Type.AssignmentToken, "set"); + } + + // TODO this and the node + + @Override + public ASTNode generateNode() throws TunkkiError { + ASTNode lhsNode = lhs.generateNode(); + + if (!(lhsNode instanceof SymbolNode)) { + ExceptionManager.raise(TunkkiError.ExceptionType.SyntaxError, getLine(), "non-symbol " + toString()); + return null; + } + + AssignmentNode node = new AssignmentNode(this, + (SymbolNode) lhsNode, + rhs.generateNode() + ); + + return node; + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/BinaryOperatorToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/BinaryOperatorToken.java new file mode 100644 index 0000000..5ca0db2 --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/BinaryOperatorToken.java @@ -0,0 +1,40 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.tokenizer.token.ArgumentInfo; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; + +import java.util.Arrays; +import java.util.List; + +/** + * Created by jan on 15.6.2016. + */ +abstract public class BinaryOperatorToken extends Token { + protected Token lhs; + protected Token rhs; + + public BinaryOperatorToken(Type type) { + super(type); + } + + public BinaryOperatorToken(Type type, String text) { + super(type, text); + } + + @Override + public Token setArguments(List args) { + lhs = args.get(0); + rhs = args.get(1); + return this; + } + + @Override + public List getChildren() { + return Arrays.asList(lhs, rhs); + } + + @Override + public ArgumentInfo getArgumentInfo() { + return new ArgumentInfo(2); + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/BlockBodyToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/BlockBodyToken.java new file mode 100644 index 0000000..35e1a29 --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/BlockBodyToken.java @@ -0,0 +1,35 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.ast.ASTNode; +import com.jantuomi.tunkki.core.parser.ast.BlockBodyNode; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; +import com.jantuomi.tunkki.exception.TunkkiError; + +import java.util.ArrayList; +import java.util.List; + +/** + * Created by jan on 16.6.2016. + */ +public class BlockBodyToken extends VarargOperatorToken { + public BlockBodyToken(String keyword) { + super(Type.FunctionBodyToken, keyword); + } + + @Override + public ASTNode generateNode() throws TunkkiError { + BlockBodyNode node = new BlockBodyNode(this); + List statements = new ArrayList<>(); + for (int i = 0; i < args.size() - 1; i++) { + Token arg = args.get(i); + statements.add(arg.generateNode()); + } + node.setArgs(statements); + return node; + } + + @Override + public Type getTerminatorTokenType() { + return Type.EndBlockToken; + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/BooleanLiteralToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/BooleanLiteralToken.java new file mode 100644 index 0000000..f1a4013 --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/BooleanLiteralToken.java @@ -0,0 +1,18 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.ast.ASTNode; +import com.jantuomi.tunkki.core.parser.ast.BooleanLiteralNode; + +/** + * Created by jan on 15.6.2016. + */ +public class BooleanLiteralToken extends TerminalToken{ + public BooleanLiteralToken(String text) { + super(Type.BooleanLiteralToken, text); + } + + @Override + public ASTNode generateNode() { + return new BooleanLiteralNode(this); + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/BranchToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/BranchToken.java new file mode 100644 index 0000000..e9f8bcb --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/BranchToken.java @@ -0,0 +1,48 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.ast.ASTNode; +import com.jantuomi.tunkki.core.parser.ast.BlockBodyNode; +import com.jantuomi.tunkki.core.parser.ast.BranchNode; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; +import com.jantuomi.tunkki.exception.TunkkiError; + +import java.util.Arrays; +import java.util.List; + +/** + * Created by jan on 21.6.2016. + */ +public class BranchToken extends VarargOperatorToken { + + private Token expression; + private BlockBodyToken branch; + + public BranchToken() { + super(Type.BranchToken, "if"); + } + + @Override + public ASTNode generateNode() throws TunkkiError { + return new BranchNode(this, + expression.generateNode(), + (BlockBodyNode) branch.generateNode() + ); + } + + @Override + public Token setArguments(List args) { + expression = args.get(0); + branch = (BlockBodyToken) args.get(1); + return this; + } + + @Override + public List getChildren() { + return Arrays.asList(expression, branch); + } + + @Override + public Type getTerminatorTokenType() { + return Type.BranchBodyToken; + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/CallEndToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/CallEndToken.java new file mode 100644 index 0000000..e90e929 --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/CallEndToken.java @@ -0,0 +1,18 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.ast.ASTNode; + +/** + * Created by jan on 19.6.2016. + */ +public class CallEndToken extends TerminalToken { + public CallEndToken() { + super(Type.CallEndToken); + } + + @Override + public ASTNode generateNode() { + return null; + } + +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/CallToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/CallToken.java new file mode 100644 index 0000000..d59ad80 --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/CallToken.java @@ -0,0 +1,40 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.ast.ASTNode; +import com.jantuomi.tunkki.core.parser.ast.CallNode; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; +import com.jantuomi.tunkki.exception.TunkkiError; + +import java.util.ArrayList; +import java.util.List; + +/** + * Created by jan on 26.7.2016. + */ +public class CallToken extends VarargOperatorToken { + public CallToken(String text, String rawText) { + super(Type.CallToken, text, rawText); + } + + @Override + public ASTNode generateNode() throws TunkkiError { + CallNode node = new CallNode(this); + node.setName(this.getText()); + + List arguments = new ArrayList<>(); + + for (Token token : args) { + ASTNode arg = token.generateNode(); + if (arg != null) { + arguments.add(arg); + } + } + node.setParameters(arguments); + return node; + } + + @Override + public Type getTerminatorTokenType() { + return Type.CallEndToken; + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/CommentToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/CommentToken.java new file mode 100644 index 0000000..b98809e --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/CommentToken.java @@ -0,0 +1,18 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.ast.ASTNode; +import com.jantuomi.tunkki.core.parser.ast.EmptyNode; + +/** + * Created by jan on 19.6.2016. + */ +public class CommentToken extends TerminalToken { + public CommentToken(String text, String rawText) { + super(Type.CommentToken, text, rawText); + } + + @Override + public ASTNode generateNode() { + return new EmptyNode(this); + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/DeclareAssignToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/DeclareAssignToken.java new file mode 100644 index 0000000..dc55896 --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/DeclareAssignToken.java @@ -0,0 +1,33 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.ast.ASTNode; +import com.jantuomi.tunkki.core.parser.ast.DeclareAssignNode; +import com.jantuomi.tunkki.core.parser.ast.SymbolNode; +import com.jantuomi.tunkki.exception.ExceptionManager; +import com.jantuomi.tunkki.exception.TunkkiError; + +/** + * Created by jan on 28.7.2016. + */ +public class DeclareAssignToken extends BinaryOperatorToken { + public DeclareAssignToken(String text) { + super(Type.DeclareAssignToken, text); + } + + @Override + public ASTNode generateNode() throws TunkkiError { + ASTNode lhsNode = lhs.generateNode(); + + if (!(lhsNode instanceof SymbolNode)) { + ExceptionManager.raise(TunkkiError.ExceptionType.SyntaxError, getLine(), "non-symbol " + toString()); + return null; + } + + DeclareAssignNode node = new DeclareAssignNode(this, + (SymbolNode) lhsNode, + rhs.generateNode() + ); + + return node; + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/DivisionToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/DivisionToken.java new file mode 100644 index 0000000..cd3100f --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/DivisionToken.java @@ -0,0 +1,23 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.ast.ASTNode; +import com.jantuomi.tunkki.core.parser.ast.DivisionNode; +import com.jantuomi.tunkki.exception.TunkkiError; + +/** + * Created by jan on 15.6.2016. + */ +public class DivisionToken extends BinaryOperatorToken { + public DivisionToken() { + super(Type.DivisionToken); + } + + @Override + public ASTNode generateNode() throws TunkkiError { + return new DivisionNode(this, + lhs.generateNode(), + rhs.generateNode() + ); + + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/DoubleLiteralToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/DoubleLiteralToken.java new file mode 100644 index 0000000..4fcf598 --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/DoubleLiteralToken.java @@ -0,0 +1,18 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.ast.ASTNode; +import com.jantuomi.tunkki.core.parser.ast.DoubleLiteralNode; + +/** + * Created by jan on 15.6.2016. + */ +public class DoubleLiteralToken extends TerminalToken { + public DoubleLiteralToken(String text, String rawText) { + super(Type.DoubleLiteralToken, text, rawText); + } + + @Override + public ASTNode generateNode() { + return new DoubleLiteralNode(this); + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/EndBlockToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/EndBlockToken.java new file mode 100644 index 0000000..9ec4b5f --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/EndBlockToken.java @@ -0,0 +1,17 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.ast.ASTNode; + +/** + * Created by jan on 16.6.2016. + */ +public class EndBlockToken extends TerminalToken { + public EndBlockToken() { + super(Type.EndBlockToken, "end"); + } + + @Override + public ASTNode generateNode() { + return null; + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/FunctionDefineToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/FunctionDefineToken.java new file mode 100644 index 0000000..ad7fa14 --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/FunctionDefineToken.java @@ -0,0 +1,50 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.ast.ASTNode; +import com.jantuomi.tunkki.core.parser.ast.BlockBodyNode; +import com.jantuomi.tunkki.core.parser.ast.FunctionDefineNode; +import com.jantuomi.tunkki.exception.TunkkiError; + +import java.util.ArrayList; +import java.util.List; + +/** + * Created by jan on 16.6.2016. + */ +public class FunctionDefineToken extends VarargOperatorToken { + public FunctionDefineToken() { + super(Type.FunctionDefineToken, "func"); + } + + @Override + public ASTNode generateNode() throws TunkkiError { + FunctionDefineNode node = new FunctionDefineNode(this); + + if (args.size() == 0) { + throw new TunkkiError(TunkkiError.ExceptionType.ExpectedDifferentTokenError, getLine(), getText()); + } + + node.setName(args.get(0).getText()); + + List funcArgs = new ArrayList<>(); + for (int i = 1; i < args.size() - 1; i++) { + funcArgs.add(args.get(i).generateNode()); + } + node.setArgs(funcArgs); + + BlockBodyNode bodyNode; + try { + bodyNode = (BlockBodyNode) args.get(args.size() - 1).generateNode(); + } + catch (Exception ex) { + throw new TunkkiError(TunkkiError.ExceptionType.ExpectedDifferentTokenError, getLine(), getText()); + } + node.setBody(bodyNode); + return node; + } + + @Override + public Type getTerminatorTokenType() { + return Type.FunctionBodyToken; + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/IntegerLiteralToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/IntegerLiteralToken.java new file mode 100644 index 0000000..c7687a8 --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/IntegerLiteralToken.java @@ -0,0 +1,18 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.ast.ASTNode; +import com.jantuomi.tunkki.core.parser.ast.IntegerLiteralNode; + +/** + * Created by jan on 15.6.2016. + */ +public class IntegerLiteralToken extends TerminalToken { + public IntegerLiteralToken(String text) { + super(Type.IntegerLiteralToken, text); + } + + @Override + public ASTNode generateNode() { + return new IntegerLiteralNode(this); + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/MultiplicationToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/MultiplicationToken.java new file mode 100644 index 0000000..e8e3721 --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/MultiplicationToken.java @@ -0,0 +1,23 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.ast.ASTNode; +import com.jantuomi.tunkki.core.parser.ast.MultiplicationNode; +import com.jantuomi.tunkki.exception.TunkkiError; + +/** + * Created by jan on 15.6.2016. + */ +public class MultiplicationToken extends BinaryOperatorToken { + public MultiplicationToken() { + super(Type.MultiplicationToken); + } + + @Override + public ASTNode generateNode() throws TunkkiError { + return new MultiplicationNode(this, + lhs.generateNode(), + rhs.generateNode() + ); + + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/NegationToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/NegationToken.java new file mode 100644 index 0000000..a094f02 --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/NegationToken.java @@ -0,0 +1,20 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.ast.ASTNode; +import com.jantuomi.tunkki.core.parser.ast.NegationNode; +import com.jantuomi.tunkki.exception.TunkkiError; + +/** + * Created by jan on 21.6.2016. + */ +public class NegationToken extends UnaryOperatorToken { + + public NegationToken() { + super(Type.NegationToken, "not"); + } + + @Override + public ASTNode generateNode() throws TunkkiError { + return new NegationNode(this, operand.generateNode()); + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/OptionalArgumentToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/OptionalArgumentToken.java new file mode 100644 index 0000000..3b16208 --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/OptionalArgumentToken.java @@ -0,0 +1,48 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.tokenizer.token.ArgumentInfo; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; + +import java.util.Arrays; +import java.util.List; + +/** + * Created by jan on 28.6.2016. + */ +public abstract class OptionalArgumentToken extends Token { + public OptionalArgumentToken(Type type, String text) { + super(type, text); + } + + protected Token optionalArgument; + + public void setOptionalArgument(Token optionalArgument) { + this.optionalArgument = optionalArgument; + } + + abstract public Token.Type getOptionalArgumentType(); + + @Override + public ArgumentInfo getArgumentInfo() { + ArgumentInfo ai = new ArgumentInfo(); + ai.setOptionalArgument(getOptionalArgumentType()); + return ai; + } + + @Override + public Token setArguments(List args) { + if (args.size() > 0) { + setOptionalArgument(args.get(0)); + } + return this; + } + + @Override + public List getChildren() { + if (optionalArgument == null) { + return Arrays.asList(); + } else { + return Arrays.asList(optionalArgument); + } + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/StringLiteralToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/StringLiteralToken.java new file mode 100644 index 0000000..de916c2 --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/StringLiteralToken.java @@ -0,0 +1,19 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.ast.ASTNode; +import com.jantuomi.tunkki.core.parser.ast.StringLiteralNode; +import com.jantuomi.tunkki.exception.TunkkiError; + +/** + * Created by jan on 21.6.2016. + */ +public class StringLiteralToken extends TerminalToken { + public StringLiteralToken(String text, String rawText) { + super(Type.StringLiteralToken, text, rawText); + } + + @Override + public ASTNode generateNode() throws TunkkiError { + return new StringLiteralNode(this); + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/SubtractionToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/SubtractionToken.java new file mode 100644 index 0000000..f5ff1b4 --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/SubtractionToken.java @@ -0,0 +1,22 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.ast.ASTNode; +import com.jantuomi.tunkki.core.parser.ast.SubtractionNode; +import com.jantuomi.tunkki.exception.TunkkiError; + +/** + * Created by jan on 16.6.2016. + */ +public class SubtractionToken extends BinaryOperatorToken { + public SubtractionToken() { + super(Type.SubtractionToken); + } + + @Override + public ASTNode generateNode() throws TunkkiError { + return new SubtractionNode(this, + lhs.generateNode(), + rhs.generateNode() + ); + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/SymbolToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/SymbolToken.java new file mode 100644 index 0000000..613ebc5 --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/SymbolToken.java @@ -0,0 +1,23 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.ast.ASTNode; +import com.jantuomi.tunkki.core.parser.ast.SymbolNode; +import com.jantuomi.tunkki.exception.TunkkiError; + +/** + * Created by jan on 16.6.2016. + */ +public class SymbolToken extends TerminalToken { + + public SymbolToken(String name) { + super(Type.SymbolToken, name); + } + + @Override + public ASTNode generateNode() throws TunkkiError { + SymbolNode node = new SymbolNode(this); + node.setName(this.getText()); + + return node; + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/TerminalToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/TerminalToken.java new file mode 100644 index 0000000..2158458 --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/TerminalToken.java @@ -0,0 +1,39 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.tokenizer.token.ArgumentInfo; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; + +import java.util.Arrays; +import java.util.List; + +/** + * Created by jan on 28.6.2016. + */ +abstract public class TerminalToken extends Token { + public TerminalToken(Type type, String text, String rawText) { + super(type, text, rawText); + } + + public TerminalToken(Type type, String text) { + super(type, text); + } + + public TerminalToken(Type type) { + super(type); + } + + @Override + public Token setArguments(List args) { + return this; + } + + @Override + public List getChildren() { + return Arrays.asList(); + } + + @Override + public ArgumentInfo getArgumentInfo() { + return new ArgumentInfo(0); + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/UnaryOperatorToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/UnaryOperatorToken.java new file mode 100644 index 0000000..2327619 --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/UnaryOperatorToken.java @@ -0,0 +1,39 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.tokenizer.token.ArgumentInfo; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; + +import java.util.Arrays; +import java.util.List; + +/** + * Created by jan on 28.6.2016. + */ +abstract public class UnaryOperatorToken extends Token { + public UnaryOperatorToken(Type type, String text) { + super(type, text); + } + + public UnaryOperatorToken(Type type) { + super(type); + } + + protected Token operand; + + + @Override + public ArgumentInfo getArgumentInfo() { + return new ArgumentInfo(1); + } + + @Override + public Token setArguments(List args) { + operand = args.get(0); + return this; + } + + @Override + public List getChildren() { + return Arrays.asList(operand); + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/VarargOperatorToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/VarargOperatorToken.java new file mode 100644 index 0000000..8119328 --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/VarargOperatorToken.java @@ -0,0 +1,49 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.tokenizer.token.ArgumentInfo; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; + +import java.util.List; + +/** + * Created by jan on 16.6.2016. + */ +abstract public class VarargOperatorToken extends Token { + + protected List args; + + public VarargOperatorToken(Type type, String text, String rawText) { + super(type, text, rawText); + } + + public VarargOperatorToken(Type type, String text) { + super(type, text); + } + + public VarargOperatorToken(Type type) { + super(type); + } + + @Override + public Token setArguments(List args) { + this.args = args; + return this; + } + + @Override + public List getChildren() { + return args; + } + + abstract public Token.Type getTerminatorTokenType(); + + @Override + public ArgumentInfo getArgumentInfo() { + ArgumentInfo ai = new ArgumentInfo(); + ai.setVariable(true); + ai.setTerminator(getTerminatorTokenType()); + return ai; + + } + +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/VariableDeclareToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/VariableDeclareToken.java new file mode 100644 index 0000000..46c5b6a --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/VariableDeclareToken.java @@ -0,0 +1,25 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.ast.ASTNode; +import com.jantuomi.tunkki.core.parser.ast.SymbolNode; +import com.jantuomi.tunkki.core.parser.ast.VariableDeclareNode; +import com.jantuomi.tunkki.exception.TunkkiError; + +/** + * Created by jan on 20.6.2016. + */ +public class VariableDeclareToken extends UnaryOperatorToken { + + + public VariableDeclareToken() { + super(Type.DeclarationToken, "decl"); + } + + @Override + public ASTNode generateNode() throws TunkkiError { + SymbolToken st = (SymbolToken) operand; + VariableDeclareNode node = new VariableDeclareNode(this, + (SymbolNode) st.generateNode()); + return node; + } +} diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/WhitespaceToken.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/WhitespaceToken.java new file mode 100644 index 0000000..e1e2941 --- /dev/null +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/token/types/WhitespaceToken.java @@ -0,0 +1,37 @@ +package com.jantuomi.tunkki.core.parser.tokenizer.token.types; + +import com.jantuomi.tunkki.core.parser.ast.ASTNode; +import com.jantuomi.tunkki.core.parser.tokenizer.token.ArgumentInfo; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; + +import java.util.Arrays; +import java.util.List; + +/** + * Created by jan on 16.6.2016. + */ +public class WhitespaceToken extends Token { + public WhitespaceToken() { + super(Type.WhitespaceToken); + } + + @Override + public ASTNode generateNode() { + return null; + } + + @Override + public Token setArguments(List args) { + return this; + } + + @Override + public List getChildren() { + return Arrays.asList(); + } + + @Override + public ArgumentInfo getArgumentInfo() { + return new ArgumentInfo(0); + } +} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/Tokenizer.java b/src/main/com/jantuomi/tunkki/core/tokenizer/Tokenizer.java deleted file mode 100644 index a055074..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/Tokenizer.java +++ /dev/null @@ -1,121 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer; - - -import com.jantuomi.tunkki.core.tokenizer.token.Token; -import com.jantuomi.tunkki.exception.ExceptionManager; -import com.jantuomi.tunkki.exception.TunkkiError; - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -/** - * Created by jan on 10.6.2016. - */ -public class Tokenizer { - private String sourceString; - - public List getTokens() { - return tokens; - } - - private List tokens = new ArrayList<>(); - - private static Map tokenRegexes = new LinkedHashMap<>(); - private static final List discardedTokenTypes = new ArrayList<>(); - private static final List erroneousTokenTypes = new ArrayList<>(); - - private static final Tokenizer instance = new Tokenizer(); - - public static final Tokenizer getInstance() { - return instance; - } - - private Tokenizer() { - tokenRegexes.put(Token.Type.CommentToken, "^\\/\\*(.*?)\\*\\/"); - - tokenRegexes.put(Token.Type.DoubleLiteralToken, "^([-+]?[0-9]*\\.?[0-9]+)f\\b"); - tokenRegexes.put(Token.Type.IntegerLiteralToken, "^([-+]?\\d+)\\b"); - tokenRegexes.put(Token.Type.BooleanLiteralToken, "^(true|false)\\b"); - tokenRegexes.put(Token.Type.StringLiteralToken, "^\"(.*?)\""); - - tokenRegexes.put(Token.Type.WhitespaceToken, "^( |\t)"); - tokenRegexes.put(Token.Type.NewlineToken, "^(\n|\r\n)"); - tokenRegexes.put(Token.Type.AdditionToken, "^(\\+)"); - tokenRegexes.put(Token.Type.SubtractionToken, "^(\\-)"); - tokenRegexes.put(Token.Type.DivisionToken, "^(\\/)"); - tokenRegexes.put(Token.Type.MultiplicationToken, "^(\\*)"); - tokenRegexes.put(Token.Type.DeclareAssignToken, "^(dset)\\b"); - tokenRegexes.put(Token.Type.AssignmentToken, "^(?)"); - tokenRegexes.put(Token.Type.LessOrEqualThanToken, "^(\\<\\=)"); - tokenRegexes.put(Token.Type.GreaterOrEqualThanToken, "^(\\>\\=)"); - tokenRegexes.put(Token.Type.EqualsToken, "^(\\=\\=)"); - tokenRegexes.put(Token.Type.NotEqualsToken, "^(\\!\\=)"); - tokenRegexes.put(Token.Type.OpenParenToken, "^(\\()"); - tokenRegexes.put(Token.Type.CallEndToken, "^(\\!)"); - tokenRegexes.put(Token.Type.FunctionDefineToken, "^(func)\\b"); - tokenRegexes.put(Token.Type.FunctionBodyToken, "^(as)\\b"); - tokenRegexes.put(Token.Type.BranchBodyToken, "^(then)\\b"); - tokenRegexes.put(Token.Type.EndBlockToken, "^(end)\\b"); - tokenRegexes.put(Token.Type.NegationToken, "^(not)\\b"); - tokenRegexes.put(Token.Type.DeclarationToken, "^(decl)\\b"); - tokenRegexes.put(Token.Type.BranchToken, "^(if)\\b"); - - tokenRegexes.put(Token.Type.CallToken, "^([a-zA-Z]+\\w*)\\?[\\s|\\!]"); - tokenRegexes.put(Token.Type.SymbolToken, "^([a-zA-Z]+\\w*)\\b"); - - discardedTokenTypes.add(Token.Type.WhitespaceToken); - discardedTokenTypes.add(Token.Type.CommentToken); - - erroneousTokenTypes.add(Token.Type.NotAToken); - } - - public List tokenize(String string) throws TunkkiError { - sourceString = string; - tokens.clear(); - int line = 1; - for (int i = 0; i < string.length();) { - - Token token = Token.makeToken(string.substring(i), tokenRegexes, line); - - if (token == null) { - i++; - continue; - } - - if (token.getTokenType() == Token.Type.NewlineToken) { - line++; - } - - if (!discardedTokenTypes.contains(token.getTokenType())) { - tokens.add(token); - } - - if (erroneousTokenTypes.contains(token.getTokenType())) { - ExceptionManager.raise(TunkkiError.ExceptionType.IllegalTokenError, line, token.getText()); - } - - String tokenRawText = token.getRawText(); - if (tokenRawText == null) { - tokenRawText = token.getText(); - } - if (tokenRawText != null) { - i += tokenRawText.length(); - } else { - i++; - } - } - return tokens; - } - - public static void printTokens(List tokens) { - System.out.println("### Tokens: ###"); - for (Token token : tokens) { - System.out.println(String.format("%-40s %s", token.getTokenType(), token.getText())); - } - System.out.println("### End ###"); - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/ArgumentInfo.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/ArgumentInfo.java deleted file mode 100644 index 5fcec7d..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/ArgumentInfo.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token; - -/** - * Created by jan on 15.6.2016. - */ -public class ArgumentInfo { - private int count = 0; - private boolean isVarargs = false; - - public Token.Type getOptionalArgument() { - return optionalArgument; - } - - public void setOptionalArgument(Token.Type optionalArgument) { - this.optionalArgument = optionalArgument; - } - - private Token.Type optionalArgument = null; - - public Token.Type getTerminator() { - return terminator; - } - - public void setTerminator(Token.Type terminator) { - this.terminator = terminator; - } - - private Token.Type terminator; - - public ArgumentInfo(int count) { - this.count = count; - } - - public ArgumentInfo() { - - } - - public void setVariable(boolean value) { - this.isVarargs = value; - } - - public boolean getVarargs() { - return isVarargs; - } - - public int getCount() { - return count; - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/Token.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/Token.java deleted file mode 100644 index 3f75566..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/Token.java +++ /dev/null @@ -1,215 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token; - -import com.jantuomi.tunkki.core.parser.ast.ASTNode; -import com.jantuomi.tunkki.exception.ExceptionManager; -import com.jantuomi.tunkki.exception.TunkkiError; -import org.apache.commons.lang3.builder.EqualsBuilder; -import org.apache.commons.lang3.builder.HashCodeBuilder; - -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - - -/** - * Created by jan on 10.6.2016. - */ -abstract public class Token { - - public void print(int indent) { - for (int i = 0; i < indent; i++) { - System.out.print("\t"); - } - System.out.println(toString()); - for (Token child : getChildren()) { - child.print(indent + 1); - } - } - - abstract public ASTNode generateNode() throws TunkkiError; - - /* Token types, ordered by precedence */ - public enum Type { - CommentToken, - - AdditionToken, - SubtractionToken, - DivisionToken, - MultiplicationToken, - - AssignmentToken, - - LessThanToken, - GreaterThanToken, - LessOrEqualThanToken, - GreaterOrEqualThanToken, - EqualsToken, - NotEqualsToken, - - OpenParenToken, - CallEndToken, - - FunctionDefineToken, - DeclarationToken, - - StringLiteralToken, - IntegerLiteralToken, - BooleanLiteralToken, - DoubleLiteralToken, - - WhitespaceToken, - NewlineToken, - - BranchToken, - SymbolToken, - CallToken, - - EndBlockToken, - FunctionBodyToken, - BranchBodyToken, - NegationToken, - - DeclareAssignToken, - NotAToken - } - - abstract public Token setArguments(List args); - - abstract public List getChildren(); - - public boolean is(Type type) { - return getTokenType() == type; - } - - public int getLine() { - return line; - } - - public void setLine(int line) { - this.line = line; - } - - private int line; - private String rawText; - private String text; - private Type type; - - public String getText() { - return text; - } - - public Type getTokenType() { - return this.type; - } - - public String getRawText() { - return rawText; - } - - public boolean isHigherPrecedenceThan(Token other) { - return getTokenType().ordinal() <= other.getTokenType().ordinal(); - } - - public Token(Type type, String text, String rawText) { - initialize(type, text, rawText); - } - - public Token(Type type, String text) { - initialize(type, text); - } - - public Token(Type type) { - initialize(type, null); - } - - private void initialize(Type type, String text, String rawText) { - this.type = type; - this.text = text; - this.rawText = rawText; - } - - private void initialize(Type type, String text) { - initialize(type, text, text); - } - - public static Token makeToken(String string, Map regexes, int line) throws TunkkiError { - Type type = Type.NotAToken; - String text = Character.toString(string.charAt(0)); - - for (Type t : regexes.keySet()) { - String regex = regexes.get(t); - Token found = matchToken(string, t, regex, line); - if (found != null) { - found.setLine(line); - return found; - } - } - - // TODO raise error - return null; - } - - public static Token matchToken(String string, Type type, String regex, int line) throws TunkkiError { - Pattern pattern = Pattern.compile(regex, Pattern.DOTALL); - Matcher matcher = pattern.matcher(string); - - if (matcher.find()) { - String lexeme = matcher.group(1); - String rawText = matcher.group(0); - - Token token = TokenFactory.create(type, lexeme, rawText); - if (token != null) { - token.setLine(line); - } - else { - ExceptionManager.raise(TunkkiError.ExceptionType.IllegalTokenError, line, - type.toString() - ); - } - - return token; - } - return null; - } - - abstract public ArgumentInfo getArgumentInfo(); - - @Override - public int hashCode() { - return new HashCodeBuilder(17, 37) - .append(text) - .append(getTokenType()) - .toHashCode(); - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof Token)) return false; - if (o == this) return true; - - Token rhs = (Token) o; - return new EqualsBuilder() - .append(getTokenType(), rhs.getTokenType()) - .append(text, rhs.text) - .isEquals(); - } - - @Override - public String toString() { - return String.format(String.format( - "%s%s", - getTokenType().toString().replace("Token", ""), - getText() == null ? "" : " " + getText() - )); - } - - public String toFormattedString() { - String textRepr = text; - return String.format("%-40s %s", getTokenType(), textRepr); - } - - public boolean isSameType(Token other) { - return type == other.type; - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/TokenFactory.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/TokenFactory.java deleted file mode 100644 index 41d73de..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/TokenFactory.java +++ /dev/null @@ -1,73 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token; - -import com.jantuomi.tunkki.core.tokenizer.token.types.*; - -/** - * Created by jan on 28.6.2016. - */ -public class TokenFactory { - private static TokenFactory instance = new TokenFactory(); - - public static TokenFactory getInstance() { - return instance; - } - - private TokenFactory() { - } - - public static Token create(Token.Type type, String text, String rawText) { - switch (type) { - case WhitespaceToken: - return new WhitespaceToken(); - case CommentToken: - return new CommentToken(text, rawText); - - // TODO placeholder whitespace instead of newline - case NewlineToken: - return new WhitespaceToken(); - - case IntegerLiteralToken: - return new IntegerLiteralToken(text); - case StringLiteralToken: - return new StringLiteralToken(text, rawText); - case BooleanLiteralToken: - return new BooleanLiteralToken(text); - case DoubleLiteralToken: - return new DoubleLiteralToken(text, rawText); - case AdditionToken: - return new AdditionToken(); - case SubtractionToken: - return new SubtractionToken(); - case MultiplicationToken: - return new MultiplicationToken(); - case DivisionToken: - return new DivisionToken(); - case FunctionDefineToken: - return new FunctionDefineToken(); - case EndBlockToken: - return new EndBlockToken(); - case FunctionBodyToken: - return new BlockBodyToken("as"); - case BranchBodyToken: - return new BlockBodyToken("then"); - case DeclarationToken: - return new VariableDeclareToken(); - case AssignmentToken: - return new AssignmentToken(); - case DeclareAssignToken: - return new DeclareAssignToken("dset"); - case SymbolToken: - return new SymbolToken(text); - case CallToken: - return new CallToken(text, rawText); - case CallEndToken: - return new CallEndToken(); - case BranchToken: - return new BranchToken(); - case NegationToken: - return new NegationToken(); - default: - return null; - } - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/AdditionToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/AdditionToken.java deleted file mode 100644 index 3934545..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/AdditionToken.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.parser.ast.ASTNode; -import com.jantuomi.tunkki.core.parser.ast.AdditionNode; -import com.jantuomi.tunkki.exception.TunkkiError; - -/** - * Created by jan on 15.6.2016. - */ -public class AdditionToken extends BinaryOperatorToken { - public AdditionToken() { - super(Type.AdditionToken); - } - - @Override - public ASTNode generateNode() throws TunkkiError { - return new AdditionNode(this, - lhs.generateNode(), - rhs.generateNode() - ); - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/AssignmentToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/AssignmentToken.java deleted file mode 100644 index 050ab22..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/AssignmentToken.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.parser.ast.ASTNode; -import com.jantuomi.tunkki.core.parser.ast.AssignmentNode; -import com.jantuomi.tunkki.core.parser.ast.SymbolNode; -import com.jantuomi.tunkki.exception.ExceptionManager; -import com.jantuomi.tunkki.exception.TunkkiError; - -/** - * Created by jan on 20.6.2016. - */ -public class AssignmentToken extends BinaryOperatorToken { - - public AssignmentToken() { - super(Type.AssignmentToken, "set"); - } - - // TODO this and the node - - @Override - public ASTNode generateNode() throws TunkkiError { - ASTNode lhsNode = lhs.generateNode(); - - if (!(lhsNode instanceof SymbolNode)) { - ExceptionManager.raise(TunkkiError.ExceptionType.SyntaxError, getLine(), "non-symbol " + toString()); - return null; - } - - AssignmentNode node = new AssignmentNode(this, - (SymbolNode) lhsNode, - rhs.generateNode() - ); - - return node; - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/BinaryOperatorToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/BinaryOperatorToken.java deleted file mode 100644 index b00ec8b..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/BinaryOperatorToken.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.tokenizer.token.ArgumentInfo; -import com.jantuomi.tunkki.core.tokenizer.token.Token; - -import java.util.Arrays; -import java.util.List; - -/** - * Created by jan on 15.6.2016. - */ -abstract public class BinaryOperatorToken extends Token { - protected Token lhs; - protected Token rhs; - - public BinaryOperatorToken(Type type) { - super(type); - } - - public BinaryOperatorToken(Type type, String text) { - super(type, text); - } - - @Override - public Token setArguments(List args) { - lhs = args.get(0); - rhs = args.get(1); - return this; - } - - @Override - public List getChildren() { - return Arrays.asList(lhs, rhs); - } - - @Override - public ArgumentInfo getArgumentInfo() { - return new ArgumentInfo(2); - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/BlockBodyToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/BlockBodyToken.java deleted file mode 100644 index 6cf19e7..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/BlockBodyToken.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.parser.ast.ASTNode; -import com.jantuomi.tunkki.core.parser.ast.BlockBodyNode; -import com.jantuomi.tunkki.core.tokenizer.token.Token; -import com.jantuomi.tunkki.exception.TunkkiError; - -import java.util.ArrayList; -import java.util.List; - -/** - * Created by jan on 16.6.2016. - */ -public class BlockBodyToken extends VarargOperatorToken { - public BlockBodyToken(String keyword) { - super(Type.FunctionBodyToken, keyword); - } - - @Override - public ASTNode generateNode() throws TunkkiError { - BlockBodyNode node = new BlockBodyNode(this); - List statements = new ArrayList<>(); - for (int i = 0; i < args.size() - 1; i++) { - Token arg = args.get(i); - statements.add(arg.generateNode()); - } - node.setArgs(statements); - return node; - } - - @Override - public Type getTerminatorTokenType() { - return Type.EndBlockToken; - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/BooleanLiteralToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/BooleanLiteralToken.java deleted file mode 100644 index 371f51b..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/BooleanLiteralToken.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.parser.ast.ASTNode; -import com.jantuomi.tunkki.core.parser.ast.BooleanLiteralNode; - -/** - * Created by jan on 15.6.2016. - */ -public class BooleanLiteralToken extends TerminalToken{ - public BooleanLiteralToken(String text) { - super(Type.BooleanLiteralToken, text); - } - - @Override - public ASTNode generateNode() { - return new BooleanLiteralNode(this); - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/BranchToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/BranchToken.java deleted file mode 100644 index 33c5565..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/BranchToken.java +++ /dev/null @@ -1,48 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.parser.ast.ASTNode; -import com.jantuomi.tunkki.core.parser.ast.BlockBodyNode; -import com.jantuomi.tunkki.core.parser.ast.BranchNode; -import com.jantuomi.tunkki.core.tokenizer.token.Token; -import com.jantuomi.tunkki.exception.TunkkiError; - -import java.util.Arrays; -import java.util.List; - -/** - * Created by jan on 21.6.2016. - */ -public class BranchToken extends VarargOperatorToken { - - private Token expression; - private BlockBodyToken branch; - - public BranchToken() { - super(Type.BranchToken, "if"); - } - - @Override - public ASTNode generateNode() throws TunkkiError { - return new BranchNode(this, - expression.generateNode(), - (BlockBodyNode) branch.generateNode() - ); - } - - @Override - public Token setArguments(List args) { - expression = args.get(0); - branch = (BlockBodyToken) args.get(1); - return this; - } - - @Override - public List getChildren() { - return Arrays.asList(expression, branch); - } - - @Override - public Type getTerminatorTokenType() { - return Type.BranchBodyToken; - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/CallEndToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/CallEndToken.java deleted file mode 100644 index 5bf30d8..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/CallEndToken.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.parser.ast.ASTNode; - -/** - * Created by jan on 19.6.2016. - */ -public class CallEndToken extends TerminalToken { - public CallEndToken() { - super(Type.CallEndToken); - } - - @Override - public ASTNode generateNode() { - return null; - } - -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/CallToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/CallToken.java deleted file mode 100644 index daa55ce..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/CallToken.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.parser.ast.ASTNode; -import com.jantuomi.tunkki.core.parser.ast.CallNode; -import com.jantuomi.tunkki.core.tokenizer.token.Token; -import com.jantuomi.tunkki.exception.TunkkiError; - -import java.util.ArrayList; -import java.util.List; - -/** - * Created by jan on 26.7.2016. - */ -public class CallToken extends VarargOperatorToken { - public CallToken(String text, String rawText) { - super(Type.CallToken, text, rawText); - } - - @Override - public ASTNode generateNode() throws TunkkiError { - CallNode node = new CallNode(this); - node.setName(this.getText()); - - List arguments = new ArrayList<>(); - - for (Token token : args) { - ASTNode arg = token.generateNode(); - if (arg != null) { - arguments.add(arg); - } - } - node.setParameters(arguments); - return node; - } - - @Override - public Type getTerminatorTokenType() { - return Type.CallEndToken; - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/CommentToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/CommentToken.java deleted file mode 100644 index f07adae..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/CommentToken.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.parser.ast.ASTNode; -import com.jantuomi.tunkki.core.parser.ast.EmptyNode; - -/** - * Created by jan on 19.6.2016. - */ -public class CommentToken extends TerminalToken { - public CommentToken(String text, String rawText) { - super(Type.CommentToken, text, rawText); - } - - @Override - public ASTNode generateNode() { - return new EmptyNode(this); - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/DeclareAssignToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/DeclareAssignToken.java deleted file mode 100644 index ae530ed..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/DeclareAssignToken.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.parser.ast.ASTNode; -import com.jantuomi.tunkki.core.parser.ast.AssignmentNode; -import com.jantuomi.tunkki.core.parser.ast.DeclareAssignNode; -import com.jantuomi.tunkki.core.parser.ast.SymbolNode; -import com.jantuomi.tunkki.exception.ExceptionManager; -import com.jantuomi.tunkki.exception.TunkkiError; - -/** - * Created by jan on 28.7.2016. - */ -public class DeclareAssignToken extends BinaryOperatorToken { - public DeclareAssignToken(String text) { - super(Type.DeclareAssignToken, text); - } - - @Override - public ASTNode generateNode() throws TunkkiError { - ASTNode lhsNode = lhs.generateNode(); - - if (!(lhsNode instanceof SymbolNode)) { - ExceptionManager.raise(TunkkiError.ExceptionType.SyntaxError, getLine(), "non-symbol " + toString()); - return null; - } - - DeclareAssignNode node = new DeclareAssignNode(this, - (SymbolNode) lhsNode, - rhs.generateNode() - ); - - return node; - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/DivisionToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/DivisionToken.java deleted file mode 100644 index 24e881b..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/DivisionToken.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.parser.ast.ASTNode; -import com.jantuomi.tunkki.core.parser.ast.DivisionNode; -import com.jantuomi.tunkki.exception.TunkkiError; - -/** - * Created by jan on 15.6.2016. - */ -public class DivisionToken extends BinaryOperatorToken { - public DivisionToken() { - super(Type.DivisionToken); - } - - @Override - public ASTNode generateNode() throws TunkkiError { - return new DivisionNode(this, - lhs.generateNode(), - rhs.generateNode() - ); - - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/DoubleLiteralToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/DoubleLiteralToken.java deleted file mode 100644 index 4929753..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/DoubleLiteralToken.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.parser.ast.ASTNode; -import com.jantuomi.tunkki.core.parser.ast.DoubleLiteralNode; - -/** - * Created by jan on 15.6.2016. - */ -public class DoubleLiteralToken extends TerminalToken { - public DoubleLiteralToken(String text, String rawText) { - super(Type.DoubleLiteralToken, text, rawText); - } - - @Override - public ASTNode generateNode() { - return new DoubleLiteralNode(this); - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/EndBlockToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/EndBlockToken.java deleted file mode 100644 index e950dcb..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/EndBlockToken.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.parser.ast.ASTNode; - -/** - * Created by jan on 16.6.2016. - */ -public class EndBlockToken extends TerminalToken { - public EndBlockToken() { - super(Type.EndBlockToken, "end"); - } - - @Override - public ASTNode generateNode() { - return null; - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/FunctionDefineToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/FunctionDefineToken.java deleted file mode 100644 index 89b8fd4..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/FunctionDefineToken.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.parser.ast.ASTNode; -import com.jantuomi.tunkki.core.parser.ast.BlockBodyNode; -import com.jantuomi.tunkki.core.parser.ast.FunctionDefineNode; -import com.jantuomi.tunkki.exception.TunkkiError; - -import java.util.ArrayList; -import java.util.List; - -/** - * Created by jan on 16.6.2016. - */ -public class FunctionDefineToken extends VarargOperatorToken { - public FunctionDefineToken() { - super(Type.FunctionDefineToken, "func"); - } - - @Override - public ASTNode generateNode() throws TunkkiError { - FunctionDefineNode node = new FunctionDefineNode(this); - - if (args.size() == 0) { - throw new TunkkiError(TunkkiError.ExceptionType.ExpectedDifferentTokenError, getLine(), getText()); - } - - node.setName(args.get(0).getText()); - - List funcArgs = new ArrayList<>(); - for (int i = 1; i < args.size() - 1; i++) { - funcArgs.add(args.get(i).generateNode()); - } - node.setArgs(funcArgs); - - BlockBodyNode bodyNode; - try { - bodyNode = (BlockBodyNode) args.get(args.size() - 1).generateNode(); - } - catch (Exception ex) { - throw new TunkkiError(TunkkiError.ExceptionType.ExpectedDifferentTokenError, getLine(), getText()); - } - node.setBody(bodyNode); - return node; - } - - @Override - public Type getTerminatorTokenType() { - return Type.FunctionBodyToken; - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/IntegerLiteralToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/IntegerLiteralToken.java deleted file mode 100644 index f8d6dda..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/IntegerLiteralToken.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.parser.ast.ASTNode; -import com.jantuomi.tunkki.core.parser.ast.IntegerLiteralNode; - -/** - * Created by jan on 15.6.2016. - */ -public class IntegerLiteralToken extends TerminalToken { - public IntegerLiteralToken(String text) { - super(Type.IntegerLiteralToken, text); - } - - @Override - public ASTNode generateNode() { - return new IntegerLiteralNode(this); - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/MultiplicationToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/MultiplicationToken.java deleted file mode 100644 index 5f01a68..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/MultiplicationToken.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.parser.ast.ASTNode; -import com.jantuomi.tunkki.core.parser.ast.MultiplicationNode; -import com.jantuomi.tunkki.exception.TunkkiError; - -/** - * Created by jan on 15.6.2016. - */ -public class MultiplicationToken extends BinaryOperatorToken { - public MultiplicationToken() { - super(Type.MultiplicationToken); - } - - @Override - public ASTNode generateNode() throws TunkkiError { - return new MultiplicationNode(this, - lhs.generateNode(), - rhs.generateNode() - ); - - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/NegationToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/NegationToken.java deleted file mode 100644 index 30dcb3b..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/NegationToken.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.parser.ast.ASTNode; -import com.jantuomi.tunkki.core.parser.ast.NegationNode; -import com.jantuomi.tunkki.exception.TunkkiError; - -/** - * Created by jan on 21.6.2016. - */ -public class NegationToken extends UnaryOperatorToken { - - public NegationToken() { - super(Type.NegationToken, "not"); - } - - @Override - public ASTNode generateNode() throws TunkkiError { - return new NegationNode(this, operand.generateNode()); - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/OptionalArgumentToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/OptionalArgumentToken.java deleted file mode 100644 index 0f233e8..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/OptionalArgumentToken.java +++ /dev/null @@ -1,48 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.tokenizer.token.ArgumentInfo; -import com.jantuomi.tunkki.core.tokenizer.token.Token; - -import java.util.Arrays; -import java.util.List; - -/** - * Created by jan on 28.6.2016. - */ -public abstract class OptionalArgumentToken extends Token { - public OptionalArgumentToken(Type type, String text) { - super(type, text); - } - - protected Token optionalArgument; - - public void setOptionalArgument(Token optionalArgument) { - this.optionalArgument = optionalArgument; - } - - abstract public Token.Type getOptionalArgumentType(); - - @Override - public ArgumentInfo getArgumentInfo() { - ArgumentInfo ai = new ArgumentInfo(); - ai.setOptionalArgument(getOptionalArgumentType()); - return ai; - } - - @Override - public Token setArguments(List args) { - if (args.size() > 0) { - setOptionalArgument(args.get(0)); - } - return this; - } - - @Override - public List getChildren() { - if (optionalArgument == null) { - return Arrays.asList(); - } else { - return Arrays.asList(optionalArgument); - } - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/StringLiteralToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/StringLiteralToken.java deleted file mode 100644 index 1f96bc8..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/StringLiteralToken.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.parser.ast.ASTNode; -import com.jantuomi.tunkki.core.parser.ast.StringLiteralNode; -import com.jantuomi.tunkki.exception.TunkkiError; - -/** - * Created by jan on 21.6.2016. - */ -public class StringLiteralToken extends TerminalToken { - public StringLiteralToken(String text, String rawText) { - super(Type.StringLiteralToken, text, rawText); - } - - @Override - public ASTNode generateNode() throws TunkkiError { - return new StringLiteralNode(this); - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/SubtractionToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/SubtractionToken.java deleted file mode 100644 index 9eb1e09..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/SubtractionToken.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.parser.ast.ASTNode; -import com.jantuomi.tunkki.core.parser.ast.SubtractionNode; -import com.jantuomi.tunkki.exception.TunkkiError; - -/** - * Created by jan on 16.6.2016. - */ -public class SubtractionToken extends BinaryOperatorToken { - public SubtractionToken() { - super(Type.SubtractionToken); - } - - @Override - public ASTNode generateNode() throws TunkkiError { - return new SubtractionNode(this, - lhs.generateNode(), - rhs.generateNode() - ); - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/SymbolToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/SymbolToken.java deleted file mode 100644 index eb98e7b..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/SymbolToken.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.parser.ast.ASTNode; -import com.jantuomi.tunkki.core.parser.ast.SymbolNode; -import com.jantuomi.tunkki.exception.TunkkiError; - -/** - * Created by jan on 16.6.2016. - */ -public class SymbolToken extends TerminalToken { - - public SymbolToken(String name) { - super(Type.SymbolToken, name); - } - - @Override - public ASTNode generateNode() throws TunkkiError { - SymbolNode node = new SymbolNode(this); - node.setName(this.getText()); - - return node; - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/TerminalToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/TerminalToken.java deleted file mode 100644 index 41195e1..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/TerminalToken.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.tokenizer.token.ArgumentInfo; -import com.jantuomi.tunkki.core.tokenizer.token.Token; - -import java.util.Arrays; -import java.util.List; - -/** - * Created by jan on 28.6.2016. - */ -abstract public class TerminalToken extends Token { - public TerminalToken(Type type, String text, String rawText) { - super(type, text, rawText); - } - - public TerminalToken(Type type, String text) { - super(type, text); - } - - public TerminalToken(Type type) { - super(type); - } - - @Override - public Token setArguments(List args) { - return this; - } - - @Override - public List getChildren() { - return Arrays.asList(); - } - - @Override - public ArgumentInfo getArgumentInfo() { - return new ArgumentInfo(0); - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/UnaryOperatorToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/UnaryOperatorToken.java deleted file mode 100644 index 301f8f2..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/UnaryOperatorToken.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.tokenizer.token.ArgumentInfo; -import com.jantuomi.tunkki.core.tokenizer.token.Token; - -import java.util.Arrays; -import java.util.List; - -/** - * Created by jan on 28.6.2016. - */ -abstract public class UnaryOperatorToken extends Token { - public UnaryOperatorToken(Type type, String text) { - super(type, text); - } - - public UnaryOperatorToken(Type type) { - super(type); - } - - protected Token operand; - - - @Override - public ArgumentInfo getArgumentInfo() { - return new ArgumentInfo(1); - } - - @Override - public Token setArguments(List args) { - operand = args.get(0); - return this; - } - - @Override - public List getChildren() { - return Arrays.asList(operand); - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/VarargOperatorToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/VarargOperatorToken.java deleted file mode 100644 index 8df7fe2..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/VarargOperatorToken.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.tokenizer.token.ArgumentInfo; -import com.jantuomi.tunkki.core.tokenizer.token.Token; - -import java.util.List; - -/** - * Created by jan on 16.6.2016. - */ -abstract public class VarargOperatorToken extends Token { - - protected List args; - - public VarargOperatorToken(Type type, String text, String rawText) { - super(type, text, rawText); - } - - public VarargOperatorToken(Type type, String text) { - super(type, text); - } - - public VarargOperatorToken(Type type) { - super(type); - } - - @Override - public Token setArguments(List args) { - this.args = args; - return this; - } - - @Override - public List getChildren() { - return args; - } - - abstract public Token.Type getTerminatorTokenType(); - - @Override - public ArgumentInfo getArgumentInfo() { - ArgumentInfo ai = new ArgumentInfo(); - ai.setVariable(true); - ai.setTerminator(getTerminatorTokenType()); - return ai; - - } - -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/VariableDeclareToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/VariableDeclareToken.java deleted file mode 100644 index d9b1783..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/VariableDeclareToken.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.parser.ast.ASTNode; -import com.jantuomi.tunkki.core.parser.ast.SymbolNode; -import com.jantuomi.tunkki.core.parser.ast.VariableDeclareNode; -import com.jantuomi.tunkki.exception.TunkkiError; - -/** - * Created by jan on 20.6.2016. - */ -public class VariableDeclareToken extends UnaryOperatorToken { - - - public VariableDeclareToken() { - super(Type.DeclarationToken, "decl"); - } - - @Override - public ASTNode generateNode() throws TunkkiError { - SymbolToken st = (SymbolToken) operand; - VariableDeclareNode node = new VariableDeclareNode(this, - (SymbolNode) st.generateNode()); - return node; - } -} diff --git a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/WhitespaceToken.java b/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/WhitespaceToken.java deleted file mode 100644 index 2938b45..0000000 --- a/src/main/com/jantuomi/tunkki/core/tokenizer/token/types/WhitespaceToken.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer.token.types; - -import com.jantuomi.tunkki.core.parser.ast.ASTNode; -import com.jantuomi.tunkki.core.tokenizer.token.ArgumentInfo; -import com.jantuomi.tunkki.core.tokenizer.token.Token; - -import java.util.Arrays; -import java.util.List; - -/** - * Created by jan on 16.6.2016. - */ -public class WhitespaceToken extends Token { - public WhitespaceToken() { - super(Type.WhitespaceToken); - } - - @Override - public ASTNode generateNode() { - return null; - } - - @Override - public Token setArguments(List args) { - return this; - } - - @Override - public List getChildren() { - return Arrays.asList(); - } - - @Override - public ArgumentInfo getArgumentInfo() { - return new ArgumentInfo(0); - } -} diff --git a/src/test/com/jantuomi/tunkki/core/parser/ParserTest.java b/src/test/com/jantuomi/tunkki/core/parser/ParserTest.java index 2e8421d..2a773cd 100644 --- a/src/test/com/jantuomi/tunkki/core/parser/ParserTest.java +++ b/src/test/com/jantuomi/tunkki/core/parser/ParserTest.java @@ -1,8 +1,8 @@ package com.jantuomi.tunkki.core.parser; -import com.jantuomi.tunkki.core.tokenizer.token.Token; -import com.jantuomi.tunkki.core.tokenizer.token.types.AdditionToken; -import com.jantuomi.tunkki.core.tokenizer.token.types.IntegerLiteralToken; +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.types.AdditionToken; +import com.jantuomi.tunkki.core.parser.tokenizer.token.types.IntegerLiteralToken; import org.junit.Test; import java.util.Arrays; diff --git a/src/test/com/jantuomi/tunkki/core/parser/tokenizer/TokenizerTest.java b/src/test/com/jantuomi/tunkki/core/parser/tokenizer/TokenizerTest.java new file mode 100644 index 0000000..1ce55b2 --- /dev/null +++ b/src/test/com/jantuomi/tunkki/core/parser/tokenizer/TokenizerTest.java @@ -0,0 +1,26 @@ +package com.jantuomi.tunkki.core.parser.tokenizer; + +import com.jantuomi.tunkki.core.parser.tokenizer.token.Token; +import com.jantuomi.tunkki.core.parser.tokenizer.token.types.IntegerLiteralToken; +import com.jantuomi.tunkki.exception.TunkkiError; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertTrue; + +/** + * Created by jan on 10.6.2016. + */ +public class TokenizerTest { + + @Test + public void testAddition() throws TunkkiError { + String testString = "1 2 +"; + List list = Tokenizer.getInstance().tokenize(testString); + + System.out.println(String.format("test string:\n%s", testString)); + Tokenizer.printTokens(list); + assertTrue(list.contains(new IntegerLiteralToken("2"))); + } +} \ No newline at end of file diff --git a/src/test/com/jantuomi/tunkki/core/tokenizer/TokenizerTest.java b/src/test/com/jantuomi/tunkki/core/tokenizer/TokenizerTest.java deleted file mode 100644 index f65830c..0000000 --- a/src/test/com/jantuomi/tunkki/core/tokenizer/TokenizerTest.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.jantuomi.tunkki.core.tokenizer; - -import com.jantuomi.tunkki.core.tokenizer.token.Token; -import com.jantuomi.tunkki.core.tokenizer.token.types.IntegerLiteralToken; -import com.jantuomi.tunkki.exception.TunkkiError; -import org.junit.Test; - -import java.util.List; - -import static org.junit.Assert.assertTrue; - -/** - * Created by jan on 10.6.2016. - */ -public class TokenizerTest { - - @Test - public void testAddition() throws TunkkiError { - String testString = "1 2 +"; - List list = Tokenizer.getInstance().tokenize(testString); - - System.out.println(String.format("test string:\n%s", testString)); - Tokenizer.printTokens(list); - assertTrue(list.contains(new IntegerLiteralToken("2"))); - } -} \ No newline at end of file -- cgit v1.3