From 48dcd488054024d98fe1e659a4029630130d1d35 Mon Sep 17 00:00:00 2001 From: Jan Tuomi Date: Wed, 22 Jun 2016 23:25:59 +0300 Subject: Add multiplication and division operations --- main/core/tokenizer/Tokenizer.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'main/core/tokenizer/Tokenizer.java') diff --git a/main/core/tokenizer/Tokenizer.java b/main/core/tokenizer/Tokenizer.java index 3501737..9ff1e4c 100644 --- a/main/core/tokenizer/Tokenizer.java +++ b/main/core/tokenizer/Tokenizer.java @@ -34,7 +34,12 @@ public class Tokenizer { private Tokenizer() { tokenRegexes.put(Token.Type.CommentToken, "^\\/\\*(.*?)\\*\\/"); + + tokenRegexes.put(Token.Type.DoubleLiteralToken, "^([-+]?[0-9]*\\.?[0-9]+)f\\b"); + tokenRegexes.put(Token.Type.IntegerLiteralToken, "^([-+]?\\d+)\\b"); + tokenRegexes.put(Token.Type.BooleanLiteralToken, "^(true|false)\\b"); tokenRegexes.put(Token.Type.StringLiteralToken, "^\"(.*?)\""); + tokenRegexes.put(Token.Type.WhitespaceToken, "^( |\t)"); tokenRegexes.put(Token.Type.NewlineToken, "^(\n|\r\n)"); tokenRegexes.put(Token.Type.AdditionToken, "^(\\+)"); @@ -57,8 +62,7 @@ public class Tokenizer { tokenRegexes.put(Token.Type.NegationToken, "^(not)\\b"); tokenRegexes.put(Token.Type.DeclarationToken, "^(decl)\\b"); tokenRegexes.put(Token.Type.BranchToken, "^(if)\\b"); - tokenRegexes.put(Token.Type.IntegerLiteralToken, "^(\\d+)"); - tokenRegexes.put(Token.Type.BooleanLiteralToken, "^(true|false)\\b"); + tokenRegexes.put(Token.Type.SymbolToken, "^([a-zA-Z]+\\w*)"); discardedTokenTypes.add(Token.Type.WhitespaceToken); -- cgit v1.3