1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
|
package com.jantuomi.interpreter.main.core.tokenizer;
import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
import com.jantuomi.interpreter.main.exception.ExceptionManager;
import com.jantuomi.interpreter.main.exception.InterpreterException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* Created by jan on 10.6.2016.
*/
public class Tokenizer {
private String sourceString;
public List<Token> getTokens() {
return tokens;
}
private List<Token> tokens = new ArrayList<>();
private static Map<Token.Type, String> tokenRegexes = new LinkedHashMap<>();
private static final List<Token.Type> discardedTokenTypes = new ArrayList<>();
private static final List<Token.Type> erroneousTokenTypes = new ArrayList<>();
private static final Tokenizer instance = new Tokenizer();
public static final Tokenizer getInstance() {
return instance;
}
private Tokenizer() {
tokenRegexes.put(Token.Type.CommentToken, "^\\/\\*(.*)\\*\\/");
tokenRegexes.put(Token.Type.StringLiteralToken, "^\"(.*?)\"");
tokenRegexes.put(Token.Type.WhitespaceToken, "^( |\t)");
tokenRegexes.put(Token.Type.NewlineToken, "^(\n|\r\n)");
tokenRegexes.put(Token.Type.AdditionToken, "^(\\+)");
tokenRegexes.put(Token.Type.SubtractionToken, "^(\\-)");
tokenRegexes.put(Token.Type.DivisionToken, "^(\\/)");
tokenRegexes.put(Token.Type.MultiplicationToken, "^(\\*)");
tokenRegexes.put(Token.Type.AssignmentToken, "^(set)\\b");
tokenRegexes.put(Token.Type.LessThanToken, "^(\\<)");
tokenRegexes.put(Token.Type.GreaterThanToken, "^(\\>)");
tokenRegexes.put(Token.Type.LessOrEqualThanToken, "^(\\<\\=)");
tokenRegexes.put(Token.Type.GreaterOrEqualThanToken, "^(\\>\\=)");
tokenRegexes.put(Token.Type.EqualsToken, "^(\\=\\=)");
tokenRegexes.put(Token.Type.NotEqualsToken, "^(\\!\\=)");
tokenRegexes.put(Token.Type.OpenParenToken, "^(\\()");
tokenRegexes.put(Token.Type.ClosedParenToken, "^(\\))");
tokenRegexes.put(Token.Type.FunctionDefineToken, "^(func)\\b");
tokenRegexes.put(Token.Type.FunctionBodyToken, "^(as)\\b");
tokenRegexes.put(Token.Type.BranchBodyToken, "^(then)\\b");
tokenRegexes.put(Token.Type.EndBlockToken, "^(end)\\b");
tokenRegexes.put(Token.Type.NegationToken, "^(not)\\b");
tokenRegexes.put(Token.Type.DeclarationToken, "^(decl)\\b");
tokenRegexes.put(Token.Type.BranchToken, "^(if)\\b");
tokenRegexes.put(Token.Type.IntegerLiteralToken, "^(\\d+)");
tokenRegexes.put(Token.Type.BooleanLiteralToken, "^(true|false)\\b");
tokenRegexes.put(Token.Type.SymbolToken, "^([a-zA-Z]+\\w*)");
discardedTokenTypes.add(Token.Type.WhitespaceToken);
discardedTokenTypes.add(Token.Type.CommentToken);
erroneousTokenTypes.add(Token.Type.NotAToken);
}
public List<Token> tokenize(String string) throws InterpreterException {
sourceString = string;
tokens.clear();
int line = 1;
for (int i = 0; i < string.length();) {
Token token = Token.makeToken(string.substring(i), tokenRegexes, line);
if (token == null) {
i++;
continue;
}
if (token.getTokenType() == Token.Type.NewlineToken) {
line++;
}
if (!discardedTokenTypes.contains(token.getTokenType())) {
tokens.add(token);
}
if (erroneousTokenTypes.contains(token.getTokenType())) {
ExceptionManager.raise(InterpreterException.ExceptionType.IllegalTokenError, line, token.getText());
}
String tokenRawText = token.getRawText();
if (tokenRawText == null) {
tokenRawText = token.getText();
}
if (tokenRawText != null) {
i += tokenRawText.length();
} else {
i++;
}
}
return tokens;
}
public static void printTokens(List<Token> tokens) {
System.out.println("### Tokens: ###");
for (Token token : tokens) {
System.out.println(String.format("%-40s %s", token.getTokenType(), token.getText()));
}
System.out.println("### End ###");
}
}
|
