1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
|
package com.jantuomi.interpreter.main.core.tokenizer.token;
import com.jantuomi.interpreter.main.core.parser.ast.ASTNode;
import com.jantuomi.interpreter.main.core.tokenizer.token.types.AdditionToken;
import com.jantuomi.interpreter.main.core.tokenizer.token.types.IntegerLiteralToken;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Created by jan on 10.6.2016.
*/
abstract public class Token {
public void print(int indent) {
for (int i = 0; i < indent; i++) {
System.out.print("\t");
}
System.out.println(toString());
for (Token child : getChildren()) {
child.print(indent + 1);
}
}
abstract public ASTNode generateNode();
/* Token types, ordered by precedence */
public enum Type {
CommentToken,
AdditionToken,
SubtractionToken,
DivisionToken,
MultiplicationToken,
AssignmentToken,
LessThanToken,
GreaterThanToken,
LessOrEqualThanToken,
GreaterOrEqualThanToken,
EqualsToken,
NotEqualsToken,
OpenParenToken,
ClosedParenToken,
FunctionDefineToken,
DeclarationToken,
StringLiteralToken,
IntegerLiteralToken,
WhitespaceToken,
NewlineToken,
EndStatementToken,
SymbolToken,
EndFunctionDefineToken,
NotAToken
}
abstract public Token setArguments(List<Token> args);
abstract public List<Token> getChildren();
public boolean is(Type type) {
return getTokenType() == type;
}
public int getLine() {
return line;
}
public void setLine(int line) {
this.line = line;
}
private int line;
private String rawText;
private String text;
private Type type;
public String getText() {
return text;
}
public Type getTokenType() {
return this.type;
}
public String getRawText() {
return rawText;
}
public boolean isHigherPrecedenceThan(Token other) {
return getTokenType().ordinal() <= other.getTokenType().ordinal();
}
public Token(Type type, String text, String rawText) {
initialize(type, text, rawText);
}
public Token(Type type, String text) {
initialize(type, text);
}
public Token(Type type) {
initialize(type, null);
}
private void initialize(Type type, String text, String rawText) {
this.type = type;
this.text = text;
this.rawText = rawText;
}
private void initialize(Type type, String text) {
initialize(type, text, text);
}
public static Token makeToken(String string, Map<Type, String> regexes, int line) {
Type type = Type.NotAToken;
String text = Character.toString(string.charAt(0));
for (Type t : regexes.keySet()) {
String regex = regexes.get(t);
Token found = matchToken(string, t, regex);
if (found != null) {
found.setLine(line);
return found;
}
}
// TODO raise error
return null;
}
public static Token matchToken(String string, Type type, String regex) {
Pattern pattern = Pattern.compile(regex, Pattern.DOTALL);
Matcher matcher = pattern.matcher(string);
if (matcher.find()) {
String text = matcher.group(1);
String rawText = matcher.group(0);
switch (type) {
case IntegerLiteralToken:
return new IntegerLiteralToken(text);
case AdditionToken:
return new AdditionToken();
default:
return null;
}
}
return null;
}
public ArgumentInfo getArgumentInfo() {
switch (getTokenType()) {
case FunctionDefineToken:
ArgumentInfo ai = new ArgumentInfo();
ai.setVariable(true);
return ai;
case AdditionToken:
case SubtractionToken:
case DivisionToken:
case MultiplicationToken:
case AssignmentToken:
case LessThanToken:
case GreaterThanToken:
case LessOrEqualThanToken:
case GreaterOrEqualThanToken:
case EqualsToken:
case NotEqualsToken:
return new ArgumentInfo(2);
case DeclarationToken:
return new ArgumentInfo(1);
default:
return new ArgumentInfo(0);
}
}
@Override
public int hashCode() {
return new HashCodeBuilder(17, 37)
.append(text)
.append(getTokenType())
.toHashCode();
}
@Override
public boolean equals(Object o) {
if (!(o instanceof Token)) return false;
if (o == this) return true;
Token rhs = (Token) o;
return new EqualsBuilder()
.append(getTokenType(), rhs.getTokenType())
.append(text, rhs.text)
.isEquals();
}
@Override
public String toString() {
return String.format(String.format("%s %s", getTokenType(), getText()));
}
public String toFormattedString() {
String textRepr = text;
return String.format("%-40s %s", getTokenType(), textRepr);
}
public boolean isSameType(Token other) {
return type == other.type;
}
}
|
