1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
|
package com.jantuomi.interpreter.main.core.tokenizer.token;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Created by jan on 10.6.2016.
*/
public class Token {
/* Token types, ordered by precedence */
public enum Type {
CommentToken,
AdditionToken,
SubtractionToken,
DivisionToken,
MultiplicationToken,
AssignmentToken,
LessThanToken,
GreaterThanToken,
LessOrEqualThanToken,
GreaterOrEqualThanToken,
EqualsToken,
NotEqualsToken,
OpenParenToken,
ClosedParenToken,
FunctionDefineToken,
DeclarationToken,
StringLiteralToken,
IntegerLiteralToken,
WhitespaceToken,
NewlineToken,
EndStatementToken,
SymbolToken,
EndFunctionDefineToken,
NotAToken
}
public boolean is(Type type) {
return getTokenType() == type;
}
public int getLine() {
return line;
}
public void setLine(int line) {
this.line = line;
}
private int line;
private String rawText;
private String text;
private Type type;
public String getText() {
return text;
}
public Type getTokenType() {
return this.type;
}
public String getRawText() {
return rawText;
}
public boolean isHigherPrecedenceThan(Token other) {
return getTokenType().ordinal() <= other.getTokenType().ordinal();
}
public Token(Type type, String text, String rawText) {
initialize(type, text, rawText);
}
public Token(Type type, String text) {
initialize(type, text);
}
public Token(Type type) {
initialize(type, null);
}
private void initialize(Type type, String text, String rawText) {
this.type = type;
this.text = text;
this.rawText = rawText;
}
private void initialize(Type type, String text) {
initialize(type, text, text);
}
public static Token makeToken(String string, Map<Type, String> regexes, int line) {
Type type = Type.NotAToken;
String text = Character.toString(string.charAt(0));
for (Type t : regexes.keySet()) {
String regex = regexes.get(t);
Token found = matchToken(string, t, regex);
if (found != null) {
found.setLine(line);
return found;
}
}
// TODO raise error
return null;
}
public static Token matchToken(String string, Type type, String regex) {
Pattern pattern = Pattern.compile(regex, Pattern.DOTALL);
Matcher matcher = pattern.matcher(string);
if (matcher.find()) {
String text = matcher.group(1);
String rawText = matcher.group(0);
return new Token(type, text, rawText);
}
return null;
}
@Override
public int hashCode() {
return new HashCodeBuilder(17, 37)
.append(text)
.append(getTokenType())
.toHashCode();
}
@Override
public boolean equals(Object o) {
if (!(o instanceof Token)) return false;
if (o == this) return true;
Token rhs = (Token) o;
return new EqualsBuilder()
.append(getTokenType(), rhs.getTokenType())
.append(text, rhs.text)
.isEquals();
}
@Override
public String toString() {
String textRepr = text;
return String.format("%-40s %s", getTokenType(), textRepr);
}
public boolean isSameType(Token other) {
return type == other.type;
}
}
|
