From 1f44fcb62a6d9c91993c13d77b528de89e8a3cdb Mon Sep 17 00:00:00 2001 From: jantuomi Date: Wed, 3 Aug 2016 15:28:29 +0300 Subject: Add support for scandinavian characters in symbols Resolves #12 --- src/main/com/jantuomi/tunkki/core/parser/tokenizer/Tokenizer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/main') diff --git a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/Tokenizer.java b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/Tokenizer.java index 4fb54bb..402aff3 100644 --- a/src/main/com/jantuomi/tunkki/core/parser/tokenizer/Tokenizer.java +++ b/src/main/com/jantuomi/tunkki/core/parser/tokenizer/Tokenizer.java @@ -65,8 +65,8 @@ public class Tokenizer { tokenRegexes.put(Token.Type.DeclarationToken, "^(decl)\\b"); tokenRegexes.put(Token.Type.BranchToken, "^(if)\\b"); - tokenRegexes.put(Token.Type.CallToken, "^([a-zA-Z]+\\w*)\\?[\\s|\\!]"); - tokenRegexes.put(Token.Type.SymbolToken, "^([a-zA-Z]+\\w*)\\b"); + tokenRegexes.put(Token.Type.CallToken, "^([a-äA-Ä]+\\w*)\\?[\\s|\\!]"); + tokenRegexes.put(Token.Type.SymbolToken, "^([a-äA-Ä]+\\w*)\\b"); discardedTokenTypes.add(Token.Type.WhitespaceToken); discardedTokenTypes.add(Token.Type.CommentToken); -- cgit v1.3