test/core/tokenizer/TokenizerTest.java


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80

package com.jantuomi.interpreter.test.core.tokenizer;

import com.jantuomi.interpreter.main.core.tokenizer.Tokenizer;
import com.jantuomi.interpreter.main.core.tokenizer.token.Token;
import org.junit.Test;

import java.util.List;

import static org.junit.Assert.assertTrue;

/**
 * Created by jan on 10.6.2016.
 */
public class TokenizerTest {

    @Test
    public void testTokenizeCommentAndSymbol() {
        String testString = "\"string literal\"\n/*comment*/\n1 2\nsymbol_Test3";
        List<Token> list = Tokenizer.getInstance().tokenize(testString);

        System.out.println(String.format("test string:\n%s", testString));
        Tokenizer.printTokens(list);
        assertTrue(list.contains(new Token(Token.Type.CommentToken, "comment")));
    }

    @Test
    public void testTokenizeMultilineString() {
        String testString = "\"this is a\nmultiline\nstring\"";
        List<Token> list = Tokenizer.getInstance().tokenize(testString);

        System.out.println(String.format("test string:\n%s", testString));
        Tokenizer.printTokens(list);
        assertTrue(list.contains(new Token(Token.Type.StringLiteralToken, "this is a\nmultiline\nstring")));
    }

    @Test
    public void testTokenizeMath() {
        String testString = "x <- (1 + 2) / 3 * 4";
        List<Token> list = Tokenizer.getInstance().tokenize(testString);

        System.out.println(String.format("test string:\n%s", testString));
        Tokenizer.printTokens(list);
        for (Token t : list) {
            if (t.isSameType(new Token(Token.Type.AssignmentToken))) {
                return;
            }
        }
        assertTrue("No assignment token found!" == null);
    }

    @Test
    public void testTokenizeDeclAndAssign() {
        String testString = "decl x\nx <- 1 + 2";
        List<Token> list = Tokenizer.getInstance().tokenize(testString);

        System.out.println(String.format("test string:\n%s", testString));
        Tokenizer.printTokens(list);
        for (Token t : list) {
            if (t.isSameType(new Token(Token.Type.DeclarationToken))) {
                return;
            }
        }
        assertTrue("No declaration token found!" == null);
    }

    @Test
    public void testTokenizeFunction() {
        String testString = "func name(arg)\nx <- 1\nend";
        List<Token> list = Tokenizer.getInstance().tokenize(testString);

        System.out.println(String.format("test string:\n%s", testString));
        Tokenizer.printTokens(list);
        for (Token t : list) {
            if (t.isSameType(new Token(Token.Type.FunctionDefineToken))) {
                return;
            }
        }
        assertTrue("No function declaration token found!" == null);
    }
}