diff options
| author | Jan Tuomi <jans.tuomi@gmail.com> | 2023-11-03 20:02:03 +0200 |
|---|---|---|
| committer | Jan Tuomi <jans.tuomi@gmail.com> | 2023-11-03 20:02:03 +0200 |
| commit | 4fbe8d2d27dd8873c1f309e15b116f8800a5e605 (patch) | |
| tree | 709d25abaa5c0a7d1ea7484750cb54765d33cd59 /src/tokenizer.py | |
| parent | abf6cbdecc8eb006a859205dee7500c2b70c0998 (diff) | |
Improve string I/O helper macros
Diffstat (limited to 'src/tokenizer.py')
| -rw-r--r-- | src/tokenizer.py | 13 |
1 files changed, 13 insertions, 0 deletions
diff --git a/src/tokenizer.py b/src/tokenizer.py index 2049461..d9c6058 100644 --- a/src/tokenizer.py +++ b/src/tokenizer.py @@ -2,6 +2,7 @@ def tokenize(line: str) -> list[str]: cur: str = "" result: list[str] = [] is_py_expr = False + is_string = False idx = 0 while idx < len(line): c = line[idx] @@ -18,6 +19,16 @@ def tokenize(line: str) -> list[str]: cur = "" elif is_py_expr: cur += c + elif not is_string and c == "\"": + cur += "\"" + is_string = True + elif is_string and c == "\"": + cur += "\"" + is_string = False + result.append(cur) + cur = "" + elif is_string: + cur += c elif c == " " or c == "\t": result.append(cur) cur = "" @@ -29,4 +40,6 @@ def tokenize(line: str) -> list[str]: if len(cur) > 0: result.append(cur) + print(result) + return [r for r in result if r != ""]
\ No newline at end of file |