From 4fbe8d2d27dd8873c1f309e15b116f8800a5e605 Mon Sep 17 00:00:00 2001
From: Jan Tuomi <jans.tuomi@gmail.com>
Date: Fri, 3 Nov 2023 20:02:03 +0200
Subject: Improve string I/O helper macros

---
 src/tokenizer.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

(limited to 'src/tokenizer.py')

diff --git a/src/tokenizer.py b/src/tokenizer.py
index 2049461..d9c6058 100644
--- a/src/tokenizer.py
+++ b/src/tokenizer.py
@@ -2,6 +2,7 @@ def tokenize(line: str) -> list[str]:
   cur: str = ""
   result: list[str] = []
   is_py_expr = False
+  is_string = False
   idx = 0
   while idx < len(line):
     c = line[idx]
@@ -18,6 +19,16 @@ def tokenize(line: str) -> list[str]:
       cur = ""
     elif is_py_expr:
       cur += c
+    elif not is_string and c == "\"":
+      cur += "\""
+      is_string = True
+    elif is_string and c == "\"":
+      cur += "\""
+      is_string = False
+      result.append(cur)
+      cur = ""
+    elif is_string:
+      cur += c
     elif c == " " or c == "\t":
       result.append(cur)
       cur = ""
@@ -29,4 +40,6 @@ def tokenize(line: str) -> list[str]:
   if len(cur) > 0:
     result.append(cur)
 
+  print(result)
+
   return [r for r in result if r != ""]
\ No newline at end of file
-- 
cgit v1.3