aboutsummaryrefslogtreecommitdiffstats
path: root/src/Tokenizer.hs
diff options
context:
space:
mode:
Diffstat (limited to 'src/Tokenizer.hs')
-rw-r--r--src/Tokenizer.hs11
1 files changed, 9 insertions, 2 deletions
diff --git a/src/Tokenizer.hs b/src/Tokenizer.hs
index 6a5cc5c..777bedb 100644
--- a/src/Tokenizer.hs
+++ b/src/Tokenizer.hs
@@ -1,5 +1,6 @@
module Tokenizer (
tokenize,
+ tokenize'
) where
import qualified Data.Bifunctor as B
@@ -81,9 +82,15 @@ _tokenize fileName acc current (x:xs)
in _tokenize fileName (token1 : token2 : acc) [] xs
| otherwise = _tokenize fileName acc (x : current) xs
+type RowN = Int
+type ColN = Int
+
tokenize :: String -> String -> LContext [Token]
-tokenize fileName src = do
- let tChars = augment 1 1 src
+tokenize = tokenize' (1, 1)
+
+tokenize' :: (RowN, ColN) -> String -> String -> LContext [Token]
+tokenize' (row, col) fileName src = do
+ let tChars = augment row col src
tokens <- _tokenize fileName [] [] tChars
return $ tokens
$> reverse