diff options
Diffstat (limited to 'src/Tokenizer.hs')
| -rw-r--r-- | src/Tokenizer.hs | 11 |
1 files changed, 9 insertions, 2 deletions
diff --git a/src/Tokenizer.hs b/src/Tokenizer.hs index 6a5cc5c..777bedb 100644 --- a/src/Tokenizer.hs +++ b/src/Tokenizer.hs @@ -1,5 +1,6 @@ module Tokenizer ( tokenize, + tokenize' ) where import qualified Data.Bifunctor as B @@ -81,9 +82,15 @@ _tokenize fileName acc current (x:xs) in _tokenize fileName (token1 : token2 : acc) [] xs | otherwise = _tokenize fileName acc (x : current) xs +type RowN = Int +type ColN = Int + tokenize :: String -> String -> LContext [Token] -tokenize fileName src = do - let tChars = augment 1 1 src +tokenize = tokenize' (1, 1) + +tokenize' :: (RowN, ColN) -> String -> String -> LContext [Token] +tokenize' (row, col) fileName src = do + let tChars = augment row col src tokens <- _tokenize fileName [] [] tChars return $ tokens $> reverse |
