diff options
Diffstat (limited to 'atk16-asm')
| -rw-r--r-- | atk16-asm/asm_eval.py | 47 | ||||
| -rw-r--r-- | atk16-asm/asm_ops.py | 154 | ||||
| -rw-r--r-- | atk16-asm/asm_pass0.py | 50 | ||||
| -rw-r--r-- | atk16-asm/asm_pass1.py | 48 | ||||
| -rw-r--r-- | atk16-asm/asm_pass2.py | 40 | ||||
| -rw-r--r-- | atk16-asm/asm_pass3.py | 78 | ||||
| -rw-r--r-- | atk16-asm/asm_pass4.py | 61 | ||||
| -rwxr-xr-x | atk16-asm/assembler.py | 82 | ||||
| -rw-r--r-- | atk16-asm/optimizer.py | 189 | ||||
| -rw-r--r-- | atk16-asm/tokenizer.py | 47 |
10 files changed, 0 insertions, 796 deletions
diff --git a/atk16-asm/asm_eval.py b/atk16-asm/asm_eval.py deleted file mode 100644 index c05fc03..0000000 --- a/atk16-asm/asm_eval.py +++ /dev/null @@ -1,47 +0,0 @@ -constants: dict[str, str] = { - # Registers - "ra": "0", - "rb": "1", - "rc": "2", - "rd": "3", - "re": "4", - "rf": "5", - "rg": "6", - "rh": "7", - # ALU instructions - "al_plus": "0", - "al_minus": "1", - "al_and": "2", - "al_or": "3", - "al_xor": "4", - "al_slr": "5", - "al_sar": "6", - "al_sll": "7", - # ALU flags - "carry": "0", - "overflow": "1", - "zero": "2", - "sign": "3", -} - -Symbols = dict[str, int] - -def check_size(bits: int, val: int) -> None: - if val >= 2 ** bits: - raise Exception(f"Value does not fit in {bits} bits: 0x{val:>04x}") - -def eval_symbol(symbols: Symbols, c: str) -> str: - if c in symbols: - return str(symbols[c]) - - if c in constants: - return constants[c] - - return c - -def eval_expr(symbols: Symbols, expr: str, bits: int = 16) -> int: - expr = expr.lower() - expr = eval_symbol(symbols, expr) - ret = eval(expr, symbols.copy()) # eval as Python expr - check_size(bits, ret) - return ret diff --git a/atk16-asm/asm_ops.py b/atk16-asm/asm_ops.py deleted file mode 100644 index 885c275..0000000 --- a/atk16-asm/asm_ops.py +++ /dev/null @@ -1,154 +0,0 @@ -from typing import Callable -from asm_eval import * -from dataclasses import dataclass - -@dataclass -class Meta: - address: int - -def make_alr(meta: Meta, symbols: Symbols, alu_op: str, left: str, right: str, target: str) -> int: - """ALR 0000 TTTL LLRR RSSS""" - target_e = eval_expr(symbols, target, bits=3) - left_e = eval_expr(symbols, left, bits=3) - right_e = eval_expr(symbols, right, bits=3) - alu_op_e = eval_expr(symbols, alu_op, bits=3) - word = (0b0000 << 12) + \ - (target_e << 9) + \ - (left_e << 6) + \ - (right_e << 3) + \ - alu_op_e - return word - -def make_ali(meta: Meta, symbols: Symbols, alu_op: str, left: str, imm: str, target: str) -> int: - """ALI 0001 TTTL LLII ISSS""" - target_e = eval_expr(symbols, target, bits=3) - left_e = eval_expr(symbols, left, bits=3) - imm_e = eval_expr(symbols, imm, bits=3) - alu_op_e = eval_expr(symbols, alu_op, bits=3) - word = (0b0001 << 12) + \ - (target_e << 9) + \ - (left_e << 6) + \ - (imm_e << 3) + \ - alu_op_e - return word - -def make_ldr(meta: Meta, symbols: Symbols, addr_reg: str, to_reg: str) -> int: - """LDR 0010 TTTR RRXX XXXX""" - to_reg_e = eval_expr(symbols, to_reg, bits=3) - addr_reg_e = eval_expr(symbols, addr_reg, bits=3) - word = (0b0010 << 12) + \ - (to_reg_e << 9) + \ - (addr_reg_e << 6) - return word - -def make_str(meta: Meta, symbols: Symbols, from_reg: str, addr_reg: str) -> int: - """STR 0011 XXXL LLRR RXXX""" - from_reg_e = eval_expr(symbols, from_reg, bits=3) - addr_reg_e = eval_expr(symbols, addr_reg, bits=3) - word = (0b0011 << 12) + \ - (addr_reg_e << 6) + \ - (from_reg_e << 3) - return word - -def make_ldi(meta: Meta, symbols: Symbols, imm: str, to_reg: str) -> int: - """LDI 0100 TTTI IIII IIII""" - to_reg_e = eval_expr(symbols, to_reg, bits=3) - imm_e = eval_expr(symbols, imm, bits=9) - word = (0b0100 << 12) + \ - (to_reg_e << 9) + \ - imm_e - return word - -def make_jpr(meta: Meta, symbols: Symbols, addr_reg: str) -> int: - """JPR 0101 XXXR RRXX XXXX""" - addr_reg_e = eval_expr(symbols, addr_reg, bits=3) - word = (0b0101 << 12) + \ - (addr_reg_e << 6) - return word - -def make_jpi(meta: Meta, symbols: Symbols, imm: str) -> int: - """JPI 0110 XXXI IIII IIII""" - imm_e = eval_expr(symbols, imm, bits=9) - imm_e = imm_e - meta.address - 1 - imm_e = imm_e & (0b111111111) - #print("symbols:", symbols) - #print(meta, imm, f"0x{imm_e:>0x}") - word = (0b0110 << 12) + \ - imm_e - return word - -def make_brr(meta: Meta, symbols: Symbols, flag_s: str, addr_reg: str) -> int: - """brr 0111 XFFR RRXX XXXX""" - flag_s_e = eval_expr(symbols, flag_s, bits=2) - addr_reg_e = eval_expr(symbols, addr_reg, bits=3) - word = (0b0111 << 12) + \ - (flag_s_e << 9) + \ - (addr_reg_e << 6) - return word - -def make_bri(meta: Meta, symbols: Symbols, flag_s: str, imm: str) -> int: - """BRI 1000 XFFI IIII IIII""" - flag_s_e = eval_expr(symbols, flag_s, bits=2) - imm_e = eval_expr(symbols, imm, bits=9) - imm_e = imm_e - meta.address - 1 - imm_e = imm_e & (0b111111111) - word = (0b1000 << 12) + \ - (flag_s_e << 9) + \ - imm_e - return word - -def make_lpc(meta: Meta, symbols: Symbols, target: str) -> int: - """LPC 1001 TTTX XXXX XXXX""" - target_e = eval_expr(symbols, target, bits=3) - word = (0b1001 << 12) + \ - (target_e << 9) - return word - -def make_rti(meta: Meta, symbols: Symbols) -> int: - """RTI 1110 XXXX XXXX XXXX""" - word = (0b1110 << 12) - return word - -def make_hlt(meta: Meta, symbols: Symbols) -> int: - """HLT 1111 XXXX XXXX XXXX""" - word = (0b1111 << 12) - return word - -OpWordDict = dict[str, Callable[..., int]] -operations: OpWordDict = { - "alr": make_alr, - "ali": make_ali, - "ldr": make_ldr, - "str": make_str, - "ldi": make_ldi, - "jpr": make_jpr, - "jpi": make_jpi, - "brr": make_brr, - "bri": make_bri, - "lpc": make_lpc, - "rti": make_rti, - "hlt": make_hlt, -} - -ExpandResult = list[list[str]] -ExpandFn = Callable[..., ExpandResult] -OpExpansionDict = dict[str, ExpandFn] - -def expand_id(*parts: str) -> ExpandResult: - return [list(parts)] - -default_expansions: OpExpansionDict = { - "alr": lambda *args: expand_id("alr", *args), - "ali": lambda *args: expand_id("ali", *args), - "ldr": lambda *args: expand_id("ldr", *args), - "str": lambda *args: expand_id("str", *args), - "ldi": lambda *args: expand_id("ldi", *args), - "jpr": lambda *args: expand_id("jpr", *args), - "jpi": lambda *args: expand_id("jpi", *args), - "brr": lambda *args: expand_id("brr", *args), - "bri": lambda *args: expand_id("bri", *args), - "lpc": lambda *args: expand_id("lpc", *args), - "rti": lambda *args: expand_id("rti", *args), - "hlt": lambda *args: expand_id("hlt", *args), -} - diff --git a/atk16-asm/asm_pass0.py b/atk16-asm/asm_pass0.py deleted file mode 100644 index f5b6b0b..0000000 --- a/atk16-asm/asm_pass0.py +++ /dev/null @@ -1,50 +0,0 @@ -from dataclasses import dataclass -import os.path -from asm_ops import * -from asm_eval import * -from tokenizer import * - -@dataclass -class Result0Line: - line_num: int - src_file: str - line: str - -@dataclass -class Result0: - lines: list[Result0Line] - -def pass_0(lines: list[str], file_name: str) -> Result0: - result_lines: list[Result0Line] = [] - - for (line_num, line) in enumerate(lines): - line = line.split(";")[0].strip() - if line == "": continue - - keyword, *args = tokenize(line) - - match keyword: - case "@include": - asm_file_name = args[0] - path = os.path.join(os.path.dirname(file_name), asm_file_name + ".atk16") - with open(path, "r") as f: - incl_lines = f.readlines() - - incl_result0 = pass_0(incl_lines, asm_file_name) - for incl_line in incl_result0.lines: - result_lines.append(Result0Line( - src_file=incl_line.src_file, - line_num=incl_line.line_num, - line=incl_line.line - )) - - case _: - result_lines.append(Result0Line( - src_file=file_name, - line_num=line_num, - line=line - )) - - return Result0( - lines=result_lines - ) diff --git a/atk16-asm/asm_pass1.py b/atk16-asm/asm_pass1.py deleted file mode 100644 index 0b06738..0000000 --- a/atk16-asm/asm_pass1.py +++ /dev/null @@ -1,48 +0,0 @@ -import importlib -import sys -import os.path -from dataclasses import dataclass -from asm_ops import * -from asm_eval import * -from asm_pass0 import * -from tokenizer import tokenize - -@dataclass -class Result1Line: - line_num: int - src_file: str - parts: list[str] - -@dataclass -class Result1: - lines: list[Result1Line] - operations: OpExpansionDict - -def pass_1(result0: Result0) -> Result1: - result_lines: list[Result1Line] = [] - operations: OpExpansionDict = default_expansions.copy() - - for line in result0.lines: - keyword, *args = tokenize(line.line) - match keyword: - case "@use": - module_name, ops = args[0].split(":") - ops_split = ops.split(",") - sys.path.append(os.path.dirname(line.src_file)) - module = importlib.import_module(module_name) - mod_expansions: OpExpansionDict = module.expansions - for op in mod_expansions: - expansion = mod_expansions[op] - if ops == "*" or op in ops_split: - operations[op] = expansion - case _: - result_lines.append(Result1Line( - src_file=line.src_file, - line_num=line.line_num, - parts=[keyword, *args] - )) - - return Result1( - operations=operations, - lines=result_lines - ) diff --git a/atk16-asm/asm_pass2.py b/atk16-asm/asm_pass2.py deleted file mode 100644 index e2ad4ec..0000000 --- a/atk16-asm/asm_pass2.py +++ /dev/null @@ -1,40 +0,0 @@ -from dataclasses import dataclass -from asm_ops import * -from asm_eval import * -from asm_pass1 import * - -@dataclass -class Result2Line: - line_num: int - src_file: str - parts: list[str] - original_parts: list[str] - -@dataclass -class Result2: - lines: list[Result2Line] - operations: OpExpansionDict - -def pass_2(result1: Result1) -> Result2: - result_lines: list[Result2Line] = [] - - for line in result1.lines: - keyword, *args = line.parts - if keyword in result1.operations: - fn = result1.operations[keyword] - output: list[list[str]] = fn(*args) - else: - output = [line.parts] - - for (idx, parts) in enumerate(output): - result_lines.append(Result2Line( - line_num=line.line_num, - src_file=line.src_file, - parts=parts, - original_parts=line.parts if idx == 0 else ["..."] - )) - - return Result2( - operations=result1.operations, - lines=result_lines - ) diff --git a/atk16-asm/asm_pass3.py b/atk16-asm/asm_pass3.py deleted file mode 100644 index 9c8cb34..0000000 --- a/atk16-asm/asm_pass3.py +++ /dev/null @@ -1,78 +0,0 @@ -from dataclasses import dataclass -from asm_ops import * -from asm_eval import * -from asm_pass2 import * - -@dataclass -class Result3Line: - line_num: int - src_file: str - address: int - parts: list[str] - original_parts: list[str] - -@dataclass -class Result3: - lines: list[Result3Line] - operations: OpExpansionDict - symbols: dict[str, int] - -def pass_3(result2: Result2) -> Result3: - result_lines: list[Result3Line] = [] - symbols: dict[str, int] = {} - address = 0 - - for line in result2.lines: - keyword, *args = line.parts - match keyword: - case "@address": - address = eval_expr(symbols, args[0]) - continue - case "@label": - label = args[0] - if label in symbols: - raise Exception(f"When defining label {label} as {address:>04x}, symbol {label} already defined as {symbols[label]:>04x}") - symbols[args[0]] = address - continue - case "@let": - symbols[args[0]] = eval_expr(symbols, args[1]) - continue - case _: - result_lines.append(Result3Line( - line_num=line.line_num, - src_file=line.src_file, - parts=line.parts, - address=address, - original_parts=line.original_parts, - )) - address += 1 - - result_lines.sort(key=lambda l: l.address) - for result_line in result_lines: - rows_with_same_addr = list(filter(lambda l: l.address == result_line.address, result_lines)) - n = len(rows_with_same_addr) - if n > 1: - formatted = format_overlapping_rows(rows_with_same_addr) - raise Exception(f"Overlapping segments: address 0x{result_line.address:>04x} has conflicting definitions:\n{formatted}") - return Result3( - operations=result2.operations, - lines=result_lines, - symbols=symbols, - ) - -def format_overlapping_rows(rows: list[Result3Line]) -> str: - longest_val = 0 - for row in rows: - joined = " ".join(row.parts) - if len(joined) > longest_val: - longest_val = len(joined) - - first_col_width = longest_val + 4 - - results: list[str] = [] - for row in rows: - joined = " ".join(row.parts) - result = joined + (first_col_width - len(joined)) * " " + f" ({row.src_file}:{row.line_num})" - results.append(result) - - return "\n".join(results) diff --git a/atk16-asm/asm_pass4.py b/atk16-asm/asm_pass4.py deleted file mode 100644 index 2fff870..0000000 --- a/atk16-asm/asm_pass4.py +++ /dev/null @@ -1,61 +0,0 @@ -from dataclasses import dataclass -from asm_ops import * -from asm_eval import * -from asm_pass3 import * - -@dataclass -class Result4Line: - line_num: int - src_file: str - address: int - word: int - text: str - original_text: str - -@dataclass -class Result4: - lines: list[Result4Line] - operations: OpExpansionDict - symbols: dict[str, int] - -def pass_4(result3: Result3) -> Result4: - result_lines: list[Result4Line] = [] - - for line in result3.lines: - keyword, *args = line.parts - meta = Meta( - address=line.address, - ) - - text = " ".join([keyword, *args]) - original_text = " ".join(line.original_parts) - - if keyword in operations: - fn = operations[keyword] - word = fn(meta, result3.symbols, *args) - result_lines.append(Result4Line( - line_num=line.line_num, - src_file=line.src_file, - address=line.address, - word=word, - text=text, - original_text=original_text, - )) - else: - try: - result_lines.append(Result4Line( - line_num=line.line_num, - src_file=line.src_file, - address=line.address, - word=eval_expr(result3.symbols, keyword), - text=text, - original_text=original_text, - )) - except: - raise Exception(f"Invalid assembly at {line.src_file}:{line.line_num + 1}\n\n{line}") - - return Result4( - operations=result3.operations, - symbols=result3.symbols, - lines=result_lines, - ) diff --git a/atk16-asm/assembler.py b/atk16-asm/assembler.py deleted file mode 100755 index c678ba0..0000000 --- a/atk16-asm/assembler.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python3 -# Assemble ATK16 assembly to bytecode - -import sys -from asm_ops import * -from asm_eval import * -from asm_pass0 import pass_0 -from asm_pass1 import pass_1 -from asm_pass2 import pass_2 -from asm_pass3 import pass_3 -from asm_pass4 import pass_4 - -if len(sys.argv) != 3: - print("usage: assembler.py <infile> <outfile> # read from file") - print(" assembler.py - <outfile> # read from stdin") - sys.exit(1) - -infile_path = sys.argv[1] -outfile_path = sys.argv[2] - -src = "" -if (infile_path == "-"): - for line in sys.stdin: - src += line -else: - with open(infile_path, "r") as f: - src = f.read() - -src_lines = src.splitlines() - -### Utils - -def parse(line: str) -> list[str]: - depth = 0 - result: list[str] = [] - acc: str = "" - for c in line: - if c.isspace() and depth == 0: - result.append(acc) - acc = "" - elif c == "(": - depth += 1 - acc += "(" - elif c == ")": - depth -= 1 - acc += ")" - else: - acc += c - - result.append(acc) - return list(filter(lambda x: len(x) > 0, result)) - -result0 = pass_0(src_lines, infile_path) -result1 = pass_1(result0) -result2 = pass_2(result1) -result3 = pass_3(result2) -result4 = pass_4(result3) - -nop = bytearray([0b1000_0000, 0]) -result = bytearray() -# initially one nop -result.extend(nop) - -for line in result4.lines: - for (symbol, symbol_value) in result4.symbols.items(): - if line.address == symbol_value: - print(f"{symbol}:") - - if len(result) < 2 * line.address + 1: - result.extend((2 * line.address + 1 - len(result)) * nop) - - out_line = f"{line.address:>08x} 0x{line.word:>04x} {line.text}" - out_spaces_n = (42 - len(out_line)) - out_spaces = out_spaces_n * " " if out_spaces_n > 0 else 4 * " " - print(f"{out_line}{out_spaces}{line.original_text}") - result[2 * line.address + 0] = ((line.word >> 8) & 0xff) - result[2 * line.address + 1] = ((line.word >> 0) & 0xff) - -with open(outfile_path, "wb") as f: - f.write(result) - -print(f"Wrote {len(result)} bytes to {outfile_path}") diff --git a/atk16-asm/optimizer.py b/atk16-asm/optimizer.py deleted file mode 100644 index 0ee2837..0000000 --- a/atk16-asm/optimizer.py +++ /dev/null @@ -1,189 +0,0 @@ -from tokenizer import tokenize - -def format_asm_row(asm: str) -> str: - if not (asm.startswith("@") or asm.startswith(";")) and not asm.startswith(" ") and len(asm) > 0: - return " " + asm - else: - return asm - -class Optimizer: - def __init__(self): - pass - - def optimize(self, asm_str: str): - asm = asm_str.split("\n") - asm = [row.strip() for row in asm] - asm = [self.strip_comment(row) for row in asm] - asm = [row for row in asm if not len(row) == 0] - asm = [tokenize(row, retain_curlies=True) for row in asm] - asm = self.compact_spu_spo_pattern(asm) - - # TODO: not safe when setting loading SP and FP - # ldr RA SP - # mov RA FP - # gets optimized to - # ldr RA FP - #asm = self.compact_target_mov_pattern(asm) - #asm = self.compact_target_mov_pattern(asm) - - # TODO: not safe at all. E.g. breaks a while True: pass loop - #asm = self.compact_mov_source_pattern(asm) - #asm = self.compact_mov_source_pattern(asm) - - asm = self.compact_spu_load_spo_pattern(asm) - # asm = self.convert_alr_to_ali(asm) - result = "\n".join([format_asm_row(" ".join(row)) for row in asm]) - - return result - - def strip_comment(self, row: str): - ret: str = "" - for c in row: - if c == ";": break - ret += c - - return ret - - def compact_spu_spo_pattern(self, asm: list[list[str]]) -> list[list[str]]: - i = 0 - result: list[list[str]] = [] - while i < len(asm): - current = asm[i] - next = asm[i + 1] if i + 1 < len(asm) else None - i += 1 - if current[0].startswith("@") or next is None: - result.append(current) - continue - - if current[0] == "spu" and next[0] == "spo": - print("=== compact_spu_spo_pattern") - print(current) - print(next) - - arg_current = current[1] - arg_next = next[1] - - print("=== arg_current:", arg_current) - print("=== arg_next:", arg_next) - - if arg_current == arg_next: - print("=== pass") - pass # remove both spu and spo - else: - print("=== mov") - mov = ["mov", arg_current, arg_next] - result.append(mov) - print("=== mov:", mov) - - print("=== last of result:", result[len(result) - 1]) - print() - i += 1 - continue - - result.append(current) - - return result - - def compact_target_mov_pattern(self, asm: list[list[str]]) -> list[list[str]]: - ops_with_target = ["ldi", "ldr", "add", "sub", "addi", "subi", "and", "or", "xor", "sll", "slr", "sar", "slli", "slri", "sari", "inc", "dec", "mov", "ali", "alr", "lpc"] - i = 0 - result: list[list[str]] = [] - while i < len(asm): - current = asm[i] - next = asm[i + 1] if i + 1 < len(asm) else None - i += 1 - if current[0].startswith("@") or next is None: - result.append(current) - continue - - if current[0] in ops_with_target and next[0] == "mov": - - op_op, op_operands, op_target_reg = current[0], current[1:len(current) - 1], current[len(current) - 1] - mov_from_reg, mov_to_reg = next[1], next[2] - - if op_target_reg == mov_from_reg: - ret = [op_op, *op_operands, mov_to_reg] - result.append(ret) - i += 1 - continue - - result.append(current) - - return result - - def compact_mov_source_pattern(self, asm: list[list[str]]) -> list[list[str]]: - ops_with_source = ["str", "ldr", "add", "sub", "addi", "subi", "and", "or", "xor", "sll", "slr", "sar", "slli", "slri", "sari", "mov", "ali", "alr"] - i = 0 - result: list[list[str]] = [] - while i < len(asm): - current = asm[i] - next = asm[i + 1] if i + 1 < len(asm) else None - i += 1 - if current[0].startswith("@") or next is None: - result.append(current) - continue - - if current[0] == "mov" and next[0] in ops_with_source: - mov_from_reg, mov_to_reg = current[1], current[2] - op_op, op_source_reg, op_operands = next[0], next[1], next[2:] - - if op_source_reg == mov_to_reg: - ret = [op_op, op_source_reg, *op_operands] - result.append(ret) - i += 1 - continue - - result.append(current) - - return result - - def compact_spu_load_spo_pattern(self, asm: list[list[str]]) -> list[list[str]]: - # spu RA - # ldi int_7 RA - # ldr RA RB - # spo RA - # OR - # spu RA - # ldi 3 RB - # spo RA - - i = 0 - result: list[list[str]] = [] - while i < len(asm): - instr0 = asm[i] - instr1 = asm[i + 1] if i + 1 < len(asm) else None - instr2 = asm[i + 2] if i + 2 < len(asm) else None - instr3 = asm[i + 3] if i + 3 < len(asm) else None - i += 1 - if instr0[0].startswith("@") or instr1 is None or instr2 is None or instr3 is None: - result.append(instr0) - continue - - if instr0[0] == "spu" and instr1[0] == "ldi" and instr2[0] == "ldr" and instr3[0] == "spo": - spu_op, spu_reg = instr0 - ldi_op, ldi_imm, ldi_target_reg = instr1 - ldr_op, ldr_from_reg, ldr_to_reg = instr2 - spo_op, spo_reg = instr3 - - if spu_reg == spo_reg and ldi_target_reg == ldr_from_reg and ldr_from_reg != ldr_to_reg: - ret0 = f"ldi {ldi_imm} {ldr_to_reg}".split() - ret1 = f"ldr {ldr_to_reg} {ldr_to_reg}".split() - result.append(ret0) - result.append(ret1) - i += 3 - continue - - elif instr0[0] == "spu" and instr1[0] == "ldi" and instr2[0] == "spo": - spu_op, spu_reg = instr0 - ldi_op, ldi_imm, ldi_target_reg = instr1 - spo_op, spo_reg = instr2 - - if spu_reg == spo_reg and ldi_target_reg != spu_reg: - ret0 = f"ldi {ldi_imm} {ldi_target_reg}".split() - result.append(ret0) - i += 2 - continue - - result.append(instr0) - - return result diff --git a/atk16-asm/tokenizer.py b/atk16-asm/tokenizer.py deleted file mode 100644 index cf31b56..0000000 --- a/atk16-asm/tokenizer.py +++ /dev/null @@ -1,47 +0,0 @@ -def tokenize(line: str, retain_curlies = False) -> list[str]: - cur: str = "" - result: list[str] = [] - is_py_expr = False - is_string = False - idx = 0 - while idx < len(line): - c = line[idx] - if not is_py_expr and c == "$": - is_py_expr = True - if retain_curlies: - cur += "${" - idx += 1 - elif is_py_expr and c == "$": - raise Exception("Unexpected start of python expr while already parsing a python expression:\n" + line) - elif not is_py_expr and c == "}": - raise Exception("Unexpected end of python expr while not parsing a python expression:\n" + line) - elif is_py_expr and c == "}": - is_py_expr = False - if retain_curlies: - cur += c - result.append(cur) - cur = "" - elif is_py_expr: - cur += c - elif not is_string and c == "\"": - cur += "\"" - is_string = True - elif is_string and c == "\"": - cur += "\"" - is_string = False - result.append(cur) - cur = "" - elif is_string: - cur += c - elif c == " " or c == "\t": - result.append(cur) - cur = "" - else: - cur += c - - idx += 1 - - if len(cur) > 0: - result.append(cur) - - return [r for r in result if r != ""]
\ No newline at end of file |
