Refactor names

10 files changed, 796 insertions, 0 deletions

diff --git a/atk16_asm/asm_eval.py b/atk16_asm/asm_eval.py
new file mode 100644
index 0000000..c05fc03
--- /dev/null
+++ b/atk16_asm/asm_eval.py
@@ -0,0 +1,47 @@
+constants: dict[str, str] = {
+  # Registers
+  "ra": "0",
+  "rb": "1",
+  "rc": "2",
+  "rd": "3",
+  "re": "4",
+  "rf": "5",
+  "rg": "6",
+  "rh": "7",
+  # ALU instructions
+  "al_plus": "0",
+  "al_minus": "1",
+  "al_and": "2",
+  "al_or": "3",
+  "al_xor": "4",
+  "al_slr": "5",
+  "al_sar": "6",
+  "al_sll": "7",
+  # ALU flags
+  "carry": "0",
+  "overflow": "1",
+  "zero": "2",
+  "sign": "3",
+}
+
+Symbols = dict[str, int]
+
+def check_size(bits: int, val: int) -> None:
+  if val >= 2 ** bits:
+    raise Exception(f"Value does not fit in {bits} bits: 0x{val:>04x}")
+
+def eval_symbol(symbols: Symbols, c: str) -> str:
+  if c in symbols:
+    return str(symbols[c])
+
+  if c in constants:
+    return constants[c]
+
+  return c
+
+def eval_expr(symbols: Symbols, expr: str, bits: int = 16) -> int:
+  expr = expr.lower()
+  expr = eval_symbol(symbols, expr)
+  ret = eval(expr, symbols.copy()) # eval as Python expr
+  check_size(bits, ret)
+  return ret
diff --git a/atk16_asm/asm_ops.py b/atk16_asm/asm_ops.py
new file mode 100644
index 0000000..885c275
--- /dev/null
+++ b/atk16_asm/asm_ops.py
@@ -0,0 +1,154 @@
+from typing import Callable
+from asm_eval import *
+from dataclasses import dataclass
+
+@dataclass
+class Meta:
+  address: int
+
+def make_alr(meta: Meta, symbols: Symbols, alu_op: str, left: str, right: str, target: str) -> int:
+  """ALR 0000 TTTL LLRR RSSS"""
+  target_e = eval_expr(symbols, target, bits=3)
+  left_e = eval_expr(symbols, left, bits=3)
+  right_e = eval_expr(symbols, right, bits=3)
+  alu_op_e = eval_expr(symbols, alu_op, bits=3)
+  word = (0b0000 << 12) + \
+              (target_e << 9) + \
+              (left_e << 6) + \
+              (right_e << 3) + \
+              alu_op_e
+  return word
+
+def make_ali(meta: Meta, symbols: Symbols, alu_op: str, left: str, imm: str, target: str) -> int:
+  """ALI 0001 TTTL LLII ISSS"""
+  target_e = eval_expr(symbols, target, bits=3)
+  left_e = eval_expr(symbols, left, bits=3)
+  imm_e = eval_expr(symbols, imm, bits=3)
+  alu_op_e = eval_expr(symbols, alu_op, bits=3)
+  word = (0b0001 << 12) + \
+              (target_e << 9) + \
+              (left_e << 6) + \
+              (imm_e << 3) + \
+              alu_op_e
+  return word
+
+def make_ldr(meta: Meta, symbols: Symbols, addr_reg: str, to_reg: str) -> int:
+  """LDR 0010 TTTR RRXX XXXX"""
+  to_reg_e = eval_expr(symbols, to_reg, bits=3)
+  addr_reg_e = eval_expr(symbols, addr_reg, bits=3)
+  word = (0b0010 << 12) + \
+              (to_reg_e << 9) + \
+              (addr_reg_e << 6)
+  return word
+
+def make_str(meta: Meta, symbols: Symbols, from_reg: str, addr_reg: str) -> int:
+  """STR 0011 XXXL LLRR RXXX"""
+  from_reg_e = eval_expr(symbols, from_reg, bits=3)
+  addr_reg_e = eval_expr(symbols, addr_reg, bits=3)
+  word = (0b0011 << 12) + \
+              (addr_reg_e << 6) + \
+              (from_reg_e << 3)
+  return word
+
+def make_ldi(meta: Meta, symbols: Symbols, imm: str, to_reg: str) -> int:
+  """LDI 0100 TTTI IIII IIII"""
+  to_reg_e = eval_expr(symbols, to_reg, bits=3)
+  imm_e = eval_expr(symbols, imm, bits=9)
+  word = (0b0100 << 12) + \
+              (to_reg_e << 9) + \
+              imm_e
+  return word
+
+def make_jpr(meta: Meta, symbols: Symbols, addr_reg: str) -> int:
+  """JPR 0101 XXXR RRXX XXXX"""
+  addr_reg_e = eval_expr(symbols, addr_reg, bits=3)
+  word = (0b0101 << 12) + \
+              (addr_reg_e << 6)
+  return word
+
+def make_jpi(meta: Meta, symbols: Symbols, imm: str) -> int:
+  """JPI 0110 XXXI IIII IIII"""
+  imm_e = eval_expr(symbols, imm, bits=9)
+  imm_e = imm_e - meta.address - 1
+  imm_e = imm_e & (0b111111111)
+  #print("symbols:", symbols)
+  #print(meta, imm, f"0x{imm_e:>0x}")
+  word = (0b0110 << 12) + \
+              imm_e
+  return word
+
+def make_brr(meta: Meta, symbols: Symbols, flag_s: str, addr_reg: str) -> int:
+  """brr  0111 XFFR RRXX XXXX"""
+  flag_s_e = eval_expr(symbols, flag_s, bits=2)
+  addr_reg_e = eval_expr(symbols, addr_reg, bits=3)
+  word = (0b0111 << 12) + \
+              (flag_s_e << 9) + \
+              (addr_reg_e << 6)
+  return word
+
+def make_bri(meta: Meta, symbols: Symbols, flag_s: str, imm: str) -> int:
+  """BRI 1000 XFFI IIII IIII"""
+  flag_s_e = eval_expr(symbols, flag_s, bits=2)
+  imm_e = eval_expr(symbols, imm, bits=9)
+  imm_e = imm_e - meta.address - 1
+  imm_e = imm_e & (0b111111111)
+  word = (0b1000 << 12) + \
+              (flag_s_e << 9) + \
+              imm_e
+  return word
+
+def make_lpc(meta: Meta, symbols: Symbols, target: str) -> int:
+  """LPC 1001 TTTX XXXX XXXX"""
+  target_e = eval_expr(symbols, target, bits=3)
+  word = (0b1001 << 12) + \
+              (target_e << 9)
+  return word
+
+def make_rti(meta: Meta, symbols: Symbols) -> int:
+  """RTI 1110 XXXX XXXX XXXX"""
+  word = (0b1110 << 12)
+  return word
+
+def make_hlt(meta: Meta, symbols: Symbols) -> int:
+  """HLT 1111 XXXX XXXX XXXX"""
+  word = (0b1111 << 12)
+  return word
+
+OpWordDict = dict[str, Callable[..., int]]
+operations: OpWordDict = {
+  "alr": make_alr,
+  "ali": make_ali,
+  "ldr": make_ldr,
+  "str": make_str,
+  "ldi": make_ldi,
+  "jpr": make_jpr,
+  "jpi": make_jpi,
+  "brr": make_brr,
+  "bri": make_bri,
+  "lpc": make_lpc,
+  "rti": make_rti,
+  "hlt": make_hlt,
+}
+
+ExpandResult = list[list[str]]
+ExpandFn = Callable[..., ExpandResult]
+OpExpansionDict = dict[str, ExpandFn]
+
+def expand_id(*parts: str) -> ExpandResult:
+  return [list(parts)]
+
+default_expansions: OpExpansionDict = {
+  "alr": lambda *args: expand_id("alr", *args),
+  "ali": lambda *args: expand_id("ali", *args),
+  "ldr": lambda *args: expand_id("ldr", *args),
+  "str": lambda *args: expand_id("str", *args),
+  "ldi": lambda *args: expand_id("ldi", *args),
+  "jpr": lambda *args: expand_id("jpr", *args),
+  "jpi": lambda *args: expand_id("jpi", *args),
+  "brr": lambda *args: expand_id("brr", *args),
+  "bri": lambda *args: expand_id("bri", *args),
+  "lpc": lambda *args: expand_id("lpc", *args),
+  "rti": lambda *args: expand_id("rti", *args),
+  "hlt": lambda *args: expand_id("hlt", *args),
+}
+
diff --git a/atk16_asm/asm_pass0.py b/atk16_asm/asm_pass0.py
new file mode 100644
index 0000000..f5b6b0b
--- /dev/null
+++ b/atk16_asm/asm_pass0.py
@@ -0,0 +1,50 @@
+from dataclasses import dataclass
+import os.path
+from asm_ops import *
+from asm_eval import *
+from tokenizer import *
+
+@dataclass
+class Result0Line:
+  line_num: int
+  src_file: str
+  line: str
+
+@dataclass
+class Result0:
+  lines: list[Result0Line]
+
+def pass_0(lines: list[str], file_name: str) -> Result0:
+  result_lines: list[Result0Line] = []
+
+  for (line_num, line) in enumerate(lines):
+    line = line.split(";")[0].strip()
+    if line == "": continue
+
+    keyword, *args = tokenize(line)
+
+    match keyword:
+      case "@include":
+        asm_file_name = args[0]
+        path = os.path.join(os.path.dirname(file_name), asm_file_name + ".atk16")
+        with open(path, "r") as f:
+          incl_lines = f.readlines()
+
+        incl_result0 = pass_0(incl_lines, asm_file_name)
+        for incl_line in incl_result0.lines:
+          result_lines.append(Result0Line(
+            src_file=incl_line.src_file,
+            line_num=incl_line.line_num,
+            line=incl_line.line
+          ))
+
+      case _:
+        result_lines.append(Result0Line(
+          src_file=file_name,
+          line_num=line_num,
+          line=line
+        ))
+
+  return Result0(
+    lines=result_lines
+  )
diff --git a/atk16_asm/asm_pass1.py b/atk16_asm/asm_pass1.py
new file mode 100644
index 0000000..0b06738
--- /dev/null
+++ b/atk16_asm/asm_pass1.py
@@ -0,0 +1,48 @@
+import importlib
+import sys
+import os.path
+from dataclasses import dataclass
+from asm_ops import *
+from asm_eval import *
+from asm_pass0 import *
+from tokenizer import tokenize
+
+@dataclass
+class Result1Line:
+  line_num: int
+  src_file: str
+  parts: list[str]
+
+@dataclass
+class Result1:
+  lines: list[Result1Line]
+  operations: OpExpansionDict
+
+def pass_1(result0: Result0) -> Result1:
+  result_lines: list[Result1Line] = []
+  operations: OpExpansionDict = default_expansions.copy()
+
+  for line in result0.lines:
+    keyword, *args = tokenize(line.line)
+    match keyword:
+      case "@use":
+        module_name, ops = args[0].split(":")
+        ops_split = ops.split(",")
+        sys.path.append(os.path.dirname(line.src_file))
+        module = importlib.import_module(module_name)
+        mod_expansions: OpExpansionDict = module.expansions
+        for op in mod_expansions:
+          expansion = mod_expansions[op]
+          if ops == "*" or op in ops_split:
+            operations[op] = expansion
+      case _:
+        result_lines.append(Result1Line(
+          src_file=line.src_file,
+          line_num=line.line_num,
+          parts=[keyword, *args]
+        ))
+
+  return Result1(
+    operations=operations,
+    lines=result_lines
+  )
diff --git a/atk16_asm/asm_pass2.py b/atk16_asm/asm_pass2.py
new file mode 100644
index 0000000..e2ad4ec
--- /dev/null
+++ b/atk16_asm/asm_pass2.py
@@ -0,0 +1,40 @@
+from dataclasses import dataclass
+from asm_ops import *
+from asm_eval import *
+from asm_pass1 import *
+
+@dataclass
+class Result2Line:
+  line_num: int
+  src_file: str
+  parts: list[str]
+  original_parts: list[str]
+
+@dataclass
+class Result2:
+  lines: list[Result2Line]
+  operations: OpExpansionDict
+
+def pass_2(result1: Result1) -> Result2:
+  result_lines: list[Result2Line] = []
+
+  for line in result1.lines:
+    keyword, *args = line.parts
+    if keyword in result1.operations:
+      fn = result1.operations[keyword]
+      output: list[list[str]] = fn(*args)
+    else:
+      output = [line.parts]
+
+    for (idx, parts) in enumerate(output):
+      result_lines.append(Result2Line(
+        line_num=line.line_num,
+        src_file=line.src_file,
+        parts=parts,
+        original_parts=line.parts if idx == 0 else ["..."]
+      ))
+
+  return Result2(
+    operations=result1.operations,
+    lines=result_lines
+  )
diff --git a/atk16_asm/asm_pass3.py b/atk16_asm/asm_pass3.py
new file mode 100644
index 0000000..9c8cb34
--- /dev/null
+++ b/atk16_asm/asm_pass3.py
@@ -0,0 +1,78 @@
+from dataclasses import dataclass
+from asm_ops import *
+from asm_eval import *
+from asm_pass2 import *
+
+@dataclass
+class Result3Line:
+  line_num: int
+  src_file: str
+  address: int
+  parts: list[str]
+  original_parts: list[str]
+
+@dataclass
+class Result3:
+  lines: list[Result3Line]
+  operations: OpExpansionDict
+  symbols: dict[str, int]
+
+def pass_3(result2: Result2) -> Result3:
+  result_lines: list[Result3Line] = []
+  symbols: dict[str, int] = {}
+  address = 0
+
+  for line in result2.lines:
+    keyword, *args = line.parts
+    match keyword:
+      case "@address":
+        address = eval_expr(symbols, args[0])
+        continue
+      case "@label":
+        label = args[0]
+        if label in symbols:
+          raise Exception(f"When defining label {label} as {address:>04x}, symbol {label} already defined as {symbols[label]:>04x}")
+        symbols[args[0]] = address
+        continue
+      case "@let":
+        symbols[args[0]] = eval_expr(symbols, args[1])
+        continue
+      case _:
+        result_lines.append(Result3Line(
+          line_num=line.line_num,
+          src_file=line.src_file,
+          parts=line.parts,
+          address=address,
+          original_parts=line.original_parts,
+        ))
+        address += 1
+
+  result_lines.sort(key=lambda l: l.address)
+  for result_line in result_lines:
+    rows_with_same_addr = list(filter(lambda l: l.address == result_line.address, result_lines))
+    n = len(rows_with_same_addr)
+    if n > 1:
+      formatted = format_overlapping_rows(rows_with_same_addr)
+      raise Exception(f"Overlapping segments: address 0x{result_line.address:>04x} has conflicting definitions:\n{formatted}")
+  return Result3(
+    operations=result2.operations,
+    lines=result_lines,
+    symbols=symbols,
+  )
+
+def format_overlapping_rows(rows: list[Result3Line]) -> str:
+  longest_val = 0
+  for row in rows:
+    joined = " ".join(row.parts)
+    if len(joined) > longest_val:
+      longest_val = len(joined)
+
+  first_col_width = longest_val + 4
+
+  results: list[str] = []
+  for row in rows:
+    joined = " ".join(row.parts)
+    result = joined + (first_col_width - len(joined)) * " " + f" ({row.src_file}:{row.line_num})"
+    results.append(result)
+
+  return "\n".join(results)
diff --git a/atk16_asm/asm_pass4.py b/atk16_asm/asm_pass4.py
new file mode 100644
index 0000000..2fff870
--- /dev/null
+++ b/atk16_asm/asm_pass4.py
@@ -0,0 +1,61 @@
+from dataclasses import dataclass
+from asm_ops import *
+from asm_eval import *
+from asm_pass3 import *
+
+@dataclass
+class Result4Line:
+  line_num: int
+  src_file: str
+  address: int
+  word: int
+  text: str
+  original_text: str
+
+@dataclass
+class Result4:
+  lines: list[Result4Line]
+  operations: OpExpansionDict
+  symbols: dict[str, int]
+
+def pass_4(result3: Result3) -> Result4:
+  result_lines: list[Result4Line] = []
+
+  for line in result3.lines:
+    keyword, *args = line.parts
+    meta = Meta(
+      address=line.address,
+    )
+
+    text = " ".join([keyword, *args])
+    original_text = " ".join(line.original_parts)
+
+    if keyword in operations:
+      fn = operations[keyword]
+      word = fn(meta, result3.symbols, *args)
+      result_lines.append(Result4Line(
+        line_num=line.line_num,
+        src_file=line.src_file,
+        address=line.address,
+        word=word,
+        text=text,
+        original_text=original_text,
+      ))
+    else:
+      try:
+        result_lines.append(Result4Line(
+          line_num=line.line_num,
+          src_file=line.src_file,
+          address=line.address,
+          word=eval_expr(result3.symbols, keyword),
+          text=text,
+          original_text=original_text,
+        ))
+      except:
+        raise Exception(f"Invalid assembly at {line.src_file}:{line.line_num + 1}\n\n{line}")
+
+  return Result4(
+    operations=result3.operations,
+    symbols=result3.symbols,
+    lines=result_lines,
+  )
diff --git a/atk16_asm/assembler.py b/atk16_asm/assembler.py
new file mode 100755
index 0000000..c678ba0
--- /dev/null
+++ b/atk16_asm/assembler.py
@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+# Assemble ATK16 assembly to bytecode
+
+import sys
+from asm_ops import *
+from asm_eval import *
+from asm_pass0 import pass_0
+from asm_pass1 import pass_1
+from asm_pass2 import pass_2
+from asm_pass3 import pass_3
+from asm_pass4 import pass_4
+
+if len(sys.argv) != 3:
+  print("usage: assembler.py <infile> <outfile> # read from file")
+  print("       assembler.py - <outfile>        # read from stdin")
+  sys.exit(1)
+
+infile_path = sys.argv[1]
+outfile_path = sys.argv[2]
+
+src = ""
+if (infile_path == "-"):
+  for line in sys.stdin:
+    src += line
+else:
+  with open(infile_path, "r") as f:
+    src = f.read()
+
+src_lines = src.splitlines()
+
+### Utils
+
+def parse(line: str) -> list[str]:
+  depth = 0
+  result: list[str] = []
+  acc: str = ""
+  for c in line:
+    if c.isspace() and depth == 0:
+      result.append(acc)
+      acc = ""
+    elif c == "(":
+      depth += 1
+      acc += "("
+    elif c == ")":
+      depth -= 1
+      acc += ")"
+    else:
+      acc += c
+
+  result.append(acc)
+  return list(filter(lambda x: len(x) > 0, result))
+
+result0 = pass_0(src_lines, infile_path)
+result1 = pass_1(result0)
+result2 = pass_2(result1)
+result3 = pass_3(result2)
+result4 = pass_4(result3)
+
+nop = bytearray([0b1000_0000, 0])
+result = bytearray()
+# initially one nop
+result.extend(nop)
+
+for line in result4.lines:
+  for (symbol, symbol_value) in result4.symbols.items():
+    if line.address == symbol_value:
+      print(f"{symbol}:")
+
+  if len(result) < 2 * line.address + 1:
+    result.extend((2 * line.address + 1 - len(result)) * nop)
+
+  out_line = f"{line.address:>08x}  0x{line.word:>04x}  {line.text}"
+  out_spaces_n = (42 - len(out_line))
+  out_spaces = out_spaces_n * " " if out_spaces_n > 0 else 4 * " "
+  print(f"{out_line}{out_spaces}{line.original_text}")
+  result[2 * line.address + 0] = ((line.word >> 8) & 0xff)
+  result[2 * line.address + 1] = ((line.word >> 0) & 0xff)
+
+with open(outfile_path, "wb") as f:
+  f.write(result)
+
+print(f"Wrote {len(result)} bytes to {outfile_path}")
diff --git a/atk16_asm/optimizer.py b/atk16_asm/optimizer.py
new file mode 100644
index 0000000..0ee2837
--- /dev/null
+++ b/atk16_asm/optimizer.py
@@ -0,0 +1,189 @@
+from tokenizer import tokenize
+
+def format_asm_row(asm: str) -> str:
+  if not (asm.startswith("@") or asm.startswith(";")) and not asm.startswith("  ") and len(asm) > 0:
+    return "  " + asm
+  else:
+    return asm
+
+class Optimizer:
+  def __init__(self):
+    pass
+
+  def optimize(self, asm_str: str):
+    asm = asm_str.split("\n")
+    asm = [row.strip() for row in asm]
+    asm = [self.strip_comment(row) for row in asm]
+    asm = [row for row in asm if not len(row) == 0]
+    asm = [tokenize(row, retain_curlies=True) for row in asm]
+    asm = self.compact_spu_spo_pattern(asm)
+
+    # TODO: not safe when setting loading SP and FP
+    #   ldr RA SP
+    #   mov RA FP
+    # gets optimized to
+    #   ldr RA FP
+    #asm = self.compact_target_mov_pattern(asm)
+    #asm = self.compact_target_mov_pattern(asm)
+
+    # TODO: not safe at all. E.g. breaks a while True: pass loop
+    #asm = self.compact_mov_source_pattern(asm)
+    #asm = self.compact_mov_source_pattern(asm)
+
+    asm = self.compact_spu_load_spo_pattern(asm)
+    # asm = self.convert_alr_to_ali(asm)
+    result = "\n".join([format_asm_row(" ".join(row)) for row in asm])
+
+    return result
+
+  def strip_comment(self, row: str):
+    ret: str = ""
+    for c in row:
+      if c == ";": break
+      ret += c
+
+    return ret
+
+  def compact_spu_spo_pattern(self, asm: list[list[str]]) -> list[list[str]]:
+    i = 0
+    result: list[list[str]] = []
+    while i < len(asm):
+      current = asm[i]
+      next = asm[i + 1] if i + 1 < len(asm) else None
+      i += 1
+      if current[0].startswith("@") or next is None:
+        result.append(current)
+        continue
+
+      if current[0] == "spu" and next[0] == "spo":
+        print("=== compact_spu_spo_pattern")
+        print(current)
+        print(next)
+
+        arg_current = current[1]
+        arg_next = next[1]
+
+        print("=== arg_current:", arg_current)
+        print("=== arg_next:", arg_next)
+
+        if arg_current == arg_next:
+          print("=== pass")
+          pass # remove both spu and spo
+        else:
+          print("=== mov")
+          mov = ["mov", arg_current, arg_next]
+          result.append(mov)
+          print("=== mov:", mov)
+
+        print("=== last of result:", result[len(result) - 1])
+        print()
+        i += 1
+        continue
+
+      result.append(current)
+
+    return result
+
+  def compact_target_mov_pattern(self, asm: list[list[str]]) -> list[list[str]]:
+    ops_with_target = ["ldi", "ldr", "add", "sub", "addi", "subi", "and", "or", "xor", "sll", "slr", "sar", "slli", "slri", "sari", "inc", "dec", "mov", "ali", "alr", "lpc"]
+    i = 0
+    result: list[list[str]] = []
+    while i < len(asm):
+      current = asm[i]
+      next = asm[i + 1] if i + 1 < len(asm) else None
+      i += 1
+      if current[0].startswith("@") or next is None:
+        result.append(current)
+        continue
+
+      if current[0] in ops_with_target and next[0] == "mov":
+
+        op_op, op_operands, op_target_reg = current[0], current[1:len(current) - 1], current[len(current) - 1]
+        mov_from_reg, mov_to_reg = next[1], next[2]
+
+        if op_target_reg == mov_from_reg:
+          ret = [op_op, *op_operands, mov_to_reg]
+          result.append(ret)
+          i += 1
+          continue
+
+      result.append(current)
+
+    return result
+
+  def compact_mov_source_pattern(self, asm: list[list[str]]) -> list[list[str]]:
+    ops_with_source = ["str", "ldr", "add", "sub", "addi", "subi", "and", "or", "xor", "sll", "slr", "sar", "slli", "slri", "sari", "mov", "ali", "alr"]
+    i = 0
+    result: list[list[str]] = []
+    while i < len(asm):
+      current = asm[i]
+      next = asm[i + 1] if i + 1 < len(asm) else None
+      i += 1
+      if current[0].startswith("@") or next is None:
+        result.append(current)
+        continue
+
+      if current[0] == "mov" and next[0] in ops_with_source:
+        mov_from_reg, mov_to_reg = current[1], current[2]
+        op_op, op_source_reg, op_operands = next[0], next[1], next[2:]
+
+        if op_source_reg == mov_to_reg:
+          ret = [op_op, op_source_reg, *op_operands]
+          result.append(ret)
+          i += 1
+          continue
+
+      result.append(current)
+
+    return result
+
+  def compact_spu_load_spo_pattern(self, asm: list[list[str]]) -> list[list[str]]:
+    #  spu RA
+    #  ldi int_7 RA
+    #  ldr RA RB
+    #  spo RA
+    # OR
+    #  spu RA
+    #  ldi 3 RB
+    #  spo RA
+
+    i = 0
+    result: list[list[str]] = []
+    while i < len(asm):
+      instr0 = asm[i]
+      instr1 = asm[i + 1] if i + 1 < len(asm) else None
+      instr2 = asm[i + 2] if i + 2 < len(asm) else None
+      instr3 = asm[i + 3] if i + 3 < len(asm) else None
+      i += 1
+      if instr0[0].startswith("@") or instr1 is None or instr2 is None or instr3 is None:
+        result.append(instr0)
+        continue
+
+      if instr0[0] == "spu" and instr1[0] == "ldi" and instr2[0] == "ldr" and instr3[0] == "spo":
+        spu_op, spu_reg = instr0
+        ldi_op, ldi_imm, ldi_target_reg = instr1
+        ldr_op, ldr_from_reg, ldr_to_reg = instr2
+        spo_op, spo_reg = instr3
+
+        if spu_reg == spo_reg and ldi_target_reg == ldr_from_reg and ldr_from_reg != ldr_to_reg:
+          ret0 = f"ldi {ldi_imm} {ldr_to_reg}".split()
+          ret1 = f"ldr {ldr_to_reg} {ldr_to_reg}".split()
+          result.append(ret0)
+          result.append(ret1)
+          i += 3
+          continue
+
+      elif instr0[0] == "spu" and instr1[0] == "ldi" and instr2[0] == "spo":
+        spu_op, spu_reg = instr0
+        ldi_op, ldi_imm, ldi_target_reg = instr1
+        spo_op, spo_reg = instr2
+
+        if spu_reg == spo_reg and ldi_target_reg != spu_reg:
+          ret0 = f"ldi {ldi_imm} {ldi_target_reg}".split()
+          result.append(ret0)
+          i += 2
+          continue
+
+      result.append(instr0)
+
+    return result
diff --git a/atk16_asm/tokenizer.py b/atk16_asm/tokenizer.py
new file mode 100644
index 0000000..cf31b56
--- /dev/null
+++ b/atk16_asm/tokenizer.py
@@ -0,0 +1,47 @@
+def tokenize(line: str, retain_curlies = False) -> list[str]:
+  cur: str = ""
+  result: list[str] = []
+  is_py_expr = False
+  is_string = False
+  idx = 0
+  while idx < len(line):
+    c = line[idx]
+    if not is_py_expr and c == "$":
+      is_py_expr = True
+      if retain_curlies:
+        cur += "${"
+      idx += 1
+    elif is_py_expr and c == "$":
+      raise Exception("Unexpected start of python expr while already parsing a python expression:\n" + line)
+    elif not is_py_expr and c == "}":
+      raise Exception("Unexpected end of python expr while not parsing a python expression:\n" + line)
+    elif is_py_expr and c == "}":
+      is_py_expr = False
+      if retain_curlies:
+        cur += c
+      result.append(cur)
+      cur = ""
+    elif is_py_expr:
+      cur += c
+    elif not is_string and c == "\"":
+      cur += "\""
+      is_string = True
+    elif is_string and c == "\"":
+      cur += "\""
+      is_string = False
+      result.append(cur)
+      cur = ""
+    elif is_string:
+      cur += c
+    elif c == " " or c == "\t":
+      result.append(cur)
+      cur = ""
+    else:
+      cur += c
+
+    idx += 1
+
+  if len(cur) > 0:
+    result.append(cur)
+
+  return [r for r in result if r != ""]
\ No newline at end of file