scripts/test-protocol-processor.py

#!/usr/bin/env python3
"""Test harness for the procedural dictation protocol processor.

Tests the deterministic symbol/number/casing engine that converts
dictated protocol words into actual syntax — no LLM needed.

Covers:
  - Symbol mapping: "dash" → "-", "dot" → ".", "slash" → "/", etc.
  - Two-word symbols: "open paren" → "(", "dash dash" → "--", etc.
  - Three-word symbols: "two redirect ampersand" → "2>&"
  - Number words: "eight zero eight zero" → "8080", "forty two" → "42"
  - Casing: "capital A" → "A", "all caps POST" → "POST"
  - Casing directives: "camel case get user" → "getUser"
  - Spacing: "space" as explicit token separator
  - Full commands: git, docker, curl, ssh, etc.

Usage:
  python3 test-protocol-processor.py                    # run full suite
  python3 test-protocol-processor.py --category git     # filter by category
  python3 test-protocol-processor.py --verbose          # show all cases, not just errors
  python3 test-protocol-processor.py --unit             # run unit tests only (no eval file)
  python3 test-protocol-processor.py --eval FILE        # use custom eval file
"""

import json
import re
import sys
import time
from pathlib import Path
from collections import defaultdict

# Import processor
sys.path.insert(0, str(Path(__file__).parent))
from importlib import import_module
_proc = import_module("procedural-processor")
process_dictation = _proc.process_dictation
consume_number = _proc.consume_number
consume_casing = _proc.consume_casing
SYMBOLS = _proc.SYMBOLS
TWO_WORD_SYMBOLS = _proc.TWO_WORD_SYMBOLS
THREE_WORD_SYMBOLS = _proc.THREE_WORD_SYMBOLS


# ── Colors ──────────────────────────────────────────────────────────────

GREEN = "\033[92m"
RED = "\033[91m"
YELLOW = "\033[93m"
DIM = "\033[2m"
BOLD = "\033[1m"
RESET = "\033[0m"


# ── Unit Tests ──────────────────────────────────────────────────────────
# These test individual features of the processor in isolation.

UNIT_TESTS = {
    "symbols_basic": [
        ("dash", "-"),
        ("dot", "."),
        ("slash", "/"),
        ("pipe", "|"),
        ("tilde", "~"),
        ("hash", "#"),
        ("at", "@"),
        ("dollar", "$"),
        ("percent", "%"),
        ("caret", "^"),
        ("ampersand", "&"),
        ("equals", "="),
        ("plus", "+"),
        ("colon", ":"),
        ("semicolon", ";"),
        ("underscore", "_"),
        ("comma", ","),
        ("backslash", "\\"),
        ("quote", '"'),
        ("backtick", "`"),
        ("star", "*"),
        ("bang", "!"),
        ("redirect", ">"),
    ],
    "symbols_synonyms": [
        ("minus", "-"),
        ("hyphen", "-"),
        ("period", "."),
        ("asterisk", "*"),
        ("hashtag", "#"),
    ],
    "two_word_symbols": [
        ("single quote", "'"),
        ("open paren", "("),
        ("close paren", ")"),
        ("open brace", "{"),
        ("close brace", "}"),
        ("open bracket", "["),
        ("close bracket", "]"),
        ("open angle", "<"),
        ("close angle", ">"),
        ("open curly", "{"),
        ("close curly", "}"),
        ("less than", "<"),
        ("question mark", "?"),
        ("dash dash", "--"),
        ("double dash", "--"),
        ("and and", "&&"),
        ("pipe pipe", "||"),
        ("dot dot", ".."),
        ("forward slash", "/"),
        ("back slash", "\\"),
        ("equals sign", "="),
        ("at sign", "@"),
        ("dollar sign", "$"),
        ("open parenthesis", "("),
        ("close parenthesis", ")"),
        ("new line", "\n"),
    ],
    "three_word_symbols": [
        ("two redirect ampersand", "2>&"),
    ],
    "numbers_single": [
        ("zero", "0"),
        ("one", "1"),
        ("five", "5"),
        ("nine", "9"),
        ("ten", "10"),
        ("twelve", "12"),
        ("nineteen", "19"),
    ],
    "numbers_compound": [
        ("twenty", "20"),
        ("forty two", "42"),
        ("ninety nine", "99"),
    ],
    "numbers_multiplier": [
        ("one hundred", "100"),
        ("three thousand", "3000"),
        ("five hundred", "500"),
    ],
    "numbers_digit_sequence": [
        ("one nine two", "192"),
        ("eight zero eight zero", "8080"),
        ("one two seven dot zero dot zero dot one", "127.0.0.1"),
        ("four four three", "443"),
    ],
    "casing_capital": [
        ("capital a", "A"),
        ("capital x", "X"),
        ("capital hello", "Hello"),
    ],
    "casing_all_caps": [
        ("all caps post", "POST"),
        ("all caps get", "GET"),
        ("all caps node", "NODE"),
    ],
    "casing_directives": [
        ("camel case get user profile", "getUserProfile"),
        ("snake case api key", "api_key"),
        ("pascal case my component", "MyComponent"),
        ("kebab case my component", "my-component"),
        ("screaming case max retries", "MAX_RETRIES"),
    ],
    "space_handling": [
        ("git space push", "git push"),
        ("ls space dash la", "ls -la"),
        ("a space b space c", "a b c"),
    ],
    "mixed_basic": [
        ("git space commit space dash m space quote hello quote", 'git commit -m "hello"'),
        ("echo space dollar all caps home", "echo $HOME"),
        ("cd space tilde slash projects", "cd ~/projects"),
        ("chmod space seven five five space script dot sh", "chmod 755 script.sh"),
    ],
    "append_redirect": [
        ("echo space hello space append space log dot txt", "echo hello >> log.txt"),
        ("cat space file space redirect space output", "cat file > output"),
    ],
}


def run_unit_tests(verbose=False):
    """Run isolated unit tests for each processor feature."""
    print(f"\n{BOLD}UNIT TESTS{RESET}")
    print("=" * 70)

    total = 0
    passed = 0
    failed_cases = []

    for group_name, cases in UNIT_TESTS.items():
        group_pass = 0
        group_total = len(cases)

        for dictated, expected in cases:
            total += 1
            got = process_dictation(dictated)

            # Normalize whitespace for comparison
            got_norm = re.sub(r'\s+', ' ', got.strip())
            exp_norm = re.sub(r'\s+', ' ', expected.strip())

            if got_norm == exp_norm:
                group_pass += 1
                passed += 1
                if verbose:
                    print(f"  {GREEN}PASS{RESET} [{group_name}] {dictated!r} → {got!r}")
            else:
                failed_cases.append((group_name, dictated, expected, got))
                print(f"  {RED}FAIL{RESET} [{group_name}] {dictated!r}")
                print(f"         expected: {expected!r}")
                print(f"         got:      {got!r}")

        if not verbose and group_pass == group_total:
            print(f"  {GREEN}PASS{RESET} {group_name}: {group_pass}/{group_total}")

    print(f"\n  Total: {passed}/{total} passed", end="")
    if failed_cases:
        print(f", {RED}{len(failed_cases)} failed{RESET}")
    else:
        print(f" {GREEN}(all pass){RESET}")

    return passed, total, failed_cases


def run_eval_tests(eval_file, category=None, verbose=False):
    """Run the processor against the eval dataset."""
    print(f"\n{BOLD}EVAL TESTS{RESET} — {eval_file}")
    print("=" * 70)

    with open(eval_file) as f:
        data = json.load(f)

    if category:
        data = [d for d in data if d.get("category", "") == category]
        print(f"  Filtered to category={category}: {len(data)} cases")

    total = len(data)
    exact = 0
    ws_match = 0
    ws_case_match = 0
    failed_cases = []
    by_category = defaultdict(lambda: {"exact": 0, "total": 0})
    timings = []

    for d in data:
        dictated = d["dictated"]
        expected = d["expected"]
        cat = d.get("category", "unknown")

        t0 = time.perf_counter()
        got = process_dictation(dictated)
        elapsed_us = (time.perf_counter() - t0) * 1_000_000
        timings.append(elapsed_us)

        got_ws = re.sub(r'\s+', ' ', got.strip())
        exp_ws = re.sub(r'\s+', ' ', expected.strip())

        is_exact = got == expected
        is_ws = got_ws == exp_ws
        is_wsc = got_ws.lower() == exp_ws.lower()

        by_category[cat]["total"] += 1

        if is_exact:
            exact += 1
            by_category[cat]["exact"] += 1
            if verbose:
                print(f"  {GREEN}PASS{RESET} [{cat}] {dictated[:60]}")
        else:
            failed_cases.append({
                "category": cat,
                "dictated": dictated,
                "expected": expected,
                "got": got,
                "ws_match": is_ws,
                "wsc_match": is_wsc,
            })
            if is_ws:
                ws_match += 1
                if verbose:
                    print(f"  {YELLOW}WS{RESET}   [{cat}] whitespace diff only")
            elif is_wsc:
                ws_case_match += 1

        if is_ws:
            ws_match += (1 if is_exact else 0)  # already counted
        if is_wsc:
            ws_case_match += (1 if is_exact else 0)

    # Recount properly
    ws_match = sum(1 for d in data if re.sub(r'\s+', ' ', process_dictation(d["dictated"]).strip()) == re.sub(r'\s+', ' ', d["expected"].strip()))
    ws_case_match = sum(1 for d in data if re.sub(r'\s+', ' ', process_dictation(d["dictated"]).strip()).lower() == re.sub(r'\s+', ' ', d["expected"].strip()).lower())

    # Summary
    print(f"\n  {BOLD}ACCURACY{RESET}")
    print(f"    Exact match:      {exact}/{total} ({exact/total*100:.1f}%)")
    print(f"    Whitespace-norm:  {ws_match}/{total} ({ws_match/total*100:.1f}%)")
    print(f"    WS+case-norm:     {ws_case_match}/{total} ({ws_case_match/total*100:.1f}%)")

    # Timing
    if timings:
        timings.sort()
        median = timings[len(timings)//2]
        p95 = timings[int(len(timings) * 0.95)]
        p99 = timings[int(len(timings) * 0.99)]
        print(f"\n  {BOLD}TIMING{RESET} (per command)")
        print(f"    Median: {median:.0f}us  P95: {p95:.0f}us  P99: {p99:.0f}us")

    # Per category
    if len(by_category) > 1:
        print(f"\n  {BOLD}BY CATEGORY{RESET}")
        for cat in sorted(by_category.keys()):
            r = by_category[cat]
            pct = r["exact"] / r["total"] * 100 if r["total"] else 0
            bar = GREEN if pct == 100 else (YELLOW if pct >= 80 else RED)
            print(f"    {cat:>14}: {bar}{r['exact']}/{r['total']}{RESET} ({pct:.0f}%)")

    # Errors
    if failed_cases:
        print(f"\n  {BOLD}FAILURES{RESET} ({len(failed_cases)}):")
        print("  " + "-" * 66)
        for e in failed_cases[:20]:
            ws_note = f" {DIM}(whitespace only){RESET}" if e["ws_match"] else ""
            wsc_note = f" {DIM}(case only){RESET}" if e["wsc_match"] and not e["ws_match"] else ""
            print(f"    [{e['category']}]{ws_note}{wsc_note}")
            print(f"      input:    {e['dictated'][:70]}")
            print(f"      expected: {e['expected'][:70]}")
            print(f"      got:      {e['got'][:70]}")

            # Show character-level diff for near-misses
            exp, got = e["expected"], e["got"]
            if len(exp) < 80 and len(got) < 80:
                diff_pos = next((i for i in range(min(len(exp), len(got))) if exp[i] != got[i]), min(len(exp), len(got)))
                if diff_pos < len(exp) or diff_pos < len(got):
                    print(f"      diff at {diff_pos}: expected {exp[diff_pos:diff_pos+10]!r} got {got[diff_pos:diff_pos+10]!r}")
            print()

        if len(failed_cases) > 20:
            print(f"    ... and {len(failed_cases) - 20} more")

    return exact, total, failed_cases


def main():
    import argparse
    parser = argparse.ArgumentParser(description="Test the procedural dictation processor")
    parser.add_argument("--unit", action="store_true", help="Run unit tests only")
    parser.add_argument("--eval", type=str, default=None, help="Eval JSON file (default: eval-independent.json)")
    parser.add_argument("--category", type=str, help="Filter eval cases by category")
    parser.add_argument("--verbose", "-v", action="store_true", help="Show all cases, not just failures")
    args = parser.parse_args()

    print("=" * 70)
    print(f"  PROTOCOL PROCESSOR TEST HARNESS")
    print("=" * 70)

    # Always run unit tests
    unit_passed, unit_total, unit_failures = run_unit_tests(verbose=args.verbose)

    eval_passed = eval_total = 0
    eval_failures = []

    if not args.unit:
        eval_file = args.eval or str(Path(__file__).parent / "eval-independent.json")
        if Path(eval_file).exists():
            eval_passed, eval_total, eval_failures = run_eval_tests(
                eval_file, category=args.category, verbose=args.verbose
            )
        else:
            print(f"\n  {YELLOW}SKIP{RESET} eval tests — {eval_file} not found")

    # Grand summary
    print(f"\n{'=' * 70}")
    print(f"  {BOLD}GRAND SUMMARY{RESET}")
    print(f"{'=' * 70}")
    print(f"  Unit tests:  {unit_passed}/{unit_total}", end="")
    print(f" {GREEN}PASS{RESET}" if not unit_failures else f" {RED}{len(unit_failures)} FAIL{RESET}")

    if eval_total:
        print(f"  Eval tests:  {eval_passed}/{eval_total} exact ({eval_passed/eval_total*100:.1f}%)", end="")
        print(f" {GREEN}PASS{RESET}" if not eval_failures else f" — {YELLOW}{len(eval_failures)} diff{RESET}")

    total_pass = unit_passed + eval_passed
    total_all = unit_total + eval_total
    print(f"  Combined:    {total_pass}/{total_all} ({total_pass/total_all*100:.1f}%)")

    sys.exit(0 if not unit_failures else 1)


if __name__ == "__main__":
    main()