rawintent 4.0.0__py3-none-any.whl

rawintent/compiler/dataset.py

@@ -0,0 +1,336 @@
+from __future__ import annotations
+
+import hashlib
+import inspect
+import json
+import random
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Iterable, Iterator
+
+from ..english_python import EnglishPython
+from .rawlang import Instruction, Program
+
+
+@dataclass(slots=True)
+class TrainingExample:
+    source: str
+    target: str
+    action: str | None = None
+    family: str | None = None
+    kind: str = "single"
+    split: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        data = {"source": self.source, "target": self.target, "action": self.action}
+        if self.family is not None:
+            data["family"] = self.family
+        if self.kind:
+            data["kind"] = self.kind
+        if self.split is not None:
+            data["split"] = self.split
+        return data
+
+
+_SAMPLE_VALUES: dict[str, list[Any]] = {
+    "text": ["hello world", "RawIntent is useful", "The quick brown fox"],
+    "value": ["hello world", {"name": "Will", "active": True}, [1, 2, 3]],
+    "key": ["secret-key", "demo-key"],
+    "url": ["https://example.com", "https://httpbin.org/get"],
+    "path": ["notes.txt", "data/example.txt", "./README.md"],
+    "source": ["source.txt", "input.txt"], "destination": ["copy.txt", "output.txt"],
+    "name": ["HOME", "PATH", "DEMO_VALUE"], "pattern": [r"\\d+", r"[A-Za-z]+"],
+    "replacement": ["X", "word"], "algorithm": ["sha256", "sha512"],
+    "expression": ["2 + 3 * 4", "sqrt(81) + 1"], "count": [3, 5], "length": [16, 24, 32],
+    "low": [1, 10], "high": [10, 100], "minimum": [1, 10], "maximum": [10, 100],
+    "start": [1, 10], "stop": [10, 100], "number": [9, 25, 144],
+    "base": [2, 3, 10], "exponent": [2, 3, 4], "a": [12, 18, 48], "b": [8, 24, 36],
+    "timeout": [20.0], "items": [["red", "blue", "green"], [1, 2, 3, 4]],
+    "values": [[1, 2, 3, 4, 5], [10, 20, 30]], "choices": [["red", "blue", "green"], ["alpha", "beta", "gamma"]],
+    "width": [40, 72], "places": [2, 4], "numerator": [1, 3], "denominator": [2, 4],
+    "year": [2028, 2032], "month": [1, 9, 12], "size": [3, 5],
+}
+
+
+def _sample_for(name: str, rng: random.Random, action: str = "") -> Any:
+    if name == "value":
+        if action.startswith(("math.", "bisect.")): return rng.choice([4, 9, 16, 25, 81])
+        if action.startswith("json.") or action.startswith(("copy.", "format.pretty", "query.build")):
+            return rng.choice([{"name": "RawIntent", "active": True}, [1, 2, 3]])
+        return rng.choice(_SAMPLE_VALUES["value"])
+    if name == "text" and action == "config.parse": return "[app]\nname=rawintent\nenabled=true"
+    if name == "text" and action == "csv.parse": return "name,score\nAda,10\nGrace,9"
+    if name == "text" and action in {"gzip.decompress", "zlib.decompress"}:
+        return "H4sIAAAAAAAC/8tIzcnJBwCGphA2BQAAAA==" if action == "gzip.decompress" else "eJzLSM3JyQcABiwCFQ=="
+    if name == "groups": return [[1, 2], ["a", "b"]]
+    if name == "parts": return ["home", "user", "file.txt"]
+    if name == "word": return "appl"
+    if name == "steps": return 2
+    if name in _SAMPLE_VALUES: return rng.choice(_SAMPLE_VALUES[name])
+    if name.endswith("_text"): return rng.choice(_SAMPLE_VALUES["text"])
+    if "url" in name: return rng.choice(_SAMPLE_VALUES["url"])
+    if "path" in name or "file" in name: return rng.choice(_SAMPLE_VALUES["path"])
+    if name in {"x", "y", "n", "k"}: return rng.randint(2, 12)
+    return "example"
+
+
+def _arguments_for_intent(action: str, intent: Any, rng: random.Random) -> dict[str, Any]:
+    sig = inspect.signature(intent.handler)
+    args: dict[str, Any] = {}
+    declared = set(intent.entities)
+    for p in sig.parameters.values():
+        if p.kind in (p.VAR_POSITIONAL, p.VAR_KEYWORD): continue
+        if p.default is inspect.Parameter.empty or p.name in declared:
+            args[p.name] = _sample_for(p.name, rng, action)
+    return args
+
+
+def _english_value(name: str, value: Any) -> str:
+    if name == "url": return str(value)
+    if name in {"low", "minimum", "start"}: return f"minimum {value}"
+    if name in {"high", "maximum", "stop"}: return f"maximum {value}"
+    if name == "count": return f"count {value}"
+    if name == "length": return f"length {value}"
+    if name in {"values", "items", "choices"}: return "values " + json.dumps(value, ensure_ascii=False, separators=(",", ":"))
+    if isinstance(value, str): return json.dumps(value, ensure_ascii=False)
+    return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
+
+
+def _realize(base: str, args: dict[str, Any]) -> str:
+    chunks = [base]
+    for name, value in args.items():
+        if name == "original_text": continue
+        chunks.append(_english_value(name, value))
+    return " ".join(chunks)
+
+
+def _drop_char(text: str, rng: random.Random) -> str:
+    c = [i for i, ch in enumerate(text) if ch.isalpha()]
+    if len(c) < 6: return text
+    i = rng.choice(c); return text[:i] + text[i+1:]
+
+
+def _swap_chars(text: str, rng: random.Random) -> str:
+    c = [i for i in range(len(text)-1) if text[i].isalpha() and text[i+1].isalpha()]
+    if not c: return text
+    i = rng.choice(c); return text[:i] + text[i+1] + text[i] + text[i+2:]
+
+
+_REPL = {
+    "pick": ["choose", "grab", "select"], "generate": ["make", "create", "produce"],
+    "calculate": ["compute", "work out", "figure out"], "random": ["random", "arbitrary"],
+    "number": ["number", "integer", "num"], "text": ["text", "string"],
+    "file": ["file", "document"], "get": ["get", "grab", "fetch"], "make": ["make", "create", "build"],
+}
+
+
+def _synonymize(text: str, rng: random.Random) -> str:
+    words = text.split()
+    for i, w in enumerate(words):
+        key = re.sub(r"[^a-z]", "", w.lower())
+        if key in _REPL and rng.random() < .7:
+            rep = rng.choice(_REPL[key]); words[i] = rep + (w[-1] if w and not w[-1].isalnum() else "")
+    return " ".join(words)
+
+
+def augment(text: str, rng: random.Random, variant: int) -> str:
+    v = variant % 18
+    if v == 0: return text
+    if v == 1: return "please " + text
+    if v == 2: return text + " please"
+    if v == 3: return _drop_char(text, rng)
+    if v == 4: return _swap_chars(text, rng)
+    if v == 5: return _synonymize(text, rng)
+    if v == 6: return text.lower()
+    if v == 7: return text.upper()
+    if v == 8: return "hey can you " + _synonymize(text, rng)
+    if v == 9: return "yo can u " + _synonymize(text, rng).replace("number", "num")
+    if v == 10: return "could you maybe " + _synonymize(text, rng)
+    if v == 11: return re.sub(r"\s+", "  ", text)
+    if v == 12: return text.rstrip(".!?") + rng.choice(["?", "!", "..."])
+    if v == 13: return "i need you to " + _synonymize(text, rng)
+    if v == 14: return _drop_char(_swap_chars(text, rng), rng)
+    if v == 15: return _synonymize(text, rng).replace("characters", "chars").replace("through", "thru")
+    if v == 16: return "basically just " + _synonymize(text, rng)
+    return "can you just " + text + " for me"
+
+
+def _family(action: str, index: int, base: str) -> str:
+    digest = hashlib.sha1(base.encode("utf-8")).hexdigest()[:8]
+    return f"{action}:phrase:{index}:{digest}"
+
+
+def _target(action: str, intent: Any, rng: random.Random) -> tuple[str, dict[str, Any]] | None:
+    args = _arguments_for_intent(action, intent, rng)
+    if "original_text" in args: return None
+    return Program([Instruction(action, args)]).to_source(), args
+
+
+def generate_examples(engine: EnglishPython | None = None, *, variants_per_phrase: int = 6, samples_per_phrase: int = 2, seed: int = 7) -> Iterator[TrainingExample]:
+    engine = engine or EnglishPython(); rng = random.Random(seed)
+    for action, intent in sorted(engine.intents.items()):
+        if intent.handler is None or action == "python.call": continue
+        for idx, base in enumerate(intent.examples):
+            for _ in range(max(1, samples_per_phrase)):
+                made = _target(action, intent, rng)
+                if made is None: continue
+                target, args = made
+                for variant in range(max(1, variants_per_phrase)):
+                    # Noise is applied only to the instruction phrase, never to literal argument values.
+                    yield TrainingExample(_realize(augment(base, rng, variant), args), target, action, _family(action, idx, base), "single")
+    for x in _composition_examples():
+        yield x
+
+
+_COMPOSITIONS = [
+    ("random_sqrt", ["pick a random number from 1 to 10 then take its square root", "choose a number one through ten and give me the square root"], 'LET n = random.integer low=1 high=10\nLET root = math.sqrt value=$n\nRETURN $root'),
+    ("password_lower", ["make a secure 24 character password then lowercase it", "generate a 24 char secure password and turn it lowercase"], 'LET password = secrets.password length=24\nLET lower = text.lower text=$password\nRETURN $lower'),
+    ("uuid_upper", ["make a uuid then uppercase it", "generate a uuid and return it in uppercase"], 'LET id = uuid.v4\nLET upper = text.upper text=$id\nRETURN $upper'),
+    ("b64_roundtrip", ["base64 encode hello world then decode it", "encode hello world as base64 and decode the result"], 'LET encoded = base64.encode text="hello world"\nLET decoded = base64.decode text=$encoded\nRETURN $decoded'),
+    ("power_sqrt", ["raise 3 to the second power then take the square root", "square three and then get the square root"], 'LET p = math.power base=3 exponent=2\nLET root = math.sqrt value=$p\nRETURN $root'),
+    ("choice_upper", ["pick one of red blue green then uppercase it", "randomly choose red blue or green and make it uppercase"], 'LET picked = random.choice items=["red","blue","green"]\nLET upper = text.upper text=$picked\nRETURN $upper'),
+    ("token_upper", ["make a secure hex token then uppercase it", "generate a 16 byte secure hex token and capitalize it"], 'LET token = secrets.token_hex length=16\nLET upper = text.upper text=$token\nRETURN $upper'),
+    ("slug_upper", ["slugify hello world then uppercase the slug", "make hello world a url slug and uppercase it"], 'LET slug = text.slug text="hello world"\nLET upper = text.upper text=$slug\nRETURN $upper'),
+    ("hex_lower", ["hex encode hello world then lowercase it", "convert hello world to hex and make the result lowercase"], 'LET encoded = hex.encode text="hello world"\nLET lower = text.lower text=$encoded\nRETURN $lower'),
+    ("url_roundtrip", ["url encode hello world then decode it", "percent encode hello world and decode the result"], 'LET encoded = url.quote text="hello world"\nLET decoded = url.unquote text=$encoded\nRETURN $decoded'),
+    ("hash_upper", ["sha256 hash hello world then uppercase the hash", "hash hello world with sha256 and capitalize the digest"], 'LET digest = hash.digest text="hello world" algorithm="sha256"\nLET upper = text.upper text=$digest\nRETURN $upper'),
+    ("uuid_lower", ["make a uuid and lowercase it", "generate uuid4 then make the result lowercase"], 'LET id = uuid.v4\nLET lower = text.lower text=$id\nRETURN $lower'),
+]
+
+
+def _composition_examples() -> list[TrainingExample]:
+    out=[]
+    for fam, phrases, target in _COMPOSITIONS:
+        for p in phrases: out.append(TrainingExample(p, target, "composition", f"composition:{fam}", "composition"))
+    return out
+
+
+_UNKNOWN_FAMILIES = [
+    ("orbit", ["calculate the orbital decay rate of this satellite", "predict how fast this spacecraft orbit will decay"]),
+    ("image", ["generate a photorealistic picture of a dragon", "draw me a watercolor mountain image"]),
+    ("video", ["render a 3d animation in blender", "make a cinematic video from this prompt"]),
+    ("database", ["query my postgres users table", "run this sql against my production database"]),
+    ("slack", ["send this message to my slack channel", "post an update in slack"]),
+    ("email", ["email my manager that i will be late", "send an email to alex"]),
+    ("calendar", ["book a meeting with the whole team next friday", "invite sam to a calendar meeting"]),
+    ("browser", ["open chrome and click the login button", "use my browser to fill this form"]),
+    ("tensorflow", ["train a tensorflow image classifier", "build me a keras neural network"]),
+    ("audio", ["transcribe this audio recording", "remove background noise from my microphone file"]),
+    ("pdf", ["digitally sign this pdf", "merge these pdf pages into a booklet"]),
+    ("docker", ["start my docker compose stack", "restart the nginx docker container"]),
+    ("aws", ["deploy this lambda function to aws", "create an s3 bucket in my account"]),
+    ("git", ["rebase my branch onto main", "push these commits to github"]),
+    ("vision", ["tell me what is in this photograph", "read the handwritten note in this image"]),
+    ("music", ["compose a piano melody", "generate a midi chord progression"]),
+    ("maps", ["route me around traffic to boston", "find the fastest driving directions"]),
+    ("weather_live", ["tell me the live weather outside right now", "is it currently raining at my house"]),
+]
+
+
+def _emit_family(engine: EnglishPython, action: str, intent: Any, base: str, idx: int, split: str, variants: int, samples: int, rng: random.Random) -> list[TrainingExample]:
+    out=[]; fam=_family(action, idx, base)
+    for _ in range(max(1, samples)):
+        made=_target(action, intent, rng)
+        if made is None: continue
+        target,args=made
+        # Evaluation uses a different augmentation region to make wording/noise less train-like.
+        offset = 0 if split == "train" else 8
+        for v in range(max(1, variants)):
+            out.append(TrainingExample(_realize(augment(base, rng, offset+v), args), target, action, fam, "single", split))
+    return out
+
+
+def generate_generalization_splits(engine: EnglishPython | None = None, *, train_variants: int = 8, eval_variants: int = 6, samples_per_phrase: int = 3, seed: int = 7, include_unknown: bool = True, include_compositions: bool = True) -> dict[str, list[TrainingExample]]:
+    """Create leakage-resistant train/validation/test sets.
+
+    Phrase families are assigned to a split *before* augmentation. Thus a held-out base phrase and all
+    of its noisy variants never occur in training. Composition families and unknown-capability concepts
+    are also split as whole families.
+    """
+    engine = engine or EnglishPython(); rng=random.Random(seed)
+    splits={"train":[], "validation":[], "test":[]}
+    for action,intent in sorted(engine.intents.items()):
+        if intent.handler is None or action == "python.call": continue
+        indices=list(range(len(intent.examples)))
+        random.Random(f"{seed}:{action}").shuffle(indices)
+        assign={i:"train" for i in indices}
+        if len(indices)>=2: assign[indices[-1]]="test"
+        if len(indices)>=3: assign[indices[-2]]="validation"
+        for i,base in enumerate(intent.examples):
+            split=assign[i]; variants=train_variants if split=="train" else eval_variants
+            splits[split].extend(_emit_family(engine, action, intent, base, i, split, variants, samples_per_phrase, rng))
+
+    if include_compositions:
+        families=list(_COMPOSITIONS); random.Random(seed+101).shuffle(families)
+        n=len(families); n_test=max(2, n//5); n_val=max(1, n//6)
+        test_names={x[0] for x in families[:n_test]}; val_names={x[0] for x in families[n_test:n_test+n_val]}
+        for fam,phrases,target in _COMPOSITIONS:
+            split="test" if fam in test_names else "validation" if fam in val_names else "train"
+            variants=train_variants if split=="train" else eval_variants
+            for phrase in phrases:
+                for v in range(max(1,variants)):
+                    splits[split].append(TrainingExample(augment(phrase,rng,(0 if split=="train" else 8)+v),target,"composition",f"composition:{fam}","composition",split))
+
+    if include_unknown:
+        unknown=list(_UNKNOWN_FAMILIES); random.Random(seed+202).shuffle(unknown)
+        n=len(unknown); n_test=max(3,n//5); n_val=max(2,n//6)
+        test_names={x[0] for x in unknown[:n_test]}; val_names={x[0] for x in unknown[n_test:n_test+n_val]}
+        for fam,phrases in _UNKNOWN_FAMILIES:
+            split="test" if fam in test_names else "validation" if fam in val_names else "train"
+            variants=train_variants if split=="train" else eval_variants
+            for phrase in phrases:
+                for v in range(max(1,variants)):
+                    splits[split].append(TrainingExample(augment(phrase,rng,(0 if split=="train" else 8)+v),"UNKNOWN_CAPABILITY","unknown",f"unknown:{fam}","unknown",split))
+
+    # Hard guarantee against family leakage.
+    fams={k:{x.family for x in v if x.family} for k,v in splits.items()}
+    if fams["train"] & fams["validation"] or fams["train"] & fams["test"] or fams["validation"] & fams["test"]:
+        raise RuntimeError("Generalization split leaked a phrase family across datasets.")
+    for k in splits: random.Random(f"{seed}:{k}:shuffle").shuffle(splits[k])
+    return splits
+
+
+def write_generalization_dataset(directory: str | Path, engine: EnglishPython | None = None, **kwargs: Any) -> dict[str, Any]:
+    directory=Path(directory); directory.mkdir(parents=True,exist_ok=True)
+    engine = engine or EnglishPython()
+    splits=generate_generalization_splits(engine, **kwargs)
+    source_sets = {name: {x.source for x in items} for name, items in splits.items()}
+    if source_sets["train"] & source_sets["validation"] or source_sets["train"] & source_sets["test"] or source_sets["validation"] & source_sets["test"]:
+        raise RuntimeError("Generalization split leaked an exact source sentence across datasets.")
+    manifest={
+        "format":"rawintent-generalization-v1",
+        "generation": dict(kwargs),
+        "registered_actions": len([i for i in engine.intents.values() if i.handler is not None and i.name != "python.call"]),
+        "counts":{},"families":{},"kinds":{},
+        "exact_source_overlap": {"train_validation": 0, "train_test": 0, "validation_test": 0},
+    }
+    for split,items in splits.items():
+        write_jsonl(directory/f"{split}.jsonl", items)
+        manifest["counts"][split]=len(items)
+        manifest["families"][split]=len({x.family for x in items if x.family})
+        kinds: dict[str,int]={}
+        for x in items: kinds[x.kind]=kinds.get(x.kind,0)+1
+        manifest["kinds"][split]=kinds
+    (directory/"manifest.json").write_text(json.dumps(manifest,indent=2),encoding="utf-8")
+    return manifest
+
+
+def write_jsonl(path: str | Path, examples: Iterable[TrainingExample]) -> int:
+    path=Path(path); path.parent.mkdir(parents=True,exist_ok=True); count=0
+    with path.open("w",encoding="utf-8") as fh:
+        for item in examples:
+            fh.write(json.dumps(item.to_dict(),ensure_ascii=False)+"\n"); count+=1
+    return count
+
+
+def read_jsonl(path: str | Path) -> list[TrainingExample]:
+    out=[]
+    with Path(path).open("r",encoding="utf-8") as fh:
+        for line_no,line in enumerate(fh,1):
+            if not line.strip(): continue
+            data=json.loads(line)
+            if not isinstance(data.get("source"),str) or not isinstance(data.get("target"),str):
+                raise ValueError(f"Line {line_no} needs string source and target fields.")
+            out.append(TrainingExample(data["source"],data["target"],data.get("action"),data.get("family"),data.get("kind","single"),data.get("split")))
+    return out

rawintent/compiler/deterministic.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from typing import Any
+
+from ..english_python import EnglishPython
+from .rawlang import Instruction, Program
+
+
+class DeterministicCompiler:
+    """Immediate compiler for commands already understood by RawIntent's rule engine."""
+
+    def __init__(self, engine: EnglishPython | None = None, **engine_options: Any) -> None:
+        self.engine = engine or EnglishPython(**engine_options)
+
+    def compile_program(self, text: str) -> Program:
+        matches = self.engine.parse_many(text)
+        statements: list[Instruction] = []
+        for match in matches:
+            if match.intent is None:
+                raise ValueError(match.message or "I couldn't understand that request.")
+            if match.negated:
+                raise ValueError(match.message or "That request says not to perform the action.")
+            if match.missing:
+                raise ValueError(match.message or f"The request is missing: {', '.join(match.missing)}")
+            statements.append(Instruction(match.intent, dict(match.entities)))
+        return Program(statements)
+
+    def compile(self, text: str) -> str:
+        return self.compile_program(text).to_source()

rawintent/compiler/evaluation.py

@@ -0,0 +1,169 @@
+from __future__ import annotations
+
+from dataclasses import asdict
+from pathlib import Path
+from typing import Any, Callable
+
+from .dataset import TrainingExample, read_jsonl
+from .deterministic import DeterministicCompiler
+from .rawlang import Instruction, Program, Reference, ReturnStatement, UnknownCapabilityStatement, parse_program
+
+
+def _value_key(value: Any) -> Any:
+    if isinstance(value, Reference):
+        return ("ref", value.root, tuple(value.path))
+    if isinstance(value, list):
+        return tuple(_value_key(v) for v in value)
+    if isinstance(value, tuple):
+        return tuple(_value_key(v) for v in value)
+    if isinstance(value, dict):
+        return tuple(sorted((str(k), _value_key(v)) for k, v in value.items()))
+    return value
+
+
+def program_signature(program: Program) -> tuple[Any, ...]:
+    out: list[Any] = []
+    for stmt in program.statements:
+        if isinstance(stmt, UnknownCapabilityStatement):
+            out.append(("unknown",))
+        elif isinstance(stmt, ReturnStatement):
+            out.append(("return", _value_key(stmt.value)))
+        else:
+            out.append(("instruction", stmt.assign, stmt.action, tuple(sorted((k, _value_key(v)) for k, v in stmt.arguments.items()))))
+    return tuple(out)
+
+
+def action_signature(program: Program) -> tuple[str, ...]:
+    actions=[]
+    for stmt in program.statements:
+        if isinstance(stmt, UnknownCapabilityStatement): actions.append("UNKNOWN_CAPABILITY")
+        elif isinstance(stmt, Instruction): actions.append(stmt.action)
+    return tuple(actions)
+
+
+def _safe_program(source: str) -> tuple[Program | None, str | None]:
+    try:
+        return parse_program(source), None
+    except Exception as exc:
+        return None, str(exc)
+
+
+def _is_unknown(program: Program | None) -> bool:
+    return bool(program and len(program.statements) == 1 and isinstance(program.statements[0], UnknownCapabilityStatement))
+
+
+def _rate(n: int, d: int) -> float:
+    return n / d if d else 0.0
+
+
+def evaluate_examples(
+    examples: list[TrainingExample],
+    compile_fn: Callable[[str], str],
+    *,
+    max_examples: int | None = None,
+    failure_limit: int = 25,
+) -> dict[str, Any]:
+    if max_examples is not None:
+        examples = examples[:max_examples]
+    totals = {"count": 0, "syntax_valid": 0, "exact": 0, "semantic": 0, "action": 0}
+    confusion = {"unknown_tp": 0, "unknown_fp": 0, "unknown_fn": 0, "unknown_tn": 0}
+    by_kind: dict[str, dict[str, int]] = {}
+    failures: list[dict[str, Any]] = []
+
+    for item in examples:
+        totals["count"] += 1
+        kind = item.kind or "single"
+        bucket = by_kind.setdefault(kind, {"count": 0, "syntax_valid": 0, "exact": 0, "semantic": 0, "action": 0})
+        bucket["count"] += 1
+        try:
+            predicted = compile_fn(item.source).strip()
+            compile_error = None
+        except Exception as exc:
+            predicted = ""
+            compile_error = str(exc)
+
+        target = item.target.strip()
+        target_program, target_error = _safe_program(target)
+        if target_error:
+            raise ValueError(f"Invalid benchmark target for {item.source!r}: {target_error}")
+        predicted_program, syntax_error = _safe_program(predicted) if predicted else (None, compile_error or "empty output")
+
+        expected_unknown = _is_unknown(target_program)
+        predicted_unknown = _is_unknown(predicted_program)
+        if expected_unknown and predicted_unknown: confusion["unknown_tp"] += 1
+        elif not expected_unknown and predicted_unknown: confusion["unknown_fp"] += 1
+        elif expected_unknown and not predicted_unknown: confusion["unknown_fn"] += 1
+        else: confusion["unknown_tn"] += 1
+
+        exact = predicted == target
+        syntax_valid = predicted_program is not None
+        semantic = bool(syntax_valid and program_signature(predicted_program) == program_signature(target_program))
+        action_match = bool(syntax_valid and action_signature(predicted_program) == action_signature(target_program))
+        for name, yes in (("syntax_valid", syntax_valid), ("exact", exact), ("semantic", semantic), ("action", action_match)):
+            if yes:
+                totals[name] += 1
+                bucket[name] += 1
+
+        if not semantic and len(failures) < failure_limit:
+            failures.append({
+                "source": item.source,
+                "target": target,
+                "prediction": predicted,
+                "kind": kind,
+                "family": item.family,
+                "syntax_error": syntax_error,
+            })
+
+    n = totals["count"]
+    tp, fp, fn = confusion["unknown_tp"], confusion["unknown_fp"], confusion["unknown_fn"]
+    precision = _rate(tp, tp + fp); recall = _rate(tp, tp + fn)
+    f1 = (2 * precision * recall / (precision + recall)) if precision + recall else 0.0
+    result: dict[str, Any] = {
+        "count": n,
+        "syntax_validity": _rate(totals["syntax_valid"], n),
+        "exact_match": _rate(totals["exact"], n),
+        "semantic_match": _rate(totals["semantic"], n),
+        "action_accuracy": _rate(totals["action"], n),
+        "unknown": {**confusion, "precision": precision, "recall": recall, "f1": f1},
+        "by_kind": {},
+        "failures": failures,
+    }
+    for kind, b in sorted(by_kind.items()):
+        d=b["count"]
+        result["by_kind"][kind]={
+            "count": d,
+            "syntax_validity": _rate(b["syntax_valid"], d),
+            "exact_match": _rate(b["exact"], d),
+            "semantic_match": _rate(b["semantic"], d),
+            "action_accuracy": _rate(b["action"], d),
+        }
+    return result
+
+
+def evaluate_dataset(
+    data_path: str | Path,
+    *,
+    checkpoint: str | Path | None = None,
+    device: str = "cpu",
+    deterministic: bool = False,
+    max_examples: int | None = None,
+    failure_limit: int = 25,
+) -> dict[str, Any]:
+    examples = read_jsonl(data_path)
+    if deterministic:
+        compiler = DeterministicCompiler()
+        compile_fn = compiler.compile
+        mode = "deterministic"
+    else:
+        if checkpoint is None:
+            raise ValueError("A neural checkpoint is required unless deterministic=True.")
+        from .compiler import NeuralCompiler
+        compiler = NeuralCompiler(checkpoint, device=device)
+        # Do not syntax-validate here; syntax validity is one of the benchmark metrics.
+        compile_fn = lambda text: compiler.compile(text, validate_syntax=False)
+        mode = "neural"
+    metrics = evaluate_examples(examples, compile_fn, max_examples=max_examples, failure_limit=failure_limit)
+    metrics["mode"] = mode
+    metrics["data"] = str(data_path)
+    if checkpoint is not None: metrics["checkpoint"] = str(checkpoint)
+    return metrics

rawintent/compiler/executor.py

@@ -0,0 +1,130 @@
+from __future__ import annotations
+
+import inspect
+from dataclasses import dataclass, field
+from typing import Any
+
+from ..english_python import EnglishPython
+from ..errors import ConfigurationError, english_exception
+from ..models import EnglishIssue
+from .rawlang import Instruction, Program, RawLangSyntaxError, Reference, ReturnStatement, UnknownCapabilityStatement, ensure_program
+
+
+@dataclass(slots=True)
+class ExecutionResult:
+    ok: bool
+    value: Any = None
+    variables: dict[str, Any] = field(default_factory=dict)
+    outputs: list[Any] = field(default_factory=list)
+    error: EnglishIssue | None = None
+    failed_line: int | None = None
+
+    @property
+    def message(self) -> str | None:
+        return self.error.message if self.error else None
+
+
+class RawLangExecutor:
+    """Validated RawLang executor backed by an :class:`EnglishPython` capability registry."""
+
+    def __init__(self, engine: EnglishPython | None = None, **engine_options: Any) -> None:
+        self.engine = engine or EnglishPython(**engine_options)
+
+    def schema(self) -> dict[str, dict[str, Any]]:
+        out: dict[str, dict[str, Any]] = {}
+        for name, intent in sorted(self.engine.intents.items()):
+            if intent.handler is None:
+                continue
+            sig = inspect.signature(intent.handler)
+            params: dict[str, Any] = {}
+            for p in sig.parameters.values():
+                if p.kind in (p.VAR_POSITIONAL, p.VAR_KEYWORD):
+                    continue
+                params[p.name] = {
+                    "required": p.default is inspect.Parameter.empty,
+                    "default": None if p.default is inspect.Parameter.empty else p.default,
+                    "annotation": None if p.annotation is inspect.Parameter.empty else str(p.annotation),
+                }
+            out[name] = {
+                "description": intent.description,
+                "parameters": params,
+            }
+        return out
+
+    def _resolve(self, value: Any, variables: dict[str, Any]) -> Any:
+        if isinstance(value, Reference):
+            if value.root not in variables:
+                raise ValueError(f"RawLang variable {value.root!r} has not been created yet.")
+            current = variables[value.root]
+            for part in value.path:
+                if isinstance(part, int):
+                    current = current[part]
+                elif isinstance(current, dict):
+                    if part not in current:
+                        raise KeyError(part)
+                    current = current[part]
+                else:
+                    current = getattr(current, part)
+            return current
+        if isinstance(value, list):
+            return [self._resolve(v, variables) for v in value]
+        if isinstance(value, tuple):
+            return tuple(self._resolve(v, variables) for v in value)
+        if isinstance(value, dict):
+            return {k: self._resolve(v, variables) for k, v in value.items()}
+        return value
+
+    def _call(self, stmt: Instruction, variables: dict[str, Any]) -> Any:
+        if stmt.action not in self.engine.intents:
+            raise ConfigurationError(
+                f"RawLang action {stmt.action!r} is not registered. Use the schema command to see available actions."
+            )
+        intent = self.engine.intents[stmt.action]
+        if intent.handler is None:
+            raise ConfigurationError(f"RawLang action {stmt.action!r} does not have an executable handler.")
+        kwargs = {k: self._resolve(v, variables) for k, v in stmt.arguments.items()}
+        sig = inspect.signature(intent.handler)
+        try:
+            bound = sig.bind(**kwargs)
+        except TypeError as exc:
+            raise TypeError(f"{stmt.action}: {exc}") from exc
+        bound.apply_defaults()
+        return intent.handler(*bound.args, **bound.kwargs)
+
+    def execute(self, program: str | Program) -> ExecutionResult:
+        try:
+            parsed = ensure_program(program)
+        except RawLangSyntaxError as exc:
+            return ExecutionResult(False, error=EnglishIssue("INVALID_RAWLANG", str(exc)))
+        except Exception as exc:
+            return ExecutionResult(False, error=english_exception(exc))
+
+        variables: dict[str, Any] = {}
+        outputs: list[Any] = []
+        last: Any = None
+        for stmt in parsed.statements:
+            try:
+                if isinstance(stmt, UnknownCapabilityStatement):
+                    return ExecutionResult(
+                        False,
+                        value=last,
+                        variables=variables,
+                        outputs=outputs,
+                        error=EnglishIssue(
+                            "UNKNOWN_CAPABILITY",
+                            "I understand the request, but this RawIntent runtime does not have a registered capability for it yet.",
+                        ),
+                        failed_line=stmt.line,
+                    )
+                if isinstance(stmt, ReturnStatement):
+                    value = self._resolve(stmt.value, variables)
+                    return ExecutionResult(True, value=value, variables=variables, outputs=outputs)
+                value = self._call(stmt, variables)
+                last = value
+                outputs.append(value)
+                if stmt.assign:
+                    variables[stmt.assign] = value
+            except Exception as exc:
+                issue = english_exception(exc, intent=stmt.action if isinstance(stmt, Instruction) else None)
+                return ExecutionResult(False, value=last, variables=variables, outputs=outputs, error=issue, failed_line=stmt.line)
+        return ExecutionResult(True, value=last, variables=variables, outputs=outputs)