rawintent 4.0.0__py3-none-any.whl

rawintent/__init__.py

@@ -0,0 +1,42 @@
+from .core import EnglishRouter, RawIntent
+from .conversation import Conversation
+from .entities import slot
+from .english_python import EnglishPython, DEFAULT_PACKS, PACK_ALIASES
+from .errors import (
+    AmbiguousRequestError,
+    CapabilityDisabledError,
+    ConfigurationError,
+    HandlerError,
+    MissingArgumentError,
+    MissingDependencyError,
+    NegatedRequestError,
+    RawIntentError,
+    UnknownRequestError,
+    ValidationError,
+    english_exception,
+)
+from .models import EnglishIssue, Intent, Match, RunResult, Slot
+from .packs import PackInfo
+from .text import normalize, normalized_text
+
+__all__ = [
+    "RawIntent", "EnglishRouter", "EnglishPython", "DEFAULT_PACKS", "PACK_ALIASES", "PackInfo",
+    "Conversation", "slot", "Slot", "Intent", "Match", "RunResult", "EnglishIssue",
+    "normalize", "normalized_text", "english_exception", "RawIntentError", "ConfigurationError",
+    "UnknownRequestError", "AmbiguousRequestError", "MissingArgumentError", "ValidationError",
+    "NegatedRequestError", "HandlerError", "CapabilityDisabledError", "MissingDependencyError",
+]
+
+__version__ = "4.0.0"
+
+# RawLang/compiler exports are dependency-free unless a neural checkpoint/model is requested.
+from .compiler import (
+    ByteTokenizer, DeterministicCompiler, ExecutionResult, Instruction, NeuralCompiler,
+    Program, RawIntentLanguage, RawLangExecutor, RawLangSyntaxError, Reference,
+    ReturnStatement, UnknownCapabilityStatement, parse_program,
+)
+__all__ += [
+    "RawIntentLanguage", "NeuralCompiler", "DeterministicCompiler", "RawLangExecutor",
+    "ExecutionResult", "Program", "Instruction", "ReturnStatement", "UnknownCapabilityStatement", "Reference",
+    "RawLangSyntaxError", "parse_program", "ByteTokenizer",
+]

rawintent/__main__.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from .english_python import EnglishPython
+
+
+def main():
+    engine = EnglishPython()
+    print("RawIntent EnglishPython demo. Type an English command, 'packs', 'actions', or 'quit'.")
+    print("Examples: random number between 1 and 10 | sha256 hash \"hello\" | calculate 2 + 3 * 4")
+    while True:
+        try:
+            text = input("> ").strip()
+        except (EOFError, KeyboardInterrupt):
+            print()
+            break
+        if text.lower() in {"quit", "exit"}:
+            break
+        if text.lower() == "packs":
+            for item in engine.pack_info():
+                marker = "loaded" if item["loaded"] else "available"
+                print(f"{item['name']}: {marker} — {item['description']}")
+            continue
+        if text.lower() == "actions":
+            for name in engine.capabilities():
+                print(name)
+            continue
+        result = engine.run_safe(text)
+        if result.ok:
+            print(result.value)
+        else:
+            print(f"{result.error.code}: {result.error.message}")
+
+
+if __name__ == "__main__":
+    main()

rawintent/compiler/__init__.py

@@ -0,0 +1,15 @@
+from .compiler import NeuralCompiler, RawIntentLanguage
+from .dataset import TrainingExample, generate_examples, generate_generalization_splits, read_jsonl, write_generalization_dataset, write_jsonl
+from .deterministic import DeterministicCompiler
+from .executor import ExecutionResult, RawLangExecutor
+from .evaluation import evaluate_dataset, evaluate_examples, program_signature
+from .rawlang import Instruction, Program, RawLangSyntaxError, Reference, ReturnStatement, UnknownCapabilityStatement, parse_program
+from .tokenizer import ByteTokenizer
+
+__all__ = [
+    "RawIntentLanguage", "NeuralCompiler", "DeterministicCompiler",
+    "RawLangExecutor", "ExecutionResult", "Program", "Instruction", "ReturnStatement",
+    "Reference", "RawLangSyntaxError", "UnknownCapabilityStatement", "parse_program", "ByteTokenizer",
+    "TrainingExample", "generate_examples", "generate_generalization_splits", "write_generalization_dataset", "read_jsonl", "write_jsonl",
+    "evaluate_dataset", "evaluate_examples", "program_signature",
+]

rawintent/compiler/cli.py

@@ -0,0 +1,179 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from ..english_python import EnglishPython
+from .compiler import RawIntentLanguage
+from .dataset import generate_examples, write_generalization_dataset, write_jsonl
+from .executor import RawLangExecutor
+
+
+def _parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="rawintent-lm", description="Train and use RawIntent's English -> RawLang compiler model.")
+    sub = p.add_subparsers(dest="command", required=True)
+
+    g = sub.add_parser("generate-data", help="Generate a bootstrap English/RawLang JSONL dataset.")
+    g.add_argument("--out", default="rawintent-train.jsonl")
+    g.add_argument("--variants", type=int, default=6)
+    g.add_argument("--samples", type=int, default=2, help="Different synthetic argument sets per registered phrase.")
+    g.add_argument("--seed", type=int, default=7)
+
+    gg = sub.add_parser("generate-generalization-data", help="Generate leakage-resistant train/validation/test data with held-out phrasings.")
+    gg.add_argument("--out-dir", default="rawintent-generalization")
+    gg.add_argument("--train-variants", type=int, default=8)
+    gg.add_argument("--eval-variants", type=int, default=6)
+    gg.add_argument("--samples", type=int, default=3)
+    gg.add_argument("--seed", type=int, default=7)
+    gg.add_argument("--no-unknown", action="store_true")
+    gg.add_argument("--no-compositions", action="store_true")
+
+    t = sub.add_parser("train", help="Train the compiler Transformer from scratch.")
+    t.add_argument("--data", required=True)
+    t.add_argument("--out", default="rawintent-compiler.pt")
+    t.add_argument("--validation-data", help="Held-out validation JSONL. Strongly recommended for generalization training.")
+    t.add_argument("--epochs", type=int, default=12)
+    t.add_argument("--batch-size", type=int, default=32)
+    t.add_argument("--lr", type=float, default=3e-4)
+    t.add_argument("--device", default="auto")
+    t.add_argument("--d-model", type=int, default=192)
+    t.add_argument("--heads", type=int, default=6)
+    t.add_argument("--layers", type=int, default=4)
+    t.add_argument("--ff", type=int, default=768)
+
+    e = sub.add_parser("evaluate", help="Measure unseen-phrasing and compositional generalization.")
+    e.add_argument("--data", required=True)
+    e.add_argument("--checkpoint")
+    e.add_argument("--deterministic", action="store_true", help="Benchmark the rule compiler instead of a neural checkpoint.")
+    e.add_argument("--device", default="cpu")
+    e.add_argument("--max-examples", type=int)
+    e.add_argument("--failure-limit", type=int, default=25)
+    e.add_argument("--out", help="Optional JSON metrics file.")
+
+    c = sub.add_parser("compile", help="Compile English to RawLang.")
+    c.add_argument("text")
+    c.add_argument("--checkpoint")
+    c.add_argument("--device", default="cpu")
+    c.add_argument("--no-fallback", action="store_true")
+
+    r = sub.add_parser("run", help="Compile and execute an English request.")
+    r.add_argument("text")
+    r.add_argument("--checkpoint")
+    r.add_argument("--device", default="cpu")
+    r.add_argument("--allow-network", action=argparse.BooleanOptionalAction, default=True)
+    r.add_argument("--allow-network-writes", action="store_true")
+    r.add_argument("--allow-file-changes", action="store_true")
+
+    x = sub.add_parser("run-rawlang", help="Execute RawLang directly.")
+    x.add_argument("source")
+    x.add_argument("--allow-network", action=argparse.BooleanOptionalAction, default=True)
+    x.add_argument("--allow-network-writes", action="store_true")
+    x.add_argument("--allow-file-changes", action="store_true")
+
+    s = sub.add_parser("schema", help="Print executable RawLang actions and parameters.")
+    s.add_argument("--json", action="store_true")
+    return p
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _parser().parse_args(argv)
+    if args.command == "generate-data":
+        count = write_jsonl(args.out, generate_examples(variants_per_phrase=args.variants, samples_per_phrase=args.samples, seed=args.seed))
+        print(f"Wrote {count} examples to {args.out}")
+        return 0
+
+    if args.command == "generate-generalization-data":
+        manifest = write_generalization_dataset(
+            args.out_dir,
+            train_variants=args.train_variants,
+            eval_variants=args.eval_variants,
+            samples_per_phrase=args.samples,
+            seed=args.seed,
+            include_unknown=not args.no_unknown,
+            include_compositions=not args.no_compositions,
+        )
+        print(json.dumps(manifest, indent=2))
+        return 0
+
+    if args.command == "train":
+        from .model import ModelConfig
+        from .training import TrainConfig, train
+        model_config = ModelConfig(
+            d_model=args.d_model,
+            nhead=args.heads,
+            num_encoder_layers=args.layers,
+            num_decoder_layers=args.layers,
+            dim_feedforward=args.ff,
+        )
+        metrics = train(
+            args.data,
+            args.out,
+            validation_path=args.validation_data,
+            model_config=model_config,
+            train_config=TrainConfig(
+                epochs=args.epochs,
+                batch_size=args.batch_size,
+                learning_rate=args.lr,
+                device=args.device,
+            ),
+        )
+        print(json.dumps(metrics, indent=2))
+        return 0
+
+    if args.command == "evaluate":
+        from .evaluation import evaluate_dataset
+        metrics = evaluate_dataset(
+            args.data,
+            checkpoint=args.checkpoint,
+            device=args.device,
+            deterministic=args.deterministic,
+            max_examples=args.max_examples,
+            failure_limit=args.failure_limit,
+        )
+        rendered = json.dumps(metrics, indent=2)
+        print(rendered)
+        if args.out:
+            Path(args.out).write_text(rendered + "\n", encoding="utf-8")
+        return 0
+
+    if args.command == "compile":
+        lang = RawIntentLanguage(args.checkpoint, device=args.device, fallback=not args.no_fallback)
+        print(lang.compile(args.text))
+        return 0
+
+    if args.command in {"run", "run-rawlang"}:
+        engine = EnglishPython(
+            allow_network=args.allow_network,
+            allow_network_writes=args.allow_network_writes,
+            allow_file_changes=args.allow_file_changes,
+        )
+        if args.command == "run":
+            result = RawIntentLanguage(args.checkpoint, device=args.device, engine=engine).run(args.text)
+        else:
+            result = RawLangExecutor(engine).execute(args.source)
+        if result.ok:
+            print(result.value)
+            return 0
+        print(f"{result.error.code}: {result.error.message}")
+        if result.failed_line:
+            print(f"RawLang line: {result.failed_line}")
+        return 2
+
+    if args.command == "schema":
+        schema = RawLangExecutor().schema()
+        if args.json:
+            print(json.dumps(schema, indent=2, default=str))
+        else:
+            for action, info in schema.items():
+                params = ", ".join(
+                    name + ("" if spec["required"] else "?")
+                    for name, spec in info["parameters"].items()
+                )
+                print(f"{action}({params})")
+        return 0
+    return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

rawintent/compiler/compiler.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from ..english_python import EnglishPython
+from .deterministic import DeterministicCompiler
+from .executor import ExecutionResult, RawLangExecutor
+from .rawlang import Instruction, Program, UnknownCapabilityStatement, parse_program
+
+
+class NeuralCompiler:
+    """Loads a trained RawCompilerModel and translates arbitrary English into RawLang."""
+
+    def __init__(self, checkpoint: str | Path, *, device: str = "cpu") -> None:
+        from .model import RawCompilerModel, compile_with_model
+        from .tokenizer import ByteTokenizer
+
+        self.model, self.metadata = RawCompilerModel.load_checkpoint(checkpoint, device=device)
+        self._compile_with_model = compile_with_model
+        self.tokenizer = ByteTokenizer()
+        self.device = device
+
+    def compile(self, text: str, *, validate_syntax: bool = True) -> str:
+        source = self._compile_with_model(self.model, text, self.tokenizer, device=self.device)
+        if validate_syntax:
+            parse_program(source)
+        return source
+
+
+class RawIntentLanguage:
+    """High-level English -> RawLang -> validated Python capabilities interface.
+
+    Without a checkpoint it uses the deterministic compiler immediately. With a checkpoint it uses
+    the custom neural compiler and can optionally fall back to deterministic parsing when generation
+    is syntactically invalid.
+    """
+
+    def __init__(
+        self,
+        checkpoint: str | Path | None = None,
+        *,
+        device: str = "cpu",
+        fallback: bool = True,
+        engine: EnglishPython | None = None,
+        **engine_options: Any,
+    ) -> None:
+        self.engine = engine or EnglishPython(**engine_options)
+        self.executor = RawLangExecutor(self.engine)
+        self.deterministic = DeterministicCompiler(self.engine)
+        self.neural = NeuralCompiler(checkpoint, device=device) if checkpoint else None
+        self.fallback = fallback
+
+    def _validate_generated_program(self, source: str) -> str:
+        import inspect
+
+        program = parse_program(source)
+        for stmt in program.statements:
+            if isinstance(stmt, UnknownCapabilityStatement):
+                continue
+            if not isinstance(stmt, Instruction):
+                continue
+            if stmt.action not in self.engine.intents:
+                raise ValueError(f"The model generated unknown RawLang action {stmt.action!r}.")
+            handler = self.engine.intents[stmt.action].handler
+            if handler is None:
+                raise ValueError(f"The model generated non-executable action {stmt.action!r}.")
+            # References are normal Python objects here, so signature binding can validate argument
+            # names and required fields without executing anything.
+            inspect.signature(handler).bind(**stmt.arguments)
+        return program.to_source()
+
+    def compile(self, text: str) -> str:
+        if self.neural is None:
+            return self.deterministic.compile(text)
+        try:
+            return self._validate_generated_program(self.neural.compile(text))
+        except Exception:
+            if not self.fallback:
+                raise
+            return self.deterministic.compile(text)
+
+    def parse(self, text: str) -> Program:
+        return parse_program(self.compile(text))
+
+    def run(self, text: str) -> ExecutionResult:
+        try:
+            rawlang = self.compile(text)
+        except Exception as exc:
+            from ..errors import english_exception
+            return ExecutionResult(False, error=english_exception(exc))
+        return self.executor.execute(rawlang)