trajectly 0.3.0rc1__tar.gz

trajectly-0.3.0rc1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Trajectly
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

trajectly-0.3.0rc1/PKG-INFO

@@ -0,0 +1,88 @@
+Metadata-Version: 2.4
+Name: trajectly
+Version: 0.3.0rc1
+Summary: Trajectory Refinement Testing (TRT) for deterministic agent CI
+Author-email: Ahmed Ashmawy <awashmawy@proton.me>
+License: MIT
+Keywords: agents,testing,regression,llm,ci
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: typer<1,>=0.12
+Requires-Dist: PyYAML<7,>=6
+Provides-Extra: dev
+Requires-Dist: pytest<9,>=8; extra == "dev"
+Requires-Dist: pytest-cov<6,>=5; extra == "dev"
+Requires-Dist: ruff<1,>=0.6; extra == "dev"
+Requires-Dist: mypy<2,>=1.11; extra == "dev"
+Requires-Dist: types-PyYAML<7,>=6.0; extra == "dev"
+Dynamic: license-file
+
+# Trajectly
+
+Deterministic regression testing for AI agents. Record a baseline, enforce contracts, catch regressions before they ship.
+
+## Install
+
+```bash
+pip install trajectly
+```
+
+## 30-Second Example
+
+Trajectly works in three steps: **record** a known-good baseline, **run** against it later, and **get a verdict**.
+
+```bash
+# Clone the repo and install dev dependencies
+git clone https://github.com/trajectly/trajectly.git
+cd trajectly
+pip install -e ".[dev]"
+
+# Set your OpenAI key (the example calls gpt-4o-mini)
+export OPENAI_API_KEY="sk-..."
+
+# 1. Record the baseline
+cd examples
+trajectly record specs/trt-support-triage-baseline.agent.yaml
+
+# 2. Run the regression variant against it
+trajectly run specs/trt-support-triage-regression.agent.yaml
+
+# 3. See what broke
+trajectly report
+```
+
+The report shows exactly **which step** failed, **why** (the regression calls `unsafe_export`, which is denied by policy), and gives you a **deterministic repro command**.
+
+## How It Works
+
+1. **Record** -- run your agent normally. Trajectly captures every tool call and LLM response as a trace.
+2. **Replay** -- re-run the agent. Trajectly replays recorded LLM responses from fixtures so results are deterministic.
+3. **Compare** -- Trajectly checks the new trace against the baseline:
+   - **Contracts**: are only allowed tools called? Are denied tools blocked?
+   - **Refinement**: does the new call sequence preserve the baseline sequence?
+4. **Verdict** -- PASS or FAIL with the exact failure step (witness index), violation code, and a copy-paste repro command.
+
+## Examples
+
+| Example | Provider | Tools | What it tests |
+|---------|----------|-------|---------------|
+| [Ticket Classifier](docs/tutorial-support-triage.md) | OpenAI | `fetch_ticket`, `store_triage` | Simple 2-tool agent with contract enforcement |
+| [Code Review Bot](docs/tutorial-code-review-bot.md) | Gemini | `fetch_pr`, `lint_code`, `post_review` | Multi-tool sequence with policy guardrails |
+
+## Documentation
+
+- [Full documentation](docs/trajectly.md) -- concepts, CLI reference, spec format, SDK reference
+- [Tutorial: Ticket Classifier](docs/tutorial-support-triage.md) -- step-by-step simple example
+- [Tutorial: Code Review Bot](docs/tutorial-code-review-bot.md) -- step-by-step medium example
+
+## License
+
+MIT

trajectly-0.3.0rc1/README.md

@@ -0,0 +1,61 @@
+# Trajectly
+
+Deterministic regression testing for AI agents. Record a baseline, enforce contracts, catch regressions before they ship.
+
+## Install
+
+```bash
+pip install trajectly
+```
+
+## 30-Second Example
+
+Trajectly works in three steps: **record** a known-good baseline, **run** against it later, and **get a verdict**.
+
+```bash
+# Clone the repo and install dev dependencies
+git clone https://github.com/trajectly/trajectly.git
+cd trajectly
+pip install -e ".[dev]"
+
+# Set your OpenAI key (the example calls gpt-4o-mini)
+export OPENAI_API_KEY="sk-..."
+
+# 1. Record the baseline
+cd examples
+trajectly record specs/trt-support-triage-baseline.agent.yaml
+
+# 2. Run the regression variant against it
+trajectly run specs/trt-support-triage-regression.agent.yaml
+
+# 3. See what broke
+trajectly report
+```
+
+The report shows exactly **which step** failed, **why** (the regression calls `unsafe_export`, which is denied by policy), and gives you a **deterministic repro command**.
+
+## How It Works
+
+1. **Record** -- run your agent normally. Trajectly captures every tool call and LLM response as a trace.
+2. **Replay** -- re-run the agent. Trajectly replays recorded LLM responses from fixtures so results are deterministic.
+3. **Compare** -- Trajectly checks the new trace against the baseline:
+   - **Contracts**: are only allowed tools called? Are denied tools blocked?
+   - **Refinement**: does the new call sequence preserve the baseline sequence?
+4. **Verdict** -- PASS or FAIL with the exact failure step (witness index), violation code, and a copy-paste repro command.
+
+## Examples
+
+| Example | Provider | Tools | What it tests |
+|---------|----------|-------|---------------|
+| [Ticket Classifier](docs/tutorial-support-triage.md) | OpenAI | `fetch_ticket`, `store_triage` | Simple 2-tool agent with contract enforcement |
+| [Code Review Bot](docs/tutorial-code-review-bot.md) | Gemini | `fetch_pr`, `lint_code`, `post_review` | Multi-tool sequence with policy guardrails |
+
+## Documentation
+
+- [Full documentation](docs/trajectly.md) -- concepts, CLI reference, spec format, SDK reference
+- [Tutorial: Ticket Classifier](docs/tutorial-support-triage.md) -- step-by-step simple example
+- [Tutorial: Code Review Bot](docs/tutorial-code-review-bot.md) -- step-by-step medium example
+
+## License
+
+MIT

trajectly-0.3.0rc1/pyproject.toml

@@ -0,0 +1,69 @@
+[build-system]
+requires = ["setuptools>=69", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "trajectly"
+version = "0.3.0rc1"
+description = "Trajectory Refinement Testing (TRT) for deterministic agent CI"
+readme = "README.md"
+requires-python = ">=3.11"
+license = { text = "MIT" }
+authors = [
+  {name = "Ahmed Ashmawy", email = "awashmawy@proton.me"}
+]
+keywords = ["agents", "testing", "regression", "llm", "ci"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Topic :: Software Development :: Testing",
+]
+dependencies = [
+  "typer>=0.12,<1",
+  "PyYAML>=6,<7",
+]
+
+[project.optional-dependencies]
+dev = [
+  "pytest>=8,<9",
+  "pytest-cov>=5,<6",
+  "ruff>=0.6,<1",
+  "mypy>=1.11,<2",
+  "types-PyYAML>=6.0,<7",
+]
+
+[project.scripts]
+trajectly = "trajectly.cli:app"
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+py-modules = ["sitecustomize"]
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+pythonpath = ["src"]
+testpaths = ["tests"]
+
+[tool.ruff]
+line-length = 120
+target-version = "py311"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "B", "UP", "N", "RUF"]
+ignore = ["B008"]
+
+[tool.mypy]
+python_version = "3.11"
+strict = true
+warn_unused_ignores = true
+warn_redundant_casts = true
+warn_unreachable = true
+show_error_codes = true
+pretty = true
+packages = ["trajectly"]

trajectly-0.3.0rc1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

trajectly-0.3.0rc1/src/sitecustomize.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+import os
+
+if os.getenv("TRAJECTLY_REPLAY_GUARD") == "1":
+    # Import-time hook used by subprocess replays. Keeping this in
+    # `sitecustomize` ensures the guard is active before user code runs.
+    from trajectly.replay_guard import activate
+
+    activate()

trajectly-0.3.0rc1/src/trajectly/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+__all__ = ["__version__"]
+
+__version__ = "0.3.0rc1"

trajectly-0.3.0rc1/src/trajectly/__main__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from trajectly.cli import app
+
+app()

trajectly-0.3.0rc1/src/trajectly/abstraction/__init__.py

@@ -0,0 +1,20 @@
+from trajectly.abstraction.pipeline import AbstractionConfig, AbstractTrace, Token, build_abstract_trace
+from trajectly.abstraction.predicates import (
+    contains_email,
+    contains_phone,
+    extract_domains,
+    extract_numeric_values,
+)
+
+# Keep this export surface stable for downstream integrations importing
+# abstraction helpers directly from `trajectly.abstraction`.
+__all__ = [
+    "AbstractTrace",
+    "AbstractionConfig",
+    "Token",
+    "build_abstract_trace",
+    "contains_email",
+    "contains_phone",
+    "extract_domains",
+    "extract_numeric_values",
+]

trajectly-0.3.0rc1/src/trajectly/abstraction/pipeline.py

@@ -0,0 +1,152 @@
+"""Deterministic abstraction pipeline (Definition 2 in trt_theory.md).
+
+Implements ``build_abstract_trace``, which maps a concrete event trace to an
+abstract representation consisting of a token stream and a predicate bag.
+
+**Determinism (Theorem 2 precondition):** The pipeline iterates events by
+index, applies a fixed token-mapping function per event type, and accumulates
+predicates in a single pass.  Output keys are sorted (``tool_calls_by_name``,
+``domains``) so the abstract trace is identical for identical inputs regardless
+of Python dict insertion order.
+
+**Abstraction homomorphism:** ``alpha(T, c) = (Tokens, Predicates)`` where
+each token preserves the event index and causal kind, and predicates aggregate
+over the full trace.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+from trajectly.abstraction.predicates import (
+    contains_email,
+    contains_phone,
+    extract_domains,
+    extract_numeric_values,
+)
+from trajectly.events import TraceEvent
+
+TokenKind = Literal[
+    "CALL",
+    "RESULT",
+    "LLM_REQUEST",
+    "LLM_RESPONSE",
+    "MESSAGE",
+    "OBSERVATION",
+    "ERROR",
+]
+
+
+@dataclass(slots=True)
+class Token:
+    event_index: int
+    kind: TokenKind
+    name: str
+    payload: dict[str, Any]
+
+
+@dataclass(slots=True)
+class AbstractionConfig:
+    ignore_call_tools: list[str] = field(default_factory=list)
+    enable_pii_detection: bool = True
+    enable_domain_extraction: bool = True
+    enable_numeric_extraction: bool = True
+
+
+@dataclass(slots=True)
+class AbstractTrace:
+    tokens: list[Token]
+    predicates: dict[str, Any]
+
+
+def _token_from_event(event: TraceEvent, event_index: int, ignore_call_tools: set[str]) -> Token | None:
+    # Event-to-token mapping is intentionally conservative: only stable,
+    # contract-relevant event types feed TRT abstraction.
+    payload = dict(event.payload)
+    if event.event_type == "tool_called":
+        tool_name = str(payload.get("tool_name", "unknown"))
+        if tool_name in ignore_call_tools:
+            return None
+        return Token(event_index=event_index, kind="CALL", name=tool_name, payload=payload)
+    if event.event_type == "tool_returned":
+        tool_name = str(payload.get("tool_name", "unknown"))
+        return Token(event_index=event_index, kind="RESULT", name=tool_name, payload=payload)
+    if event.event_type == "llm_called":
+        provider = str(payload.get("provider", "unknown"))
+        model = str(payload.get("model", "unknown"))
+        return Token(event_index=event_index, kind="LLM_REQUEST", name=f"{provider}:{model}", payload=payload)
+    if event.event_type == "llm_returned":
+        provider = str(payload.get("provider", "unknown"))
+        model = str(payload.get("model", "unknown"))
+        return Token(event_index=event_index, kind="LLM_RESPONSE", name=f"{provider}:{model}", payload=payload)
+    if event.event_type == "agent_step":
+        name = str(payload.get("name", "step"))
+        return Token(event_index=event_index, kind="MESSAGE", name=name, payload=payload)
+    if event.event_type == "run_finished":
+        return Token(event_index=event_index, kind="OBSERVATION", name="run_finished", payload=payload)
+    return None
+
+
+def build_abstract_trace(
+    events: list[TraceEvent],
+    *,
+    config: AbstractionConfig | None = None,
+) -> AbstractTrace:
+    cfg = config or AbstractionConfig()
+    ignore_call_tools = set(cfg.ignore_call_tools)
+    tokens: list[Token] = []
+
+    for index, event in enumerate(events):
+        token = _token_from_event(event, index, ignore_call_tools)
+        if token is not None:
+            tokens.append(token)
+
+    # Predicate bag shape is fixed for deterministic report payloads.
+    predicates: dict[str, Any] = {
+        "tool_calls_total": sum(1 for token in tokens if token.kind == "CALL"),
+        "tool_calls_by_name": {},
+        "domains": [],
+        "pii": {"email": False, "phone": False},
+        "max_numeric_value": None,
+        "refund_count": 0,
+    }
+
+    tool_counts: dict[str, int] = {}
+    domains: set[str] = set()
+    numeric_values: list[float] = []
+    has_email = False
+    has_phone = False
+    refund_count = 0
+
+    for token in tokens:
+        # Predicates are derived in a single deterministic pass so witness-level
+        # checks can be reproduced exactly in CI replay.
+        if token.kind == "CALL":
+            tool_counts[token.name] = tool_counts.get(token.name, 0) + 1
+            if "refund" in token.name.lower():
+                refund_count += 1
+
+        if cfg.enable_domain_extraction:
+            domains.update(extract_domains(token.payload))
+        if cfg.enable_numeric_extraction:
+            numeric_values.extend(extract_numeric_values(token.payload))
+        if cfg.enable_pii_detection:
+            has_email = has_email or contains_email(token.payload)
+            has_phone = has_phone or contains_phone(token.payload)
+
+    predicates["tool_calls_by_name"] = dict(sorted(tool_counts.items()))
+    predicates["refund_count"] = refund_count
+    predicates["domains"] = sorted(domains)
+    predicates["pii"] = {"email": has_email, "phone": has_phone}
+    predicates["max_numeric_value"] = max(numeric_values) if numeric_values else None
+
+    return AbstractTrace(tokens=tokens, predicates=predicates)
+
+
+__all__ = [
+    "AbstractTrace",
+    "AbstractionConfig",
+    "Token",
+    "build_abstract_trace",
+]

trajectly-0.3.0rc1/src/trajectly/abstraction/predicates.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable
+from typing import Any
+from urllib.parse import urlparse
+
+EMAIL_RE = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b")
+PHONE_RE = re.compile(r"\b(?:\+?1[-.\s]?)?(?:\(?\d{3}\)?[-.\s]?)?\d{3}[-.\s]?\d{4}\b")
+URL_RE = re.compile(r"https?://[^\s)]+")
+
+
+def _walk_strings(value: Any) -> Iterable[str]:
+    # Predicate extraction intentionally walks only serializable payload-like
+    # shapes to keep abstraction deterministic and side-effect free.
+    if isinstance(value, str):
+        yield value
+        return
+    if isinstance(value, dict):
+        for item in value.values():
+            yield from _walk_strings(item)
+        return
+    if isinstance(value, list | tuple):
+        for item in value:
+            yield from _walk_strings(item)
+
+
+def contains_email(value: Any) -> bool:
+    return any(EMAIL_RE.search(text) for text in _walk_strings(value))
+
+
+def contains_phone(value: Any) -> bool:
+    return any(PHONE_RE.search(text) for text in _walk_strings(value))
+
+
+def extract_domains(value: Any) -> list[str]:
+    domains: set[str] = set()
+    for text in _walk_strings(value):
+        candidates = [text, *URL_RE.findall(text)]
+        for candidate in candidates:
+            parsed = urlparse(candidate)
+            host = parsed.hostname
+            if host:
+                domains.add(host.lower())
+    return sorted(domains)
+
+
+def extract_numeric_values(value: Any) -> list[float]:
+    numbers: list[float] = []
+    if isinstance(value, int | float):
+        return [float(value)]
+    if isinstance(value, dict):
+        for item in value.values():
+            numbers.extend(extract_numeric_values(item))
+        return numbers
+    if isinstance(value, list | tuple):
+        for item in value:
+            numbers.extend(extract_numeric_values(item))
+        return numbers
+    return numbers
+
+
+__all__ = [
+    "contains_email",
+    "contains_phone",
+    "extract_domains",
+    "extract_numeric_values",
+]

trajectly-0.3.0rc1/src/trajectly/benchmark.py

@@ -0,0 +1,78 @@
+"""
+TRT performance benchmark harness (QA-T007).
+
+Runs TRT run_specs in a minimal workspace repeatedly and reports wall-clock times.
+Deterministic and offline-safe (replay-only; no network).
+"""
+
+from __future__ import annotations
+
+import tempfile
+import time
+from pathlib import Path
+from typing import Any
+
+from trajectly.constants import EXIT_SUCCESS
+from trajectly.engine import initialize_workspace, record_specs, run_specs
+
+
+def _write(path: Path, body: str) -> None:
+    path.write_text(body.strip() + "\n", encoding="utf-8")
+
+
+def _setup_workspace(root: Path) -> Path:
+    """Create minimal TRT workspace with one spec and agent; record baseline. Returns spec path."""
+    initialize_workspace(root)
+    agent = root / "agent.py"
+    _write(agent, "print('ok')")
+    spec = root / "bench.agent.yaml"
+    _write(
+        spec,
+        """
+schema_version: "0.3"
+name: bench
+command: python agent.py
+workdir: .
+strict: true
+""",
+    )
+    outcome = record_specs(targets=[str(spec)], project_root=root)
+    if outcome.exit_code != EXIT_SUCCESS:
+        raise RuntimeError(f"record_specs failed: {outcome.errors}")
+    return spec
+
+
+def run_benchmark(iterations: int = 5) -> dict[str, Any]:
+    """Run TRT run_specs `iterations` times in a fresh workspace; return timings and summary."""
+    times_s: list[float] = []
+    with tempfile.TemporaryDirectory(prefix="trajectly_bench_") as tmp:
+        root = Path(tmp)
+        spec = _setup_workspace(root)
+        for _ in range(iterations):
+            t0 = time.perf_counter()
+            outcome = run_specs(targets=[str(spec)], project_root=root)
+            t1 = time.perf_counter()
+            if outcome.exit_code != EXIT_SUCCESS:
+                raise RuntimeError(f"run_specs failed: {outcome.errors}")
+            times_s.append(t1 - t0)
+    n = len(times_s)
+    return {
+        "runs": [{"wall_s": round(t, 6)} for t in times_s],
+        "summary": {
+            "n": n,
+            "mean_s": round(sum(times_s) / n, 6),
+            "min_s": round(min(times_s), 6),
+            "max_s": round(max(times_s), 6),
+        },
+    }
+
+
+def to_md(data: dict[str, Any]) -> str:
+    """Short Markdown summary of benchmark result."""
+    s = data["summary"]
+    return (
+        "## TRT benchmark summary\n\n"
+        f"- **Runs:** {s['n']}\n"
+        f"- **Mean:** {s['mean_s']} s\n"
+        f"- **Min / Max:** {s['min_s']} s / {s['max_s']} s\n"
+    )

trajectly-0.3.0rc1/src/trajectly/canonical.py

@@ -0,0 +1,17 @@
+from trajectly.normalize.canonical import (
+    DEFAULT_CANONICAL_NORMALIZER,
+    CanonicalNormalizer,
+    canonical_dumps,
+    normalize_for_json,
+    sha256_of_data,
+    sha256_of_subset,
+)
+
+__all__ = [
+    "DEFAULT_CANONICAL_NORMALIZER",
+    "CanonicalNormalizer",
+    "canonical_dumps",
+    "normalize_for_json",
+    "sha256_of_data",
+    "sha256_of_subset",
+]