agent-trajectory-diff 0.3.0__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/.gitignore +1 -0
  2. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/PKG-INFO +21 -1
  3. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/README.md +20 -0
  4. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/__init__.py +1 -1
  5. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/cli.py +161 -3
  6. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/config.py +1 -0
  7. agent_trajectory_diff-0.4.0/src/agentdiff/governance.py +86 -0
  8. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/models/report.py +5 -0
  9. agent_trajectory_diff-0.4.0/src/agentdiff/recorder.py +158 -0
  10. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/reporters/pr.py +30 -5
  11. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/reporters/terminal.py +9 -2
  12. agent_trajectory_diff-0.4.0/src/agentdiff/staleness.py +61 -0
  13. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/LICENSE +0 -0
  14. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/pyproject.toml +0 -0
  15. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/__main__.py +0 -0
  16. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/adapters/__init__.py +0 -0
  17. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/adapters/_iso.py +0 -0
  18. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/adapters/_messages.py +0 -0
  19. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/adapters/base.py +0 -0
  20. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/adapters/crewai.py +0 -0
  21. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/adapters/generic.py +0 -0
  22. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/adapters/langfuse.py +0 -0
  23. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/adapters/langgraph.py +0 -0
  24. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/adapters/langsmith.py +0 -0
  25. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/adapters/openai_agents.py +0 -0
  26. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/adapters/openinference.py +0 -0
  27. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/adapters/registry.py +0 -0
  28. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/ci/baseline.py +0 -0
  29. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/ci/github.py +0 -0
  30. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/engine/__init__.py +0 -0
  31. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/engine/aligner.py +0 -0
  32. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/engine/benchmark.py +0 -0
  33. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/engine/comparator.py +0 -0
  34. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/engine/explanations.py +0 -0
  35. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/engine/loop_detector.py +0 -0
  36. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/engine/metrics.py +0 -0
  37. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/engine/suite.py +0 -0
  38. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/engine/tree.py +0 -0
  39. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/loader.py +0 -0
  40. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/models/__init__.py +0 -0
  41. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/models/step.py +0 -0
  42. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/models/trace.py +0 -0
  43. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/py.typed +0 -0
  44. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/pytest_plugin.py +0 -0
  45. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/reporters/__init__.py +0 -0
  46. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/reporters/markdown.py +0 -0
  47. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/testing/__init__.py +0 -0
  48. {agent_trajectory_diff-0.3.0 → agent_trajectory_diff-0.4.0}/src/agentdiff/testing/assertions.py +0 -0
@@ -82,3 +82,4 @@ website/.env.production.local
82
82
 
83
83
  # pytest-benchmark local results
84
84
  .benchmarks/
85
+ _record_demo/
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: agent-trajectory-diff
3
- Version: 0.3.0
3
+ Version: 0.4.0
4
4
  Summary: A developer-first package to evaluate and find regressions in agent trajectories
5
5
  Project-URL: Homepage, https://github.com/lostmartian/agentdiff
6
6
  Project-URL: Repository, https://github.com/lostmartian/agentdiff
@@ -60,8 +60,26 @@ Or using `uv`:
60
60
  uv add agent-trajectory-diff
61
61
  ```
62
62
 
63
+ Enable tab-completion for the CLI (bash/zsh/fish/powershell):
64
+ ```bash
65
+ agentdiff --install-completion
66
+ ```
67
+ > Tip: for a global `agentdiff` command without activating a venv, use `uv tool install agent-trajectory-diff` — then completion works anywhere.
68
+
63
69
  ## Quickstart
64
70
 
71
+ ### 0. No trace yet? Record one.
72
+
73
+ Point `record` at any callable (your agent's entry function) and it captures a canonical trace:
74
+
75
+ ```bash
76
+ agentdiff record my_agent:run --input '{"question": "What is AgentDiff?"}' --out traces/run.json
77
+ ```
78
+
79
+ - `--input` takes a JSON object (passed as kwargs) or `@file.json`
80
+ - A failed run is still recorded — diff it to see exactly what broke
81
+ - Then compare: `agentdiff traces/baseline.json traces/run.json`
82
+
65
83
  ### 1. CLI Usage
66
84
 
67
85
  Compare two trajectory JSON traces from your terminal:
@@ -78,6 +96,8 @@ Options:
78
96
  - `--max-cost-delta`: Maximum cost increase percentage allowed.
79
97
  - `--baseline, -b PATH`: Compare against a persistent baseline trace file (see [Baseline workflow](#baseline-workflow)).
80
98
  - `--update-baseline`: Overwrite the persistent baseline with the candidate after a clean diff.
99
+ - `--baseline-config PATH`: The `agentdiff.toml` the **baseline** was recorded with. When gate values differ from this run's config, the report flags the change (Goodhart guard — see [Gate governance](https://agentdiff.lostmartian.in/docs/configuration#gate-governance-goodhart-guard)).
100
+ - `--stale-days N`: Warn via `--explain` when the baseline file is older than N days (default: config `stale_baseline_days` or 30 — advisory only).
81
101
  - `--config PATH`: Load defaults from an `agentdiff.toml` (auto-discovered if not given).
82
102
 
83
103
  #### Config-as-code (`agentdiff.toml`)
@@ -40,8 +40,26 @@ Or using `uv`:
40
40
  uv add agent-trajectory-diff
41
41
  ```
42
42
 
43
+ Enable tab-completion for the CLI (bash/zsh/fish/powershell):
44
+ ```bash
45
+ agentdiff --install-completion
46
+ ```
47
+ > Tip: for a global `agentdiff` command without activating a venv, use `uv tool install agent-trajectory-diff` — then completion works anywhere.
48
+
43
49
  ## Quickstart
44
50
 
51
+ ### 0. No trace yet? Record one.
52
+
53
+ Point `record` at any callable (your agent's entry function) and it captures a canonical trace:
54
+
55
+ ```bash
56
+ agentdiff record my_agent:run --input '{"question": "What is AgentDiff?"}' --out traces/run.json
57
+ ```
58
+
59
+ - `--input` takes a JSON object (passed as kwargs) or `@file.json`
60
+ - A failed run is still recorded — diff it to see exactly what broke
61
+ - Then compare: `agentdiff traces/baseline.json traces/run.json`
62
+
45
63
  ### 1. CLI Usage
46
64
 
47
65
  Compare two trajectory JSON traces from your terminal:
@@ -58,6 +76,8 @@ Options:
58
76
  - `--max-cost-delta`: Maximum cost increase percentage allowed.
59
77
  - `--baseline, -b PATH`: Compare against a persistent baseline trace file (see [Baseline workflow](#baseline-workflow)).
60
78
  - `--update-baseline`: Overwrite the persistent baseline with the candidate after a clean diff.
79
+ - `--baseline-config PATH`: The `agentdiff.toml` the **baseline** was recorded with. When gate values differ from this run's config, the report flags the change (Goodhart guard — see [Gate governance](https://agentdiff.lostmartian.in/docs/configuration#gate-governance-goodhart-guard)).
80
+ - `--stale-days N`: Warn via `--explain` when the baseline file is older than N days (default: config `stale_baseline_days` or 30 — advisory only).
61
81
  - `--config PATH`: Load defaults from an `agentdiff.toml` (auto-discovered if not given).
62
82
 
63
83
  #### Config-as-code (`agentdiff.toml`)
@@ -32,7 +32,7 @@ Typical flow::
32
32
  assert_no_regressions(report)
33
33
  """
34
34
 
35
- __version__ = "0.3.0"
35
+ __version__ = "0.4.0"
36
36
 
37
37
  from agentdiff.adapters import (
38
38
  BaseAdapter,
@@ -2,25 +2,101 @@ import json
2
2
  import os
3
3
  import shutil
4
4
  import sys
5
+ from pathlib import Path
5
6
 
6
7
  import typer
7
8
 
8
9
  from agentdiff.ci.baseline import decide_rotation
9
10
  from agentdiff.ci.github import post_pr_comment
10
- from agentdiff.config import AgentDiffConfig, load_config
11
+ from agentdiff.config import AgentDiffConfig, find_config_file, load_config
11
12
  from agentdiff.engine.comparator import compare
12
13
  from agentdiff.engine.explanations import format_explanations, locate_culprit
13
14
  from agentdiff.engine.tree import render_tree
15
+ from agentdiff.governance import diff_gate_thresholds, provenance_line
14
16
  from agentdiff.loader import load_trace
17
+ from agentdiff.models.step import StepStatus
18
+ from agentdiff.recorder import record_run, save_trace
15
19
  from agentdiff.reporters.markdown import generate_markdown
16
20
  from agentdiff.reporters.pr import generate_pr_markdown
17
21
  from agentdiff.reporters.terminal import print_report
22
+ from agentdiff.staleness import check_baseline_staleness
18
23
 
19
24
  app = typer.Typer(
20
25
  help="AgentDiff CLI - Compare multi-turn agent execution trajectories."
21
26
  )
22
27
 
23
28
 
29
+ @app.command(name="record")
30
+ def record(
31
+ target: str = typer.Argument(
32
+ ...,
33
+ help="Callable to run, as 'module:function' or 'module:Class.method'.",
34
+ ),
35
+ out: str = typer.Option(
36
+ ...,
37
+ "--out",
38
+ "-o",
39
+ help="Path to write the captured trace JSON (e.g. traces/run.json).",
40
+ ),
41
+ input_json: str | None = typer.Option(
42
+ None,
43
+ "--input",
44
+ "-i",
45
+ help="JSON object passed to the callable as kwargs (or a single positional arg if not an object). Use '@file.json' to read from a file.",
46
+ ),
47
+ name: str | None = typer.Option(
48
+ None,
49
+ "--name",
50
+ help="Agent name recorded in the trace (default: function name).",
51
+ ),
52
+ ):
53
+ """Runs an agent callable once and captures its trajectory as a trace.
54
+
55
+ Example:
56
+ agentdiff record my_agent:run --input '{"question": "hi"}' --out traces/run.json
57
+
58
+ The captured trace is a canonical AgentDiff JSON, ready for:
59
+
60
+ agentdiff diff traces/baseline.json traces/run.json
61
+ """
62
+ task_input: dict | None = None
63
+ if input_json:
64
+ try:
65
+ if input_json.startswith("@"):
66
+ task_input = json.loads(
67
+ Path(input_json[1:]).read_text(encoding="utf-8")
68
+ )
69
+ else:
70
+ task_input = json.loads(input_json)
71
+ except (json.JSONDecodeError, FileNotFoundError) as e:
72
+ typer.echo(f"Invalid --input: {e}", err=True)
73
+ sys.exit(2)
74
+ if not isinstance(task_input, dict):
75
+ task_input = {"input": task_input}
76
+
77
+ try:
78
+ trace = record_run(target, task_input=task_input, agent_name=name)
79
+ except ValueError as e:
80
+ typer.echo(f"Error: {e}", err=True)
81
+ sys.exit(2)
82
+ except Exception as e:
83
+ typer.echo(f"Recording failed: {e}", err=True)
84
+ sys.exit(3)
85
+
86
+ path = save_trace(trace, out)
87
+
88
+ if trace.steps[0].status == StepStatus.ERROR:
89
+ typer.echo(
90
+ f"Recorded FAILED run → {path}\n error: {trace.steps[0].error_message}",
91
+ err=True,
92
+ )
93
+ sys.exit(1)
94
+
95
+ typer.echo(f"Recorded run → {path}")
96
+ typer.echo(f" agent: {trace.agent_name} · latency: {trace.total_latency_ms:.0f}ms")
97
+ typer.echo(f"Next: agentdiff diff <baseline> {path}")
98
+
99
+
24
100
  def _resolve_cli(cfg: AgentDiffConfig, **values):
25
101
  """Returns config-provided values for any option left at its sentinel."""
26
102
  resolved = {}
@@ -106,9 +182,34 @@ def diff(
106
182
  max_drift: float = typer.Option(
107
183
  0.05, help="Max TDI a clean run may have for staged auto-rotation."
108
184
  ),
185
+ baseline_config: str | None = typer.Option(
186
+ None,
187
+ "--baseline-config",
188
+ help="Path to the agentdiff.toml the BASELINE was recorded with. When gate values differ from this run's config, the report flags the change (Goodhart guard).",
189
+ ),
190
+ stale_days: int | None = typer.Option(
191
+ None,
192
+ "--stale-days",
193
+ help="Warn (with --explain) when the baseline file is older than this many days (default: config or 30).",
194
+ ),
109
195
  ):
110
196
  """Compares baseline and candidate agent trajectories."""
111
197
  cfg = load_config(config)
198
+ config_source = config or (str(find_config_file()) if find_config_file() else None)
199
+ threshold_changes: list = []
200
+ if baseline_config:
201
+ try:
202
+ baseline_cfg = load_config(baseline_config)
203
+ except FileNotFoundError as e:
204
+ typer.echo(f"Baseline config not found: {e}", err=True)
205
+ sys.exit(2)
206
+ threshold_changes = diff_gate_thresholds(baseline_cfg, cfg)
207
+ gate_provenance = provenance_line(cfg, config_source)
208
+ stale_days = (
209
+ stale_days
210
+ if stale_days is not None
211
+ else getattr(cfg.cli, "stale_baseline_days", 30)
212
+ )
112
213
  values = _resolve_cli(
113
214
  cfg,
114
215
  adapter=adapter,
@@ -156,6 +257,11 @@ def diff(
156
257
  else:
157
258
  actual_baseline = baseline_path
158
259
 
260
+ # F7 — surface baseline age so stale baselines are re-recorded deliberately
261
+ staleness = check_baseline_staleness(
262
+ actual_baseline, stale_after_days=stale_days
263
+ )
264
+
159
265
  baseline = load_trace(actual_baseline, adapter)
160
266
  except (json.JSONDecodeError, ValueError, FileNotFoundError) as e:
161
267
  typer.echo(f"Error loading or parsing trace: {e}", err=True)
@@ -194,10 +300,11 @@ def diff(
194
300
  # For console printing we write directly, but we can capture it or format differently if output_file is active
195
301
  if output_file:
196
302
  # If writing terminal format to file, output the text summary representation
197
- output_content = report.summary()
303
+ output_content = report.summary() + f"\n{gate_provenance}\n"
198
304
  else:
199
- print_report(report)
305
+ print_report(report, gate_provenance=gate_provenance)
200
306
  elif format.lower() == "json":
307
+ report.gate_provenance = gate_provenance
201
308
  output_content = report.model_dump_json(indent=2)
202
309
  if not output_file:
203
310
  typer.echo(output_content)
@@ -212,6 +319,8 @@ def diff(
212
319
  max_loops=max_loops,
213
320
  max_cost_delta=max_cost_delta,
214
321
  max_recovery_ratio=max_recovery_ratio,
322
+ threshold_changes=threshold_changes,
323
+ gate_provenance=gate_provenance,
215
324
  )
216
325
  if not output_file:
217
326
  typer.echo(output_content)
@@ -225,6 +334,12 @@ def diff(
225
334
  culprit = locate_culprit(report)
226
335
  if culprit:
227
336
  typer.echo("\n" + culprit.render())
337
+ if staleness.is_stale:
338
+ typer.echo(f"\n! {staleness.render()}")
339
+ if threshold_changes:
340
+ typer.echo("\nGate thresholds changed vs baseline config:")
341
+ for change in threshold_changes:
342
+ typer.echo(f" ! {change.render()}")
228
343
 
229
344
  # Print the collapsed divergence tree when requested
230
345
  if tree:
@@ -238,6 +353,8 @@ def diff(
238
353
  max_loops=max_loops,
239
354
  max_cost_delta=max_cost_delta,
240
355
  max_recovery_ratio=max_recovery_ratio,
356
+ threshold_changes=threshold_changes,
357
+ gate_provenance=gate_provenance,
241
358
  )
242
359
  comment = post_pr_comment(body, pr)
243
360
  url = comment.get("html_url")
@@ -285,5 +402,46 @@ def main():
285
402
  sys.exit(3)
286
403
 
287
404
 
405
+ def _install_diff_default() -> None:
406
+ """E1 compat: `agentdiff base.json cand.json` (no subcommand) means `diff`.
407
+
408
+ With two commands registered, Typer requires an explicit subcommand. We
409
+ patch the built click.Group so a leading positional that isn't a known
410
+ command is treated as `diff ...`. Works for the real CLI and CliRunner.
411
+ """
412
+ import typer.main
413
+
414
+ original_get_command = typer.main.get_command
415
+
416
+ def get_command_with_default(typer_app):
417
+ group = original_get_command(typer_app)
418
+ if not getattr(group, "_diff_default", False):
419
+ original_parse_args = group.parse_args
420
+
421
+ def parse_args(ctx, args):
422
+ if (
423
+ args
424
+ and args[0] not in group.commands
425
+ and not args[0].startswith("-")
426
+ ):
427
+ args = ["diff", *args]
428
+ original_parse_args(ctx, args)
429
+
430
+ group.parse_args = parse_args
431
+ group._diff_default = True
432
+ return group
433
+
434
+ typer.main.get_command = get_command_with_default
435
+
436
+ # typer.testing binds get_command at import time — patch its reference too
437
+ import typer.testing as testing_module
438
+
439
+ if getattr(testing_module, "_get_command", None) is original_get_command:
440
+ testing_module._get_command = get_command_with_default
441
+
442
+
443
+ _install_diff_default()
444
+
445
+
288
446
  if __name__ == "__main__":
289
447
  main()
@@ -69,6 +69,7 @@ class CliConfig:
69
69
  max_cost_delta: float = 10.0
70
70
  max_wasted_effort: float = 0.1
71
71
  max_recovery_ratio: float | None = None
72
+ stale_baseline_days: int = 30
72
73
 
73
74
 
74
75
  @dataclass
@@ -0,0 +1,86 @@
1
+ """Gate governance — detect when the gate itself changed (G6/G7).
2
+
3
+ Goodhart guard: a threshold tuned until CI goes green stops being a control.
4
+ These helpers make threshold changes visible *next to the diff they let
5
+ through*, so loosening the gate is as reviewable as the code it guards.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from dataclasses import dataclass
11
+
12
+ from agentdiff.config import AgentDiffConfig
13
+
14
+ # The gate knobs that decide pass/fail in the CLI diff path, in display order.
15
+ _GATED_KEYS: tuple[str, ...] = (
16
+ "max_divergence",
17
+ "max_loops",
18
+ "max_cost_delta",
19
+ "max_recovery_ratio",
20
+ )
21
+
22
+ # Defaults used when neither config nor flag provides a value (mirrors cli.py).
23
+ _EFFECTIVE_DEFAULTS: dict[str, float | int | None] = {
24
+ "max_divergence": 0.3,
25
+ "max_loops": 0,
26
+ "max_cost_delta": 10.0,
27
+ "max_recovery_ratio": None,
28
+ }
29
+
30
+
31
+ @dataclass(frozen=True)
32
+ class ThresholdChange:
33
+ """One gate knob that differs between the baseline config and this run."""
34
+
35
+ gate: str
36
+ old: float | int | None
37
+ new: float | int | None
38
+
39
+ def render(self) -> str:
40
+ return f"{self.gate}: `{self.old}` → `{self.new}`"
41
+
42
+
43
+ def effective_gates(cfg: AgentDiffConfig) -> dict[str, float | int | None]:
44
+ """Resolves the effective gate values from a config (config or defaults)."""
45
+ resolved: dict[str, float | int | None] = {}
46
+ for key in _GATED_KEYS:
47
+ value = getattr(cfg.cli, key, None)
48
+ if value is None:
49
+ value = _EFFECTIVE_DEFAULTS[key]
50
+ resolved[key] = value
51
+ return resolved
52
+
53
+
54
+ def diff_gate_thresholds(
55
+ baseline_cfg: AgentDiffConfig, candidate_cfg: AgentDiffConfig
56
+ ) -> list[ThresholdChange]:
57
+ """Returns gate knobs that differ between two configs, display-ordered."""
58
+ old_gates = effective_gates(baseline_cfg)
59
+ new_gates = effective_gates(candidate_cfg)
60
+ return [
61
+ ThresholdChange(gate=key, old=old_gates[key], new=new_gates[key])
62
+ for key in _GATED_KEYS
63
+ if old_gates[key] != new_gates[key]
64
+ ]
65
+
66
+
67
+ def provenance_line(cfg: AgentDiffConfig, config_path: str | None) -> str:
68
+ """G7 — one-line, self-describing gate summary for any report.
69
+
70
+ Names the active thresholds and where they came from, so every diff
71
+ answers "what rules judged me?" without opening the config.
72
+ """
73
+ gates = effective_gates(cfg)
74
+ source = (
75
+ f"agentdiff.toml ({config_path})"
76
+ if config_path
77
+ else "defaults (no agentdiff.toml found)"
78
+ )
79
+ parts = [
80
+ f"max_divergence={gates['max_divergence']}",
81
+ f"max_loops={gates['max_loops']}",
82
+ f"max_cost_delta={gates['max_cost_delta']}%",
83
+ ]
84
+ if gates["max_recovery_ratio"] is not None:
85
+ parts.append(f"max_recovery_ratio={gates['max_recovery_ratio']}")
86
+ return f"Gate: {', '.join(parts)} — source: {source}"
@@ -37,6 +37,11 @@ class DiffReport(BaseModel):
37
37
  recovery_step_ratio: float = 0.0
38
38
  step_diffs: list[StepDiff] = Field(default_factory=list)
39
39
  passed: bool = True
40
+ gate_provenance: str | None = Field(
41
+ default=None,
42
+ description="G7: one-line summary of active gate thresholds and their "
43
+ "source (agentdiff.toml path or defaults). Populated by the CLI.",
44
+ )
40
45
 
41
46
  def summary(self) -> str:
42
47
  """Returns a string summarizing the comparison report."""
@@ -0,0 +1,158 @@
1
+ """Record an agent run and capture its trajectory as an AgentDiff trace.
2
+
3
+ E3 — the ``record`` subcommand closes the biggest onboarding gap: "how do I
4
+ get a trace?". Point it at any callable that returns a dict (or a string,
5
+ which is wrapped as ``{"output": ...}``), and AgentDiff:
6
+
7
+ 1. imports the callable (``module:function`` or ``module.Class.method``),
8
+ 2. times its execution,
9
+ 3. captures the return value as the final output,
10
+ 4. writes a canonical Generic-format trace JSON ready for ``agentdiff diff``.
11
+
12
+ The callable's *internal* steps (tool calls, LLM turns) are opaque to us —
13
+ frameworks that expose those should export their native traces through the
14
+ adapters instead. ``record`` is for agents (and plain functions) with no
15
+ telemetry: one deterministic step per run, so diffs catch output/behavior
16
+ changes, loops and cost deltas come from repeated runs, and the pytest
17
+ plugin works unchanged.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import importlib
23
+ import json
24
+ import sys
25
+ import time
26
+ from collections.abc import Callable
27
+ from datetime import datetime, timezone
28
+ from pathlib import Path
29
+ from typing import Any
30
+
31
+ from agentdiff.models.step import StepStatus, StepType, TokenUsage, TraceStep
32
+ from agentdiff.models.trace import AgentTrace
33
+
34
+ _RECORD_STEP_ID = "recorded-run"
35
+
36
+
37
+ def resolve_callable(target: str) -> Callable[..., Any]:
38
+ """Imports and returns the callable described by ``module:function``.
39
+
40
+ Also accepts ``module:Class.method`` (unbound — called with no args only
41
+ if it does not require self, e.g. a ``@staticmethod``) and ``module:function``
42
+ where ``function`` is any zero-arg callable object.
43
+ """
44
+ if ":" not in target:
45
+ raise ValueError(
46
+ f"Invalid target {target!r}. Use 'module:function' or 'module:Class.method'."
47
+ )
48
+ module_path, _, attr_path = target.partition(":")
49
+
50
+ # The user runs this from their project root; their agent module lives there.
51
+ # pytest does the same thing with rootdir insertion.
52
+ cwd = str(Path.cwd())
53
+ if cwd not in sys.path:
54
+ sys.path.insert(0, cwd)
55
+
56
+ try:
57
+ module = importlib.import_module(module_path)
58
+ except ImportError as e:
59
+ raise ValueError(f"Cannot import module {module_path!r}: {e}") from e
60
+
61
+ obj: Any = module
62
+ for part in attr_path.split("."):
63
+ try:
64
+ obj = getattr(obj, part)
65
+ except AttributeError as e:
66
+ raise ValueError(
67
+ f"Cannot resolve {attr_path!r} in module {module_path!r}: {e}"
68
+ ) from e
69
+
70
+ if not callable(obj):
71
+ raise ValueError(
72
+ f"Target {target!r} resolved to a non-callable ({type(obj).__name__})."
73
+ )
74
+ return obj
75
+
76
+
77
+ def record_run(
78
+ target: str,
79
+ task_input: dict[str, Any] | None = None,
80
+ agent_name: str | None = None,
81
+ ) -> AgentTrace:
82
+ """Runs ``target`` once and captures the execution as an :class:`AgentTrace`.
83
+
84
+ The callable is invoked with ``task_input`` unpacked as keyword arguments
85
+ when it is a dict, or as a single positional argument otherwise.
86
+ """
87
+ fn = resolve_callable(target)
88
+ task = task_input if task_input is not None else {}
89
+ name = agent_name or target.split(":")[-1]
90
+
91
+ started = time.perf_counter()
92
+ error_message: str | None = None
93
+ status = StepStatus.SUCCESS
94
+ result: Any = None
95
+ try:
96
+ if isinstance(task, dict) and task:
97
+ result = fn(**task)
98
+ elif isinstance(task, dict):
99
+ result = fn()
100
+ else:
101
+ result = fn(task)
102
+ except Exception as e:
103
+ status = StepStatus.ERROR
104
+ error_message = f"{type(e).__name__}: {e}"
105
+ latency_ms = (time.perf_counter() - started) * 1000.0
106
+
107
+ if result is None or isinstance(result, dict):
108
+ output: dict[str, Any] | None = result if isinstance(result, dict) else None
109
+ if result is not None and not isinstance(result, dict):
110
+ output = {"output": result}
111
+ elif isinstance(result, str):
112
+ output = {"output": result}
113
+ else:
114
+ output = {"output": _safe_serialize(result)}
115
+
116
+ step = TraceStep(
117
+ step_id=_RECORD_STEP_ID,
118
+ parent_id=None,
119
+ step_index=0,
120
+ step_type=StepType.TOOL_CALL,
121
+ name=name,
122
+ input_payload=task
123
+ if isinstance(task, dict)
124
+ else {"input": _safe_serialize(task)},
125
+ output_payload=output,
126
+ status=status,
127
+ error_message=error_message,
128
+ latency_ms=latency_ms,
129
+ tokens=TokenUsage(),
130
+ )
131
+
132
+ trace = AgentTrace(
133
+ trace_id=f"recorded-{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S')}-{abs(hash(target)) % 10000:04d}",
134
+ agent_name=name,
135
+ task_input=task if isinstance(task, dict) else {"input": _safe_serialize(task)},
136
+ final_output=output if status == StepStatus.SUCCESS else None,
137
+ steps=[step],
138
+ total_latency_ms=latency_ms,
139
+ metadata={"recorded_from": target, "recorder": "agentdiff record"},
140
+ )
141
+ return trace
142
+
143
+
144
+ def _safe_serialize(value: Any) -> Any:
145
+ """Best-effort JSON-safe conversion for arbitrary return values."""
146
+ try:
147
+ json.dumps(value)
148
+ return value
149
+ except (TypeError, ValueError):
150
+ return repr(value)
151
+
152
+
153
+ def save_trace(trace: AgentTrace, out_path: str | Path) -> Path:
154
+ """Writes the trace as canonical AgentDiff JSON; returns the resolved path."""
155
+ path = Path(out_path)
156
+ path.parent.mkdir(parents=True, exist_ok=True)
157
+ path.write_text(trace.model_dump_json(indent=2), encoding="utf-8")
158
+ return path
@@ -2,6 +2,7 @@ from __future__ import annotations
2
2
 
3
3
  from agentdiff.engine.explanations import locate_culprit
4
4
  from agentdiff.engine.tree import render_tree
5
+ from agentdiff.governance import ThresholdChange
5
6
  from agentdiff.models.report import DiffReport
6
7
 
7
8
  DEFAULT_MAX_DIVERGENCE = 0.3
@@ -14,12 +15,18 @@ def generate_pr_markdown(
14
15
  max_loops: int = 0,
15
16
  max_cost_delta: float = DEFAULT_MAX_COST_DELTA,
16
17
  max_recovery_ratio: float | None = None,
18
+ threshold_changes: list[ThresholdChange] | None = None,
19
+ gate_provenance: str | None = None,
17
20
  ) -> str:
18
21
  """Renders a compact, PR-ready markdown comment.
19
22
 
20
23
  Summary status + gate thresholds, the collapsed divergence tree, and the
21
24
  root-cause step — everything a reviewer needs without the full diff.
22
25
  The Recovery Step Ratio row only appears when a threshold is provided.
26
+
27
+ When ``threshold_changes`` is non-empty (G6), a warning block renders
28
+ above the gate table: the gate itself moved in this PR, so the diff
29
+ above was judged against looser/tighter rules than the baseline had.
23
30
  """
24
31
  status = "⛔ **FAILED**" if not report.passed else "✅ **PASSED**"
25
32
 
@@ -28,12 +35,27 @@ def generate_pr_markdown(
28
35
  "",
29
36
  f"**Status:** {status}",
30
37
  "",
31
- "| Gate | Value | Threshold |",
32
- "| :--- | :--- | :--- |",
33
- f"| TDI | `{report.trajectory_divergence_index:.4f}` | ≤ `{max_divergence}` |",
34
- f"| Loops | `{len(report.loops_detected)}` | ≤ `{max_loops}` |",
35
- f"| Cost delta | `{report.cost_delta_percentage:+.2f}%` | ≤ `{max_cost_delta}%` |",
36
38
  ]
39
+
40
+ if threshold_changes:
41
+ lines.append("> [!WARNING]")
42
+ lines.append(
43
+ "> **Gate thresholds changed in this PR** — the diff below was judged against this PR's rules, not the baseline's."
44
+ )
45
+ lines.append(">")
46
+ for change in threshold_changes:
47
+ lines.append(f"> - {change.render()}")
48
+ lines.append("")
49
+
50
+ lines.extend(
51
+ [
52
+ "| Gate | Value | Threshold |",
53
+ "| :--- | :--- | :--- |",
54
+ f"| TDI | `{report.trajectory_divergence_index:.4f}` | ≤ `{max_divergence}` |",
55
+ f"| Loops | `{len(report.loops_detected)}` | ≤ `{max_loops}` |",
56
+ f"| Cost delta | `{report.cost_delta_percentage:+.2f}%` | ≤ `{max_cost_delta}%` |",
57
+ ]
58
+ )
37
59
  if max_recovery_ratio is not None:
38
60
  lines.append(
39
61
  f"| Recovery Step Ratio | `{report.recovery_step_ratio:.2f} "
@@ -72,4 +94,7 @@ def generate_pr_markdown(
72
94
  )
73
95
  lines.append("")
74
96
 
97
+ if gate_provenance:
98
+ lines.append(f"<sub>{gate_provenance}</sub>")
99
+
75
100
  return "\n".join(lines)
@@ -63,8 +63,13 @@ def render_diff_table(report: DiffReport) -> Table:
63
63
  return table
64
64
 
65
65
 
66
- def print_report(report: DiffReport):
67
- """Outputs the complete DiffReport to the terminal."""
66
+ def print_report(report: DiffReport, gate_provenance: str | None = None):
67
+ """Outputs the complete DiffReport to the terminal.
68
+
69
+ ``gate_provenance`` (G7) appends a one-line, self-describing gate summary
70
+ — active thresholds and their source — so the report answers "what rules
71
+ judged me?" without opening the config.
72
+ """
68
73
  console = Console()
69
74
 
70
75
  status_str = (
@@ -87,6 +92,8 @@ def print_report(report: DiffReport):
87
92
  f" • Token Delta: {report.token_delta_percentage:+.2f}%\n"
88
93
  f" • Cost Delta: {report.cost_delta_percentage:+.2f}%"
89
94
  )
95
+ if gate_provenance:
96
+ summary_text += f"\n\n[dim]{gate_provenance}[/dim]"
90
97
 
91
98
  panel = Panel(
92
99
  summary_text,
@@ -0,0 +1,61 @@
1
+ """F7 — stale-baseline detection (regressional Goodhart guard).
2
+
3
+ Baselines age silently: a golden trace from three sprints ago still anchors
4
+ every diff, but the product it represented no longer exists. This module
5
+ makes that age visible so teams re-record baselines deliberately instead of
6
+ losing trust in a gate that "just got noisy".
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import os
12
+ import time
13
+ from dataclasses import dataclass
14
+
15
+ DEFAULT_STALE_DAYS = 30
16
+
17
+
18
+ @dataclass(frozen=True)
19
+ class Staleness:
20
+ """How old a baseline file is, and whether that crosses the stale line."""
21
+
22
+ age_days: float
23
+ stale_after_days: int
24
+ missing: bool = False
25
+
26
+ @property
27
+ def is_stale(self) -> bool:
28
+ return not self.missing and self.age_days > self.stale_after_days
29
+
30
+ def render(self) -> str:
31
+ if self.missing:
32
+ return ""
33
+ age = f"{self.age_days:.0f} days" if self.age_days >= 1 else "less than a day"
34
+ line = f"Baseline is {age} old (last modified {time.strftime('%Y-%m-%d', time.localtime(self._mtime))})."
35
+ if self.is_stale:
36
+ line += (
37
+ f" Stale threshold is {self.stale_after_days} days - consider re-recording"
38
+ " it if the agent's expected behavior has legitimately changed"
39
+ " (agentdiff record ... --update-baseline)."
40
+ )
41
+ return line
42
+
43
+ _mtime: float = 0.0
44
+
45
+
46
+ def check_baseline_staleness(
47
+ baseline_path: str | os.PathLike,
48
+ stale_after_days: int = DEFAULT_STALE_DAYS,
49
+ ) -> Staleness:
50
+ """Inspects the baseline file's modification time.
51
+
52
+ Missing files return ``missing=True`` with ``is_stale=False`` - a missing
53
+ baseline is a load error, not a staleness problem.
54
+ """
55
+ try:
56
+ mtime = os.path.getmtime(baseline_path)
57
+ except OSError:
58
+ return Staleness(age_days=0.0, stale_after_days=stale_after_days, missing=True)
59
+
60
+ age_days = max(0.0, (time.time() - mtime) / 86400)
61
+ return Staleness(age_days=age_days, stale_after_days=stale_after_days, _mtime=mtime)