agentloop-profiler 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. agentloop/__init__.py +136 -0
  2. agentloop/ablation_protocol.py +229 -0
  3. agentloop/ablations.py +564 -0
  4. agentloop/aggregate_cli.py +88 -0
  5. agentloop/aggregate_stats.py +175 -0
  6. agentloop/aggregates.py +532 -0
  7. agentloop/audit.py +39 -0
  8. agentloop/autoinstrument.py +145 -0
  9. agentloop/batch_experiments.py +295 -0
  10. agentloop/batching.py +95 -0
  11. agentloop/budget_types.py +335 -0
  12. agentloop/budgets.py +539 -0
  13. agentloop/calibration.py +872 -0
  14. agentloop/checkpoint_state.py +306 -0
  15. agentloop/checkpoint_store.py +86 -0
  16. agentloop/ci.py +202 -0
  17. agentloop/cli.py +1314 -0
  18. agentloop/client.py +228 -0
  19. agentloop/completion.py +345 -0
  20. agentloop/completion_types.py +199 -0
  21. agentloop/config.py +50 -0
  22. agentloop/context_controls.py +443 -0
  23. agentloop/context_experiments.py +337 -0
  24. agentloop/context_types.py +216 -0
  25. agentloop/costs.py +667 -0
  26. agentloop/cursor_validation.py +48 -0
  27. agentloop/decorators.py +445 -0
  28. agentloop/demo.py +300 -0
  29. agentloop/doctor.py +341 -0
  30. agentloop/drift.py +361 -0
  31. agentloop/drift_cli.py +147 -0
  32. agentloop/drift_types.py +110 -0
  33. agentloop/drift_windows.py +340 -0
  34. agentloop/entrypoint.py +201 -0
  35. agentloop/estimates.py +133 -0
  36. agentloop/events.py +75 -0
  37. agentloop/experiment_artifacts.py +80 -0
  38. agentloop/experiment_observations.py +169 -0
  39. agentloop/experiment_reports.py +545 -0
  40. agentloop/experiment_types.py +355 -0
  41. agentloop/experiments.py +392 -0
  42. agentloop/exporters.py +117 -0
  43. agentloop/finding_benchmark_types.py +131 -0
  44. agentloop/finding_benchmarks.py +424 -0
  45. agentloop/findings.py +419 -0
  46. agentloop/graph.py +464 -0
  47. agentloop/harness.py +825 -0
  48. agentloop/harness_bench/__init__.py +1 -0
  49. agentloop/harness_bench/cli.py +77 -0
  50. agentloop/harness_bench/drivers.py +96 -0
  51. agentloop/harness_bench/offline.py +507 -0
  52. agentloop/harness_evidence.py +585 -0
  53. agentloop/harness_usage.py +91 -0
  54. agentloop/html_report.py +860 -0
  55. agentloop/integrations/__init__.py +1 -0
  56. agentloop/integrations/crewai.py +144 -0
  57. agentloop/integrations/harbor/__init__.py +1 -0
  58. agentloop/integrations/harbor/atif.py +982 -0
  59. agentloop/integrations/harbor/atif_validation.py +329 -0
  60. agentloop/integrations/harbor/cli.py +207 -0
  61. agentloop/integrations/harbor/manifest.py +86 -0
  62. agentloop/integrations/harbor/trial_evidence.py +196 -0
  63. agentloop/integrations/harbor/trials.py +1109 -0
  64. agentloop/integrations/harbor/verifier.py +118 -0
  65. agentloop/integrations/langgraph.py +150 -0
  66. agentloop/integrations/langgraph_harness.py +466 -0
  67. agentloop/integrations/langgraph_recovery.py +366 -0
  68. agentloop/integrations/omnigent/__init__.py +1 -0
  69. agentloop/integrations/omnigent/cli.py +78 -0
  70. agentloop/integrations/omnigent/evidence.py +93 -0
  71. agentloop/integrations/omnigent/telemetry.py +422 -0
  72. agentloop/integrations/openai.py +415 -0
  73. agentloop/integrations/openai_agents.py +291 -0
  74. agentloop/integrations/sdk.py +82 -0
  75. agentloop/integrations/vercel_ai.py +83 -0
  76. agentloop/interoperability/__init__.py +5 -0
  77. agentloop/interoperability/artifacts.py +112 -0
  78. agentloop/interoperability/cli.py +59 -0
  79. agentloop/interoperability/cohort_evidence.py +175 -0
  80. agentloop/interoperability/cohorts.py +681 -0
  81. agentloop/interoperability/contracts.py +427 -0
  82. agentloop/interoperability/coordination.py +554 -0
  83. agentloop/interoperability/evidence.py +215 -0
  84. agentloop/interoperability/otlp_jsonl.py +775 -0
  85. agentloop/interoperability/otlp_validation.py +496 -0
  86. agentloop/interoperability/registry.py +561 -0
  87. agentloop/interoperability/validation.py +333 -0
  88. agentloop/intervention_service.py +110 -0
  89. agentloop/intervention_store.py +118 -0
  90. agentloop/interventions.py +190 -0
  91. agentloop/issues.py +90 -0
  92. agentloop/judgment_adapters.py +125 -0
  93. agentloop/judgment_benchmark_types.py +98 -0
  94. agentloop/judgment_benchmarks.py +606 -0
  95. agentloop/judgment_types.py +241 -0
  96. agentloop/judgment_views.py +38 -0
  97. agentloop/judgments.py +342 -0
  98. agentloop/loop_guards.py +274 -0
  99. agentloop/loop_types.py +66 -0
  100. agentloop/markdown.py +102 -0
  101. agentloop/metrics.py +342 -0
  102. agentloop/migrations.py +303 -0
  103. agentloop/model_routing.py +470 -0
  104. agentloop/onboarding.py +315 -0
  105. agentloop/operations.py +44 -0
  106. agentloop/optimizer.py +199 -0
  107. agentloop/otel.py +715 -0
  108. agentloop/otel_ids.py +81 -0
  109. agentloop/otel_semantics.py +134 -0
  110. agentloop/parallelism.py +141 -0
  111. agentloop/patches.py +603 -0
  112. agentloop/plan_export.py +126 -0
  113. agentloop/policy_promotion.py +325 -0
  114. agentloop/promotion_evidence.py +304 -0
  115. agentloop/promotion_types.py +209 -0
  116. agentloop/quality.py +446 -0
  117. agentloop/ranking.py +529 -0
  118. agentloop/recovery_evidence.py +96 -0
  119. agentloop/replay.py +753 -0
  120. agentloop/retention.py +375 -0
  121. agentloop/retention_types.py +113 -0
  122. agentloop/reuse_experiments.py +247 -0
  123. agentloop/routing_checks.py +116 -0
  124. agentloop/routing_types.py +241 -0
  125. agentloop/rules.py +396 -0
  126. agentloop/runtime.py +286 -0
  127. agentloop/savings.py +158 -0
  128. agentloop/scheduling_plan.py +99 -0
  129. agentloop/scheduling_types.py +169 -0
  130. agentloop/schema.py +257 -0
  131. agentloop/semantic_rules.py +200 -0
  132. agentloop/semantic_waste.py +345 -0
  133. agentloop/semantic_waste_types.py +88 -0
  134. agentloop/server.py +494 -0
  135. agentloop/store.py +1429 -0
  136. agentloop/structural_experiments.py +431 -0
  137. agentloop/structural_types.py +113 -0
  138. agentloop/structured_quality.py +1020 -0
  139. agentloop/studies.py +629 -0
  140. agentloop/study_statistics.py +58 -0
  141. agentloop/substitution_evidence.py +199 -0
  142. agentloop/substitution_export.py +190 -0
  143. agentloop/substitution_reports.py +407 -0
  144. agentloop/substitution_types.py +284 -0
  145. agentloop/substitutions.py +267 -0
  146. agentloop/timing.py +142 -0
  147. agentloop/tokens.py +188 -0
  148. agentloop/tool_scheduling.py +410 -0
  149. agentloop/trace_sources.py +31 -0
  150. agentloop/tracer.py +494 -0
  151. agentloop/value.py +306 -0
  152. agentloop/version.py +3 -0
  153. agentloop/workflow_types.py +264 -0
  154. agentloop/workflows.py +284 -0
  155. agentloop_profiler-0.1.0.dist-info/METADATA +513 -0
  156. agentloop_profiler-0.1.0.dist-info/RECORD +166 -0
  157. agentloop_profiler-0.1.0.dist-info/WHEEL +5 -0
  158. agentloop_profiler-0.1.0.dist-info/entry_points.txt +2 -0
  159. agentloop_profiler-0.1.0.dist-info/licenses/LICENSE +201 -0
  160. agentloop_profiler-0.1.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +101 -0
  161. agentloop_profiler-0.1.0.dist-info/top_level.txt +2 -0
  162. dashboard/__init__.py +5 -0
  163. dashboard/app.py +922 -0
  164. dashboard/streamlit_compat.py +63 -0
  165. dashboard/trace_upload.py +57 -0
  166. dashboard/value_view.py +105 -0
agentloop/harness.py ADDED
@@ -0,0 +1,825 @@
1
+ """Opt-in lifecycle controls for explicitly wrapped Python callables.
2
+
3
+ Policies are synchronous trusted application code. The harness is neither a
4
+ sandbox nor an interceptor for calls outside its wrappers. See docs/HARNESS.md.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import inspect
10
+ import json
11
+ import math
12
+ import re
13
+ from asyncio import CancelledError
14
+ from collections.abc import Callable, Mapping
15
+ from contextvars import ContextVar
16
+ from dataclasses import dataclass, field
17
+ from functools import wraps
18
+ from hashlib import sha256
19
+ from threading import RLock
20
+ from time import perf_counter_ns
21
+ from types import MappingProxyType
22
+ from typing import Any, TypeVar
23
+ from uuid import uuid4
24
+ from weakref import ReferenceType, ref
25
+
26
+ from agentloop.budget_types import BudgetSnapshot, DispatchOptions, ResourceUsage
27
+ from agentloop.events import new_run_id, utc_now_iso
28
+ from agentloop.harness_evidence import (
29
+ METADATA_KEY,
30
+ HarnessEvidenceError,
31
+ append_records,
32
+ canonical_json,
33
+ decision_id,
34
+ empty_evidence,
35
+ policy_snapshot,
36
+ records_for_hook,
37
+ valid_decision_id,
38
+ )
39
+ from agentloop.harness_usage import ObservedGenerator, UsageTracker
40
+ from agentloop.tracer import AgentTrace, current_event_id, current_trace
41
+
42
+ CONTRACT_VERSION = "1.0"
43
+ BOUNDARIES = frozenset({"model", "tool", "iteration", "completion"})
44
+ PHASES = frozenset({"before", "after"})
45
+ ACTIONS = frozenset({"continue", "deny", "stop", "escalate"})
46
+ EXECUTION_KINDS = frozenset({"sync", "async", "generator", "async_generator"})
47
+ _PRECEDENCE = {"continue": 0, "deny": 1, "stop": 2, "escalate": 3}
48
+ _IDENTIFIER = re.compile(r"[A-Za-z0-9_.:-]{1,96}\Z")
49
+ _IN_POLICY: ContextVar[bool] = ContextVar("agentloop_in_harness_policy", default=False)
50
+ F = TypeVar("F", bound=Callable[..., Any])
51
+
52
+
53
+ def _identifier(value: str, label: str) -> None:
54
+ if not isinstance(value, str) or not _IDENTIFIER.fullmatch(value):
55
+ raise ValueError(f"{label} must be a bounded identifier, not free-form content")
56
+
57
+
58
+ def _reference(value: str, label: str) -> None:
59
+ if not isinstance(value, str) or not value:
60
+ raise ValueError(f"{label} must be a nonempty string")
61
+
62
+
63
+ def _json_value(value: Any) -> Any:
64
+ if isinstance(value, Mapping):
65
+ if any(not isinstance(key, str) for key in value):
66
+ raise ValueError("configuration keys must be strings")
67
+ return {key: _json_value(item) for key, item in value.items()}
68
+ if isinstance(value, (list, tuple)):
69
+ return [_json_value(item) for item in value]
70
+ if value is None or type(value) in {str, bool, int}:
71
+ return value
72
+ if type(value) is float and math.isfinite(value):
73
+ return value
74
+ raise ValueError("configuration must contain only finite JSON values")
75
+
76
+
77
+ def _freeze(value: Any) -> Any:
78
+ if isinstance(value, dict):
79
+ return MappingProxyType({key: _freeze(item) for key, item in value.items()})
80
+ if isinstance(value, list):
81
+ return tuple(_freeze(item) for item in value)
82
+ return value
83
+
84
+
85
+ def _hash(value: Any) -> str:
86
+ return sha256(
87
+ json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode()
88
+ ).hexdigest()
89
+
90
+
91
+ def _callable_kind(function: Callable[..., Any]) -> str:
92
+ for kind, check in (
93
+ ("async_generator", inspect.isasyncgenfunction),
94
+ ("generator", inspect.isgeneratorfunction),
95
+ ("async", inspect.iscoroutinefunction),
96
+ ):
97
+ if check(function) or check(getattr(function, "__call__", None)):
98
+ return kind
99
+ return "sync"
100
+
101
+
102
+ @dataclass(frozen=True, order=True)
103
+ class Hook:
104
+ """One caller-defined dispatch boundary and lifecycle phase."""
105
+
106
+ boundary: str
107
+ phase: str = "before"
108
+
109
+ def __post_init__(self) -> None:
110
+ if self.boundary not in BOUNDARIES or self.phase not in PHASES:
111
+ raise ValueError("unsupported harness hook")
112
+
113
+
114
+ ALL_HOOKS = frozenset(Hook(boundary, phase) for boundary in BOUNDARIES for phase in PHASES)
115
+
116
+
117
+ @dataclass(frozen=True)
118
+ class Decision:
119
+ """A bounded proposal with no arguments, outputs, or exception details."""
120
+
121
+ action: str = "continue"
122
+ reason_code: str = "allowed"
123
+ evidence_refs: tuple[str, ...] = ()
124
+ retry_of: str | None = None
125
+ budget_snapshot: BudgetSnapshot | None = None
126
+ feedback: str | None = None
127
+
128
+ def __post_init__(self) -> None:
129
+ if self.action not in ACTIONS:
130
+ raise ValueError("unsupported harness action")
131
+ _identifier(self.reason_code, "reason_code")
132
+ if self.feedback is not None and (
133
+ not isinstance(self.feedback, str) or not self.feedback or len(self.feedback) > 256
134
+ ):
135
+ raise ValueError("feedback must be nonempty text of at most 256 characters")
136
+ if not isinstance(self.evidence_refs, (tuple, list, set, frozenset)):
137
+ raise ValueError("evidence_refs must be a collection of identifiers")
138
+ refs = tuple(self.evidence_refs)
139
+ for value in refs:
140
+ _reference(value, "evidence reference")
141
+ object.__setattr__(self, "evidence_refs", tuple(sorted(set(refs))))
142
+ if self.retry_of is not None and not valid_decision_id(self.retry_of):
143
+ raise ValueError("retry_of must reference a harness decision ID")
144
+ if self.budget_snapshot is not None:
145
+ if type(self.budget_snapshot) is not BudgetSnapshot:
146
+ raise ValueError("budget_snapshot must be a BudgetSnapshot")
147
+ BudgetSnapshot.from_dict(self.budget_snapshot.to_dict())
148
+
149
+
150
+ @dataclass(frozen=True)
151
+ class HookContext:
152
+ """Safe call identity plus one policy's synchronized, run-local state."""
153
+
154
+ run_id: str
155
+ call_id: str
156
+ branch_id: str
157
+ hook: Hook
158
+ mode: str
159
+ status: str
160
+ trace_id: str | None
161
+ parent_span_id: str | None
162
+ configuration: Mapping[str, Any]
163
+ state: dict[str, Any] = field(repr=False, compare=False)
164
+ dispatched: bool
165
+ decision_id: str = ""
166
+ dispatch: DispatchOptions = field(default_factory=DispatchOptions)
167
+ usage: ResourceUsage | None = None
168
+ usage_error: str | None = None
169
+
170
+
171
+ @dataclass(frozen=True)
172
+ class Policy:
173
+ """Versioned callback and immutable declarations checked before enforcement."""
174
+
175
+ policy_id: str
176
+ version: str
177
+ evaluate: Callable[[HookContext], Decision] = field(repr=False, compare=False)
178
+ hooks: frozenset[Hook] = field(default_factory=lambda: frozenset({Hook("model")}))
179
+ actions: frozenset[str] = field(default_factory=lambda: frozenset({"continue"}))
180
+ priority: int = 0
181
+ configuration: Mapping[str, Any] = field(default_factory=dict, repr=False)
182
+ config_hash: str = field(init=False)
183
+
184
+ def __post_init__(self) -> None:
185
+ _identifier(self.policy_id, "policy_id")
186
+ _identifier(self.version, "policy version")
187
+ if not callable(self.evaluate) or _callable_kind(self.evaluate) != "sync":
188
+ raise ValueError("policy evaluate must be a synchronous callable")
189
+ if type(self.priority) is not int:
190
+ raise ValueError("policy priority must be an integer")
191
+ hooks, actions = frozenset(self.hooks), frozenset(self.actions)
192
+ if not hooks or any(not isinstance(hook, Hook) for hook in hooks):
193
+ raise ValueError("policy must declare supported Hook objects")
194
+ if not actions or not actions <= ACTIONS:
195
+ raise ValueError("policy must declare supported actions")
196
+ if "deny" in actions and not any(hook.phase == "before" for hook in hooks):
197
+ raise ValueError("deny requires a before hook")
198
+ if not isinstance(self.configuration, Mapping):
199
+ raise ValueError("policy configuration must be a JSON object")
200
+ configuration = _json_value(self.configuration)
201
+ object.__setattr__(self, "hooks", hooks)
202
+ object.__setattr__(self, "actions", actions)
203
+ object.__setattr__(self, "configuration", _freeze(configuration))
204
+ object.__setattr__(
205
+ self,
206
+ "config_hash",
207
+ _hash(
208
+ {
209
+ "contract_version": CONTRACT_VERSION,
210
+ "policy_id": self.policy_id,
211
+ "version": self.version,
212
+ "priority": self.priority,
213
+ "hooks": [(hook.boundary, hook.phase) for hook in sorted(hooks)],
214
+ "actions": sorted(actions),
215
+ "configuration": configuration,
216
+ }
217
+ ),
218
+ )
219
+
220
+
221
+ @dataclass(frozen=True)
222
+ class HarnessConfig:
223
+ """A deterministic, immutable policy configuration; controls default to off."""
224
+
225
+ mode: str = "disabled"
226
+ policies: tuple[Policy, ...] = ()
227
+ schema_version: str = CONTRACT_VERSION
228
+ capture_policy_configuration: bool = False
229
+ config_hash: str = field(init=False)
230
+
231
+ def __post_init__(self) -> None:
232
+ if self.mode not in {"disabled", "shadow", "enforce"}:
233
+ raise ValueError("mode must be disabled, shadow, or enforce")
234
+ if self.schema_version != CONTRACT_VERSION:
235
+ raise ValueError("unsupported harness configuration version")
236
+ if type(self.capture_policy_configuration) is not bool:
237
+ raise ValueError("capture_policy_configuration must be a boolean")
238
+ policies = tuple(self.policies)
239
+ if any(not isinstance(policy, Policy) for policy in policies):
240
+ raise ValueError("policies must be Policy objects")
241
+ if len({policy.policy_id for policy in policies}) != len(policies):
242
+ raise ValueError("policy IDs must be unique")
243
+ policies = tuple(sorted(policies, key=lambda policy: (policy.priority, policy.policy_id)))
244
+ object.__setattr__(self, "policies", policies)
245
+ object.__setattr__(
246
+ self,
247
+ "config_hash",
248
+ _hash(
249
+ {
250
+ "schema_version": self.schema_version,
251
+ "mode": self.mode,
252
+ "policies": [policy.config_hash for policy in policies],
253
+ **(
254
+ {"capture_policy_configuration": True}
255
+ if self.capture_policy_configuration
256
+ else {}
257
+ ),
258
+ }
259
+ ),
260
+ )
261
+
262
+
263
+ @dataclass(frozen=True)
264
+ class AdapterCapabilities:
265
+ """Explicit supported hooks, actions, and callable lifecycles."""
266
+
267
+ name: str
268
+ hooks: frozenset[Hook]
269
+ actions: frozenset[str]
270
+ execution_kinds: frozenset[str]
271
+
272
+ def __post_init__(self) -> None:
273
+ _identifier(self.name, "adapter name")
274
+ for key in ("hooks", "actions", "execution_kinds"):
275
+ object.__setattr__(self, key, frozenset(getattr(self, key)))
276
+ if not self.hooks <= ALL_HOOKS or not self.actions <= ACTIONS:
277
+ raise ValueError("unsupported adapter capability")
278
+ if not self.execution_kinds <= EXECUTION_KINDS:
279
+ raise ValueError("unsupported callable lifecycle")
280
+
281
+
282
+ PYTHON_CAPABILITIES = AdapterCapabilities("python", ALL_HOOKS, ACTIONS, EXECUTION_KINDS)
283
+
284
+
285
+ @dataclass(frozen=True)
286
+ class PolicyResult:
287
+ """One policy proposal, including safe policy-error evidence."""
288
+
289
+ policy_id: str
290
+ policy_version: str
291
+ config_hash: str
292
+ decision: Decision
293
+ failed: bool = False
294
+ started_at: str | None = None
295
+ duration_ms: float | None = None
296
+
297
+
298
+ @dataclass(frozen=True)
299
+ class HookResult:
300
+ """In-memory proposals and resolution, not a measured intervention outcome."""
301
+
302
+ run_id: str
303
+ call_id: str
304
+ branch_id: str
305
+ hook: Hook
306
+ mode: str
307
+ status: str
308
+ trace_id: str | None
309
+ parent_span_id: str | None
310
+ proposals: tuple[PolicyResult, ...]
311
+ action: str
312
+ applied: bool
313
+ dispatched: bool
314
+ started_at: str | None = None
315
+ duration_ms: float | None = None
316
+ sequence: int | None = None
317
+
318
+
319
+ class HarnessControlError(RuntimeError):
320
+ """Base for explicit control flow that the host application must handle."""
321
+
322
+ def __init__(self, result: HookResult) -> None:
323
+ self.result = result
324
+ super().__init__(f"Harness {result.action} at {result.hook.boundary}/{result.hook.phase}")
325
+
326
+
327
+ class HarnessDeniedError(HarnessControlError):
328
+ """The protected callable was not admitted."""
329
+
330
+
331
+ class HarnessStoppedError(HarnessControlError):
332
+ """The run no longer admits new work; existing work is not interrupted."""
333
+
334
+
335
+ class HarnessEscalationError(HarnessControlError):
336
+ """The run requires host intervention before further work can be admitted."""
337
+
338
+
339
+ @dataclass(frozen=True)
340
+ class Harness:
341
+ """Factory for independent runs sharing immutable settings, never run state."""
342
+
343
+ config: HarnessConfig = field(default_factory=HarnessConfig)
344
+ capabilities: AdapterCapabilities = PYTHON_CAPABILITIES
345
+ _policies_by_hook: Mapping[Hook, tuple[Policy, ...]] = field(
346
+ init=False, repr=False, compare=False
347
+ )
348
+ _snapshot_json: Mapping[str, str] = field(init=False, repr=False, compare=False)
349
+ _system_snapshot_json: str = field(init=False, repr=False, compare=False)
350
+
351
+ def __post_init__(self) -> None:
352
+ if not isinstance(self.config, HarnessConfig) or not isinstance(
353
+ self.capabilities, AdapterCapabilities
354
+ ):
355
+ raise ValueError("harness requires configuration and adapter capabilities")
356
+ if (
357
+ self.config.mode == "enforce"
358
+ and not {"continue", "stop", "escalate"} <= self.capabilities.actions
359
+ ):
360
+ raise ValueError("adapter actions must support fail-closed stop and escalation")
361
+ if self.config.mode != "disabled":
362
+ for policy in self.config.policies:
363
+ if not policy.hooks <= self.capabilities.hooks:
364
+ raise ValueError("adapter does not support every declared policy hook")
365
+ if not policy.actions <= self.capabilities.actions:
366
+ raise ValueError("adapter does not support every declared policy action")
367
+ object.__setattr__(
368
+ self,
369
+ "_policies_by_hook",
370
+ MappingProxyType(
371
+ {
372
+ hook: tuple(policy for policy in self.config.policies if hook in policy.hooks)
373
+ for hook in ALL_HOOKS
374
+ }
375
+ ),
376
+ )
377
+ snapshots = {
378
+ policy.config_hash: canonical_json(
379
+ policy_snapshot(
380
+ policy, include_configuration=self.config.capture_policy_configuration
381
+ )
382
+ )
383
+ for policy in self.config.policies
384
+ }
385
+ system = {
386
+ "policy_id": "agentloop.harness",
387
+ "version": CONTRACT_VERSION,
388
+ "config_hash": _hash(
389
+ {
390
+ "harness_config_hash": self.config.config_hash,
391
+ "adapter": self.capabilities.name,
392
+ "hooks": [
393
+ (hook.boundary, hook.phase) for hook in sorted(self.capabilities.hooks)
394
+ ],
395
+ "actions": sorted(self.capabilities.actions),
396
+ "execution_kinds": sorted(self.capabilities.execution_kinds),
397
+ }
398
+ ),
399
+ "priority": 0,
400
+ "hooks": [
401
+ {"boundary": hook.boundary, "phase": hook.phase} for hook in sorted(ALL_HOOKS)
402
+ ],
403
+ "actions": sorted(ACTIONS),
404
+ "configuration_capture": "redacted",
405
+ "configuration": None,
406
+ }
407
+ serialized = canonical_json(system)
408
+ snapshots[system["config_hash"]] = serialized
409
+ object.__setattr__(self, "_snapshot_json", MappingProxyType(snapshots))
410
+ object.__setattr__(self, "_system_snapshot_json", serialized)
411
+
412
+ def start_run(self, run_id: str | None = None) -> HarnessRun:
413
+ """Create fresh state, using the active trace's identity when available."""
414
+ trace = current_trace()
415
+ identity = run_id if run_id is not None else trace.run_id if trace else new_run_id()
416
+ _reference(identity, "run_id")
417
+ return HarnessRun(self, identity)
418
+
419
+
420
+ @dataclass(frozen=True)
421
+ class _Invocation:
422
+ call_id: str
423
+ boundary: str
424
+ branch_id: str
425
+ trace_id: str | None
426
+ parent_span_id: str | None
427
+ trace: AgentTrace | None = field(repr=False, compare=False)
428
+ dispatch: DispatchOptions
429
+ usage: UsageTracker = field(repr=False, compare=False)
430
+
431
+
432
+ @dataclass(frozen=True)
433
+ class _WrapperMarker:
434
+ run: HarnessRun
435
+ boundary: str
436
+ branch_id: str
437
+ function: ReferenceType
438
+ dispatch: DispatchOptions
439
+ usage_reader: Callable | None
440
+ error_usage_reader: Callable | None
441
+
442
+
443
+ def _failure_status(exc: BaseException) -> str:
444
+ if isinstance(exc, GeneratorExit):
445
+ return "closed"
446
+ if isinstance(exc, CancelledError):
447
+ return "cancelled"
448
+ return "error"
449
+
450
+
451
+ def _invoke_usage_reader(reader: Callable, value: Any):
452
+ token = _IN_POLICY.set(True)
453
+ try:
454
+ return reader(value)
455
+ finally:
456
+ _IN_POLICY.reset(token)
457
+
458
+
459
+ class HarnessRun:
460
+ """Explicit run ownership with serialized hooks and concurrent protected work."""
461
+
462
+ def __init__(self, harness: Harness, run_id: str) -> None:
463
+ self._harness = harness
464
+ self._run_id = run_id
465
+ self._lock = RLock()
466
+ self._states: dict[str, dict[str, Any]] = {}
467
+ self._results: list[HookResult] = []
468
+ self._stopped = False
469
+ self._evidence = empty_evidence()
470
+ self._snapshots = {key: json.loads(value) for key, value in harness._snapshot_json.items()}
471
+ self._system_snapshot = json.loads(harness._system_snapshot_json)
472
+
473
+ @property
474
+ def harness(self) -> Harness:
475
+ """The immutable factory/configuration captured when this run was created."""
476
+ return self._harness
477
+
478
+ @property
479
+ def run_id(self) -> str:
480
+ """The host's stable identity for this run."""
481
+ return self._run_id
482
+
483
+ @property
484
+ def results(self) -> tuple[HookResult, ...]:
485
+ """Return an immutable snapshot of recorded hooks."""
486
+ with self._lock:
487
+ return tuple(self._results)
488
+
489
+ @property
490
+ def stopped(self) -> bool:
491
+ """Whether enforcement has stopped new admissions for this run."""
492
+ with self._lock:
493
+ return self._stopped
494
+
495
+ def export_evidence(self) -> dict[str, Any]:
496
+ """Return a detached artifact, including any safe trace-capture diagnostics."""
497
+ with self._lock:
498
+ return json.loads(canonical_json(self._evidence))
499
+
500
+ def _record_evidence(self, invocation: _Invocation, result: HookResult) -> None:
501
+ records = records_for_hook(result, self.harness.config.config_hash, self._system_snapshot)
502
+ append_records(self._evidence, records, self._snapshots)
503
+ if invocation.trace is None:
504
+ return
505
+ try:
506
+ envelope = invocation.trace.metadata.setdefault(METADATA_KEY, empty_evidence())
507
+ append_records(envelope, records, self._snapshots)
508
+ except (HarnessEvidenceError, AttributeError, TypeError, KeyError):
509
+ self._evidence["capture_errors"].append(
510
+ {
511
+ "call_id": invocation.call_id,
512
+ "phase": result.hook.phase,
513
+ "category": "trace_evidence_error",
514
+ }
515
+ )
516
+ if self.harness.config.mode == "enforce":
517
+ self._stopped = True
518
+ raise HarnessEvidenceError("trace evidence capture failed") from None
519
+
520
+ def _hook(
521
+ self, invocation: _Invocation, phase: str, status: str, *, dispatched: bool = False
522
+ ) -> HookResult:
523
+ hook = Hook(invocation.boundary, phase)
524
+ config = self.harness.config
525
+ started_at = utc_now_iso()
526
+ started_ns = perf_counter_ns()
527
+ with self._lock:
528
+ proposals = []
529
+ fatal: BaseException | None = None
530
+ action = "stop" if self._stopped and phase == "before" else "continue"
531
+ if action == "continue":
532
+ for policy in self.harness._policies_by_hook[hook]:
533
+ context = HookContext(
534
+ self.run_id,
535
+ invocation.call_id,
536
+ invocation.branch_id,
537
+ hook,
538
+ config.mode,
539
+ status,
540
+ invocation.trace_id,
541
+ invocation.parent_span_id,
542
+ policy.configuration,
543
+ self._states.setdefault(policy.policy_id, {}),
544
+ dispatched,
545
+ decision_id(
546
+ self.run_id,
547
+ invocation.call_id,
548
+ hook.boundary,
549
+ hook.phase,
550
+ policy.policy_id,
551
+ ),
552
+ invocation.dispatch,
553
+ invocation.usage.latest,
554
+ invocation.usage.error,
555
+ )
556
+ token = _IN_POLICY.set(True)
557
+ failed = False
558
+ policy_started_at = utc_now_iso()
559
+ policy_started_ns = perf_counter_ns()
560
+ try:
561
+ decision = policy.evaluate(context)
562
+ if (
563
+ isinstance(decision, Decision)
564
+ and decision.budget_snapshot is not None
565
+ and config.mode == "shadow"
566
+ ):
567
+ budget = decision.budget_snapshot.to_dict()
568
+ if any(
569
+ budget[key] == "best_effort"
570
+ for key in ("spend_enforcement", "token_enforcement")
571
+ ):
572
+ raise ValueError("shadow budget evidence cannot claim enforcement")
573
+ if (
574
+ not isinstance(decision, Decision)
575
+ or decision.action not in policy.actions
576
+ or (phase == "after" and decision.action == "deny")
577
+ or decision.retry_of == context.decision_id
578
+ ):
579
+ if inspect.iscoroutine(decision):
580
+ decision.close()
581
+ raise ValueError("policy returned an undeclared decision")
582
+ except BaseException as exc:
583
+ decision = Decision("escalate", "policy_error")
584
+ failed = True
585
+ if not isinstance(exc, Exception):
586
+ fatal = exc
587
+ finally:
588
+ _IN_POLICY.reset(token)
589
+ proposals.append(
590
+ PolicyResult(
591
+ policy.policy_id,
592
+ policy.version,
593
+ policy.config_hash,
594
+ decision,
595
+ failed,
596
+ policy_started_at,
597
+ max(0, perf_counter_ns() - policy_started_ns) / 1_000_000,
598
+ )
599
+ )
600
+ if _PRECEDENCE[decision.action] > _PRECEDENCE[action]:
601
+ action = decision.action
602
+ if fatal is not None:
603
+ break
604
+ if config.mode == "enforce" and action in {"stop", "escalate"}:
605
+ self._stopped = True
606
+ result = HookResult(
607
+ self.run_id,
608
+ invocation.call_id,
609
+ invocation.branch_id,
610
+ hook,
611
+ config.mode,
612
+ status,
613
+ invocation.trace_id,
614
+ invocation.parent_span_id,
615
+ tuple(proposals),
616
+ action,
617
+ config.mode == "enforce" and action != "continue",
618
+ dispatched,
619
+ started_at,
620
+ max(0, perf_counter_ns() - started_ns) / 1_000_000,
621
+ len(self._results),
622
+ )
623
+ self._results.append(result)
624
+ try:
625
+ self._record_evidence(invocation, result)
626
+ except HarnessEvidenceError:
627
+ if fatal is None:
628
+ raise
629
+ if fatal is not None:
630
+ raise fatal
631
+ return result
632
+
633
+ @staticmethod
634
+ def _apply(result: HookResult) -> None:
635
+ if not result.applied:
636
+ return
637
+ error = {
638
+ "deny": HarnessDeniedError,
639
+ "stop": HarnessStoppedError,
640
+ "escalate": HarnessEscalationError,
641
+ }
642
+ raise error[result.action](result)
643
+
644
+ def _begin(
645
+ self,
646
+ boundary: str,
647
+ branch_id: str,
648
+ dispatch: DispatchOptions,
649
+ usage_reader: Callable | None,
650
+ error_usage_reader: Callable | None = None,
651
+ ) -> _Invocation:
652
+ if _IN_POLICY.get():
653
+ raise RuntimeError("protected dispatch from a policy is unsupported")
654
+ trace = current_trace()
655
+ invocation = _Invocation(
656
+ f"call_{uuid4().hex}",
657
+ boundary,
658
+ branch_id,
659
+ trace.run_id if trace else None,
660
+ current_event_id() if trace else None,
661
+ trace,
662
+ dispatch,
663
+ UsageTracker(usage_reader, _invoke_usage_reader, error_usage_reader),
664
+ )
665
+ try:
666
+ self._apply(self._hook(invocation, "before", "pending"))
667
+ except BaseException as exc:
668
+ status = "denied" if isinstance(exc, HarnessControlError) else _failure_status(exc)
669
+ self._finish(invocation, status, exc, dispatched=False)
670
+ raise
671
+ return invocation
672
+
673
+ def _finish(
674
+ self,
675
+ invocation: _Invocation,
676
+ status: str,
677
+ original: BaseException | None = None,
678
+ *,
679
+ dispatched: bool = True,
680
+ ) -> None:
681
+ try:
682
+ self._apply(self._hook(invocation, "after", status, dispatched=dispatched))
683
+ except BaseException:
684
+ if original is None:
685
+ raise
686
+ # Preserve an original failure/cancellation. The hook result has
687
+ # already recorded the failure and stopped future enforced work.
688
+
689
+ def wrap(
690
+ self,
691
+ function: F,
692
+ *,
693
+ boundary: str,
694
+ branch_id: str = "main",
695
+ dispatch: DispatchOptions | None = None,
696
+ usage_reader: Callable[[Any], ResourceUsage | None] | None = None,
697
+ error_usage_reader: Callable[[BaseException], ResourceUsage | None] | None = None,
698
+ ) -> F:
699
+ """Wrap a dispatch, optionally reading exclusive usage from SDK failures.
700
+
701
+ Error readers run only after dispatch and cannot replace the original
702
+ failure/cancellation. They receive an exception locally, never in hooks.
703
+ """
704
+ if not callable(function):
705
+ raise ValueError("protected work must be callable")
706
+ Hook(boundary)
707
+ _reference(branch_id, "branch_id")
708
+ options = DispatchOptions() if dispatch is None else dispatch
709
+ if type(options) is not DispatchOptions:
710
+ raise ValueError("dispatch must be DispatchOptions")
711
+ if usage_reader is not None and (
712
+ not callable(usage_reader) or _callable_kind(usage_reader) != "sync"
713
+ ):
714
+ raise ValueError("usage_reader must be a synchronous callable")
715
+ if error_usage_reader is not None and (
716
+ not callable(error_usage_reader) or _callable_kind(error_usage_reader) != "sync"
717
+ ):
718
+ raise ValueError("error_usage_reader must be a synchronous callable")
719
+ if self.harness.config.mode == "disabled":
720
+ return function
721
+ marker = getattr(function, "__agentloop_harness__", None)
722
+ if (
723
+ isinstance(marker, _WrapperMarker)
724
+ and marker.run is self
725
+ and marker.boundary == boundary
726
+ and marker.branch_id == branch_id
727
+ and marker.function() is function
728
+ ):
729
+ if (
730
+ marker.dispatch == options
731
+ and marker.usage_reader is usage_reader
732
+ and marker.error_usage_reader is error_usage_reader
733
+ ):
734
+ return function
735
+ raise ValueError("wrap the original callable to change dispatch or usage options")
736
+ kind = _callable_kind(function)
737
+ if kind not in self.harness.capabilities.execution_kinds:
738
+ raise ValueError("adapter does not support this callable lifecycle")
739
+ if not {Hook(boundary), Hook(boundary, "after")} <= self.harness.capabilities.hooks:
740
+ raise ValueError("adapter does not support this dispatch boundary's lifecycle")
741
+
742
+ if kind == "async_generator":
743
+
744
+ @wraps(function)
745
+ async def wrapped(*args, **kwargs):
746
+ invocation = self._begin(
747
+ boundary, branch_id, options, usage_reader, error_usage_reader
748
+ )
749
+ try:
750
+ generator = function(*args, **kwargs)
751
+ item = await generator.__anext__()
752
+ while True:
753
+ invocation.usage.observe(item)
754
+ try:
755
+ sent = yield item
756
+ except GeneratorExit:
757
+ await generator.aclose()
758
+ raise
759
+ except BaseException as exc:
760
+ item = await generator.athrow(exc)
761
+ else:
762
+ item = await generator.asend(sent)
763
+ except StopAsyncIteration:
764
+ self._finish(invocation, "ok")
765
+ except BaseException as exc:
766
+ invocation.usage.observe_error(exc)
767
+ self._finish(invocation, _failure_status(exc), exc)
768
+ raise
769
+ elif kind == "generator":
770
+
771
+ @wraps(function)
772
+ def wrapped(*args, **kwargs):
773
+ invocation = self._begin(
774
+ boundary, branch_id, options, usage_reader, error_usage_reader
775
+ )
776
+ try:
777
+ source = function(*args, **kwargs)
778
+ result = yield from (
779
+ source
780
+ if usage_reader is None
781
+ else ObservedGenerator(source, invocation.usage)
782
+ )
783
+ except BaseException as exc:
784
+ invocation.usage.observe_error(exc)
785
+ self._finish(invocation, _failure_status(exc), exc)
786
+ raise
787
+ self._finish(invocation, "ok")
788
+ return result
789
+ elif kind == "async":
790
+
791
+ @wraps(function)
792
+ async def wrapped(*args, **kwargs):
793
+ invocation = self._begin(
794
+ boundary, branch_id, options, usage_reader, error_usage_reader
795
+ )
796
+ try:
797
+ result = await function(*args, **kwargs)
798
+ invocation.usage.observe(result)
799
+ except BaseException as exc:
800
+ invocation.usage.observe_error(exc)
801
+ self._finish(invocation, _failure_status(exc), exc)
802
+ raise
803
+ self._finish(invocation, "ok")
804
+ return result
805
+ else:
806
+
807
+ @wraps(function)
808
+ def wrapped(*args, **kwargs):
809
+ invocation = self._begin(
810
+ boundary, branch_id, options, usage_reader, error_usage_reader
811
+ )
812
+ try:
813
+ result = function(*args, **kwargs)
814
+ invocation.usage.observe(result)
815
+ except BaseException as exc:
816
+ invocation.usage.observe_error(exc)
817
+ self._finish(invocation, _failure_status(exc), exc)
818
+ raise
819
+ self._finish(invocation, "ok")
820
+ return result
821
+
822
+ wrapped.__agentloop_harness__ = _WrapperMarker(
823
+ self, boundary, branch_id, ref(wrapped), options, usage_reader, error_usage_reader
824
+ )
825
+ return wrapped # type: ignore[return-value]