agentloop-profiler 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentloop/__init__.py +136 -0
- agentloop/ablation_protocol.py +229 -0
- agentloop/ablations.py +564 -0
- agentloop/aggregate_cli.py +88 -0
- agentloop/aggregate_stats.py +175 -0
- agentloop/aggregates.py +532 -0
- agentloop/audit.py +39 -0
- agentloop/autoinstrument.py +145 -0
- agentloop/batch_experiments.py +295 -0
- agentloop/batching.py +95 -0
- agentloop/budget_types.py +335 -0
- agentloop/budgets.py +539 -0
- agentloop/calibration.py +872 -0
- agentloop/checkpoint_state.py +306 -0
- agentloop/checkpoint_store.py +86 -0
- agentloop/ci.py +202 -0
- agentloop/cli.py +1314 -0
- agentloop/client.py +228 -0
- agentloop/completion.py +345 -0
- agentloop/completion_types.py +199 -0
- agentloop/config.py +50 -0
- agentloop/context_controls.py +443 -0
- agentloop/context_experiments.py +337 -0
- agentloop/context_types.py +216 -0
- agentloop/costs.py +667 -0
- agentloop/cursor_validation.py +48 -0
- agentloop/decorators.py +445 -0
- agentloop/demo.py +300 -0
- agentloop/doctor.py +341 -0
- agentloop/drift.py +361 -0
- agentloop/drift_cli.py +147 -0
- agentloop/drift_types.py +110 -0
- agentloop/drift_windows.py +340 -0
- agentloop/entrypoint.py +201 -0
- agentloop/estimates.py +133 -0
- agentloop/events.py +75 -0
- agentloop/experiment_artifacts.py +80 -0
- agentloop/experiment_observations.py +169 -0
- agentloop/experiment_reports.py +545 -0
- agentloop/experiment_types.py +355 -0
- agentloop/experiments.py +392 -0
- agentloop/exporters.py +117 -0
- agentloop/finding_benchmark_types.py +131 -0
- agentloop/finding_benchmarks.py +424 -0
- agentloop/findings.py +419 -0
- agentloop/graph.py +464 -0
- agentloop/harness.py +825 -0
- agentloop/harness_bench/__init__.py +1 -0
- agentloop/harness_bench/cli.py +77 -0
- agentloop/harness_bench/drivers.py +96 -0
- agentloop/harness_bench/offline.py +507 -0
- agentloop/harness_evidence.py +585 -0
- agentloop/harness_usage.py +91 -0
- agentloop/html_report.py +860 -0
- agentloop/integrations/__init__.py +1 -0
- agentloop/integrations/crewai.py +144 -0
- agentloop/integrations/harbor/__init__.py +1 -0
- agentloop/integrations/harbor/atif.py +982 -0
- agentloop/integrations/harbor/atif_validation.py +329 -0
- agentloop/integrations/harbor/cli.py +207 -0
- agentloop/integrations/harbor/manifest.py +86 -0
- agentloop/integrations/harbor/trial_evidence.py +196 -0
- agentloop/integrations/harbor/trials.py +1109 -0
- agentloop/integrations/harbor/verifier.py +118 -0
- agentloop/integrations/langgraph.py +150 -0
- agentloop/integrations/langgraph_harness.py +466 -0
- agentloop/integrations/langgraph_recovery.py +366 -0
- agentloop/integrations/omnigent/__init__.py +1 -0
- agentloop/integrations/omnigent/cli.py +78 -0
- agentloop/integrations/omnigent/evidence.py +93 -0
- agentloop/integrations/omnigent/telemetry.py +422 -0
- agentloop/integrations/openai.py +415 -0
- agentloop/integrations/openai_agents.py +291 -0
- agentloop/integrations/sdk.py +82 -0
- agentloop/integrations/vercel_ai.py +83 -0
- agentloop/interoperability/__init__.py +5 -0
- agentloop/interoperability/artifacts.py +112 -0
- agentloop/interoperability/cli.py +59 -0
- agentloop/interoperability/cohort_evidence.py +175 -0
- agentloop/interoperability/cohorts.py +681 -0
- agentloop/interoperability/contracts.py +427 -0
- agentloop/interoperability/coordination.py +554 -0
- agentloop/interoperability/evidence.py +215 -0
- agentloop/interoperability/otlp_jsonl.py +775 -0
- agentloop/interoperability/otlp_validation.py +496 -0
- agentloop/interoperability/registry.py +561 -0
- agentloop/interoperability/validation.py +333 -0
- agentloop/intervention_service.py +110 -0
- agentloop/intervention_store.py +118 -0
- agentloop/interventions.py +190 -0
- agentloop/issues.py +90 -0
- agentloop/judgment_adapters.py +125 -0
- agentloop/judgment_benchmark_types.py +98 -0
- agentloop/judgment_benchmarks.py +606 -0
- agentloop/judgment_types.py +241 -0
- agentloop/judgment_views.py +38 -0
- agentloop/judgments.py +342 -0
- agentloop/loop_guards.py +274 -0
- agentloop/loop_types.py +66 -0
- agentloop/markdown.py +102 -0
- agentloop/metrics.py +342 -0
- agentloop/migrations.py +303 -0
- agentloop/model_routing.py +470 -0
- agentloop/onboarding.py +315 -0
- agentloop/operations.py +44 -0
- agentloop/optimizer.py +199 -0
- agentloop/otel.py +715 -0
- agentloop/otel_ids.py +81 -0
- agentloop/otel_semantics.py +134 -0
- agentloop/parallelism.py +141 -0
- agentloop/patches.py +603 -0
- agentloop/plan_export.py +126 -0
- agentloop/policy_promotion.py +325 -0
- agentloop/promotion_evidence.py +304 -0
- agentloop/promotion_types.py +209 -0
- agentloop/quality.py +446 -0
- agentloop/ranking.py +529 -0
- agentloop/recovery_evidence.py +96 -0
- agentloop/replay.py +753 -0
- agentloop/retention.py +375 -0
- agentloop/retention_types.py +113 -0
- agentloop/reuse_experiments.py +247 -0
- agentloop/routing_checks.py +116 -0
- agentloop/routing_types.py +241 -0
- agentloop/rules.py +396 -0
- agentloop/runtime.py +286 -0
- agentloop/savings.py +158 -0
- agentloop/scheduling_plan.py +99 -0
- agentloop/scheduling_types.py +169 -0
- agentloop/schema.py +257 -0
- agentloop/semantic_rules.py +200 -0
- agentloop/semantic_waste.py +345 -0
- agentloop/semantic_waste_types.py +88 -0
- agentloop/server.py +494 -0
- agentloop/store.py +1429 -0
- agentloop/structural_experiments.py +431 -0
- agentloop/structural_types.py +113 -0
- agentloop/structured_quality.py +1020 -0
- agentloop/studies.py +629 -0
- agentloop/study_statistics.py +58 -0
- agentloop/substitution_evidence.py +199 -0
- agentloop/substitution_export.py +190 -0
- agentloop/substitution_reports.py +407 -0
- agentloop/substitution_types.py +284 -0
- agentloop/substitutions.py +267 -0
- agentloop/timing.py +142 -0
- agentloop/tokens.py +188 -0
- agentloop/tool_scheduling.py +410 -0
- agentloop/trace_sources.py +31 -0
- agentloop/tracer.py +494 -0
- agentloop/value.py +306 -0
- agentloop/version.py +3 -0
- agentloop/workflow_types.py +264 -0
- agentloop/workflows.py +284 -0
- agentloop_profiler-0.1.0.dist-info/METADATA +513 -0
- agentloop_profiler-0.1.0.dist-info/RECORD +166 -0
- agentloop_profiler-0.1.0.dist-info/WHEEL +5 -0
- agentloop_profiler-0.1.0.dist-info/entry_points.txt +2 -0
- agentloop_profiler-0.1.0.dist-info/licenses/LICENSE +201 -0
- agentloop_profiler-0.1.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +101 -0
- agentloop_profiler-0.1.0.dist-info/top_level.txt +2 -0
- dashboard/__init__.py +5 -0
- dashboard/app.py +922 -0
- dashboard/streamlit_compat.py +63 -0
- dashboard/trace_upload.py +57 -0
- dashboard/value_view.py +105 -0
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
"""Owned completion candidates and explicitly versioned trusted check contracts."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from dataclasses import asdict, dataclass, field
|
|
7
|
+
from threading import Event
|
|
8
|
+
from time import monotonic
|
|
9
|
+
from typing import Callable
|
|
10
|
+
|
|
11
|
+
from agentloop.budget_types import DispatchOptions
|
|
12
|
+
from agentloop.context_types import canonical, fingerprint, identifier, synchronous
|
|
13
|
+
|
|
14
|
+
COMPLETION_KEY = "agentloop.completion"
|
|
15
|
+
COMPLETION_VERSION = "1.0"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True, init=False)
|
|
19
|
+
class CompletionCandidate:
|
|
20
|
+
_json: str = field(repr=False)
|
|
21
|
+
present: bool
|
|
22
|
+
already_streamed: bool
|
|
23
|
+
|
|
24
|
+
def __init__(self, value=None, *, present=True, already_streamed=False):
|
|
25
|
+
if type(present) is not bool or type(already_streamed) is not bool:
|
|
26
|
+
raise ValueError("candidate flags must be booleans")
|
|
27
|
+
encoded = canonical(value)
|
|
28
|
+
if len(encoded.encode("utf-8")) > 1_000_000:
|
|
29
|
+
raise ValueError("completion candidate exceeds one million bytes")
|
|
30
|
+
object.__setattr__(self, "_json", encoded)
|
|
31
|
+
object.__setattr__(self, "present", present)
|
|
32
|
+
object.__setattr__(self, "already_streamed", already_streamed)
|
|
33
|
+
|
|
34
|
+
@property
|
|
35
|
+
def value(self):
|
|
36
|
+
return json.loads(self._json)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True)
|
|
40
|
+
class CheckResult:
|
|
41
|
+
passed: bool
|
|
42
|
+
feedback: str = field(default="", repr=False)
|
|
43
|
+
|
|
44
|
+
def __post_init__(self):
|
|
45
|
+
if type(self.passed) is not bool:
|
|
46
|
+
raise ValueError("check pass must be boolean")
|
|
47
|
+
if not isinstance(self.feedback, str) or len(self.feedback) > 1024:
|
|
48
|
+
raise ValueError("check feedback must be at most 1024 characters")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass(frozen=True)
|
|
52
|
+
class VerificationFeedback:
|
|
53
|
+
check_id: str
|
|
54
|
+
status: str
|
|
55
|
+
detail: str = field(default="", repr=False)
|
|
56
|
+
trust_class: str = field(default="verification_data", init=False)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass(frozen=True)
|
|
60
|
+
class VerificationContext:
|
|
61
|
+
deadline: float
|
|
62
|
+
cancel_event: Event | None = field(default=None, repr=False)
|
|
63
|
+
|
|
64
|
+
@property
|
|
65
|
+
def remaining_s(self):
|
|
66
|
+
return max(0.0, self.deadline - monotonic())
|
|
67
|
+
|
|
68
|
+
@property
|
|
69
|
+
def cancellation_requested(self):
|
|
70
|
+
return self.cancel_event is not None and self.cancel_event.is_set()
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass(frozen=True)
|
|
74
|
+
class CompletionCheck:
|
|
75
|
+
check_id: str
|
|
76
|
+
version: str
|
|
77
|
+
invoke: Callable = field(repr=False, compare=False)
|
|
78
|
+
configuration_hash: str | None = None
|
|
79
|
+
dispatch: DispatchOptions = field(default_factory=DispatchOptions)
|
|
80
|
+
usage_reader: Callable | None = field(default=None, repr=False, compare=False)
|
|
81
|
+
error_usage_reader: Callable | None = field(default=None, repr=False, compare=False)
|
|
82
|
+
|
|
83
|
+
def __post_init__(self):
|
|
84
|
+
identifier(self.check_id, "check_id")
|
|
85
|
+
identifier(self.version, "check version")
|
|
86
|
+
_callbacks(self)
|
|
87
|
+
if self.configuration_hash is not None and (
|
|
88
|
+
not isinstance(self.configuration_hash, str)
|
|
89
|
+
or len(self.configuration_hash) != 64
|
|
90
|
+
or any(char not in "0123456789abcdef" for char in self.configuration_hash)
|
|
91
|
+
):
|
|
92
|
+
raise ValueError("configuration_hash must be a SHA-256 digest")
|
|
93
|
+
|
|
94
|
+
def declaration(self):
|
|
95
|
+
return {
|
|
96
|
+
"id": self.check_id,
|
|
97
|
+
"version": self.version,
|
|
98
|
+
"configuration_hash": self.configuration_hash,
|
|
99
|
+
"dispatch_hash": fingerprint(asdict(self.dispatch)),
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@dataclass(frozen=True)
|
|
104
|
+
class RepairBackend:
|
|
105
|
+
invoke: Callable = field(repr=False, compare=False)
|
|
106
|
+
boundary: str = "model"
|
|
107
|
+
dispatch: DispatchOptions = field(default_factory=DispatchOptions)
|
|
108
|
+
usage_reader: Callable | None = field(default=None, repr=False, compare=False)
|
|
109
|
+
error_usage_reader: Callable | None = field(default=None, repr=False, compare=False)
|
|
110
|
+
|
|
111
|
+
def __post_init__(self):
|
|
112
|
+
_callbacks(self)
|
|
113
|
+
if self.boundary not in {"model", "tool"}:
|
|
114
|
+
raise ValueError("repair boundary must be model or tool")
|
|
115
|
+
if self.dispatch.step is None or not self.dispatch.step.retry_safe:
|
|
116
|
+
raise ValueError("repair requires explicit retry-safe StepInfo")
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _callbacks(value):
|
|
120
|
+
if not synchronous(value.invoke):
|
|
121
|
+
raise ValueError("completion callbacks must be synchronous")
|
|
122
|
+
if type(value.dispatch) is not DispatchOptions:
|
|
123
|
+
raise ValueError("dispatch must be DispatchOptions")
|
|
124
|
+
for reader in (value.usage_reader, value.error_usage_reader):
|
|
125
|
+
if reader is not None and not synchronous(reader):
|
|
126
|
+
raise ValueError("usage readers must be synchronous")
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@dataclass(frozen=True)
|
|
130
|
+
class CompletionConfig:
|
|
131
|
+
policy_id: str
|
|
132
|
+
version: str
|
|
133
|
+
branch_id: str
|
|
134
|
+
checks: tuple[CompletionCheck, ...]
|
|
135
|
+
max_repairs: int = 0
|
|
136
|
+
verification_timeout_s: float = 5.0
|
|
137
|
+
total_timeout_s: float = 30.0
|
|
138
|
+
on_failure: str = "stop"
|
|
139
|
+
|
|
140
|
+
def __post_init__(self):
|
|
141
|
+
for name in ("policy_id", "version", "branch_id"):
|
|
142
|
+
identifier(getattr(self, name), name)
|
|
143
|
+
if not isinstance(self.checks, (list, tuple)) or not 1 <= len(self.checks) <= 16:
|
|
144
|
+
raise ValueError("configure between one and sixteen completion checks")
|
|
145
|
+
if any(type(check) is not CompletionCheck for check in self.checks):
|
|
146
|
+
raise ValueError("checks must be CompletionCheck objects")
|
|
147
|
+
if len({check.check_id for check in self.checks}) != len(self.checks):
|
|
148
|
+
raise ValueError("check IDs must be unique")
|
|
149
|
+
object.__setattr__(self, "checks", tuple(self.checks))
|
|
150
|
+
if type(self.max_repairs) is not int or not 0 <= self.max_repairs <= 8:
|
|
151
|
+
raise ValueError("max_repairs must be between zero and eight")
|
|
152
|
+
for timeout in (self.verification_timeout_s, self.total_timeout_s):
|
|
153
|
+
if type(timeout) not in {int, float} or not 0 < timeout < float("inf"):
|
|
154
|
+
raise ValueError("completion deadlines must be finite and positive")
|
|
155
|
+
if self.on_failure not in {"stop", "escalate"}:
|
|
156
|
+
raise ValueError("on_failure must be stop or escalate")
|
|
157
|
+
|
|
158
|
+
def to_dict(self):
|
|
159
|
+
return {
|
|
160
|
+
"schema_version": COMPLETION_VERSION,
|
|
161
|
+
"branch_id": self.branch_id,
|
|
162
|
+
"checks": [check.declaration() for check in self.checks],
|
|
163
|
+
"max_repairs": self.max_repairs,
|
|
164
|
+
"verification_timeout_s": self.verification_timeout_s,
|
|
165
|
+
"total_timeout_s": self.total_timeout_s,
|
|
166
|
+
"on_failure": self.on_failure,
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
@dataclass(frozen=True)
|
|
171
|
+
class CompletionResult:
|
|
172
|
+
status: str
|
|
173
|
+
candidate: CompletionCandidate = field(repr=False)
|
|
174
|
+
verified: bool
|
|
175
|
+
repairs: int
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def quality_check(check_id, version, fixture):
|
|
179
|
+
"""Bind an owned deterministic quality fixture; never import executable scorers."""
|
|
180
|
+
from agentloop.quality import score_output, validate_quality_fixtures
|
|
181
|
+
|
|
182
|
+
encoded = canonical(fixture)
|
|
183
|
+
owned = json.loads(encoded)
|
|
184
|
+
if (
|
|
185
|
+
not isinstance(owned, dict)
|
|
186
|
+
or not isinstance(owned.get("scorer", {}), dict)
|
|
187
|
+
or owned.get("scorer", {}).get("type") == "custom"
|
|
188
|
+
):
|
|
189
|
+
raise ValueError("completion quality checks require deterministic fixtures")
|
|
190
|
+
validate_quality_fixtures([owned])
|
|
191
|
+
|
|
192
|
+
def invoke(candidate, context):
|
|
193
|
+
item = json.loads(encoded)
|
|
194
|
+
result = score_output(candidate.value, item, item.get("scorer", {"type": "exact_match"}))
|
|
195
|
+
return CheckResult(
|
|
196
|
+
result["passed"], "configured quality criterion failed" if not result["passed"] else ""
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
return CompletionCheck(check_id, version, invoke, fingerprint(owned))
|
agentloop/config.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
|
|
5
|
+
_POSTGRES_URL_ENV_VARS = ("AGENTLOOP_DATABASE_URL", "DATABASE_URL")
|
|
6
|
+
_POSTGRES_LIBPQ_SOURCE_ENV_VARS = ("PGHOST", "PGHOSTADDR", "PGSERVICE")
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def get_api_key() -> str | None:
|
|
10
|
+
return os.getenv("AGENTLOOP_API_KEY")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def get_admin_api_key() -> str | None:
|
|
14
|
+
return os.getenv("AGENTLOOP_ADMIN_API_KEY")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def require_api_key() -> bool:
|
|
18
|
+
return os.getenv("AGENTLOOP_REQUIRE_API_KEY", "false").lower() in {"1", "true", "yes", "on"}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def get_api_url() -> str:
|
|
22
|
+
return os.getenv("AGENTLOOP_API_URL", "http://127.0.0.1:8000")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def get_cors_origins() -> list[str]:
|
|
26
|
+
value = os.getenv("AGENTLOOP_CORS_ORIGINS", "")
|
|
27
|
+
return [origin.strip() for origin in value.split(",") if origin.strip()]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def get_postgres_dsn() -> str | None:
|
|
31
|
+
"""Return the explicit Postgres DSN, preserving AgentLoop override precedence."""
|
|
32
|
+
for name in _POSTGRES_URL_ENV_VARS:
|
|
33
|
+
value = os.getenv(name)
|
|
34
|
+
if value:
|
|
35
|
+
return value
|
|
36
|
+
return None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def get_postgres_password_file() -> str | None:
|
|
40
|
+
"""Return the optional file containing a libpq password."""
|
|
41
|
+
return os.getenv("AGENTLOOP_POSTGRES_PASSWORD_FILE") or None
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def postgres_connection_source() -> str | None:
|
|
45
|
+
"""Describe the configured Postgres connection source without exposing secrets."""
|
|
46
|
+
if get_postgres_dsn():
|
|
47
|
+
return "database URL"
|
|
48
|
+
if any(os.getenv(name) for name in _POSTGRES_LIBPQ_SOURCE_ENV_VARS):
|
|
49
|
+
return "libpq environment"
|
|
50
|
+
return None
|
|
@@ -0,0 +1,443 @@
|
|
|
1
|
+
"""Opt-in tool-result summaries after supported before-model admission hooks."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import inspect
|
|
6
|
+
from collections.abc import Iterator
|
|
7
|
+
from contextvars import ContextVar
|
|
8
|
+
from copy import deepcopy
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from threading import RLock
|
|
11
|
+
from time import perf_counter
|
|
12
|
+
from uuid import uuid4
|
|
13
|
+
|
|
14
|
+
from agentloop.budget_types import DispatchOptions
|
|
15
|
+
from agentloop.context_types import (
|
|
16
|
+
CONTEXT_KEY,
|
|
17
|
+
CONTEXT_VERSION,
|
|
18
|
+
EXACT_COUNTS,
|
|
19
|
+
ContextAdapter,
|
|
20
|
+
ContextPolicyConfig,
|
|
21
|
+
ContextRequest,
|
|
22
|
+
ContextTokenCount,
|
|
23
|
+
ContextTransformError,
|
|
24
|
+
SummaryBackend,
|
|
25
|
+
SummaryInput,
|
|
26
|
+
SummaryResult,
|
|
27
|
+
fingerprint,
|
|
28
|
+
synchronous,
|
|
29
|
+
)
|
|
30
|
+
from agentloop.harness import Decision, HarnessControlError, HarnessRun, Hook, Policy
|
|
31
|
+
from agentloop.tracer import current_trace
|
|
32
|
+
|
|
33
|
+
_ACTIVE: ContextVar[dict | None] = ContextVar("agentloop_context_preparation", default=None)
|
|
34
|
+
_EVIDENCE_LOCK = RLock()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass(frozen=True)
|
|
38
|
+
class _ContextAdmission:
|
|
39
|
+
config: ContextPolicyConfig
|
|
40
|
+
|
|
41
|
+
def __call__(self, context):
|
|
42
|
+
if context.branch_id != self.config.branch_id:
|
|
43
|
+
return Decision("continue", "context_not_selected")
|
|
44
|
+
active = _ACTIVE.get()
|
|
45
|
+
if (
|
|
46
|
+
active is None
|
|
47
|
+
or active["used"]
|
|
48
|
+
or active["config"] != self.config
|
|
49
|
+
or active["run_id"] != context.run_id
|
|
50
|
+
):
|
|
51
|
+
return Decision("deny", "context_adapter_required")
|
|
52
|
+
# One-shot binding: nested generic calls cannot borrow this admission,
|
|
53
|
+
# even when a caller reuses a run label across different HarnessRuns.
|
|
54
|
+
active["used"] = True
|
|
55
|
+
active["receipt"]["harness_call_id"] = context.call_id
|
|
56
|
+
return Decision("continue", "context_preparation_admitted")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def context_policy(config: ContextPolicyConfig):
|
|
60
|
+
if type(config) is not ContextPolicyConfig:
|
|
61
|
+
raise ValueError("context policy requires ContextPolicyConfig")
|
|
62
|
+
return Policy(
|
|
63
|
+
config.policy_id,
|
|
64
|
+
config.version,
|
|
65
|
+
_ContextAdmission(config),
|
|
66
|
+
hooks=frozenset({Hook("model")}),
|
|
67
|
+
actions=frozenset({"continue", "deny"}),
|
|
68
|
+
priority=50,
|
|
69
|
+
configuration={"transformation": "summarize_optional_tool_results", **config.to_dict()},
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _tool_results(payload):
|
|
74
|
+
messages = payload.get("messages")
|
|
75
|
+
if not isinstance(messages, list) or not messages or len(messages) > 512:
|
|
76
|
+
raise ContextTransformError("invalid_history")
|
|
77
|
+
pending, declared, results = set(), set(), {}
|
|
78
|
+
for index, message in enumerate(messages):
|
|
79
|
+
if (
|
|
80
|
+
not isinstance(message, dict)
|
|
81
|
+
or not isinstance(message.get("role"), str)
|
|
82
|
+
or message["role"]
|
|
83
|
+
not in {
|
|
84
|
+
"system",
|
|
85
|
+
"developer",
|
|
86
|
+
"user",
|
|
87
|
+
"assistant",
|
|
88
|
+
"tool",
|
|
89
|
+
}
|
|
90
|
+
):
|
|
91
|
+
raise ContextTransformError("invalid_role")
|
|
92
|
+
role = message["role"]
|
|
93
|
+
calls = message.get("tool_calls")
|
|
94
|
+
if role != "assistant" and calls is not None:
|
|
95
|
+
raise ContextTransformError("invalid_tool_calls")
|
|
96
|
+
content = message.get("content")
|
|
97
|
+
if not isinstance(content, str) and not (role == "assistant" and content is None and calls):
|
|
98
|
+
raise ContextTransformError("unsupported_content")
|
|
99
|
+
if role == "tool":
|
|
100
|
+
identity = message.get("tool_call_id")
|
|
101
|
+
if not isinstance(identity, str) or identity not in pending or identity in results:
|
|
102
|
+
raise ContextTransformError("unpaired_tool_result")
|
|
103
|
+
pending.remove(identity)
|
|
104
|
+
results[identity] = index
|
|
105
|
+
else:
|
|
106
|
+
if pending:
|
|
107
|
+
raise ContextTransformError("missing_tool_result")
|
|
108
|
+
if message.get("tool_call_id") is not None:
|
|
109
|
+
raise ContextTransformError("invalid_tool_reference")
|
|
110
|
+
if calls is not None:
|
|
111
|
+
if not isinstance(calls, list) or not calls:
|
|
112
|
+
raise ContextTransformError("invalid_tool_calls")
|
|
113
|
+
for call in calls:
|
|
114
|
+
if (
|
|
115
|
+
not isinstance(call, dict)
|
|
116
|
+
or not isinstance(call.get("id"), str)
|
|
117
|
+
or not call["id"]
|
|
118
|
+
or call["id"] in declared
|
|
119
|
+
):
|
|
120
|
+
raise ContextTransformError("ambiguous_tool_call")
|
|
121
|
+
declared.add(call["id"])
|
|
122
|
+
pending.add(call["id"])
|
|
123
|
+
if pending:
|
|
124
|
+
raise ContextTransformError("missing_tool_result")
|
|
125
|
+
return results
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
class ContextModel:
|
|
129
|
+
"""Explicit synchronous model adapter; summary calls share the supplied run.
|
|
130
|
+
|
|
131
|
+
The provider receives a fresh JSON payload. The summarizer is trusted host
|
|
132
|
+
code representing one model invocation, with normalized exclusive usage.
|
|
133
|
+
Policies see only safe hook metadata; content never enters HookContext.
|
|
134
|
+
"""
|
|
135
|
+
|
|
136
|
+
def __init__(
|
|
137
|
+
self,
|
|
138
|
+
run,
|
|
139
|
+
provider,
|
|
140
|
+
*,
|
|
141
|
+
config,
|
|
142
|
+
adapter,
|
|
143
|
+
summarizer,
|
|
144
|
+
token_counter=None,
|
|
145
|
+
dispatch=None,
|
|
146
|
+
usage_reader=None,
|
|
147
|
+
max_records=256,
|
|
148
|
+
):
|
|
149
|
+
if (
|
|
150
|
+
not isinstance(run, HarnessRun)
|
|
151
|
+
or type(config) is not ContextPolicyConfig
|
|
152
|
+
or type(adapter) is not ContextAdapter
|
|
153
|
+
or type(summarizer) is not SummaryBackend
|
|
154
|
+
):
|
|
155
|
+
raise ValueError(
|
|
156
|
+
"typed run, context configuration, adapter and summarizer are required"
|
|
157
|
+
)
|
|
158
|
+
if not synchronous(provider) or (
|
|
159
|
+
token_counter is not None and not synchronous(token_counter)
|
|
160
|
+
):
|
|
161
|
+
raise ValueError("context adapter supports synchronous providers and counters")
|
|
162
|
+
if type(max_records) is not int or not 1 <= max_records <= 10000:
|
|
163
|
+
raise ValueError("max_records must be bounded")
|
|
164
|
+
expected = context_policy(config)
|
|
165
|
+
if run.harness.config.mode != "disabled" and not any(
|
|
166
|
+
policy.config_hash == expected.config_hash
|
|
167
|
+
and type(policy.evaluate) is _ContextAdmission
|
|
168
|
+
for policy in run.harness.config.policies
|
|
169
|
+
):
|
|
170
|
+
raise ValueError("register the matching context_policy in the harness before binding")
|
|
171
|
+
self.run, self.provider = run, provider
|
|
172
|
+
self.config, self.adapter, self.summarizer = config, adapter, summarizer
|
|
173
|
+
self.token_counter, self.max_records = token_counter, max_records
|
|
174
|
+
self._policy_hash = expected.config_hash
|
|
175
|
+
self._records = {}
|
|
176
|
+
self._lock = RLock()
|
|
177
|
+
self._capture_failed = False
|
|
178
|
+
self._summary = run.wrap(
|
|
179
|
+
self._invoke_summary,
|
|
180
|
+
boundary="model",
|
|
181
|
+
branch_id=config.branch_id + ":summary",
|
|
182
|
+
dispatch=summarizer.dispatch,
|
|
183
|
+
usage_reader=lambda result: result.usage if type(result) is SummaryResult else None,
|
|
184
|
+
)
|
|
185
|
+
self._model = run.wrap(
|
|
186
|
+
self._dispatch,
|
|
187
|
+
boundary="model",
|
|
188
|
+
branch_id=config.branch_id,
|
|
189
|
+
dispatch=DispatchOptions() if dispatch is None else dispatch,
|
|
190
|
+
usage_reader=usage_reader,
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
def export_evidence(self):
|
|
194
|
+
with self._lock:
|
|
195
|
+
return {"schema_version": CONTEXT_VERSION, "records": deepcopy(self._records)}
|
|
196
|
+
|
|
197
|
+
def _count(self, payload):
|
|
198
|
+
if self.token_counter is None:
|
|
199
|
+
return ContextTokenCount()
|
|
200
|
+
try:
|
|
201
|
+
count = self.token_counter(deepcopy(payload))
|
|
202
|
+
except Exception:
|
|
203
|
+
raise ContextTransformError("token_counter_failed") from None
|
|
204
|
+
if type(count) is not ContextTokenCount:
|
|
205
|
+
raise ContextTransformError("invalid_token_count")
|
|
206
|
+
return count
|
|
207
|
+
|
|
208
|
+
def _fits(self, count):
|
|
209
|
+
limit = self.config.max_input_tokens
|
|
210
|
+
return limit is None or (
|
|
211
|
+
count.provenance in EXACT_COUNTS and count.value is not None and count.value <= limit
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
def _invoke_summary(self, request, receipt):
|
|
215
|
+
receipt["summary_invocations"] += 1
|
|
216
|
+
result = self.summarizer.summarize(request)
|
|
217
|
+
if type(result) is SummaryResult:
|
|
218
|
+
usage = None if result.usage is None else result.usage.to_dict()
|
|
219
|
+
if usage is not None and usage.get("usage_id") is not None:
|
|
220
|
+
usage["usage_id"] = "sha256:" + fingerprint(usage["usage_id"])
|
|
221
|
+
receipt["summary_usage"].append(usage)
|
|
222
|
+
return result
|
|
223
|
+
|
|
224
|
+
def _prepare(self, request, receipt):
|
|
225
|
+
original = request.payload
|
|
226
|
+
if not self.adapter.before_model or self.adapter.format != "text_tool_messages_v1":
|
|
227
|
+
raise ContextTransformError("unsupported_adapter")
|
|
228
|
+
if original.get("stream") is not None and original.get("stream") is not False:
|
|
229
|
+
raise ContextTransformError("streaming_not_supported")
|
|
230
|
+
results = _tool_results(original)
|
|
231
|
+
optional, protected = (
|
|
232
|
+
set(request.optional_tool_results),
|
|
233
|
+
set(request.protected_tool_results),
|
|
234
|
+
)
|
|
235
|
+
if not (optional | protected) <= results.keys():
|
|
236
|
+
raise ContextTransformError("unknown_tool_selection")
|
|
237
|
+
before = self._count(original)
|
|
238
|
+
receipt["original_tokens"] = before.to_dict()
|
|
239
|
+
targets = [
|
|
240
|
+
identity
|
|
241
|
+
for identity, index in results.items()
|
|
242
|
+
if identity in optional - protected
|
|
243
|
+
and len(original["messages"][index]["content"]) > self.config.max_summary_chars
|
|
244
|
+
]
|
|
245
|
+
if len(targets) > self.config.max_summaries:
|
|
246
|
+
raise ContextTransformError("summary_count_limit")
|
|
247
|
+
receipt["target_count"] = len(targets)
|
|
248
|
+
receipt["protected_result_count"] = len(protected)
|
|
249
|
+
if self.run.harness.config.mode == "shadow":
|
|
250
|
+
receipt["status"] = "proposed" if targets else "unchanged"
|
|
251
|
+
receipt["effective_tokens"] = before.to_dict()
|
|
252
|
+
return original
|
|
253
|
+
candidate = deepcopy(original)
|
|
254
|
+
for identity in targets:
|
|
255
|
+
index = results[identity]
|
|
256
|
+
text = original["messages"][index]["content"]
|
|
257
|
+
receipt["summary_attempts"] += 1
|
|
258
|
+
try:
|
|
259
|
+
summary = self._summary(
|
|
260
|
+
SummaryInput(text, self.config.max_summary_chars, fingerprint(text)), receipt
|
|
261
|
+
)
|
|
262
|
+
except HarnessControlError:
|
|
263
|
+
raise
|
|
264
|
+
except Exception:
|
|
265
|
+
raise ContextTransformError("summary_failed") from None
|
|
266
|
+
if (
|
|
267
|
+
type(summary) is not SummaryResult
|
|
268
|
+
or not summary.text.strip()
|
|
269
|
+
or len(summary.text) > self.config.max_summary_chars
|
|
270
|
+
or len(summary.text) >= len(text)
|
|
271
|
+
):
|
|
272
|
+
raise ContextTransformError("invalid_summary")
|
|
273
|
+
# Trust class, tool reference, call declaration and every other
|
|
274
|
+
# message/option remain byte-for-byte values from the owned input.
|
|
275
|
+
candidate["messages"][index]["content"] = summary.text
|
|
276
|
+
_tool_results(candidate)
|
|
277
|
+
after = self._count(candidate)
|
|
278
|
+
receipt["proposed_tokens"] = after.to_dict()
|
|
279
|
+
if (
|
|
280
|
+
before.value is not None
|
|
281
|
+
and after.value is not None
|
|
282
|
+
and (before.provenance != after.provenance or before.reference != after.reference)
|
|
283
|
+
):
|
|
284
|
+
raise ContextTransformError("token_counter_changed")
|
|
285
|
+
if not self._fits(after):
|
|
286
|
+
raise ContextTransformError("context_limit_unverified")
|
|
287
|
+
if (
|
|
288
|
+
targets
|
|
289
|
+
and before.provenance in EXACT_COUNTS
|
|
290
|
+
and after.provenance in EXACT_COUNTS
|
|
291
|
+
and after.value >= before.value
|
|
292
|
+
):
|
|
293
|
+
raise ContextTransformError("tokens_not_reduced")
|
|
294
|
+
receipt.update(
|
|
295
|
+
status="applied" if targets else "unchanged",
|
|
296
|
+
transformed_hash=fingerprint(candidate),
|
|
297
|
+
effective_tokens=after.to_dict(),
|
|
298
|
+
)
|
|
299
|
+
return candidate
|
|
300
|
+
|
|
301
|
+
def _dispatch(self, request, receipt, start):
|
|
302
|
+
original = request.payload
|
|
303
|
+
try:
|
|
304
|
+
prepared = self._prepare(request, receipt)
|
|
305
|
+
except ContextTransformError as exc:
|
|
306
|
+
receipt["error_code"] = exc.reason_code
|
|
307
|
+
if self.run.harness.config.mode == "shadow":
|
|
308
|
+
prepared = original
|
|
309
|
+
receipt["status"] = "invalid_proposal"
|
|
310
|
+
elif (
|
|
311
|
+
self.config.on_invalid == "original"
|
|
312
|
+
and not self.run.stopped
|
|
313
|
+
and exc.reason_code != "streaming_not_supported"
|
|
314
|
+
and self._fits(ContextTokenCount(**receipt["original_tokens"]))
|
|
315
|
+
):
|
|
316
|
+
prepared = original
|
|
317
|
+
receipt["status"] = "restored_original"
|
|
318
|
+
receipt["effective_tokens"] = receipt["original_tokens"]
|
|
319
|
+
else:
|
|
320
|
+
raise
|
|
321
|
+
receipt["effective_hash"] = fingerprint(prepared)
|
|
322
|
+
receipt["preparation_ms"] = (perf_counter() - start) * 1000
|
|
323
|
+
receipt["provider_invoked"] = True
|
|
324
|
+
result = self.provider(deepcopy(prepared))
|
|
325
|
+
if self.run.harness.config.mode == "enforce" and (
|
|
326
|
+
isinstance(result, Iterator) or inspect.isawaitable(result)
|
|
327
|
+
):
|
|
328
|
+
close = getattr(result, "close", None)
|
|
329
|
+
if callable(close):
|
|
330
|
+
close()
|
|
331
|
+
raise ContextTransformError("unexpected_streaming_result")
|
|
332
|
+
return result
|
|
333
|
+
|
|
334
|
+
def __call__(self, request):
|
|
335
|
+
if type(request) is not ContextRequest:
|
|
336
|
+
raise TypeError("context model requires ContextRequest")
|
|
337
|
+
if self.run.harness.config.mode == "disabled":
|
|
338
|
+
return self.provider(request.payload)
|
|
339
|
+
if _ACTIVE.get() is not None:
|
|
340
|
+
raise ContextTransformError("recursive_context_transform")
|
|
341
|
+
identity = "ctx_" + uuid4().hex
|
|
342
|
+
receipt = {
|
|
343
|
+
"record_id": identity,
|
|
344
|
+
"policy_id": self.config.policy_id,
|
|
345
|
+
"policy_version": self.config.version,
|
|
346
|
+
"policy_config_hash": self._policy_hash,
|
|
347
|
+
"adapter": self.adapter.name,
|
|
348
|
+
"mode": self.run.harness.config.mode,
|
|
349
|
+
"quality_ref": self.config.quality_ref,
|
|
350
|
+
"summarizer_ref": self.summarizer.reference,
|
|
351
|
+
"harness_run_id": self.run.run_id,
|
|
352
|
+
"harness_call_id": None,
|
|
353
|
+
"original_hash": fingerprint(request.payload),
|
|
354
|
+
"transformed_hash": None,
|
|
355
|
+
"effective_hash": None,
|
|
356
|
+
"status": "pending",
|
|
357
|
+
"error_code": None,
|
|
358
|
+
"summary_attempts": 0,
|
|
359
|
+
"summary_invocations": 0,
|
|
360
|
+
"summary_usage": [],
|
|
361
|
+
"provider_invoked": False,
|
|
362
|
+
"original_tokens": ContextTokenCount().to_dict(),
|
|
363
|
+
"proposed_tokens": ContextTokenCount().to_dict(),
|
|
364
|
+
"effective_tokens": ContextTokenCount().to_dict(),
|
|
365
|
+
"preparation_ms": None,
|
|
366
|
+
"provider_cache_outcome": "unobserved",
|
|
367
|
+
}
|
|
368
|
+
trace = current_trace()
|
|
369
|
+
with self._lock, _EVIDENCE_LOCK:
|
|
370
|
+
if self._capture_failed:
|
|
371
|
+
raise ContextTransformError("invalid_trace_evidence")
|
|
372
|
+
if len(self._records) >= self.max_records:
|
|
373
|
+
raise ContextTransformError("evidence_limit")
|
|
374
|
+
if trace is not None:
|
|
375
|
+
envelope = trace.metadata.setdefault(
|
|
376
|
+
CONTEXT_KEY, {"schema_version": CONTEXT_VERSION, "records": {}}
|
|
377
|
+
)
|
|
378
|
+
if (
|
|
379
|
+
not isinstance(envelope, dict)
|
|
380
|
+
or envelope.get("schema_version") != CONTEXT_VERSION
|
|
381
|
+
or not isinstance(envelope.get("records"), dict)
|
|
382
|
+
or len(envelope["records"]) >= 10000
|
|
383
|
+
):
|
|
384
|
+
raise ContextTransformError("invalid_trace_evidence")
|
|
385
|
+
envelope["records"][identity] = deepcopy(receipt)
|
|
386
|
+
self._records[identity] = deepcopy(receipt)
|
|
387
|
+
start = perf_counter()
|
|
388
|
+
token = _ACTIVE.set(
|
|
389
|
+
{"config": self.config, "run_id": self.run.run_id, "receipt": receipt, "used": False}
|
|
390
|
+
)
|
|
391
|
+
failure = None
|
|
392
|
+
try:
|
|
393
|
+
return self._model(request, receipt, start)
|
|
394
|
+
except BaseException as exc:
|
|
395
|
+
failure = exc
|
|
396
|
+
receipt["status"] = (
|
|
397
|
+
"cancelled"
|
|
398
|
+
if not isinstance(exc, Exception)
|
|
399
|
+
else "provider_failed"
|
|
400
|
+
if receipt["provider_invoked"]
|
|
401
|
+
else "blocked"
|
|
402
|
+
if isinstance(exc, HarnessControlError)
|
|
403
|
+
else "failed"
|
|
404
|
+
if isinstance(exc, Exception)
|
|
405
|
+
else "cancelled"
|
|
406
|
+
)
|
|
407
|
+
receipt["error_code"] = (
|
|
408
|
+
"cancelled"
|
|
409
|
+
if not isinstance(exc, Exception)
|
|
410
|
+
else exc.reason_code
|
|
411
|
+
if isinstance(exc, ContextTransformError)
|
|
412
|
+
else "harness_control"
|
|
413
|
+
if isinstance(exc, HarnessControlError)
|
|
414
|
+
else "provider_error"
|
|
415
|
+
if receipt["provider_invoked"]
|
|
416
|
+
else "preparation_error"
|
|
417
|
+
if isinstance(exc, Exception)
|
|
418
|
+
else "cancelled"
|
|
419
|
+
)
|
|
420
|
+
raise
|
|
421
|
+
finally:
|
|
422
|
+
_ACTIVE.reset(token)
|
|
423
|
+
if receipt["preparation_ms"] is None:
|
|
424
|
+
receipt["preparation_ms"] = (perf_counter() - start) * 1000
|
|
425
|
+
with self._lock, _EVIDENCE_LOCK:
|
|
426
|
+
capture_error = False
|
|
427
|
+
if trace is not None:
|
|
428
|
+
try:
|
|
429
|
+
current = trace.metadata[CONTEXT_KEY]
|
|
430
|
+
if (
|
|
431
|
+
current is not envelope
|
|
432
|
+
or not isinstance(current["records"], dict)
|
|
433
|
+
or current.get("schema_version") != CONTEXT_VERSION
|
|
434
|
+
):
|
|
435
|
+
raise ValueError("capture changed")
|
|
436
|
+
current["records"][identity] = deepcopy(receipt)
|
|
437
|
+
except (KeyError, TypeError, ValueError):
|
|
438
|
+
capture_error = True
|
|
439
|
+
self._capture_failed = True
|
|
440
|
+
receipt["capture_error"] = "invalid_trace_evidence"
|
|
441
|
+
self._records[identity] = deepcopy(receipt)
|
|
442
|
+
if capture_error and failure is None:
|
|
443
|
+
raise ContextTransformError("invalid_trace_evidence")
|