atif-sql 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- atif_analytics/__init__.py +9 -0
- atif_analytics/application/__init__.py +3 -0
- atif_analytics/application/analyze.py +158 -0
- atif_analytics/application/prompts.py +750 -0
- atif_analytics/application/use_cases/__init__.py +3 -0
- atif_analytics/application/use_cases/_shared.py +264 -0
- atif_analytics/application/use_cases/classify.py +430 -0
- atif_analytics/application/use_cases/conflicts.py +448 -0
- atif_analytics/application/use_cases/friction.py +634 -0
- atif_analytics/application/use_cases/perceived.py +451 -0
- atif_analytics/domain/__init__.py +3 -0
- atif_analytics/domain/authorship.py +166 -0
- atif_analytics/domain/config.py +30 -0
- atif_analytics/domain/costs.py +50 -0
- atif_analytics/domain/friction.py +117 -0
- atif_analytics/domain/layout.py +94 -0
- atif_analytics/domain/models.py +385 -0
- atif_analytics/domain/ports.py +100 -0
- atif_analytics/domain/transcript.py +359 -0
- atif_analytics/infrastructure/__init__.py +3 -0
- atif_analytics/infrastructure/corpus_reader.py +546 -0
- atif_analytics/infrastructure/parquet_cache.py +186 -0
- atif_analytics/infrastructure/settings.py +114 -0
- atif_analytics/infrastructure/sqlite_state/__init__.py +3 -0
- atif_analytics/infrastructure/sqlite_state/checkpointer.py +326 -0
- atif_analytics/infrastructure/sqlite_state/retry_queue.py +296 -0
- atif_analytics/py.typed +0 -0
- atif_cli/__init__.py +3 -0
- atif_cli/__main__.py +8 -0
- atif_cli/app.py +2459 -0
- atif_cli/converter_adapter.py +151 -0
- atif_cli/corpus.py +409 -0
- atif_cli/cron.py +311 -0
- atif_cli/duck_errors.py +97 -0
- atif_cli/embed_lake.py +101 -0
- atif_cli/errors.py +89 -0
- atif_cli/lake.py +373 -0
- atif_cli/lake_sessions.py +270 -0
- atif_cli/output.py +297 -0
- atif_cli/py.typed +0 -0
- atif_converter/__init__.py +3 -0
- atif_converter/application/__init__.py +3 -0
- atif_converter/application/convert_and_audit.py +294 -0
- atif_converter/application/convert_codex.py +210 -0
- atif_converter/domain/__init__.py +9 -0
- atif_converter/domain/agents.py +40 -0
- atif_converter/domain/atif/__init__.py +55 -0
- atif_converter/domain/atif/agent.py +37 -0
- atif_converter/domain/atif/content.py +161 -0
- atif_converter/domain/atif/final_metrics.py +45 -0
- atif_converter/domain/atif/metrics.py +49 -0
- atif_converter/domain/atif/observation.py +21 -0
- atif_converter/domain/atif/observation_result.py +47 -0
- atif_converter/domain/atif/step.py +144 -0
- atif_converter/domain/atif/subagent_trajectory_ref.py +90 -0
- atif_converter/domain/atif/tool_call.py +36 -0
- atif_converter/domain/atif/trajectory.py +211 -0
- atif_converter/domain/atif/trajectory_validator.py +299 -0
- atif_converter/domain/blobs.py +510 -0
- atif_converter/domain/claude_code_conversion.py +1077 -0
- atif_converter/domain/codex_conversion.py +1036 -0
- atif_converter/domain/codex_edges.py +192 -0
- atif_converter/domain/codex_enrichment.py +419 -0
- atif_converter/domain/codex_fidelity.py +124 -0
- atif_converter/domain/codex_nested_calls.py +165 -0
- atif_converter/domain/edges.py +113 -0
- atif_converter/domain/enrichment.py +412 -0
- atif_converter/domain/errors.py +60 -0
- atif_converter/domain/fidelity.py +171 -0
- atif_converter/domain/model_prices.json +8538 -0
- atif_converter/domain/pricing.py +790 -0
- atif_converter/domain/result_signals.py +774 -0
- atif_converter/domain/schema_version.py +90 -0
- atif_converter/domain/session_events.py +408 -0
- atif_converter/infrastructure/__init__.py +3 -0
- atif_converter/infrastructure/census.py +89 -0
- atif_converter/infrastructure/claude_code_converter.py +127 -0
- atif_converter/infrastructure/codex_adapter.py +132 -0
- atif_converter/infrastructure/codex_census.py +100 -0
- atif_converter/infrastructure/codex_converter.py +82 -0
- atif_converter/infrastructure/harbor_adapter.py +187 -0
- atif_converter/infrastructure/raw_records.py +426 -0
- atif_converter/infrastructure/source_archive.py +154 -0
- atif_converter/py.typed +0 -0
- atif_corpus/__init__.py +13 -0
- atif_corpus/application/__init__.py +7 -0
- atif_corpus/application/materialize.py +1826 -0
- atif_corpus/domain/__init__.py +9 -0
- atif_corpus/domain/agents.py +33 -0
- atif_corpus/domain/generation.py +34 -0
- atif_corpus/domain/layout.py +168 -0
- atif_corpus/domain/ports.py +175 -0
- atif_corpus/domain/session_id.py +68 -0
- atif_corpus/domain/sessions.py +228 -0
- atif_corpus/domain/slug.py +72 -0
- atif_corpus/domain/source_layout.py +143 -0
- atif_corpus/domain/watermark.py +76 -0
- atif_corpus/infrastructure/__init__.py +7 -0
- atif_corpus/infrastructure/atomic.py +266 -0
- atif_corpus/infrastructure/compress_artifacts.py +161 -0
- atif_corpus/infrastructure/fake_converter.py +174 -0
- atif_corpus/infrastructure/scanner.py +316 -0
- atif_corpus/infrastructure/settings.py +130 -0
- atif_corpus/infrastructure/source_archive.py +202 -0
- atif_corpus/py.typed +0 -0
- atif_duck/__init__.py +43 -0
- atif_duck/domain/__init__.py +32 -0
- atif_duck/domain/artifacts.py +66 -0
- atif_duck/domain/authorship.py +165 -0
- atif_duck/domain/catalog.py +626 -0
- atif_duck/domain/columnar.py +149 -0
- atif_duck/domain/embedding_guard.py +88 -0
- atif_duck/domain/examples.py +265 -0
- atif_duck/domain/lake.py +427 -0
- atif_duck/domain/lake_sessions.py +97 -0
- atif_duck/domain/raw_readers.py +140 -0
- atif_duck/domain/session_id.py +69 -0
- atif_duck/domain/sql_literal.py +50 -0
- atif_duck/infrastructure/__init__.py +34 -0
- atif_duck/infrastructure/analytics.py +469 -0
- atif_duck/infrastructure/authorship.py +139 -0
- atif_duck/infrastructure/columnar.py +641 -0
- atif_duck/infrastructure/lake.py +1383 -0
- atif_duck/infrastructure/lake_sessions.py +194 -0
- atif_duck/infrastructure/lake_settings.py +63 -0
- atif_duck/infrastructure/lake_steps.py +225 -0
- atif_duck/infrastructure/projections.py +221 -0
- atif_duck/infrastructure/registry.py +2194 -0
- atif_duck/infrastructure/stored_artifacts.py +115 -0
- atif_duck/py.typed +0 -0
- atif_embed/__init__.py +12 -0
- atif_embed/application/__init__.py +3 -0
- atif_embed/application/embed.py +293 -0
- atif_embed/application/prune.py +80 -0
- atif_embed/domain/__init__.py +3 -0
- atif_embed/domain/discovery.py +70 -0
- atif_embed/domain/embedding_guard.py +83 -0
- atif_embed/domain/errors.py +94 -0
- atif_embed/domain/ports.py +223 -0
- atif_embed/domain/sql_literal.py +49 -0
- atif_embed/domain/text_stamp.py +71 -0
- atif_embed/infrastructure/__init__.py +3 -0
- atif_embed/infrastructure/cohere_bedrock.py +433 -0
- atif_embed/infrastructure/corpus_text_rows.py +273 -0
- atif_embed/infrastructure/gemma_local.py +355 -0
- atif_embed/infrastructure/lake_text_rows.py +203 -0
- atif_embed/infrastructure/lance_store.py +440 -0
- atif_embed/infrastructure/providers.py +30 -0
- atif_embed/infrastructure/settings.py +154 -0
- atif_embed/py.typed +0 -0
- atif_models/__init__.py +14 -0
- atif_models/domain/__init__.py +9 -0
- atif_models/domain/ports.py +143 -0
- atif_models/domain/registry.py +151 -0
- atif_models/domain/schema.py +109 -0
- atif_models/infrastructure/__init__.py +8 -0
- atif_models/infrastructure/openai_bedrock.py +314 -0
- atif_models/infrastructure/settings.py +85 -0
- atif_models/py.typed +0 -0
- atif_sql-0.2.0.dist-info/METADATA +272 -0
- atif_sql-0.2.0.dist-info/RECORD +164 -0
- atif_sql-0.2.0.dist-info/WHEEL +4 -0
- atif_sql-0.2.0.dist-info/entry_points.txt +2 -0
- atif_sql-0.2.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,451 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
|
|
3
|
+
"""Perceived-error detection pipeline (LangSmith's Perceived Error, ours).
|
|
4
|
+
|
|
5
|
+
Implements LangSmith's published "Perceived Error" definition on the
|
|
6
|
+
conflicts-pipeline chassis.
|
|
7
|
+
|
|
8
|
+
Transcript text is sent to a third-party model provider on this path: a
|
|
9
|
+
session's full rendered body, tool arguments and tool results included,
|
|
10
|
+
leaves this machine.
|
|
11
|
+
|
|
12
|
+
Reads complete session transcripts WITH uuid headers (``include_uuids=True``
|
|
13
|
+
puts each text step's uuid — the FIRST source_uuid, per CONTRACT-V2 — in
|
|
14
|
+
the header as ``[uuid=... role ts]``) and emits one row per user-perceived
|
|
15
|
+
agent error, anchored on the USER turn where the perception surfaces.
|
|
16
|
+
Clean sessions produce ZERO rows.
|
|
17
|
+
|
|
18
|
+
Eligibility (LangSmith parity): a session needs >= 2 human-AI message
|
|
19
|
+
pairs before it can be judged — perceiving an error requires the human
|
|
20
|
+
RESPONDING to AI output. Only HUMAN turns open a pair (Stop hook feedback,
|
|
21
|
+
task notifications and harness text do not), so a turn audit (no human
|
|
22
|
+
turn) and a one-shot job (one) are never eligible. Ineligible sessions are checkpointed as skipped
|
|
23
|
+
(zero LLM calls) so they never re-render; if the session later grows, the
|
|
24
|
+
advancing (last_ts, mtime) bounds re-admit it through the normal
|
|
25
|
+
checkpoint path. Two more LangSmith runtime traits carry over through
|
|
26
|
+
existing machinery: idle-gated evaluation (our corpus quiescence — the
|
|
27
|
+
materializer only finalizes settled sessions) and "skipped/failed units
|
|
28
|
+
are never billed" (checkpoint + blocked-units retry gate).
|
|
29
|
+
|
|
30
|
+
The uuid-validity guard (conflicts precedent) validates the model's
|
|
31
|
+
returned ``turn_uuid`` values against the session's HUMAN-turn header uuids
|
|
32
|
+
before they land in the parquet — the documented invariant is a user turn
|
|
33
|
+
the human wrote, so an assistant uuid, a Stop hook's uuid (or a
|
|
34
|
+
hallucinated one) is dropped at the source (logged, reason 'non-user
|
|
35
|
+
turn_uuid'). The guard
|
|
36
|
+
FAILS CLOSED: a session that rendered no user-turn header uuids at all has
|
|
37
|
+
no universe to validate against, so its rows are dropped rather than
|
|
38
|
+
admitted unverified. Refusals land in the ``analytics/refusals`` sidecar (a
|
|
39
|
+
sentinel row would poison the uuid-keyed joins).
|
|
40
|
+
|
|
41
|
+
Output schema (per parquet shard)::
|
|
42
|
+
|
|
43
|
+
session_id VARCHAR
|
|
44
|
+
turn_uuid VARCHAR (NOT NULL — validated: user-turn header uuid)
|
|
45
|
+
signal VARCHAR (enum: correction|repeated_request|
|
|
46
|
+
rejected_action|contradictory_response|
|
|
47
|
+
acknowledged_mistake|
|
|
48
|
+
persistent_misunderstanding|
|
|
49
|
+
unresolved_outcome)
|
|
50
|
+
severity VARCHAR (enum: minor|moderate|major)
|
|
51
|
+
evidence VARCHAR
|
|
52
|
+
agent_error_summary VARCHAR
|
|
53
|
+
confidence DOUBLE
|
|
54
|
+
detected_at TIMESTAMP
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
from __future__ import annotations
|
|
58
|
+
|
|
59
|
+
import asyncio
|
|
60
|
+
import time
|
|
61
|
+
from datetime import UTC, datetime
|
|
62
|
+
from typing import TYPE_CHECKING, Any
|
|
63
|
+
|
|
64
|
+
import polars as pl
|
|
65
|
+
from loguru import logger
|
|
66
|
+
|
|
67
|
+
from atif_analytics.application.prompts import PERCEIVED_SYSTEM_PROMPT
|
|
68
|
+
from atif_analytics.application.use_cases._shared import (
|
|
69
|
+
RunBudget,
|
|
70
|
+
build_cache,
|
|
71
|
+
build_checkpoint,
|
|
72
|
+
build_provider,
|
|
73
|
+
build_retry_queue,
|
|
74
|
+
gather_under_budget,
|
|
75
|
+
pipeline_usage,
|
|
76
|
+
write_chunk_size,
|
|
77
|
+
write_refusal_audit,
|
|
78
|
+
)
|
|
79
|
+
from atif_analytics.domain.costs import estimate_cost_tokens, tokens_for_chars
|
|
80
|
+
from atif_analytics.domain.models import PerceivedErrorsResult
|
|
81
|
+
from atif_analytics.domain.transcript import human_ai_pair_count, is_human_turn
|
|
82
|
+
from atif_analytics.infrastructure.corpus_reader import CorpusReader
|
|
83
|
+
from atif_analytics.infrastructure.sqlite_state import checkpointer
|
|
84
|
+
|
|
85
|
+
if TYPE_CHECKING:
|
|
86
|
+
from atif_analytics.infrastructure.parquet_cache import ParquetCache
|
|
87
|
+
from atif_analytics.infrastructure.settings import AnalyticsSettings
|
|
88
|
+
from atif_analytics.infrastructure.sqlite_state.checkpointer import SqliteCheckpoint
|
|
89
|
+
from atif_analytics.infrastructure.sqlite_state.retry_queue import SqliteRetryQueue
|
|
90
|
+
from atif_models.domain.ports import LlmStructuredProvider
|
|
91
|
+
from atif_models.domain.registry import ModelSpec
|
|
92
|
+
|
|
93
|
+
_PARQUET_SCHEMA: dict[str, Any] = {
|
|
94
|
+
"session_id": pl.Utf8,
|
|
95
|
+
"turn_uuid": pl.Utf8,
|
|
96
|
+
"signal": pl.Utf8,
|
|
97
|
+
"severity": pl.Utf8,
|
|
98
|
+
"evidence": pl.Utf8,
|
|
99
|
+
"agent_error_summary": pl.Utf8,
|
|
100
|
+
"confidence": pl.Float64,
|
|
101
|
+
"detected_at": pl.Datetime("us", "UTC"),
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
#: LangSmith eligibility floor: judging a perceived error requires the human
|
|
105
|
+
#: responding to AI output at least twice.
|
|
106
|
+
_MIN_HUMAN_AI_PAIRS = 2
|
|
107
|
+
|
|
108
|
+
#: Dry-run OUTPUT budget per session (structured array, conflicts-sized).
|
|
109
|
+
#: Input is MEASURED from the rendered uuid-headed transcript (chars/4).
|
|
110
|
+
_AVG_OUT_TOKENS = 400
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _already_done(cache: ParquetCache) -> set[str]:
|
|
114
|
+
"""Session ids already present in the cache."""
|
|
115
|
+
done_df = cache.read_all(columns=["session_id"])
|
|
116
|
+
if done_df is not None and done_df.height > 0:
|
|
117
|
+
return set(done_df["session_id"].to_list())
|
|
118
|
+
return set()
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _eligible(reader: CorpusReader, session_id: str) -> bool:
|
|
122
|
+
"""LangSmith parity gate: >= 2 completed human-AI message pairs."""
|
|
123
|
+
return human_ai_pair_count(reader.load_steps(session_id)) >= _MIN_HUMAN_AI_PAIRS
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _user_turn_uuids(reader: CorpusReader, session_id: str) -> set[str]:
|
|
127
|
+
"""The human-turn header uuids the model actually saw.
|
|
128
|
+
|
|
129
|
+
Exactly the ``[uuid=... user ...]`` headers :func:`render_session_text`
|
|
130
|
+
emits under ``include_uuids`` (machine-written user-role steps render
|
|
131
|
+
under their author label instead), so this is the validity universe for
|
|
132
|
+
a returned ``turn_uuid``.
|
|
133
|
+
"""
|
|
134
|
+
return {s.uuid for s in reader.load_steps(session_id) if is_human_turn(s) and s.text and s.uuid}
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
async def _perceived_async(
|
|
138
|
+
settings: AnalyticsSettings,
|
|
139
|
+
*,
|
|
140
|
+
reader: CorpusReader,
|
|
141
|
+
provider: LlmStructuredProvider,
|
|
142
|
+
since_days: int | None,
|
|
143
|
+
limit: int | None,
|
|
144
|
+
cache: ParquetCache,
|
|
145
|
+
checkpoint: SqliteCheckpoint,
|
|
146
|
+
retry: SqliteRetryQueue,
|
|
147
|
+
budget: RunBudget | None,
|
|
148
|
+
) -> int:
|
|
149
|
+
"""Async implementation behind :func:`detect_perceived_errors`."""
|
|
150
|
+
already = _already_done(cache)
|
|
151
|
+
|
|
152
|
+
bounds = reader.session_bounds(since_days=since_days, limit=limit)
|
|
153
|
+
unchanged_pending, skipped = checkpointer.filter_unchanged(
|
|
154
|
+
((sid, lt, mt) for sid, (lt, mt) in bounds.items()),
|
|
155
|
+
pipeline="perceived",
|
|
156
|
+
checkpoint_db_path=checkpoint.db_path,
|
|
157
|
+
)
|
|
158
|
+
keep = set(unchanged_pending)
|
|
159
|
+
|
|
160
|
+
retry_ids = set(retry.drain(pipeline="perceived"))
|
|
161
|
+
if retry_ids:
|
|
162
|
+
logger.info("perceived: draining {} retry-queue entries", len(retry_ids))
|
|
163
|
+
keep |= retry_ids
|
|
164
|
+
|
|
165
|
+
# Retry queue is the single re-admission gate for failed units (see
|
|
166
|
+
# classify): a live entry blocks dispatch until drain says it's due.
|
|
167
|
+
blocked = retry.blocked_units(pipeline="perceived")
|
|
168
|
+
if blocked & keep:
|
|
169
|
+
logger.info(
|
|
170
|
+
"perceived: skipping {} sessions with live retry entries (backoff/attempts cap)",
|
|
171
|
+
len(blocked & keep),
|
|
172
|
+
)
|
|
173
|
+
keep -= blocked
|
|
174
|
+
|
|
175
|
+
# Session ceiling applied DURING the walk, not after: bounds is
|
|
176
|
+
# newest-first and the cap keeps the freshest sessions. The walk stops
|
|
177
|
+
# admitting at the cap, so it costs O(cap) eligibility probes and
|
|
178
|
+
# renders rather than O(corpus) — a session that cannot run this tick
|
|
179
|
+
# never has its steps parsed or its transcript (up to 800K chars) built.
|
|
180
|
+
max_sessions = settings.llm_max_sessions_per_run
|
|
181
|
+
pending: list[tuple[str, str]] = []
|
|
182
|
+
# Captured while this session's steps are still hot in the reader's
|
|
183
|
+
# bounded memo — a later re-walk would re-parse the trajectory.
|
|
184
|
+
valid_uuids: dict[str, set[str]] = {}
|
|
185
|
+
ineligible: list[str] = []
|
|
186
|
+
empty_sids: list[str] = []
|
|
187
|
+
deferred = 0
|
|
188
|
+
for sid in bounds:
|
|
189
|
+
if sid in already and sid not in retry_ids:
|
|
190
|
+
continue
|
|
191
|
+
if sid not in keep:
|
|
192
|
+
continue
|
|
193
|
+
if len(pending) >= max_sessions:
|
|
194
|
+
deferred += 1
|
|
195
|
+
continue
|
|
196
|
+
# LangSmith eligibility gate: < 2 human-AI pairs -> checkpoint as
|
|
197
|
+
# skipped (never billed, never re-rendered until the session grows
|
|
198
|
+
# and its bounds advance past the checkpoint).
|
|
199
|
+
if not _eligible(reader, sid):
|
|
200
|
+
ineligible.append(sid)
|
|
201
|
+
continue
|
|
202
|
+
text = reader.session_text(sid, include_uuids=True)
|
|
203
|
+
if not text:
|
|
204
|
+
empty_sids.append(sid)
|
|
205
|
+
continue
|
|
206
|
+
pending.append((sid, text))
|
|
207
|
+
valid_uuids[sid] = _user_turn_uuids(reader, sid)
|
|
208
|
+
|
|
209
|
+
if ineligible or empty_sids:
|
|
210
|
+
checkpoint.mark_completed(
|
|
211
|
+
pipeline="perceived",
|
|
212
|
+
rows=[(sid, *bounds.get(sid, (None, None))) for sid in (*ineligible, *empty_sids)],
|
|
213
|
+
)
|
|
214
|
+
logger.info(
|
|
215
|
+
"perceived: checkpointed {} ineligible (<{} human-AI pairs) and "
|
|
216
|
+
"{} empty-transcript sessions as skipped",
|
|
217
|
+
len(ineligible),
|
|
218
|
+
_MIN_HUMAN_AI_PAIRS,
|
|
219
|
+
len(empty_sids),
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
if deferred:
|
|
223
|
+
logger.warning(
|
|
224
|
+
"perceived: capped at {} sessions (llm_max_sessions_per_run); {} deferred",
|
|
225
|
+
max_sessions,
|
|
226
|
+
deferred,
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
if not pending:
|
|
230
|
+
logger.info("perceived: no pending sessions (skipped={} via checkpoint)", skipped)
|
|
231
|
+
return 0
|
|
232
|
+
if skipped:
|
|
233
|
+
logger.info("perceived: skipped {} sessions via checkpoint", skipped)
|
|
234
|
+
|
|
235
|
+
chunk_size = write_chunk_size(settings.batch_size)
|
|
236
|
+
logger.info("perceived: {} pending sessions", len(pending))
|
|
237
|
+
|
|
238
|
+
written_errors = 0
|
|
239
|
+
processed_sessions = 0
|
|
240
|
+
i = 0
|
|
241
|
+
chunk_index = 0
|
|
242
|
+
total_chunks = (len(pending) + chunk_size - 1) // chunk_size
|
|
243
|
+
while i < len(pending):
|
|
244
|
+
chunk_index += 1
|
|
245
|
+
t0 = time.monotonic()
|
|
246
|
+
# Budget-checked sub-batches; ``i`` advances by what was actually
|
|
247
|
+
# SENT so a mid-chunk stop leaves the remainder unstamped.
|
|
248
|
+
chunk, results = await gather_under_budget(
|
|
249
|
+
pending[i : i + chunk_size],
|
|
250
|
+
make_coro=lambda unit: provider.classify_structured(
|
|
251
|
+
system=PERCEIVED_SYSTEM_PROMPT,
|
|
252
|
+
prompt=unit[1],
|
|
253
|
+
schema=PerceivedErrorsResult,
|
|
254
|
+
),
|
|
255
|
+
budget=budget,
|
|
256
|
+
)
|
|
257
|
+
if not chunk:
|
|
258
|
+
logger.warning(
|
|
259
|
+
"perceived: cost ceiling hit (${:.2f} >= ${:.2f}) — "
|
|
260
|
+
"aborting with {} sessions unstarted (nothing stamped)",
|
|
261
|
+
budget.spent_usd() if budget is not None else 0.0,
|
|
262
|
+
budget.max_cost_usd if budget is not None else 0.0,
|
|
263
|
+
len(pending) - i,
|
|
264
|
+
)
|
|
265
|
+
break
|
|
266
|
+
i += len(chunk)
|
|
267
|
+
now = datetime.now(UTC)
|
|
268
|
+
|
|
269
|
+
rows: list[dict[str, Any]] = []
|
|
270
|
+
errors = 0
|
|
271
|
+
ok_sids: set[str] = set()
|
|
272
|
+
refused: list[tuple[str, str]] = []
|
|
273
|
+
for (sid, _), res in zip(chunk, results, strict=True):
|
|
274
|
+
if isinstance(res, BaseException):
|
|
275
|
+
from atif_models.domain.ports import RefusalError
|
|
276
|
+
|
|
277
|
+
if isinstance(res, RefusalError):
|
|
278
|
+
# Terminal: zero error rows (a sentinel row would poison
|
|
279
|
+
# the uuid-keyed conversation-time joins), but the
|
|
280
|
+
# refusal lands as a durable audit row in the sidecar.
|
|
281
|
+
logger.info("perceived: {} refused (terminal) — audit sidecar row", sid)
|
|
282
|
+
ok_sids.add(sid)
|
|
283
|
+
refused.append((sid, str(res)))
|
|
284
|
+
continue
|
|
285
|
+
errors += 1
|
|
286
|
+
logger.warning("perceived: {} failed (queued for retry): {}", sid, res)
|
|
287
|
+
retry.enqueue(pipeline="perceived", unit_id=sid, error=str(res))
|
|
288
|
+
continue
|
|
289
|
+
ok_sids.add(sid)
|
|
290
|
+
for e in res.errors:
|
|
291
|
+
turn_uuid = e.turn_uuid
|
|
292
|
+
if not turn_uuid:
|
|
293
|
+
logger.warning("perceived: {} returned an empty turn_uuid — skipping", sid)
|
|
294
|
+
continue
|
|
295
|
+
# uuid-validity guard (conflicts precedent): the schema's
|
|
296
|
+
# invariant is a USER-turn anchor, so anything outside the
|
|
297
|
+
# user-role text-step header uuids — an assistant turn's
|
|
298
|
+
# uuid or a hallucinated one — can never be recovered on
|
|
299
|
+
# the conversation-time axis this pipeline joins on. FAIL
|
|
300
|
+
# CLOSED: an empty universe means no returned uuid can be
|
|
301
|
+
# confirmed, and an unverified uuid in perceived_errors is
|
|
302
|
+
# indistinguishable from a verified one downstream.
|
|
303
|
+
session_uuids = valid_uuids.get(sid) or set()
|
|
304
|
+
if turn_uuid not in session_uuids:
|
|
305
|
+
logger.warning(
|
|
306
|
+
"perceived: {} returned a non-user turn_uuid {!r} — "
|
|
307
|
+
"dropping row (signal={})",
|
|
308
|
+
sid,
|
|
309
|
+
turn_uuid,
|
|
310
|
+
e.signal,
|
|
311
|
+
)
|
|
312
|
+
continue
|
|
313
|
+
rows.append(
|
|
314
|
+
{
|
|
315
|
+
"session_id": sid,
|
|
316
|
+
"turn_uuid": turn_uuid,
|
|
317
|
+
"signal": e.signal,
|
|
318
|
+
"severity": e.severity,
|
|
319
|
+
"evidence": e.evidence,
|
|
320
|
+
"agent_error_summary": e.agent_error_summary,
|
|
321
|
+
"confidence": float(e.confidence),
|
|
322
|
+
"detected_at": now,
|
|
323
|
+
}
|
|
324
|
+
)
|
|
325
|
+
if rows:
|
|
326
|
+
cache.write_part(pl.DataFrame(rows, schema=_PARQUET_SCHEMA))
|
|
327
|
+
if refused:
|
|
328
|
+
write_refusal_audit(
|
|
329
|
+
settings.layout().refusals_dir, pipeline="perceived", refusals=refused
|
|
330
|
+
)
|
|
331
|
+
if ok_sids:
|
|
332
|
+
checkpoint.mark_completed(
|
|
333
|
+
pipeline="perceived",
|
|
334
|
+
rows=[(sid, *bounds.get(sid, (None, None))) for sid in ok_sids],
|
|
335
|
+
)
|
|
336
|
+
retry.mark_done(pipeline="perceived", unit_ids=list(ok_sids))
|
|
337
|
+
written_errors += len(rows)
|
|
338
|
+
processed_sessions += len(ok_sids)
|
|
339
|
+
logger.info(
|
|
340
|
+
"perceived chunk {}/{}: {} sessions, {} error rows, {} failures, {:.1f}s",
|
|
341
|
+
chunk_index,
|
|
342
|
+
total_chunks,
|
|
343
|
+
len(ok_sids),
|
|
344
|
+
len(rows),
|
|
345
|
+
errors,
|
|
346
|
+
time.monotonic() - t0,
|
|
347
|
+
)
|
|
348
|
+
|
|
349
|
+
logger.info(
|
|
350
|
+
"perceived: processed {} sessions, wrote {} error rows",
|
|
351
|
+
processed_sessions,
|
|
352
|
+
written_errors,
|
|
353
|
+
)
|
|
354
|
+
return processed_sessions
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def detect_perceived_errors(
|
|
358
|
+
settings: AnalyticsSettings,
|
|
359
|
+
*,
|
|
360
|
+
since_days: int | None = None,
|
|
361
|
+
limit: int | None = None,
|
|
362
|
+
dry_run: bool = True,
|
|
363
|
+
reader: CorpusReader | None = None,
|
|
364
|
+
provider: LlmStructuredProvider | None = None,
|
|
365
|
+
spec: ModelSpec | None = None,
|
|
366
|
+
cache: ParquetCache | None = None,
|
|
367
|
+
checkpoint: SqliteCheckpoint | None = None,
|
|
368
|
+
retry: SqliteRetryQueue | None = None,
|
|
369
|
+
budget: RunBudget | None = None,
|
|
370
|
+
) -> int | dict[str, Any]:
|
|
371
|
+
"""Detect user-perceived errors per session; return sessions processed.
|
|
372
|
+
|
|
373
|
+
The returned int counts SESSIONS, not error rows: a clean session and
|
|
374
|
+
one with three perceived errors both count as one (and both are
|
|
375
|
+
checkpointed so a rerun is a no-op). Ineligible sessions (< 2 human-AI
|
|
376
|
+
pairs) are checkpointed as skipped without an LLM call and do not
|
|
377
|
+
count. ``dry_run`` (the default) returns a plan dict instead.
|
|
378
|
+
"""
|
|
379
|
+
layout = settings.layout()
|
|
380
|
+
reader = (
|
|
381
|
+
reader
|
|
382
|
+
if reader is not None
|
|
383
|
+
else CorpusReader(settings.corpus_root, caps=settings.transcript_caps())
|
|
384
|
+
)
|
|
385
|
+
cache = cache if cache is not None else build_cache(layout.perceived_errors_dir)
|
|
386
|
+
|
|
387
|
+
if provider is None or spec is None:
|
|
388
|
+
provider, spec = build_provider(settings, "perceived")
|
|
389
|
+
|
|
390
|
+
if dry_run:
|
|
391
|
+
# Measured estimation: input tokens from the actual rendered
|
|
392
|
+
# uuid-headed transcript (chars/4), eligibility-gated and capped
|
|
393
|
+
# like the real run.
|
|
394
|
+
already = _already_done(cache)
|
|
395
|
+
bounds = reader.session_bounds(since_days=since_days, limit=limit)
|
|
396
|
+
pending = [sid for sid in bounds if sid not in already]
|
|
397
|
+
eligible = [sid for sid in pending if _eligible(reader, sid)]
|
|
398
|
+
max_sessions = settings.llm_max_sessions_per_run
|
|
399
|
+
capped = eligible[:max_sessions]
|
|
400
|
+
in_tokens = sum(
|
|
401
|
+
tokens_for_chars(len(text)) for text in reader.session_texts(capped, include_uuids=True)
|
|
402
|
+
)
|
|
403
|
+
out_tokens = len(capped) * _AVG_OUT_TOKENS
|
|
404
|
+
pricing = (spec.pricing_in or 0.0, spec.pricing_out or 0.0)
|
|
405
|
+
cost = estimate_cost_tokens(in_tokens, out_tokens, pricing)
|
|
406
|
+
logger.info(
|
|
407
|
+
"perceived --dry-run: {} sessions pending, {} eligible "
|
|
408
|
+
"({} would run under the cap), estimated cost ~${:.2f}",
|
|
409
|
+
len(pending),
|
|
410
|
+
len(eligible),
|
|
411
|
+
len(capped),
|
|
412
|
+
cost,
|
|
413
|
+
)
|
|
414
|
+
return {
|
|
415
|
+
"pipeline": "perceived",
|
|
416
|
+
"candidates": len(pending),
|
|
417
|
+
"eligible_candidates": len(eligible),
|
|
418
|
+
"capped_candidates": len(capped),
|
|
419
|
+
"llm_calls": len(capped),
|
|
420
|
+
"estimated_input_tokens": in_tokens,
|
|
421
|
+
"estimated_output_tokens": out_tokens,
|
|
422
|
+
"estimated_cost_usd": round(cost, 4),
|
|
423
|
+
"max_sessions_per_run": max_sessions,
|
|
424
|
+
"max_cost_usd_per_run": settings.llm_max_cost_usd_per_run,
|
|
425
|
+
"model": spec.model_id,
|
|
426
|
+
"since_days": since_days,
|
|
427
|
+
"limit": limit,
|
|
428
|
+
"dry_run": True,
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
checkpoint = checkpoint if checkpoint is not None else build_checkpoint(layout.state_db_path)
|
|
432
|
+
retry = retry if retry is not None else build_retry_queue(layout.state_db_path)
|
|
433
|
+
if budget is not None:
|
|
434
|
+
budget.watch(provider, spec)
|
|
435
|
+
with pipeline_usage(provider, spec, "perceived"):
|
|
436
|
+
return asyncio.run(
|
|
437
|
+
_perceived_async(
|
|
438
|
+
settings,
|
|
439
|
+
reader=reader,
|
|
440
|
+
provider=provider,
|
|
441
|
+
since_days=since_days,
|
|
442
|
+
limit=limit,
|
|
443
|
+
cache=cache,
|
|
444
|
+
checkpoint=checkpoint,
|
|
445
|
+
retry=retry,
|
|
446
|
+
budget=budget,
|
|
447
|
+
)
|
|
448
|
+
)
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
__all__ = ["detect_perceived_errors"]
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
|
|
3
|
+
"""Who wrote a user-role step, and what kind of session the steps make.
|
|
4
|
+
|
|
5
|
+
This is the atif-analytics TWIN of ``atif_duck.domain.authorship``. The
|
|
6
|
+
independence contract forbids importing atif-duck, so the rule tables below
|
|
7
|
+
are a copy, and ``packages/atif-duck/tests/test_authorship_twin_pin.py``
|
|
8
|
+
reads this file as source text and fails on any difference from the
|
|
9
|
+
canonical ones. Edit both or neither.
|
|
10
|
+
|
|
11
|
+
Claude Code and Codex deliver machine-written text in the USER role: Stop
|
|
12
|
+
hook feedback, task notifications, retry and continuation nudges, skill
|
|
13
|
+
bodies, slash-command wrappers, screenshot metadata, and the prompts of
|
|
14
|
+
automated review sessions. The LLM pipelines read a session through
|
|
15
|
+
:func:`step_author` so that only ``human`` steps count as the user speaking:
|
|
16
|
+
friction candidates, the perceived-error pair gate and uuid universe, the
|
|
17
|
+
rendered transcript's role labels, and the session-kind gate every pipeline
|
|
18
|
+
applies before it spends anything.
|
|
19
|
+
|
|
20
|
+
Matching is a prefix test over the message with leading whitespace
|
|
21
|
+
stripped; first rule wins; an empty message is ``harness``. A step flagged
|
|
22
|
+
``is_compact_summary`` is ``harness`` whatever its text; the corpus reader
|
|
23
|
+
applies that when it projects a step (mirroring atif-duck's
|
|
24
|
+
``user_steps.author``), so :func:`step_author` itself sees only source and
|
|
25
|
+
text, like the SQL macro.
|
|
26
|
+
|
|
27
|
+
Pure domain module: stdlib only.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
from typing import TYPE_CHECKING, Literal
|
|
33
|
+
|
|
34
|
+
if TYPE_CHECKING:
|
|
35
|
+
from collections.abc import Iterable
|
|
36
|
+
|
|
37
|
+
AUTHOR_VALUES: tuple[str, ...] = (
|
|
38
|
+
"human",
|
|
39
|
+
"stop_hook",
|
|
40
|
+
"task_notification",
|
|
41
|
+
"harness",
|
|
42
|
+
"audit_prompt",
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
#: Characters stripped from the left of a message before matching.
|
|
46
|
+
AUTHOR_STRIP_CHARS: str = " \t\r\n"
|
|
47
|
+
|
|
48
|
+
#: ``(author, prefix)``, first match wins. Every prefix was read off real
|
|
49
|
+
#: Claude Code and Codex corpora in September 2026 as a most-common opening of
|
|
50
|
+
#: user-role steps; the comment beside each says what writes it.
|
|
51
|
+
AUTHOR_PREFIX_RULES: tuple[tuple[str, str], ...] = (
|
|
52
|
+
# An automated review session: a hook that audits another agent's turn
|
|
53
|
+
# opens a fresh session whose whole content is its prompt and a verdict.
|
|
54
|
+
("audit_prompt", "You are auditing an agent turn"),
|
|
55
|
+
# Claude Code Stop / SubagentStop hooks that blocked the agent's stop.
|
|
56
|
+
("stop_hook", "Stop hook feedback"),
|
|
57
|
+
("stop_hook", "SubagentStop hook feedback"),
|
|
58
|
+
# Background task completion notices.
|
|
59
|
+
("task_notification", "<task-notification>"),
|
|
60
|
+
("task_notification", "[SYSTEM NOTIFICATION"),
|
|
61
|
+
# Claude Code bookkeeping: interrupts, continuation and retry nudges,
|
|
62
|
+
# compaction summaries, skill bodies, image metadata for a screenshot the
|
|
63
|
+
# agent read, workflow-computed tasks, reminders and hook output.
|
|
64
|
+
("harness", "[Request interrupted"),
|
|
65
|
+
("harness", "Continue from where you left off"),
|
|
66
|
+
("harness", "Your previous attempt hit a transient"),
|
|
67
|
+
("harness", "This session is being continued from a previous conversation"),
|
|
68
|
+
("harness", "Base directory for this skill:"),
|
|
69
|
+
("harness", "[Image: original "),
|
|
70
|
+
("harness", '{"type": "image"'),
|
|
71
|
+
("harness", "[Workflow harness"),
|
|
72
|
+
("harness", "[structured-output-enforce]"),
|
|
73
|
+
("harness", "<system-reminder>"),
|
|
74
|
+
("harness", "<user-prompt-submit-hook>"),
|
|
75
|
+
("harness", "UserPromptSubmit operation blocked by hook"),
|
|
76
|
+
("harness", "<fork-boilerplate>"),
|
|
77
|
+
("harness", "Another Claude session sent a message"),
|
|
78
|
+
("harness", "The coordinator sent a message"),
|
|
79
|
+
# Slash-command and local-shell wrappers.
|
|
80
|
+
("harness", "<command-name>"),
|
|
81
|
+
("harness", "<command-message>"),
|
|
82
|
+
("harness", "<command-args>"),
|
|
83
|
+
("harness", "<local-command-stdout>"),
|
|
84
|
+
("harness", "<local-command-stderr>"),
|
|
85
|
+
("harness", "<local-command-caveat>"),
|
|
86
|
+
("harness", "Caveat: The messages below were generated"),
|
|
87
|
+
("harness", "<bash-input>"),
|
|
88
|
+
("harness", "<bash-stdout>"),
|
|
89
|
+
("harness", "<bash-stderr>"),
|
|
90
|
+
# Codex CLI context injections.
|
|
91
|
+
("harness", "<environment_context>"),
|
|
92
|
+
("harness", "<codex_internal_context"),
|
|
93
|
+
("harness", "<user_instructions>"),
|
|
94
|
+
("harness", "# AGENTS.md instructions"),
|
|
95
|
+
("harness", "<turn_aborted>"),
|
|
96
|
+
("harness", "The following is the Codex agent history"),
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
#: Prefixes that mark an interrupted turn (``session_outcomes.interrupts``).
|
|
100
|
+
#: Both are also ``harness`` rules above, so an interrupt never counts as a
|
|
101
|
+
#: human turn.
|
|
102
|
+
INTERRUPT_PREFIXES: tuple[str, ...] = ("[Request interrupted", "<turn_aborted>")
|
|
103
|
+
|
|
104
|
+
#: ``session_outcomes.kind`` values (atif-duck's view computes the same).
|
|
105
|
+
SessionKind = Literal["interactive", "one_shot_job", "turn_audit"]
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def step_author(source: str, message: str | None) -> str | None:
|
|
109
|
+
"""The author label for one step; ``None`` when ``source`` is not ``user``.
|
|
110
|
+
|
|
111
|
+
``source`` is the ATIF ``Step.source`` (``user`` / ``agent`` /
|
|
112
|
+
``system``). Mirrors atif-duck's ``step_author(src, msg)`` macro.
|
|
113
|
+
"""
|
|
114
|
+
if source != "user":
|
|
115
|
+
return None
|
|
116
|
+
text = (message or "").lstrip(AUTHOR_STRIP_CHARS)
|
|
117
|
+
if not text:
|
|
118
|
+
return "harness"
|
|
119
|
+
for author, prefix in AUTHOR_PREFIX_RULES:
|
|
120
|
+
if text.startswith(prefix):
|
|
121
|
+
return author
|
|
122
|
+
return "human"
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def is_interrupt(message: str | None) -> bool:
|
|
126
|
+
"""True when ``message`` opens with one of :data:`INTERRUPT_PREFIXES`."""
|
|
127
|
+
return (message or "").lstrip(AUTHOR_STRIP_CHARS).startswith(INTERRUPT_PREFIXES)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def session_kind(first_user_author: str | None, human_turns: int) -> SessionKind:
|
|
131
|
+
"""``turn_audit`` | ``one_shot_job`` | ``interactive``, as ``session_outcomes`` computes it.
|
|
132
|
+
|
|
133
|
+
``first_user_author`` is the author of the session's first main-chain
|
|
134
|
+
user step (``None`` when it has none); ``human_turns`` counts main-chain
|
|
135
|
+
human steps.
|
|
136
|
+
"""
|
|
137
|
+
if first_user_author == "audit_prompt":
|
|
138
|
+
return "turn_audit"
|
|
139
|
+
if human_turns <= 1:
|
|
140
|
+
return "one_shot_job"
|
|
141
|
+
return "interactive"
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def kind_of(authors: Iterable[str]) -> SessionKind:
|
|
145
|
+
""":func:`session_kind` over one session's main-chain user-step authors, in order."""
|
|
146
|
+
first: str | None = None
|
|
147
|
+
humans = 0
|
|
148
|
+
for author in authors:
|
|
149
|
+
if first is None:
|
|
150
|
+
first = author
|
|
151
|
+
if author == "human":
|
|
152
|
+
humans += 1
|
|
153
|
+
return session_kind(first, humans)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
__all__ = [
|
|
157
|
+
"AUTHOR_PREFIX_RULES",
|
|
158
|
+
"AUTHOR_STRIP_CHARS",
|
|
159
|
+
"AUTHOR_VALUES",
|
|
160
|
+
"INTERRUPT_PREFIXES",
|
|
161
|
+
"SessionKind",
|
|
162
|
+
"is_interrupt",
|
|
163
|
+
"kind_of",
|
|
164
|
+
"session_kind",
|
|
165
|
+
"step_author",
|
|
166
|
+
]
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
|
|
3
|
+
"""Per-pipeline config value-objects (pure, frozen dataclasses).
|
|
4
|
+
|
|
5
|
+
The transcript caps, carved into a small frozen dataclass so the renderer
|
|
6
|
+
never sees a model id or a corpus path. Stdlib-only; the env-driven
|
|
7
|
+
``AnalyticsSettings`` in ``infrastructure.settings`` projects down into it.
|
|
8
|
+
(The clustering, community and c-TF-IDF configs left with the structural
|
|
9
|
+
pipelines on 2026-09-27.)
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass(frozen=True, slots=True)
|
|
18
|
+
class TranscriptCaps:
|
|
19
|
+
"""Character caps for session-transcript assembly.
|
|
20
|
+
|
|
21
|
+
The per-tool-result clip bounds arbitrarily large Bash / file-read
|
|
22
|
+
outputs; the total cap keeps an assembled session under the model
|
|
23
|
+
context window.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
session_text_total_max_chars: int = 800_000
|
|
27
|
+
session_text_tool_result_max_chars: int = 50_000
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
__all__ = ["TranscriptCaps"]
|