atif-sql 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- atif_analytics/__init__.py +9 -0
- atif_analytics/application/__init__.py +3 -0
- atif_analytics/application/analyze.py +158 -0
- atif_analytics/application/prompts.py +750 -0
- atif_analytics/application/use_cases/__init__.py +3 -0
- atif_analytics/application/use_cases/_shared.py +264 -0
- atif_analytics/application/use_cases/classify.py +430 -0
- atif_analytics/application/use_cases/conflicts.py +448 -0
- atif_analytics/application/use_cases/friction.py +634 -0
- atif_analytics/application/use_cases/perceived.py +451 -0
- atif_analytics/domain/__init__.py +3 -0
- atif_analytics/domain/authorship.py +166 -0
- atif_analytics/domain/config.py +30 -0
- atif_analytics/domain/costs.py +50 -0
- atif_analytics/domain/friction.py +117 -0
- atif_analytics/domain/layout.py +94 -0
- atif_analytics/domain/models.py +385 -0
- atif_analytics/domain/ports.py +100 -0
- atif_analytics/domain/transcript.py +359 -0
- atif_analytics/infrastructure/__init__.py +3 -0
- atif_analytics/infrastructure/corpus_reader.py +546 -0
- atif_analytics/infrastructure/parquet_cache.py +186 -0
- atif_analytics/infrastructure/settings.py +114 -0
- atif_analytics/infrastructure/sqlite_state/__init__.py +3 -0
- atif_analytics/infrastructure/sqlite_state/checkpointer.py +326 -0
- atif_analytics/infrastructure/sqlite_state/retry_queue.py +296 -0
- atif_analytics/py.typed +0 -0
- atif_cli/__init__.py +3 -0
- atif_cli/__main__.py +8 -0
- atif_cli/app.py +2459 -0
- atif_cli/converter_adapter.py +151 -0
- atif_cli/corpus.py +409 -0
- atif_cli/cron.py +311 -0
- atif_cli/duck_errors.py +97 -0
- atif_cli/embed_lake.py +101 -0
- atif_cli/errors.py +89 -0
- atif_cli/lake.py +373 -0
- atif_cli/lake_sessions.py +270 -0
- atif_cli/output.py +297 -0
- atif_cli/py.typed +0 -0
- atif_converter/__init__.py +3 -0
- atif_converter/application/__init__.py +3 -0
- atif_converter/application/convert_and_audit.py +294 -0
- atif_converter/application/convert_codex.py +210 -0
- atif_converter/domain/__init__.py +9 -0
- atif_converter/domain/agents.py +40 -0
- atif_converter/domain/atif/__init__.py +55 -0
- atif_converter/domain/atif/agent.py +37 -0
- atif_converter/domain/atif/content.py +161 -0
- atif_converter/domain/atif/final_metrics.py +45 -0
- atif_converter/domain/atif/metrics.py +49 -0
- atif_converter/domain/atif/observation.py +21 -0
- atif_converter/domain/atif/observation_result.py +47 -0
- atif_converter/domain/atif/step.py +144 -0
- atif_converter/domain/atif/subagent_trajectory_ref.py +90 -0
- atif_converter/domain/atif/tool_call.py +36 -0
- atif_converter/domain/atif/trajectory.py +211 -0
- atif_converter/domain/atif/trajectory_validator.py +299 -0
- atif_converter/domain/blobs.py +510 -0
- atif_converter/domain/claude_code_conversion.py +1077 -0
- atif_converter/domain/codex_conversion.py +1036 -0
- atif_converter/domain/codex_edges.py +192 -0
- atif_converter/domain/codex_enrichment.py +419 -0
- atif_converter/domain/codex_fidelity.py +124 -0
- atif_converter/domain/codex_nested_calls.py +165 -0
- atif_converter/domain/edges.py +113 -0
- atif_converter/domain/enrichment.py +412 -0
- atif_converter/domain/errors.py +60 -0
- atif_converter/domain/fidelity.py +171 -0
- atif_converter/domain/model_prices.json +8538 -0
- atif_converter/domain/pricing.py +790 -0
- atif_converter/domain/result_signals.py +774 -0
- atif_converter/domain/schema_version.py +90 -0
- atif_converter/domain/session_events.py +408 -0
- atif_converter/infrastructure/__init__.py +3 -0
- atif_converter/infrastructure/census.py +89 -0
- atif_converter/infrastructure/claude_code_converter.py +127 -0
- atif_converter/infrastructure/codex_adapter.py +132 -0
- atif_converter/infrastructure/codex_census.py +100 -0
- atif_converter/infrastructure/codex_converter.py +82 -0
- atif_converter/infrastructure/harbor_adapter.py +187 -0
- atif_converter/infrastructure/raw_records.py +426 -0
- atif_converter/infrastructure/source_archive.py +154 -0
- atif_converter/py.typed +0 -0
- atif_corpus/__init__.py +13 -0
- atif_corpus/application/__init__.py +7 -0
- atif_corpus/application/materialize.py +1826 -0
- atif_corpus/domain/__init__.py +9 -0
- atif_corpus/domain/agents.py +33 -0
- atif_corpus/domain/generation.py +34 -0
- atif_corpus/domain/layout.py +168 -0
- atif_corpus/domain/ports.py +175 -0
- atif_corpus/domain/session_id.py +68 -0
- atif_corpus/domain/sessions.py +228 -0
- atif_corpus/domain/slug.py +72 -0
- atif_corpus/domain/source_layout.py +143 -0
- atif_corpus/domain/watermark.py +76 -0
- atif_corpus/infrastructure/__init__.py +7 -0
- atif_corpus/infrastructure/atomic.py +266 -0
- atif_corpus/infrastructure/compress_artifacts.py +161 -0
- atif_corpus/infrastructure/fake_converter.py +174 -0
- atif_corpus/infrastructure/scanner.py +316 -0
- atif_corpus/infrastructure/settings.py +130 -0
- atif_corpus/infrastructure/source_archive.py +202 -0
- atif_corpus/py.typed +0 -0
- atif_duck/__init__.py +43 -0
- atif_duck/domain/__init__.py +32 -0
- atif_duck/domain/artifacts.py +66 -0
- atif_duck/domain/authorship.py +165 -0
- atif_duck/domain/catalog.py +626 -0
- atif_duck/domain/columnar.py +149 -0
- atif_duck/domain/embedding_guard.py +88 -0
- atif_duck/domain/examples.py +265 -0
- atif_duck/domain/lake.py +427 -0
- atif_duck/domain/lake_sessions.py +97 -0
- atif_duck/domain/raw_readers.py +140 -0
- atif_duck/domain/session_id.py +69 -0
- atif_duck/domain/sql_literal.py +50 -0
- atif_duck/infrastructure/__init__.py +34 -0
- atif_duck/infrastructure/analytics.py +469 -0
- atif_duck/infrastructure/authorship.py +139 -0
- atif_duck/infrastructure/columnar.py +641 -0
- atif_duck/infrastructure/lake.py +1383 -0
- atif_duck/infrastructure/lake_sessions.py +194 -0
- atif_duck/infrastructure/lake_settings.py +63 -0
- atif_duck/infrastructure/lake_steps.py +225 -0
- atif_duck/infrastructure/projections.py +221 -0
- atif_duck/infrastructure/registry.py +2194 -0
- atif_duck/infrastructure/stored_artifacts.py +115 -0
- atif_duck/py.typed +0 -0
- atif_embed/__init__.py +12 -0
- atif_embed/application/__init__.py +3 -0
- atif_embed/application/embed.py +293 -0
- atif_embed/application/prune.py +80 -0
- atif_embed/domain/__init__.py +3 -0
- atif_embed/domain/discovery.py +70 -0
- atif_embed/domain/embedding_guard.py +83 -0
- atif_embed/domain/errors.py +94 -0
- atif_embed/domain/ports.py +223 -0
- atif_embed/domain/sql_literal.py +49 -0
- atif_embed/domain/text_stamp.py +71 -0
- atif_embed/infrastructure/__init__.py +3 -0
- atif_embed/infrastructure/cohere_bedrock.py +433 -0
- atif_embed/infrastructure/corpus_text_rows.py +273 -0
- atif_embed/infrastructure/gemma_local.py +355 -0
- atif_embed/infrastructure/lake_text_rows.py +203 -0
- atif_embed/infrastructure/lance_store.py +440 -0
- atif_embed/infrastructure/providers.py +30 -0
- atif_embed/infrastructure/settings.py +154 -0
- atif_embed/py.typed +0 -0
- atif_models/__init__.py +14 -0
- atif_models/domain/__init__.py +9 -0
- atif_models/domain/ports.py +143 -0
- atif_models/domain/registry.py +151 -0
- atif_models/domain/schema.py +109 -0
- atif_models/infrastructure/__init__.py +8 -0
- atif_models/infrastructure/openai_bedrock.py +314 -0
- atif_models/infrastructure/settings.py +85 -0
- atif_models/py.typed +0 -0
- atif_sql-0.2.0.dist-info/METADATA +272 -0
- atif_sql-0.2.0.dist-info/RECORD +164 -0
- atif_sql-0.2.0.dist-info/WHEEL +4 -0
- atif_sql-0.2.0.dist-info/entry_points.txt +2 -0
- atif_sql-0.2.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,750 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
|
|
3
|
+
"""Task-framing system prompts for the structured-output classifiers.
|
|
4
|
+
|
|
5
|
+
Every "Claude Code" phrase describing the DATA is kept verbatim, because the
|
|
6
|
+
corpus IS Claude Code transcripts.
|
|
7
|
+
|
|
8
|
+
Adaptation for the GPT-5.6 strict-structured-output backend (CONTRACT-V2:
|
|
9
|
+
"adapting ONLY Claude-specific phrasing that misleads a GPT model"; asserted
|
|
10
|
+
in ``tests/test_prompts.py``): ``_CLASSIFIER_APPENDIX`` <output_rules> says
|
|
11
|
+
"The response_format json_schema strict mode enforces the schema" where the
|
|
12
|
+
ported text said "Bedrock's output_config.format enforces the schema".
|
|
13
|
+
output_config is the Anthropic-on-Bedrock wire shape; naming it to a GPT
|
|
14
|
+
model asserts a mechanism that doesn't exist on its path.
|
|
15
|
+
|
|
16
|
+
Two house edits on 2026-09-27, both asserted in ``tests/test_prompts.py``:
|
|
17
|
+
the classify prompt asks for work category and goal only (the autonomy and
|
|
18
|
+
success labels were dropped with their schema fields), and the three
|
|
19
|
+
transcript prompts carry ``_TRANSCRIPT_ROLES``, which names the author
|
|
20
|
+
labels the renderer now puts on machine-written user-role text. The
|
|
21
|
+
trajectory prompt left with its pipeline.
|
|
22
|
+
|
|
23
|
+
``PERCEIVED_SYSTEM_PROMPT`` is not a port. It implements LangSmith's
|
|
24
|
+
published "Perceived Error" definition
|
|
25
|
+
(docs.langchain.com/langsmith/tuned-evaluators) in the house prompt style
|
|
26
|
+
(instructions/context/calibration/examples/anti_patterns + the shared
|
|
27
|
+
classifier appendix). Its two house anti-patterns beyond that definition,
|
|
28
|
+
both asserted in ``tests/test_prompts.py``: (a) text QUOTED or PASTED
|
|
29
|
+
inside a turn (reviews, transcripts under analysis, log excerpts) is not
|
|
30
|
+
THIS session's user perceiving anything; (b) an orchestrator re-issuing a
|
|
31
|
+
task packet or retrying an identical instruction after an infrastructure
|
|
32
|
+
failure is not repeated_request.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
from __future__ import annotations
|
|
36
|
+
|
|
37
|
+
_CLASSIFY_SYSTEM_PROMPT_BODY = """\
|
|
38
|
+
<instructions>
|
|
39
|
+
You are an offline post-hoc analyst classifying complete Claude Code coding
|
|
40
|
+
sessions. The user message contains the full session transcript (user turns,
|
|
41
|
+
assistant turns, tool calls, and tool results) already concatenated.
|
|
42
|
+
|
|
43
|
+
Emit exactly one JSON object matching the schema: the work category, the
|
|
44
|
+
user's goal, and a self-assessed confidence. No surrounding prose, no
|
|
45
|
+
markdown fences.
|
|
46
|
+
</instructions>
|
|
47
|
+
|
|
48
|
+
<context>
|
|
49
|
+
How to read the transcript:
|
|
50
|
+
|
|
51
|
+
- The opening user message states or implies the goal; later user turns
|
|
52
|
+
refine it.
|
|
53
|
+
- Tool calls plus tool results are the strongest evidence of what kind of
|
|
54
|
+
work happened — read past chitchat to the actions.
|
|
55
|
+
|
|
56
|
+
Work category cues:
|
|
57
|
+
|
|
58
|
+
- sde: code, tests, refactors, CI failures, debugging, package management,
|
|
59
|
+
type errors, lint output, anything in src/ or tests/. Default for any
|
|
60
|
+
coding-tool session.
|
|
61
|
+
- admin: scheduling, calendar, expense reports, low-signal email triage,
|
|
62
|
+
routine ops with no code changes.
|
|
63
|
+
- strategy_business: business analysis, competitive landscape, strategic
|
|
64
|
+
memos, proposals, market sizing. Reading and writing strategy documents.
|
|
65
|
+
- events: speaker prep, agenda building, event logistics.
|
|
66
|
+
- thought_leadership: writing for external audiences (blog posts,
|
|
67
|
+
conference abstracts, LinkedIn). Polished prose, not internal docs.
|
|
68
|
+
- other: only when nothing else fits. Sessions that mix sde plus a second
|
|
69
|
+
category should pick the one with more turns / tool calls.
|
|
70
|
+
</context>
|
|
71
|
+
|
|
72
|
+
<calibration>
|
|
73
|
+
- goal must be one sentence in present tense, paraphrasing the user — not
|
|
74
|
+
a literal quote, not two goals concatenated with "and".
|
|
75
|
+
- Use confidence < 0.5 when the category is genuinely mixed or the goal
|
|
76
|
+
is unclear. Do not manufacture certainty to fill the schema.
|
|
77
|
+
</calibration>
|
|
78
|
+
|
|
79
|
+
<examples>
|
|
80
|
+
<example>
|
|
81
|
+
<input>A 4-hour session where the user opens with "implement Phase 2 of the
|
|
82
|
+
auth migration", the agent runs ~80 tool calls, the user replies "ok",
|
|
83
|
+
"good", "ship it" between long agent runs.</input>
|
|
84
|
+
<output>work_category=sde, goal="Implement Phase 2 of the auth
|
|
85
|
+
migration.", confidence=0.9</output>
|
|
86
|
+
</example>
|
|
87
|
+
<example>
|
|
88
|
+
<input>A 2-hour session of strategic memo work — user dictates section
|
|
89
|
+
outlines, agent drafts, user rewrites paragraphs heavily.</input>
|
|
90
|
+
<output>work_category=strategy_business, goal="Draft the strategy memo
|
|
91
|
+
section by section.", confidence=0.85</output>
|
|
92
|
+
</example>
|
|
93
|
+
<example>
|
|
94
|
+
<input>A session that opens with "schedule a 1:1 with X", the agent calls
|
|
95
|
+
calendar, finds slots, user picks one, agent books.</input>
|
|
96
|
+
<output>work_category=admin, goal="Schedule a 1:1 with X.",
|
|
97
|
+
confidence=0.95</output>
|
|
98
|
+
</example>
|
|
99
|
+
</examples>
|
|
100
|
+
|
|
101
|
+
<anti_patterns>
|
|
102
|
+
- Don't infer goals from agent actions. The user's opening message is
|
|
103
|
+
the ground truth for goal. If the agent went on a tangent, the goal is
|
|
104
|
+
still what the user asked for.
|
|
105
|
+
- Don't judge whether the goal was met. That is not a field.
|
|
106
|
+
</anti_patterns>
|
|
107
|
+
"""
|
|
108
|
+
|
|
109
|
+
_CONFLICTS_SYSTEM_PROMPT_BODY = """\
|
|
110
|
+
<instructions>
|
|
111
|
+
You analyze a complete Claude Code coding session for STANCE CONFLICTS —
|
|
112
|
+
moments where the user and the agent (or the agent's own reasoning) hold
|
|
113
|
+
mutually-exclusive positions on the same substantive question.
|
|
114
|
+
|
|
115
|
+
Emit exactly one JSON object with a conflicts array. Each entry has SEVEN
|
|
116
|
+
fields:
|
|
117
|
+
- turn_a_uuid (string) — UUID of one of the turns whose stance clashes,
|
|
118
|
+
copied verbatim from the ``[uuid=...]`` headers in the bound transcript.
|
|
119
|
+
- turn_b_uuid (string) — UUID of the opposing turn. Must differ from
|
|
120
|
+
turn_a_uuid. Same verbatim-copy rule.
|
|
121
|
+
- conflict_kind (enum: disagreement | correction | reversal | impasse).
|
|
122
|
+
- severity (enum: low | medium | high).
|
|
123
|
+
- agent_position (one-sentence summary in the agent's own framing).
|
|
124
|
+
- user_position (one-sentence summary in the user's own framing).
|
|
125
|
+
- confidence (0.0-1.0).
|
|
126
|
+
|
|
127
|
+
An empty conflicts array is valid and common — sessions with no
|
|
128
|
+
conflicts produce zero rows downstream.
|
|
129
|
+
|
|
130
|
+
Output JSON only. No surrounding prose, no markdown fences.
|
|
131
|
+
</instructions>
|
|
132
|
+
|
|
133
|
+
<context>
|
|
134
|
+
What counts as a conflict:
|
|
135
|
+
|
|
136
|
+
- Two stances on the same technical decision held by different parties,
|
|
137
|
+
or by the same party at different points. "Use Sonnet" vs "use Opus".
|
|
138
|
+
"Ship the simple version now" vs "wait for the architectural cleanup".
|
|
139
|
+
"Cache the embeddings" vs "rebuild from parquet every run".
|
|
140
|
+
- Two stances on a strategic / scope decision: "rename the field" vs
|
|
141
|
+
"keep the field name and shift semantics". "One bundled PR" vs
|
|
142
|
+
"split into three". "Fix it on this branch" vs "open a follow-up".
|
|
143
|
+
- The conflict must be SUBSTANTIVE — measurable consequences, not style.
|
|
144
|
+
|
|
145
|
+
conflict_kind semantics:
|
|
146
|
+
|
|
147
|
+
- disagreement: two parties hold opposing positions and discuss them
|
|
148
|
+
without one explicitly telling the other they're wrong. The
|
|
149
|
+
prototypical "stance A vs stance B" debate.
|
|
150
|
+
- correction: one party explicitly tells the other their answer or
|
|
151
|
+
action was wrong ("no, not that, do X instead", "that's not what I
|
|
152
|
+
asked for"). Different from disagreement: the corrector treats the
|
|
153
|
+
other's stance as a mistake to be overwritten, not a position to
|
|
154
|
+
argue against.
|
|
155
|
+
- reversal: the SAME party flips their own earlier position ("actually
|
|
156
|
+
let's NOT do X", "scratch that, going the other way"). Both turns
|
|
157
|
+
are spoken by the same role.
|
|
158
|
+
- impasse: both sides restate their positions across multiple exchanges
|
|
159
|
+
without converging, and the topic stalls. Distinct from "unresolved":
|
|
160
|
+
impasse implies repeated re-statement, not just running out of time.
|
|
161
|
+
|
|
162
|
+
severity semantics:
|
|
163
|
+
|
|
164
|
+
- low: a minor course nudge with little downstream impact.
|
|
165
|
+
- medium: changes the implementation approach or scope but stays inside
|
|
166
|
+
the original goal.
|
|
167
|
+
- high: blocks progress, reverses a major decision, or fundamentally
|
|
168
|
+
changes the goal.
|
|
169
|
+
|
|
170
|
+
Identification heuristics:
|
|
171
|
+
|
|
172
|
+
1. Strongest signal is structural: stance A proposed at one turn,
|
|
173
|
+
counter-stance B held at another turn. Without two distinct turns
|
|
174
|
+
holding opposing positions, you don't have a conflict.
|
|
175
|
+
2. Verbal markers: "but I think", "actually I'd argue", "I disagree",
|
|
176
|
+
"the other side of that is", "alternatively", "no, not that".
|
|
177
|
+
3. Skip agent's internal monologue ("on one hand X, on the other Y") when
|
|
178
|
+
the agent immediately picks one — that's deliberation. Only count when
|
|
179
|
+
two distinct turns hold the opposing stances.
|
|
180
|
+
4. Pull turn_a_uuid / turn_b_uuid from the literal ``[uuid=...]`` headers
|
|
181
|
+
in the transcript — never invent or paraphrase.
|
|
182
|
+
</context>
|
|
183
|
+
|
|
184
|
+
<calibration>
|
|
185
|
+
When in doubt, return an empty conflicts array. False positives pollute
|
|
186
|
+
the corpus more than missed conflicts hurt — downstream views
|
|
187
|
+
(session_conflicts) are used by humans to find interesting decision
|
|
188
|
+
points, and noise drowns signal.
|
|
189
|
+
|
|
190
|
+
Typical coding session has 0 conflicts. Typical strategy / planning
|
|
191
|
+
session has 0-2. Sessions with 3+ conflicts exist but are rare;
|
|
192
|
+
double-check your output if you're emitting that many.
|
|
193
|
+
</calibration>
|
|
194
|
+
|
|
195
|
+
<examples>
|
|
196
|
+
<example>
|
|
197
|
+
<input>User wants to optimize a slow query. At [uuid=t1] agent proposes
|
|
198
|
+
"denormalize the table". At [uuid=t2] user counters: "no, let's add a
|
|
199
|
+
covering index instead — I don't want to touch the schema". Agent
|
|
200
|
+
accepts the index approach.</input>
|
|
201
|
+
<output>conflicts=[{turn_a_uuid: "t1", turn_b_uuid: "t2",
|
|
202
|
+
conflict_kind: "correction", severity: "medium",
|
|
203
|
+
agent_position: "Denormalize the table to make the query faster.",
|
|
204
|
+
user_position: "Keep the schema; add a covering index instead.",
|
|
205
|
+
confidence: 0.9}]</output>
|
|
206
|
+
</example>
|
|
207
|
+
<example>
|
|
208
|
+
<input>User proposes a 3-step plan. Agent says "I think step 2 is risky
|
|
209
|
+
because of X — should we add a rollback first?" User agrees, plan
|
|
210
|
+
becomes 4 steps. Both proceed.</input>
|
|
211
|
+
<output>conflicts=[]. Agent flagged a risk, user incorporated it. No
|
|
212
|
+
counter-stance held.</output>
|
|
213
|
+
</example>
|
|
214
|
+
<example>
|
|
215
|
+
<input>At [uuid=u1] user leans toward "ship simple version now". At
|
|
216
|
+
[uuid=a1] agent leans toward "wait for architectural cleanup". They
|
|
217
|
+
go back and forth multiple times without converging; session ends
|
|
218
|
+
with user saying "let me think about it".</input>
|
|
219
|
+
<output>conflicts=[{turn_a_uuid: "u1", turn_b_uuid: "a1",
|
|
220
|
+
conflict_kind: "impasse", severity: "high",
|
|
221
|
+
agent_position: "Wait for the architectural cleanup so we don't ship debt.",
|
|
222
|
+
user_position: "Ship the simple version now to unblock users.",
|
|
223
|
+
confidence: 0.85}]</output>
|
|
224
|
+
</example>
|
|
225
|
+
<example>
|
|
226
|
+
<input>At [uuid=u3] user says "let's go with GitHub Actions". At
|
|
227
|
+
[uuid=u9] same user later says "actually scratch that, stick with
|
|
228
|
+
CodeBuild — Actions doesn't have the IAM role we need".</input>
|
|
229
|
+
<output>conflicts=[{turn_a_uuid: "u3", turn_b_uuid: "u9",
|
|
230
|
+
conflict_kind: "reversal", severity: "medium",
|
|
231
|
+
agent_position: "Switch to GitHub Actions.",
|
|
232
|
+
user_position: "Stick with CodeBuild for the IAM role.",
|
|
233
|
+
confidence: 0.9}]</output>
|
|
234
|
+
</example>
|
|
235
|
+
<example>
|
|
236
|
+
<input>At [uuid=a4] agent says "I'll use cosine similarity for the
|
|
237
|
+
nearest-neighbor lookup". At [uuid=u5] user objects: "no, use dot
|
|
238
|
+
product — the embeddings are already L2-normalized so it's the same
|
|
239
|
+
math but cheaper". Agent agrees, switches to dot product.</input>
|
|
240
|
+
<output>conflicts=[{turn_a_uuid: "a4", turn_b_uuid: "u5",
|
|
241
|
+
conflict_kind: "correction", severity: "low",
|
|
242
|
+
agent_position: "Use cosine similarity for the nearest-neighbor lookup.",
|
|
243
|
+
user_position: "Use dot product since embeddings are L2-normalized.",
|
|
244
|
+
confidence: 0.85}]</output>
|
|
245
|
+
</example>
|
|
246
|
+
<example>
|
|
247
|
+
<input>At [uuid=u2] user says "let's deprecate the v1 API endpoint". At
|
|
248
|
+
[uuid=a3] agent pushes back: "we still have customers on v1; we should
|
|
249
|
+
co-exist for at least one quarter". User considers it but doesn't
|
|
250
|
+
agree or disagree — pivots to a different topic. Topic never returns.</input>
|
|
251
|
+
<output>conflicts=[{turn_a_uuid: "u2", turn_b_uuid: "a3",
|
|
252
|
+
conflict_kind: "disagreement", severity: "medium",
|
|
253
|
+
agent_position: "Keep v1 alive for one quarter to avoid customer impact.",
|
|
254
|
+
user_position: "Deprecate the v1 API endpoint.",
|
|
255
|
+
confidence: 0.7}]</output>
|
|
256
|
+
</example>
|
|
257
|
+
<example>
|
|
258
|
+
<input>Brief disagreement about which CI config to use. User pivots to
|
|
259
|
+
a different topic without engaging. Never returns to the CI question
|
|
260
|
+
and the agent doesn't restate its position either.</input>
|
|
261
|
+
<output>conflicts=[]. A single unreciprocated remark is not enough —
|
|
262
|
+
need two distinct turns each holding their position.</output>
|
|
263
|
+
</example>
|
|
264
|
+
</examples>
|
|
265
|
+
|
|
266
|
+
<anti_patterns>
|
|
267
|
+
- Don't count collaboration as conflict. Agent proposes a plan, user
|
|
268
|
+
agrees with caveats and the agent adapts. That's collaboration.
|
|
269
|
+
- Don't count agent deliberation. Agent considers two approaches in its
|
|
270
|
+
own reasoning, then picks one with the user's blessing. That's
|
|
271
|
+
deliberation, not conflict.
|
|
272
|
+
- Don't count surface-level pushback that the user immediately retracts.
|
|
273
|
+
("wait, isn't that broken? — oh you're right, never mind") is not a
|
|
274
|
+
conflict; it's a question the agent satisfied.
|
|
275
|
+
- Don't count style / formatting disagreements ("I'd phrase that
|
|
276
|
+
differently", "use semicolons not commas", "this comment should be
|
|
277
|
+
one line"). Style preferences with no consequence to behaviour are
|
|
278
|
+
not conflicts.
|
|
279
|
+
- Don't count accepted risk. Agent flags risk, user accepts it. That's
|
|
280
|
+
a noted caveat, not a conflict — both parties end up agreeing on the
|
|
281
|
+
same plan, they just acknowledge the risk.
|
|
282
|
+
- Don't count iteration. Two failed attempts at the same task (agent
|
|
283
|
+
tried X, then Y, both failed) are iteration, not conflict — neither
|
|
284
|
+
attempt represents a held stance the other side opposed.
|
|
285
|
+
- Don't count tooling preferences without consequence ("I'd use jq here"
|
|
286
|
+
vs "I'd use python -c"). If the underlying behaviour is identical,
|
|
287
|
+
the choice is bikeshed, not stance.
|
|
288
|
+
- Don't count the agent's hedging as a stance. "I could do X, but Y is
|
|
289
|
+
also reasonable" is not a position the agent committed to. A real
|
|
290
|
+
agent_position is a sentence the agent would defend if challenged.
|
|
291
|
+
- Don't count clarifying questions as conflicts. The user asking "why
|
|
292
|
+
did you choose X?" is gathering context, not opposing X — unless the
|
|
293
|
+
agent's answer fails to land and the user then explicitly disagrees.
|
|
294
|
+
- Don't count one-off tone slips. A single curt user message ("no, do
|
|
295
|
+
it the other way") with no prior agent stance to oppose is just a
|
|
296
|
+
command, not a conflict pair.
|
|
297
|
+
- Don't manufacture a conflict to fill the array. If the session is a
|
|
298
|
+
smooth collaboration with no opposing stances, return [] confidently.
|
|
299
|
+
Empty arrays are the correct answer for the majority of sessions.
|
|
300
|
+
- Never invent turn UUIDs. If you cannot identify two specific turns
|
|
301
|
+
in the bound transcript whose stances clash, return an empty array.
|
|
302
|
+
An invented UUID is worse than a missed conflict.
|
|
303
|
+
- Never use a turn UUID twice in the same conflict pair. turn_a_uuid
|
|
304
|
+
and turn_b_uuid must always differ — even in a reversal, the two
|
|
305
|
+
flips are at distinct turns.
|
|
306
|
+
- Never set confidence > 0.5 when the rationale relies on inferring
|
|
307
|
+
unstated positions. If you have to read between the lines, the cue
|
|
308
|
+
is too weak to claim high confidence.
|
|
309
|
+
</anti_patterns>
|
|
310
|
+
|
|
311
|
+
<calibration_notes>
|
|
312
|
+
The downstream conflicts_summary view counts rows per session; a single
|
|
313
|
+
inflated row drowns the signal more than a missed pair would. False
|
|
314
|
+
negatives are recoverable (the pair-scanner pass in v1.1 will catch
|
|
315
|
+
them); false positives are not. When in genuine doubt between
|
|
316
|
+
"borderline conflict at confidence 0.4" and "no conflict", prefer the
|
|
317
|
+
empty array.
|
|
318
|
+
|
|
319
|
+
severity is also calibration-sensitive:
|
|
320
|
+
- 'low' is the right call when the conflict is purely about *how* to
|
|
321
|
+
achieve an agreed-upon outcome and either approach would land the
|
|
322
|
+
same goal.
|
|
323
|
+
- 'medium' applies when the choice changes the implementation shape
|
|
324
|
+
(different file structure, different dependency, different schema).
|
|
325
|
+
- 'high' is reserved for conflicts that change *what* gets shipped or
|
|
326
|
+
block the session from progressing. If you find yourself stamping
|
|
327
|
+
'high' on more than one pair per session, double-check both — that
|
|
328
|
+
density of high-severity conflict is genuinely rare.
|
|
329
|
+
</calibration_notes>
|
|
330
|
+
"""
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
_USER_FRICTION_SYSTEM_PROMPT_BODY = """\
|
|
334
|
+
<instructions>
|
|
335
|
+
You classify ONE short user message from a Claude Code coding session for
|
|
336
|
+
friction signals — cues that the human is impatient, confused,
|
|
337
|
+
interrupting the agent, correcting it, or asking for something the agent
|
|
338
|
+
should have provided proactively but didn't.
|
|
339
|
+
|
|
340
|
+
The message is presented in isolation. You will not see prior turns or
|
|
341
|
+
the agent response that preceded it. Make the call from the message
|
|
342
|
+
text alone.
|
|
343
|
+
|
|
344
|
+
Emit exactly one JSON object with three fields: label (one of the seven
|
|
345
|
+
values below), rationale (one short sentence naming the cue), and
|
|
346
|
+
confidence (0.0-1.0). Output JSON only. No surrounding prose, no
|
|
347
|
+
markdown fences.
|
|
348
|
+
</instructions>
|
|
349
|
+
|
|
350
|
+
<context>
|
|
351
|
+
Label semantics:
|
|
352
|
+
|
|
353
|
+
- status_ping: progress / ETA query.
|
|
354
|
+
Triggers: "how's it going?", "any update?", "where are we?",
|
|
355
|
+
"still working?", "what's your eta?", "are you alive?"
|
|
356
|
+
NOT triggers: "where does the config live?" (technical question),
|
|
357
|
+
"where are we in the migration plan?" (substantive scope question).
|
|
358
|
+
|
|
359
|
+
- unmet_expectation: short question pointing at something the agent
|
|
360
|
+
should have produced.
|
|
361
|
+
Triggers: bare one-word questions ending in "?": "screenshot?",
|
|
362
|
+
"tests?", "diff?", "link?", "logs?", "stacktrace?".
|
|
363
|
+
NOT triggers: "what's the type of X?" (substantive),
|
|
364
|
+
"tests for which file?" (clarification, not friction).
|
|
365
|
+
|
|
366
|
+
- confusion: user signals they don't follow the output or state.
|
|
367
|
+
Triggers: "what does that mean?", "I don't get it", "huh?",
|
|
368
|
+
"why did you do X?" (when X already happened), "wait, what?"
|
|
369
|
+
NOT triggers: a calm question about a future action, a request for
|
|
370
|
+
explanation ("explain that step please" — neutral instruction).
|
|
371
|
+
|
|
372
|
+
- interruption: user cuts the agent off or pivots mid-task.
|
|
373
|
+
Triggers: "wait", "stop", "hold on", "pause", "actually...",
|
|
374
|
+
"before you do that", "nvm", "never mind".
|
|
375
|
+
NOT triggers: "wait until tests pass" (instruction, not interrupt),
|
|
376
|
+
"stop the server" (action request).
|
|
377
|
+
|
|
378
|
+
- correction: explicit "you got it wrong".
|
|
379
|
+
Triggers: "no, not that", "that's wrong", "nope", "try again",
|
|
380
|
+
"you're doing it wrong", "incorrect".
|
|
381
|
+
NOT triggers: "actually let me clarify" (re-framing, not correcting),
|
|
382
|
+
technical bug reports ("X returns None instead of []" — substantive).
|
|
383
|
+
|
|
384
|
+
- frustration: terse annoyance or sarcasm.
|
|
385
|
+
Triggers: "ugh", "seriously?", "are you kidding", "really?",
|
|
386
|
+
"come on".
|
|
387
|
+
NOT triggers: a curt but neutral instruction.
|
|
388
|
+
|
|
389
|
+
- none: ordinary task turn. THIS IS THE MAJORITY CLASS — use it
|
|
390
|
+
aggressively. Anything that's a substantive instruction, a plain
|
|
391
|
+
technical question, an acknowledgement, a routing decision, or text
|
|
392
|
+
the user typed to advance the task is none. The threshold for
|
|
393
|
+
friction is high.
|
|
394
|
+
</context>
|
|
395
|
+
|
|
396
|
+
<calibration>
|
|
397
|
+
- confidence < 0.5 is correct when the message is genuinely ambiguous
|
|
398
|
+
between none and a friction label. Don't manufacture certainty.
|
|
399
|
+
- confidence > 0.8 requires an unambiguous cue you can name in the
|
|
400
|
+
rationale field.
|
|
401
|
+
- For obvious cases ("ugh"), 0.95 is fine.
|
|
402
|
+
</calibration>
|
|
403
|
+
|
|
404
|
+
<examples>
|
|
405
|
+
<example>
|
|
406
|
+
<input>screenshot?</input>
|
|
407
|
+
<output>label=unmet_expectation, confidence=0.7. Bare one-word
|
|
408
|
+
question pointing at a missed artifact.</output>
|
|
409
|
+
</example>
|
|
410
|
+
<example>
|
|
411
|
+
<input>stop</input>
|
|
412
|
+
<output>label=interruption, confidence=0.95. Hard interruption keyword
|
|
413
|
+
as the entire message.</output>
|
|
414
|
+
</example>
|
|
415
|
+
<example>
|
|
416
|
+
<input>delete that file</input>
|
|
417
|
+
<output>label=none, confidence=0.9. Bare instruction, not friction.</output>
|
|
418
|
+
</example>
|
|
419
|
+
<example>
|
|
420
|
+
<input>ugh</input>
|
|
421
|
+
<output>label=frustration, confidence=0.95. Unambiguous annoyance.</output>
|
|
422
|
+
</example>
|
|
423
|
+
<example>
|
|
424
|
+
<input>why did you do that?</input>
|
|
425
|
+
<output>label=confusion, confidence=0.85. Questioning a completed
|
|
426
|
+
action.</output>
|
|
427
|
+
</example>
|
|
428
|
+
<example>
|
|
429
|
+
<input>where does the config live?</input>
|
|
430
|
+
<output>label=none, confidence=0.9. Substantive technical question.</output>
|
|
431
|
+
</example>
|
|
432
|
+
<example>
|
|
433
|
+
<input>nope, try again</input>
|
|
434
|
+
<output>label=correction, confidence=0.95. Explicit rejection plus
|
|
435
|
+
redo.</output>
|
|
436
|
+
</example>
|
|
437
|
+
<example>
|
|
438
|
+
<input>tests for the auth module</input>
|
|
439
|
+
<output>label=none, confidence=0.9. Substantive instruction — what
|
|
440
|
+
tests, not a bare "tests?".</output>
|
|
441
|
+
</example>
|
|
442
|
+
</examples>
|
|
443
|
+
|
|
444
|
+
<anti_patterns>
|
|
445
|
+
- A bare instruction is none, even if it sounds curt. "delete that file"
|
|
446
|
+
is not correction. "add a test for X" is not unmet_expectation.
|
|
447
|
+
- A short technical question is none. "what's the type?" /
|
|
448
|
+
"where is X?" are not friction signals. Friction requires affect or
|
|
449
|
+
implicit complaint.
|
|
450
|
+
- Don't flag based on tone alone. "ok" is none, even if you imagine
|
|
451
|
+
it's sarcastic — without surrounding context you can't tell, so
|
|
452
|
+
default to none.
|
|
453
|
+
- Claude Code injects two strings as user-role messages that look like
|
|
454
|
+
friction but are CLI bookkeeping: "Continue from where you left off."
|
|
455
|
+
and "[Request interrupted by user for tool use]". Both should be
|
|
456
|
+
none. (They're filtered upstream so you'll rarely see them, but be
|
|
457
|
+
safe.)
|
|
458
|
+
</anti_patterns>
|
|
459
|
+
"""
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
_PERCEIVED_SYSTEM_PROMPT_BODY = """\
|
|
463
|
+
<instructions>
|
|
464
|
+
You analyze a complete Claude Code coding session for PERCEIVED ERRORS —
|
|
465
|
+
moments where the USER perceives that the agent made a mistake,
|
|
466
|
+
misunderstood a request, or took the interaction in the wrong direction.
|
|
467
|
+
|
|
468
|
+
You judge PERCEPTION, not objective correctness. An agent can be
|
|
469
|
+
technically right and still be perceived as wrong (report it); an agent
|
|
470
|
+
can be objectively wrong without the user ever noticing (do NOT report
|
|
471
|
+
it — there is no user-visible evidence).
|
|
472
|
+
|
|
473
|
+
Emit exactly one JSON object with an errors array. Each entry has SIX
|
|
474
|
+
fields:
|
|
475
|
+
- turn_uuid (string) — UUID of the USER turn where the perception
|
|
476
|
+
surfaces, copied verbatim from the ``[uuid=...]`` headers in the bound
|
|
477
|
+
transcript. For inferred signals with no single user cue turn, use the
|
|
478
|
+
user turn closest after the agent's error.
|
|
479
|
+
- signal (enum: correction | repeated_request | rejected_action |
|
|
480
|
+
contradictory_response | acknowledged_mistake |
|
|
481
|
+
persistent_misunderstanding | unresolved_outcome).
|
|
482
|
+
- severity (enum: minor | moderate | major).
|
|
483
|
+
- evidence (the user's words — a short verbatim quote or close
|
|
484
|
+
paraphrase, <= 280 chars).
|
|
485
|
+
- agent_error_summary (one sentence: what the agent got wrong from the
|
|
486
|
+
user's perspective, <= 280 chars).
|
|
487
|
+
- confidence (0.0-1.0).
|
|
488
|
+
|
|
489
|
+
An empty errors array is valid and common — it means a clean session.
|
|
490
|
+
|
|
491
|
+
Output JSON only. No surrounding prose, no markdown fences.
|
|
492
|
+
</instructions>
|
|
493
|
+
|
|
494
|
+
<context>
|
|
495
|
+
Signal semantics — three EXPLICIT signals (the user says so):
|
|
496
|
+
|
|
497
|
+
- correction: the user tells the agent its answer or action was wrong,
|
|
498
|
+
or restates what they actually meant ("no, not that file", "that's
|
|
499
|
+
wrong", "what I meant was...", "you misunderstood — I wanted X").
|
|
500
|
+
- repeated_request: the user asks for the same thing AGAIN because the
|
|
501
|
+
first response missed it. The repeat must point at the same unmet
|
|
502
|
+
need, not a new instance of a similar task.
|
|
503
|
+
- rejected_action: the user declines, reverts, or interrupts an action
|
|
504
|
+
the agent took or proposed ("undo that", "stop — don't push",
|
|
505
|
+
"revert the rename", rejecting a proposed plan as off-target).
|
|
506
|
+
|
|
507
|
+
Four INFERRED signals (the conversation shape shows it):
|
|
508
|
+
|
|
509
|
+
- contradictory_response: the agent contradicts its own earlier
|
|
510
|
+
statement and the user is exposed to both ("tests pass" then later
|
|
511
|
+
"the tests were failing all along" with no new information).
|
|
512
|
+
- acknowledged_mistake: the agent itself admits an error in front of
|
|
513
|
+
the user ("you're right, I misread the schema", "my mistake — that
|
|
514
|
+
flag doesn't exist").
|
|
515
|
+
- persistent_misunderstanding: across MULTIPLE turns the agent keeps
|
|
516
|
+
answering a different question than the one the user is asking,
|
|
517
|
+
despite the user re-asking or rephrasing.
|
|
518
|
+
- unresolved_outcome: the session ends with the user's stated problem
|
|
519
|
+
visibly unsolved after the agent's attempts — the user walks away
|
|
520
|
+
without the thing they came for, and the transcript shows they
|
|
521
|
+
noticed (trailing frustration, giving up, "never mind I'll do it
|
|
522
|
+
myself").
|
|
523
|
+
|
|
524
|
+
severity — calibrated for coding sessions, where mild steering is
|
|
525
|
+
routine:
|
|
526
|
+
|
|
527
|
+
- minor: a one-turn nudge; the user corrected and work resumed
|
|
528
|
+
immediately. THE COMMON CASE — most perceived errors in coding
|
|
529
|
+
sessions are minor.
|
|
530
|
+
- moderate: the error cost visible rework — repeated exchanges on the
|
|
531
|
+
same point, an action undone and redone, a detour of several turns.
|
|
532
|
+
- major: the error derailed the session — a wrong direction sustained
|
|
533
|
+
across many turns, a destructive action rejected too late, or the
|
|
534
|
+
user giving up on the goal.
|
|
535
|
+
|
|
536
|
+
Identification heuristics:
|
|
537
|
+
|
|
538
|
+
1. Anchor every error on USER-visible evidence. The evidence field
|
|
539
|
+
quotes the user's words (or, for acknowledged_mistake /
|
|
540
|
+
contradictory_response, the agent text the user visibly saw).
|
|
541
|
+
2. Pull turn_uuid from the literal ``[uuid=...]`` headers — never
|
|
542
|
+
invent or paraphrase. It must be a USER turn.
|
|
543
|
+
3. One perceived error = one row, even when the user corrects the same
|
|
544
|
+
mistake twice; pick the turn where perception FIRST surfaces.
|
|
545
|
+
4. Distinct mistakes get distinct rows. A session can have several.
|
|
546
|
+
</context>
|
|
547
|
+
|
|
548
|
+
<calibration>
|
|
549
|
+
- When in doubt, leave it out. False positives drown the review queue
|
|
550
|
+
this pipeline feeds; a missed borderline nudge costs little.
|
|
551
|
+
- Typical smooth coding session has 0 perceived errors. A session with
|
|
552
|
+
active back-and-forth steering typically has 0-2. If you're emitting
|
|
553
|
+
4+, re-check that you aren't counting ordinary iteration.
|
|
554
|
+
- Confidence < 0.5 when the cue could equally be the user changing
|
|
555
|
+
their mind or ordinary exploratory steering.
|
|
556
|
+
- Confidence > 0.85 only when the user's words explicitly name the
|
|
557
|
+
agent's mistake ("you edited the wrong file").
|
|
558
|
+
- severity skews minor in coding sessions: users steer agents
|
|
559
|
+
constantly, and a fast correction that lands is a minor error even
|
|
560
|
+
when the user's tone is curt.
|
|
561
|
+
</calibration>
|
|
562
|
+
|
|
563
|
+
<examples>
|
|
564
|
+
<example>
|
|
565
|
+
<input>User asks the agent to rename a config KEY. At [uuid=a2] the agent
|
|
566
|
+
renames the config FILE. At [uuid=u3] user: "no — I meant the key
|
|
567
|
+
inside the file, not the file itself. Put the filename back." Agent
|
|
568
|
+
reverts and fixes the key.</input>
|
|
569
|
+
<output>errors=[{turn_uuid: "u3", signal: "correction", severity: "minor",
|
|
570
|
+
evidence: "no — I meant the key inside the file, not the file itself.",
|
|
571
|
+
agent_error_summary: "The agent renamed the config file when the user asked to rename a key inside it.",
|
|
572
|
+
confidence: 0.9}]</output>
|
|
573
|
+
</example>
|
|
574
|
+
<example>
|
|
575
|
+
<input>At [uuid=u1] user asks for a summary table of test failures. The
|
|
576
|
+
agent replies with prose. At [uuid=u4] user: "can you give me the table
|
|
577
|
+
of failures I asked for?" — same request, restated because the first
|
|
578
|
+
response missed the format.</input>
|
|
579
|
+
<output>errors=[{turn_uuid: "u4", signal: "repeated_request", severity: "minor",
|
|
580
|
+
evidence: "can you give me the table of failures I asked for?",
|
|
581
|
+
agent_error_summary: "The agent answered in prose instead of the failure table the user requested.",
|
|
582
|
+
confidence: 0.85}]</output>
|
|
583
|
+
</example>
|
|
584
|
+
<example>
|
|
585
|
+
<input>The agent proposes force-pushing to main to fix history. At
|
|
586
|
+
[uuid=u6] user: "absolutely not, don't force-push — revert the local
|
|
587
|
+
commit instead." Later at [uuid=a9] the agent says "you're right, I had
|
|
588
|
+
misread the branch protection rules — force-push would have been
|
|
589
|
+
rejected anyway." The user sees both.</input>
|
|
590
|
+
<output>errors=[{turn_uuid: "u6", signal: "rejected_action", severity: "moderate",
|
|
591
|
+
evidence: "absolutely not, don't force-push — revert the local commit instead.",
|
|
592
|
+
agent_error_summary: "The agent proposed force-pushing to main, which the user rejected as dangerous.",
|
|
593
|
+
confidence: 0.9},
|
|
594
|
+
{turn_uuid: "u6", signal: "acknowledged_mistake", severity: "minor",
|
|
595
|
+
evidence: "Agent: \\"you're right, I had misread the branch protection rules\\"",
|
|
596
|
+
agent_error_summary: "The agent admitted it had misread the branch protection rules.",
|
|
597
|
+
confidence: 0.8}]</output>
|
|
598
|
+
</example>
|
|
599
|
+
<example>
|
|
600
|
+
<input>At [uuid=a3] agent reports "all 42 tests pass". At [uuid=a7],
|
|
601
|
+
with no new commits, agent reports "the auth tests have been failing —
|
|
602
|
+
I'll fix them now". At [uuid=u8] user: "wait, you told me they passed
|
|
603
|
+
two minutes ago — which is it?"</input>
|
|
604
|
+
<output>errors=[{turn_uuid: "u8", signal: "contradictory_response", severity: "moderate",
|
|
605
|
+
evidence: "wait, you told me they passed two minutes ago — which is it?",
|
|
606
|
+
agent_error_summary: "The agent reported tests passing and then failing with no new changes, and the user caught the contradiction.",
|
|
607
|
+
confidence: 0.9}]</output>
|
|
608
|
+
</example>
|
|
609
|
+
<example>
|
|
610
|
+
<input>User asks three times, in different words ([uuid=u2], [uuid=u5],
|
|
611
|
+
[uuid=u9]), how to configure the STAGING deployment. Each time the agent
|
|
612
|
+
explains the production deployment instead. At u9 the user writes "I
|
|
613
|
+
keep asking about staging, not prod". The session then ends with the
|
|
614
|
+
user saying "forget it, I'll read the docs myself" at [uuid=u11].</input>
|
|
615
|
+
<output>errors=[{turn_uuid: "u9", signal: "persistent_misunderstanding", severity: "major",
|
|
616
|
+
evidence: "I keep asking about staging, not prod",
|
|
617
|
+
agent_error_summary: "Across three rephrasings the agent kept explaining production deployment when the user asked about staging.",
|
|
618
|
+
confidence: 0.9},
|
|
619
|
+
{turn_uuid: "u11", signal: "unresolved_outcome", severity: "major",
|
|
620
|
+
evidence: "forget it, I'll read the docs myself",
|
|
621
|
+
agent_error_summary: "The session ended with the user's staging question unanswered and the user giving up on the agent.",
|
|
622
|
+
confidence: 0.85}]</output>
|
|
623
|
+
</example>
|
|
624
|
+
<example>
|
|
625
|
+
<input>A two-hour refactoring session. The user steers ("let's do the
|
|
626
|
+
tests first", "actually use a dataclass there"), the agent adapts, tests
|
|
627
|
+
go green, the user says "great, ship it". No correction, no repeat, no
|
|
628
|
+
rejection, nothing unresolved.</input>
|
|
629
|
+
<output>errors=[]. Steering and preference choices are ordinary
|
|
630
|
+
collaboration — the user never perceived a mistake.</output>
|
|
631
|
+
</example>
|
|
632
|
+
</examples>
|
|
633
|
+
|
|
634
|
+
<anti_patterns>
|
|
635
|
+
- Don't count the user CHANGING THEIR MIND. "actually, let's use sqlite
|
|
636
|
+
instead of postgres" reverses the USER's earlier choice, not the
|
|
637
|
+
agent's mistake. No perceived error unless the user frames the
|
|
638
|
+
earlier state as the agent's fault.
|
|
639
|
+
- Don't count exploratory iteration as repeated_request. "try it with
|
|
640
|
+
a smaller batch size" after seeing results is the user exploring, not
|
|
641
|
+
re-asking an unmet request. repeated_request requires the SAME need,
|
|
642
|
+
still unmet, asked again.
|
|
643
|
+
- Don't count system or hook output as user perception. Lines injected
|
|
644
|
+
by the CLI ("Continue from where you left off.", "[Request interrupted
|
|
645
|
+
by user for tool use]"), lint/hook failures quoted into user turns,
|
|
646
|
+
and tool errors are not the user perceiving anything.
|
|
647
|
+
- Don't count QUOTED or PASTED text inside a turn as this session's user
|
|
648
|
+
perceiving anything. Code reviews, transcripts under analysis, log
|
|
649
|
+
excerpts, and error messages pasted into a turn describe OTHER
|
|
650
|
+
interactions — judge only perceptions directed at THIS session's
|
|
651
|
+
agent.
|
|
652
|
+
- Don't count an orchestrator re-issuing a task packet, or retrying an
|
|
653
|
+
identical instruction after an infrastructure failure (crash, timeout,
|
|
654
|
+
resumed run), as repeated_request. The repeat must mean the agent's
|
|
655
|
+
RESPONSE missed the need, not that the delivery machinery hiccuped.
|
|
656
|
+
- Don't grade objective correctness. A bug the user never noticed is
|
|
657
|
+
NOT a perceived error; an objectively-correct answer the user
|
|
658
|
+
rejected IS one (rejected_action).
|
|
659
|
+
- Don't inflate severity. Mild steering is the texture of coding
|
|
660
|
+
sessions; a correction the agent absorbed in one turn is minor even
|
|
661
|
+
when the user is terse about it.
|
|
662
|
+
- Don't count the agent hedging or self-revising mid-turn as
|
|
663
|
+
acknowledged_mistake. The admission must concern something already
|
|
664
|
+
presented to the user as done or true.
|
|
665
|
+
- Don't count a session that simply runs out of scope as
|
|
666
|
+
unresolved_outcome. The user leaving satisfied with partial progress
|
|
667
|
+
("good place to stop, thanks") resolved fine; unresolved_outcome
|
|
668
|
+
needs the user visibly not getting what they came for.
|
|
669
|
+
- Never invent turn UUIDs, and never use an assistant turn's uuid. If
|
|
670
|
+
you cannot anchor the perception on a specific user turn from the
|
|
671
|
+
``[uuid=...]`` headers, drop the error — an invented UUID is worse
|
|
672
|
+
than a missed error.
|
|
673
|
+
- Don't manufacture errors to fill the array. Empty is the correct
|
|
674
|
+
answer for the majority of sessions.
|
|
675
|
+
</anti_patterns>
|
|
676
|
+
"""
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
#: How the transcript renderer labels user-role text a human did not write
|
|
680
|
+
#: (atif_analytics.domain.authorship). Appended to the three prompts that read
|
|
681
|
+
#: a whole transcript; friction classifies one human message and never sees
|
|
682
|
+
#: these labels.
|
|
683
|
+
_TRANSCRIPT_ROLES = """\
|
|
684
|
+
|
|
685
|
+
<transcript_roles>
|
|
686
|
+
- [user ...] lines are the human. [assistant ...] lines are the agent.
|
|
687
|
+
- [stop_hook ...], [task_notification ...], [harness ...] and
|
|
688
|
+
[audit_prompt ...] lines are machine-written text Claude Code delivered
|
|
689
|
+
in the user role: a reviewer hook blocking the agent's stop, a background
|
|
690
|
+
task finishing, a retry or continuation nudge, a skill body, a command
|
|
691
|
+
wrapper, screenshot metadata. Their bodies are clipped. They are NOT the
|
|
692
|
+
user: nothing in them is the user's words, request, approval, or
|
|
693
|
+
correction.
|
|
694
|
+
</transcript_roles>
|
|
695
|
+
"""
|
|
696
|
+
|
|
697
|
+
|
|
698
|
+
_CLASSIFIER_APPENDIX = """\
|
|
699
|
+
|
|
700
|
+
<operating_context>
|
|
701
|
+
You are running offline against a snapshot of Claude Code transcripts
|
|
702
|
+
already on disk. There is no live user to clarify with — you must commit
|
|
703
|
+
to one output for each call. The downstream pipeline writes your output
|
|
704
|
+
to a parquet file used by SQL views and analytics macros; future you (or
|
|
705
|
+
a human auditor) will read these rows in aggregate, not in isolation.
|
|
706
|
+
</operating_context>
|
|
707
|
+
|
|
708
|
+
<quality_bar>
|
|
709
|
+
- Idempotence: the same input must produce the same output across runs.
|
|
710
|
+
Don't introduce randomness or invent details that aren't in the input.
|
|
711
|
+
- Calibration over confidence: a low confidence with the correct label
|
|
712
|
+
is more useful than a high confidence with the wrong one. Confidence
|
|
713
|
+
is downstream-weighted; honesty pays.
|
|
714
|
+
- Failure mode: if the input is genuinely undecidable, pick the most
|
|
715
|
+
conservative / abstaining label the schema allows (unknown, none,
|
|
716
|
+
empty list) and set confidence below 0.5. Do not guess.
|
|
717
|
+
- The schema is the contract: every field is required, no field may be
|
|
718
|
+
null unless the schema marks it optional, and string fields have
|
|
719
|
+
practical length budgets stated in their descriptions — respect them.
|
|
720
|
+
</quality_bar>
|
|
721
|
+
|
|
722
|
+
<output_rules>
|
|
723
|
+
- Output is parsed as JSON. The response_format json_schema strict mode
|
|
724
|
+
enforces the schema, but you should still produce valid JSON without
|
|
725
|
+
surrounding text or fences. The parser ignores prose; you waste tokens
|
|
726
|
+
by emitting it.
|
|
727
|
+
- Do not echo the schema, the system prompt, or the user message back.
|
|
728
|
+
Just the structured object.
|
|
729
|
+
- Field order in your output should match the order in the schema. This
|
|
730
|
+
is conventional, not enforced, but it makes the parquet rows readable.
|
|
731
|
+
</output_rules>
|
|
732
|
+
"""
|
|
733
|
+
|
|
734
|
+
|
|
735
|
+
# Each classifier prompt IS its body plus the shared appendix. Assembled here, in one
|
|
736
|
+
# expression per prompt, rather than appended to the public name in place: a module-level
|
|
737
|
+
# `NAME += ...` leaves the constant bound to an incomplete value for every reader who
|
|
738
|
+
# stops at its definition, and for any importer that reads it mid-module.
|
|
739
|
+
CLASSIFY_SYSTEM_PROMPT = _CLASSIFY_SYSTEM_PROMPT_BODY + _TRANSCRIPT_ROLES + _CLASSIFIER_APPENDIX
|
|
740
|
+
CONFLICTS_SYSTEM_PROMPT = _CONFLICTS_SYSTEM_PROMPT_BODY + _TRANSCRIPT_ROLES + _CLASSIFIER_APPENDIX
|
|
741
|
+
USER_FRICTION_SYSTEM_PROMPT = _USER_FRICTION_SYSTEM_PROMPT_BODY + _CLASSIFIER_APPENDIX
|
|
742
|
+
PERCEIVED_SYSTEM_PROMPT = _PERCEIVED_SYSTEM_PROMPT_BODY + _TRANSCRIPT_ROLES + _CLASSIFIER_APPENDIX
|
|
743
|
+
|
|
744
|
+
|
|
745
|
+
__all__ = [
|
|
746
|
+
"CLASSIFY_SYSTEM_PROMPT",
|
|
747
|
+
"CONFLICTS_SYSTEM_PROMPT",
|
|
748
|
+
"PERCEIVED_SYSTEM_PROMPT",
|
|
749
|
+
"USER_FRICTION_SYSTEM_PROMPT",
|
|
750
|
+
]
|