atif-sql 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- atif_analytics/__init__.py +9 -0
- atif_analytics/application/__init__.py +3 -0
- atif_analytics/application/analyze.py +158 -0
- atif_analytics/application/prompts.py +750 -0
- atif_analytics/application/use_cases/__init__.py +3 -0
- atif_analytics/application/use_cases/_shared.py +264 -0
- atif_analytics/application/use_cases/classify.py +430 -0
- atif_analytics/application/use_cases/conflicts.py +448 -0
- atif_analytics/application/use_cases/friction.py +634 -0
- atif_analytics/application/use_cases/perceived.py +451 -0
- atif_analytics/domain/__init__.py +3 -0
- atif_analytics/domain/authorship.py +166 -0
- atif_analytics/domain/config.py +30 -0
- atif_analytics/domain/costs.py +50 -0
- atif_analytics/domain/friction.py +117 -0
- atif_analytics/domain/layout.py +94 -0
- atif_analytics/domain/models.py +385 -0
- atif_analytics/domain/ports.py +100 -0
- atif_analytics/domain/transcript.py +359 -0
- atif_analytics/infrastructure/__init__.py +3 -0
- atif_analytics/infrastructure/corpus_reader.py +546 -0
- atif_analytics/infrastructure/parquet_cache.py +186 -0
- atif_analytics/infrastructure/settings.py +114 -0
- atif_analytics/infrastructure/sqlite_state/__init__.py +3 -0
- atif_analytics/infrastructure/sqlite_state/checkpointer.py +326 -0
- atif_analytics/infrastructure/sqlite_state/retry_queue.py +296 -0
- atif_analytics/py.typed +0 -0
- atif_cli/__init__.py +3 -0
- atif_cli/__main__.py +8 -0
- atif_cli/app.py +2459 -0
- atif_cli/converter_adapter.py +151 -0
- atif_cli/corpus.py +409 -0
- atif_cli/cron.py +311 -0
- atif_cli/duck_errors.py +97 -0
- atif_cli/embed_lake.py +101 -0
- atif_cli/errors.py +89 -0
- atif_cli/lake.py +373 -0
- atif_cli/lake_sessions.py +270 -0
- atif_cli/output.py +297 -0
- atif_cli/py.typed +0 -0
- atif_converter/__init__.py +3 -0
- atif_converter/application/__init__.py +3 -0
- atif_converter/application/convert_and_audit.py +294 -0
- atif_converter/application/convert_codex.py +210 -0
- atif_converter/domain/__init__.py +9 -0
- atif_converter/domain/agents.py +40 -0
- atif_converter/domain/atif/__init__.py +55 -0
- atif_converter/domain/atif/agent.py +37 -0
- atif_converter/domain/atif/content.py +161 -0
- atif_converter/domain/atif/final_metrics.py +45 -0
- atif_converter/domain/atif/metrics.py +49 -0
- atif_converter/domain/atif/observation.py +21 -0
- atif_converter/domain/atif/observation_result.py +47 -0
- atif_converter/domain/atif/step.py +144 -0
- atif_converter/domain/atif/subagent_trajectory_ref.py +90 -0
- atif_converter/domain/atif/tool_call.py +36 -0
- atif_converter/domain/atif/trajectory.py +211 -0
- atif_converter/domain/atif/trajectory_validator.py +299 -0
- atif_converter/domain/blobs.py +510 -0
- atif_converter/domain/claude_code_conversion.py +1077 -0
- atif_converter/domain/codex_conversion.py +1036 -0
- atif_converter/domain/codex_edges.py +192 -0
- atif_converter/domain/codex_enrichment.py +419 -0
- atif_converter/domain/codex_fidelity.py +124 -0
- atif_converter/domain/codex_nested_calls.py +165 -0
- atif_converter/domain/edges.py +113 -0
- atif_converter/domain/enrichment.py +412 -0
- atif_converter/domain/errors.py +60 -0
- atif_converter/domain/fidelity.py +171 -0
- atif_converter/domain/model_prices.json +8538 -0
- atif_converter/domain/pricing.py +790 -0
- atif_converter/domain/result_signals.py +774 -0
- atif_converter/domain/schema_version.py +90 -0
- atif_converter/domain/session_events.py +408 -0
- atif_converter/infrastructure/__init__.py +3 -0
- atif_converter/infrastructure/census.py +89 -0
- atif_converter/infrastructure/claude_code_converter.py +127 -0
- atif_converter/infrastructure/codex_adapter.py +132 -0
- atif_converter/infrastructure/codex_census.py +100 -0
- atif_converter/infrastructure/codex_converter.py +82 -0
- atif_converter/infrastructure/harbor_adapter.py +187 -0
- atif_converter/infrastructure/raw_records.py +426 -0
- atif_converter/infrastructure/source_archive.py +154 -0
- atif_converter/py.typed +0 -0
- atif_corpus/__init__.py +13 -0
- atif_corpus/application/__init__.py +7 -0
- atif_corpus/application/materialize.py +1826 -0
- atif_corpus/domain/__init__.py +9 -0
- atif_corpus/domain/agents.py +33 -0
- atif_corpus/domain/generation.py +34 -0
- atif_corpus/domain/layout.py +168 -0
- atif_corpus/domain/ports.py +175 -0
- atif_corpus/domain/session_id.py +68 -0
- atif_corpus/domain/sessions.py +228 -0
- atif_corpus/domain/slug.py +72 -0
- atif_corpus/domain/source_layout.py +143 -0
- atif_corpus/domain/watermark.py +76 -0
- atif_corpus/infrastructure/__init__.py +7 -0
- atif_corpus/infrastructure/atomic.py +266 -0
- atif_corpus/infrastructure/compress_artifacts.py +161 -0
- atif_corpus/infrastructure/fake_converter.py +174 -0
- atif_corpus/infrastructure/scanner.py +316 -0
- atif_corpus/infrastructure/settings.py +130 -0
- atif_corpus/infrastructure/source_archive.py +202 -0
- atif_corpus/py.typed +0 -0
- atif_duck/__init__.py +43 -0
- atif_duck/domain/__init__.py +32 -0
- atif_duck/domain/artifacts.py +66 -0
- atif_duck/domain/authorship.py +165 -0
- atif_duck/domain/catalog.py +626 -0
- atif_duck/domain/columnar.py +149 -0
- atif_duck/domain/embedding_guard.py +88 -0
- atif_duck/domain/examples.py +265 -0
- atif_duck/domain/lake.py +427 -0
- atif_duck/domain/lake_sessions.py +97 -0
- atif_duck/domain/raw_readers.py +140 -0
- atif_duck/domain/session_id.py +69 -0
- atif_duck/domain/sql_literal.py +50 -0
- atif_duck/infrastructure/__init__.py +34 -0
- atif_duck/infrastructure/analytics.py +469 -0
- atif_duck/infrastructure/authorship.py +139 -0
- atif_duck/infrastructure/columnar.py +641 -0
- atif_duck/infrastructure/lake.py +1383 -0
- atif_duck/infrastructure/lake_sessions.py +194 -0
- atif_duck/infrastructure/lake_settings.py +63 -0
- atif_duck/infrastructure/lake_steps.py +225 -0
- atif_duck/infrastructure/projections.py +221 -0
- atif_duck/infrastructure/registry.py +2194 -0
- atif_duck/infrastructure/stored_artifacts.py +115 -0
- atif_duck/py.typed +0 -0
- atif_embed/__init__.py +12 -0
- atif_embed/application/__init__.py +3 -0
- atif_embed/application/embed.py +293 -0
- atif_embed/application/prune.py +80 -0
- atif_embed/domain/__init__.py +3 -0
- atif_embed/domain/discovery.py +70 -0
- atif_embed/domain/embedding_guard.py +83 -0
- atif_embed/domain/errors.py +94 -0
- atif_embed/domain/ports.py +223 -0
- atif_embed/domain/sql_literal.py +49 -0
- atif_embed/domain/text_stamp.py +71 -0
- atif_embed/infrastructure/__init__.py +3 -0
- atif_embed/infrastructure/cohere_bedrock.py +433 -0
- atif_embed/infrastructure/corpus_text_rows.py +273 -0
- atif_embed/infrastructure/gemma_local.py +355 -0
- atif_embed/infrastructure/lake_text_rows.py +203 -0
- atif_embed/infrastructure/lance_store.py +440 -0
- atif_embed/infrastructure/providers.py +30 -0
- atif_embed/infrastructure/settings.py +154 -0
- atif_embed/py.typed +0 -0
- atif_models/__init__.py +14 -0
- atif_models/domain/__init__.py +9 -0
- atif_models/domain/ports.py +143 -0
- atif_models/domain/registry.py +151 -0
- atif_models/domain/schema.py +109 -0
- atif_models/infrastructure/__init__.py +8 -0
- atif_models/infrastructure/openai_bedrock.py +314 -0
- atif_models/infrastructure/settings.py +85 -0
- atif_models/py.typed +0 -0
- atif_sql-0.2.0.dist-info/METADATA +272 -0
- atif_sql-0.2.0.dist-info/RECORD +164 -0
- atif_sql-0.2.0.dist-info/WHEEL +4 -0
- atif_sql-0.2.0.dist-info/entry_points.txt +2 -0
- atif_sql-0.2.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,1077 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
# Ported from harbor 0.24.0, src/harbor/agents/installed/claude_code.py
|
|
3
|
+
# (Apache-2.0, Copyright the Harbor authors), onto the ATIF models vendored in
|
|
4
|
+
# atif_converter.domain.atif, so atif-converter doesn't depend on harbor.
|
|
5
|
+
|
|
6
|
+
"""Claude Code session records -> ATIF trajectory, as a pure function.
|
|
7
|
+
|
|
8
|
+
This is ``ClaudeCode._convert_events_to_trajectory`` and its helpers, lifted
|
|
9
|
+
out of harbor's agent class so the conversion runs over ALREADY-PARSED records
|
|
10
|
+
with no file I/O and no harbor import. The ATIF data classes (harbor's
|
|
11
|
+
``harbor.models.trajectories``, vendored as :mod:`atif_converter.domain.atif`)
|
|
12
|
+
are the contract the output is expressed in, which is why this domain module
|
|
13
|
+
imports them. harbor's structure and names
|
|
14
|
+
are kept recognizable on purpose, so a future upstream diff can be re-applied
|
|
15
|
+
by hand; the one structural change is that the body of
|
|
16
|
+
``_convert_events_to_trajectory`` is split into helpers around the
|
|
17
|
+
:class:`_NormalizationState` it shared, because the repo's function-size
|
|
18
|
+
ratchets are set at today's worst function and this one was larger.
|
|
19
|
+
|
|
20
|
+
INPUT SHAPE. harbor takes a session DIRECTORY and reads
|
|
21
|
+
``sorted(glob("*.jsonl")) + sorted(rglob("subagents/*.jsonl"))``, one main
|
|
22
|
+
transcript followed by every staged side-file, in file-name order. Nothing in
|
|
23
|
+
the conversion uses a file name after that: the side-file names decide only
|
|
24
|
+
the READ ORDER, which matters because uuid de-duplication keeps the first
|
|
25
|
+
occurrence and the timestamp sort is stable. :func:`convert_claude_code_records`
|
|
26
|
+
therefore takes the main transcript's records in file order plus a mapping of
|
|
27
|
+
staged side-file NAME (``agent-abc.jsonl``,
|
|
28
|
+
``workflows__wf_123__agent-def.jsonl`` for a workflow-nested file, exactly as
|
|
29
|
+
the old adapter staged them flat) to that file's records, and reads the
|
|
30
|
+
side-files in sorted-name order.
|
|
31
|
+
|
|
32
|
+
OMITTED: ``_parse_total_cost_from_stream_json``. It reads
|
|
33
|
+
``<logs_dir>/claude-code.txt``, the stdout tee of a harbor-driven run. Our
|
|
34
|
+
adapter always handed harbor an EMPTY temp ``logs_dir``, so ``read_text``
|
|
35
|
+
raised ``OSError`` and the method returned ``None`` on every call, which sent
|
|
36
|
+
every trajectory down the ``litellm`` estimate path. That fallback ordering is
|
|
37
|
+
kept verbatim: ``total_cost_usd`` is the litellm estimate or ``None``, and
|
|
38
|
+
``final_metrics.extra["cost_source"] == "litellm_estimate"`` whenever the
|
|
39
|
+
estimate priced the session. Since harbor 0.23.0 every priced step also carries
|
|
40
|
+
its own ``metrics.cost_usd`` and ``metrics.extra["cost_source"]``. The estimate itself now comes from
|
|
41
|
+
:mod:`atif_converter.domain.pricing`, which repeats ``litellm.cost_per_token``'s
|
|
42
|
+
arithmetic over a vendored copy of litellm's price data without litellm being
|
|
43
|
+
installed; the floats are identical, so the label stays. Also not ported: ``_session_dirs`` (log-dir
|
|
44
|
+
discovery, replaced by the infrastructure reader), and ``_session_text`` /
|
|
45
|
+
``_session_tool_result_content`` (they serve ``atif_to_native_trajectory``, the
|
|
46
|
+
REVERSE conversion, not this one).
|
|
47
|
+
|
|
48
|
+
``self.logger`` is loguru's ``logger``; ``self.model_name`` is the
|
|
49
|
+
``model_name`` parameter (our adapter never set one, so the default ``None``
|
|
50
|
+
reproduces the behavior we had).
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
from __future__ import annotations
|
|
54
|
+
|
|
55
|
+
import base64
|
|
56
|
+
import contextlib
|
|
57
|
+
import json
|
|
58
|
+
from collections.abc import Mapping, Sequence
|
|
59
|
+
from dataclasses import dataclass, field
|
|
60
|
+
from typing import Any
|
|
61
|
+
|
|
62
|
+
from loguru import logger
|
|
63
|
+
|
|
64
|
+
from atif_converter.domain import pricing
|
|
65
|
+
from atif_converter.domain.agents import AgentSource
|
|
66
|
+
from atif_converter.domain.atif import (
|
|
67
|
+
Agent,
|
|
68
|
+
FinalMetrics,
|
|
69
|
+
Metrics,
|
|
70
|
+
Observation,
|
|
71
|
+
ObservationResult,
|
|
72
|
+
Step,
|
|
73
|
+
ToolCall,
|
|
74
|
+
Trajectory,
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
#: The schema version harbor 0.23.0 stamps on a Claude Code trajectory.
|
|
78
|
+
SCHEMA_VERSION = "ATIF-v1.7"
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _convert_event_to_step(event: dict[str, Any], step_id: int, model_name: str | None) -> Step:
|
|
82
|
+
"""Convert a normalized Claude Code event dictionary into an ATIF step."""
|
|
83
|
+
kind = event.get("kind")
|
|
84
|
+
timestamp = event.get("timestamp")
|
|
85
|
+
|
|
86
|
+
if kind == "message":
|
|
87
|
+
role = event.get("role", "user")
|
|
88
|
+
text = event.get("text", "")
|
|
89
|
+
reasoning = event.get("reasoning")
|
|
90
|
+
metrics = event.get("metrics")
|
|
91
|
+
extra = event.get("extra")
|
|
92
|
+
event_model_name = event.get("model_name")
|
|
93
|
+
|
|
94
|
+
if role == "assistant":
|
|
95
|
+
source = "agent"
|
|
96
|
+
elif role == "user":
|
|
97
|
+
source = "user"
|
|
98
|
+
else:
|
|
99
|
+
source = "system"
|
|
100
|
+
|
|
101
|
+
step = Step(
|
|
102
|
+
step_id=step_id,
|
|
103
|
+
timestamp=timestamp,
|
|
104
|
+
source=source,
|
|
105
|
+
message=text,
|
|
106
|
+
llm_call_count=1 if source == "agent" else None,
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
if source == "agent":
|
|
110
|
+
if reasoning:
|
|
111
|
+
step.reasoning_content = reasoning
|
|
112
|
+
if event_model_name:
|
|
113
|
+
step.model_name = event_model_name
|
|
114
|
+
elif model_name:
|
|
115
|
+
step.model_name = model_name
|
|
116
|
+
|
|
117
|
+
if metrics:
|
|
118
|
+
step.metrics = metrics
|
|
119
|
+
if extra:
|
|
120
|
+
step.extra = extra
|
|
121
|
+
|
|
122
|
+
return step
|
|
123
|
+
|
|
124
|
+
if kind == "agent_step":
|
|
125
|
+
text = event.get("text") or ""
|
|
126
|
+
reasoning = event.get("reasoning")
|
|
127
|
+
metrics = event.get("metrics")
|
|
128
|
+
extra = event.get("extra")
|
|
129
|
+
step_model_name = event.get("model_name") or model_name
|
|
130
|
+
tool_specs = event.get("tool_calls") or []
|
|
131
|
+
|
|
132
|
+
tool_calls: list[ToolCall] = []
|
|
133
|
+
results: list[ObservationResult] = []
|
|
134
|
+
for spec in tool_specs:
|
|
135
|
+
spec_call_id = spec.get("call_id")
|
|
136
|
+
if not spec_call_id:
|
|
137
|
+
continue
|
|
138
|
+
tool_calls.append(
|
|
139
|
+
ToolCall(
|
|
140
|
+
tool_call_id=spec_call_id,
|
|
141
|
+
function_name=spec.get("tool_name") or "",
|
|
142
|
+
arguments=spec.get("arguments") or {},
|
|
143
|
+
extra=spec.get("extra"),
|
|
144
|
+
)
|
|
145
|
+
)
|
|
146
|
+
if spec.get("output") is not None:
|
|
147
|
+
results.append(
|
|
148
|
+
ObservationResult(
|
|
149
|
+
source_call_id=spec_call_id,
|
|
150
|
+
content=spec.get("output"),
|
|
151
|
+
subagent_trajectory_ref=None,
|
|
152
|
+
extra=spec.get("result_extra"),
|
|
153
|
+
)
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
step = Step(
|
|
157
|
+
step_id=step_id,
|
|
158
|
+
timestamp=timestamp,
|
|
159
|
+
source="agent",
|
|
160
|
+
message=text,
|
|
161
|
+
tool_calls=tool_calls or None,
|
|
162
|
+
observation=Observation(results=results) if results else None,
|
|
163
|
+
llm_call_count=1,
|
|
164
|
+
)
|
|
165
|
+
if reasoning:
|
|
166
|
+
step.reasoning_content = reasoning
|
|
167
|
+
if step_model_name:
|
|
168
|
+
step.model_name = step_model_name
|
|
169
|
+
if metrics:
|
|
170
|
+
step.metrics = metrics
|
|
171
|
+
if extra:
|
|
172
|
+
step.extra = extra
|
|
173
|
+
|
|
174
|
+
return step
|
|
175
|
+
|
|
176
|
+
if kind == "tool_call":
|
|
177
|
+
call_id = event.get("call_id")
|
|
178
|
+
tool_name = event.get("tool_name")
|
|
179
|
+
if not call_id or not tool_name:
|
|
180
|
+
msg = "Tool call event missing call_id or tool_name"
|
|
181
|
+
raise ValueError(msg)
|
|
182
|
+
|
|
183
|
+
arguments = event.get("arguments") or {}
|
|
184
|
+
raw_arguments = event.get("raw_arguments")
|
|
185
|
+
reasoning = event.get("reasoning")
|
|
186
|
+
metrics = event.get("metrics")
|
|
187
|
+
extra = event.get("extra")
|
|
188
|
+
status = event.get("status")
|
|
189
|
+
message = event.get("message")
|
|
190
|
+
output = event.get("output")
|
|
191
|
+
metadata = event.get("metadata")
|
|
192
|
+
step_model_name = event.get("model_name") or model_name
|
|
193
|
+
|
|
194
|
+
tool_call = ToolCall(
|
|
195
|
+
tool_call_id=call_id,
|
|
196
|
+
function_name=tool_name,
|
|
197
|
+
arguments=arguments,
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
observation_result = ObservationResult(
|
|
201
|
+
source_call_id=call_id,
|
|
202
|
+
content=output,
|
|
203
|
+
subagent_trajectory_ref=None,
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
observation = Observation(results=[observation_result]) if output is not None else None
|
|
207
|
+
|
|
208
|
+
extra = extra or {}
|
|
209
|
+
updates = {
|
|
210
|
+
"metadata": metadata,
|
|
211
|
+
"raw_arguments": raw_arguments,
|
|
212
|
+
"status": status,
|
|
213
|
+
}
|
|
214
|
+
for key, value in updates.items():
|
|
215
|
+
if value is not None:
|
|
216
|
+
extra.setdefault(key, value)
|
|
217
|
+
|
|
218
|
+
step = Step(
|
|
219
|
+
step_id=step_id,
|
|
220
|
+
timestamp=timestamp,
|
|
221
|
+
source="agent",
|
|
222
|
+
message=message or "",
|
|
223
|
+
tool_calls=[tool_call],
|
|
224
|
+
observation=observation,
|
|
225
|
+
llm_call_count=1,
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
if step_model_name:
|
|
229
|
+
step.model_name = step_model_name
|
|
230
|
+
if reasoning:
|
|
231
|
+
step.reasoning_content = reasoning
|
|
232
|
+
if metrics:
|
|
233
|
+
step.metrics = metrics
|
|
234
|
+
if extra:
|
|
235
|
+
step.extra = extra
|
|
236
|
+
|
|
237
|
+
return step
|
|
238
|
+
|
|
239
|
+
msg = f"Unsupported event kind '{kind}'"
|
|
240
|
+
raise ValueError(msg)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _stringify(value: Any) -> str:
|
|
244
|
+
if isinstance(value, str):
|
|
245
|
+
return value
|
|
246
|
+
try:
|
|
247
|
+
dumped = json.dumps(value, ensure_ascii=False)
|
|
248
|
+
except TypeError:
|
|
249
|
+
return str(value)
|
|
250
|
+
return dumped
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _extract_text_reasoning_tool_uses(
|
|
254
|
+
content: Any,
|
|
255
|
+
) -> tuple[str, str | None, list[dict[str, Any]]]:
|
|
256
|
+
if isinstance(content, str):
|
|
257
|
+
text = content.strip()
|
|
258
|
+
return text, None, []
|
|
259
|
+
|
|
260
|
+
text_parts: list[str] = []
|
|
261
|
+
reasoning_parts: list[str] = []
|
|
262
|
+
tool_blocks: list[dict[str, Any]] = []
|
|
263
|
+
|
|
264
|
+
if isinstance(content, list):
|
|
265
|
+
for block in content:
|
|
266
|
+
if not isinstance(block, dict):
|
|
267
|
+
text_parts.append(_stringify(block))
|
|
268
|
+
continue
|
|
269
|
+
|
|
270
|
+
block_type = block.get("type")
|
|
271
|
+
if block_type == "tool_use":
|
|
272
|
+
tool_blocks.append(block)
|
|
273
|
+
continue
|
|
274
|
+
|
|
275
|
+
if block_type in {"thinking", "reasoning", "analysis"}:
|
|
276
|
+
text_value = (
|
|
277
|
+
block.get("text") if block.get("text") is not None else block.get("thinking")
|
|
278
|
+
)
|
|
279
|
+
if isinstance(text_value, str):
|
|
280
|
+
reasoning_parts.append(text_value.strip())
|
|
281
|
+
else:
|
|
282
|
+
reasoning_parts.append(_stringify(text_value))
|
|
283
|
+
continue
|
|
284
|
+
|
|
285
|
+
if block_type == "redacted_thinking":
|
|
286
|
+
# Anthropic redacted_thinking blocks carry encrypted reasoning
|
|
287
|
+
# in `data` that clients cannot decrypt. OpenRouter mis-uses
|
|
288
|
+
# this envelope to pass through PLAIN reasoning from
|
|
289
|
+
# non-Anthropic models: `data` is `openrouter.reasoning:<b64>`
|
|
290
|
+
# whose base64 decodes to JSON with a `text` field. Surface
|
|
291
|
+
# that text as reasoning; drop genuine ciphertext rather than
|
|
292
|
+
# dumping the raw envelope into the human-readable message.
|
|
293
|
+
data = block.get("data")
|
|
294
|
+
if isinstance(data, str) and data.startswith("openrouter.reasoning:"):
|
|
295
|
+
with contextlib.suppress(ValueError, json.JSONDecodeError):
|
|
296
|
+
payload = data[len("openrouter.reasoning:") :]
|
|
297
|
+
decoded = base64.b64decode(payload + "==").decode("utf-8", "replace")
|
|
298
|
+
inner = json.loads(decoded)
|
|
299
|
+
inner_text = inner.get("text")
|
|
300
|
+
if isinstance(inner_text, str):
|
|
301
|
+
reasoning_parts.append(inner_text.strip())
|
|
302
|
+
continue
|
|
303
|
+
|
|
304
|
+
if block_type == "code" and isinstance(block.get("code"), str):
|
|
305
|
+
text_parts.append(block["code"])
|
|
306
|
+
continue
|
|
307
|
+
|
|
308
|
+
text_value = block.get("text")
|
|
309
|
+
if isinstance(text_value, str):
|
|
310
|
+
text_parts.append(text_value)
|
|
311
|
+
else:
|
|
312
|
+
text_parts.append(_stringify(block))
|
|
313
|
+
elif content is not None:
|
|
314
|
+
text_parts.append(_stringify(content))
|
|
315
|
+
|
|
316
|
+
text = "\n\n".join(part.strip() for part in text_parts if part and str(part).strip())
|
|
317
|
+
reasoning = "\n\n".join(part.strip() for part in reasoning_parts if part and str(part).strip())
|
|
318
|
+
|
|
319
|
+
return text, (reasoning or None), tool_blocks
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def _build_metrics(usage: Any) -> Metrics | None:
|
|
323
|
+
if not isinstance(usage, dict):
|
|
324
|
+
return None
|
|
325
|
+
|
|
326
|
+
# ``or 0`` rather than a ``get`` default: an interrupted streaming response
|
|
327
|
+
# can leave a usage field present but ``None``, and ``get(key, 0)`` only
|
|
328
|
+
# falls back when the key is absent. Without this guard the arithmetic
|
|
329
|
+
# below raises ``TypeError`` and aborts the whole conversion.
|
|
330
|
+
cached_tokens = usage.get("cache_read_input_tokens") or 0
|
|
331
|
+
creation = usage.get("cache_creation_input_tokens") or 0
|
|
332
|
+
input_tokens = usage.get("input_tokens") or 0
|
|
333
|
+
# Align with Anthropic session totals: input + cache read + cache creation.
|
|
334
|
+
prompt_tokens = input_tokens + cached_tokens + creation
|
|
335
|
+
completion_tokens = usage.get("output_tokens") or 0
|
|
336
|
+
|
|
337
|
+
extra: dict[str, Any] = {}
|
|
338
|
+
for key, value in usage.items():
|
|
339
|
+
if key in {"input_tokens", "output_tokens"}:
|
|
340
|
+
continue
|
|
341
|
+
extra[key] = value
|
|
342
|
+
|
|
343
|
+
# harbor then tests ``prompt_tokens is None and completion_tokens is None
|
|
344
|
+
# and cached_tokens is None and not extra`` and returns None; every one of
|
|
345
|
+
# those is an int after the ``or 0`` above, so the branch is unreachable
|
|
346
|
+
# and is left out here (ty rejects an always-false condition).
|
|
347
|
+
|
|
348
|
+
return Metrics(
|
|
349
|
+
prompt_tokens=prompt_tokens,
|
|
350
|
+
completion_tokens=completion_tokens,
|
|
351
|
+
cached_tokens=cached_tokens,
|
|
352
|
+
cost_usd=None,
|
|
353
|
+
extra=extra or None,
|
|
354
|
+
)
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def _format_tool_result(
|
|
358
|
+
block: dict[str, Any], tool_use_result: Any
|
|
359
|
+
) -> tuple[str | None, dict[str, Any] | None]:
|
|
360
|
+
parts: list[str] = []
|
|
361
|
+
|
|
362
|
+
content = block.get("content")
|
|
363
|
+
if isinstance(content, str):
|
|
364
|
+
if content.strip():
|
|
365
|
+
parts.append(content.strip())
|
|
366
|
+
elif isinstance(content, list):
|
|
367
|
+
for item in content:
|
|
368
|
+
text_value = _stringify(item)
|
|
369
|
+
if text_value.strip():
|
|
370
|
+
parts.append(text_value.strip())
|
|
371
|
+
elif content not in (None, ""):
|
|
372
|
+
parts.append(_stringify(content))
|
|
373
|
+
|
|
374
|
+
metadata: dict[str, Any] | None = None
|
|
375
|
+
if tool_use_result and isinstance(tool_use_result, dict):
|
|
376
|
+
metadata = {"tool_use_result": tool_use_result}
|
|
377
|
+
stdout = tool_use_result.get("stdout")
|
|
378
|
+
stderr = tool_use_result.get("stderr")
|
|
379
|
+
exit_code = tool_use_result.get("exitCode") or tool_use_result.get("exit_code")
|
|
380
|
+
interrupted = tool_use_result.get("interrupted")
|
|
381
|
+
is_image = tool_use_result.get("isImage")
|
|
382
|
+
|
|
383
|
+
formatted_chunks: list[str] = []
|
|
384
|
+
if stdout:
|
|
385
|
+
formatted_chunks.append(f"[stdout]\n{stdout}".rstrip())
|
|
386
|
+
if stderr:
|
|
387
|
+
formatted_chunks.append(f"[stderr]\n{stderr}".rstrip())
|
|
388
|
+
if exit_code not in (None, 0):
|
|
389
|
+
formatted_chunks.append(f"[exit_code] {exit_code}")
|
|
390
|
+
if interrupted:
|
|
391
|
+
formatted_chunks.append(f"[interrupted] {interrupted}")
|
|
392
|
+
if is_image:
|
|
393
|
+
formatted_chunks.append(f"[is_image] {is_image}")
|
|
394
|
+
|
|
395
|
+
remaining_meta = {
|
|
396
|
+
key: value
|
|
397
|
+
for key, value in tool_use_result.items()
|
|
398
|
+
if key
|
|
399
|
+
not in {
|
|
400
|
+
"stdout",
|
|
401
|
+
"stderr",
|
|
402
|
+
"exitCode",
|
|
403
|
+
"exit_code",
|
|
404
|
+
"interrupted",
|
|
405
|
+
"isImage",
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
if remaining_meta:
|
|
409
|
+
formatted_chunks.append(f"[metadata] {json.dumps(remaining_meta, ensure_ascii=False)}")
|
|
410
|
+
|
|
411
|
+
if formatted_chunks:
|
|
412
|
+
parts.append("\n".join(chunk for chunk in formatted_chunks if chunk))
|
|
413
|
+
|
|
414
|
+
if block.get("is_error") is True:
|
|
415
|
+
parts.append("[error] tool reported failure")
|
|
416
|
+
metadata = metadata or {}
|
|
417
|
+
metadata["is_error"] = True
|
|
418
|
+
|
|
419
|
+
if metadata is not None:
|
|
420
|
+
metadata.setdefault("raw_tool_result", block)
|
|
421
|
+
|
|
422
|
+
result_text = "\n\n".join(part for part in parts if part).strip()
|
|
423
|
+
return (result_text or None), metadata
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def _estimate_step_costs(steps: list[Step]) -> float | None:
|
|
427
|
+
"""Price each step, writing ``metrics.cost_usd`` on it, and return the sum.
|
|
428
|
+
|
|
429
|
+
harbor 0.23.0's ``_estimate_step_costs``: every step with token usage that
|
|
430
|
+
can be priced gets ``metrics.cost_usd`` and ``metrics.extra["cost_source"]``,
|
|
431
|
+
and the sum is the session estimate used when Claude omits its result event.
|
|
432
|
+
The sum is ``None`` when any step with usage could not be priced, since a
|
|
433
|
+
partial sum would understate the total. harbor priced each step through
|
|
434
|
+
``litellm.cost_per_token``; the call goes through
|
|
435
|
+
:mod:`atif_converter.domain.pricing`, which returns the same floats from a
|
|
436
|
+
vendored copy of litellm's price data. A pricing failure leaves that step
|
|
437
|
+
unpriced, as a litellm error did in harbor.
|
|
438
|
+
"""
|
|
439
|
+
total_cost = 0.0
|
|
440
|
+
priced_any_step = False
|
|
441
|
+
priced_all_steps = True
|
|
442
|
+
for step in steps:
|
|
443
|
+
metrics = step.metrics
|
|
444
|
+
if metrics is None:
|
|
445
|
+
continue
|
|
446
|
+
|
|
447
|
+
prompt_tokens = metrics.prompt_tokens or 0
|
|
448
|
+
completion_tokens = metrics.completion_tokens or 0
|
|
449
|
+
if prompt_tokens <= 0 and completion_tokens <= 0:
|
|
450
|
+
continue
|
|
451
|
+
if not step.model_name:
|
|
452
|
+
logger.debug("Cannot estimate Claude cost without a step model")
|
|
453
|
+
priced_all_steps = False
|
|
454
|
+
continue
|
|
455
|
+
|
|
456
|
+
extra = metrics.extra or {}
|
|
457
|
+
cache_creation_tokens = extra.get("cache_creation_input_tokens")
|
|
458
|
+
if not isinstance(cache_creation_tokens, int):
|
|
459
|
+
cache_creation_tokens = 0
|
|
460
|
+
cache_read_tokens = extra.get("cache_read_input_tokens")
|
|
461
|
+
if not isinstance(cache_read_tokens, int):
|
|
462
|
+
cache_read_tokens = metrics.cached_tokens or 0
|
|
463
|
+
service_tier = extra.get("service_tier")
|
|
464
|
+
if not isinstance(service_tier, str):
|
|
465
|
+
service_tier = None
|
|
466
|
+
|
|
467
|
+
try:
|
|
468
|
+
priced = pricing.priced_cost_per_token(
|
|
469
|
+
model=step.model_name,
|
|
470
|
+
prompt_tokens=prompt_tokens,
|
|
471
|
+
completion_tokens=completion_tokens,
|
|
472
|
+
cache_creation_input_tokens=cache_creation_tokens,
|
|
473
|
+
cache_read_input_tokens=cache_read_tokens,
|
|
474
|
+
service_tier=service_tier,
|
|
475
|
+
)
|
|
476
|
+
except Exception as exc: # noqa: BLE001 — harbor swallowed every pricing error into "unpriced"
|
|
477
|
+
logger.debug(
|
|
478
|
+
"Cannot estimate Claude cost for model '{}': {}",
|
|
479
|
+
step.model_name,
|
|
480
|
+
exc,
|
|
481
|
+
)
|
|
482
|
+
priced_all_steps = False
|
|
483
|
+
continue
|
|
484
|
+
if priced is None:
|
|
485
|
+
# DELIBERATE DIVERGENCE from harbor, which writes litellm's (0.0, 0.0)
|
|
486
|
+
# for a model with no rates as a $0 step and sums it into a $0
|
|
487
|
+
# session. The step stays unpriced (no ``cost_usd``, no label) and
|
|
488
|
+
# the session estimate is unknown, as a litellm error makes it.
|
|
489
|
+
logger.debug("No price for Claude model '{}'; step left unpriced", step.model_name)
|
|
490
|
+
priced_all_steps = False
|
|
491
|
+
continue
|
|
492
|
+
|
|
493
|
+
prompt_cost, completion_cost = priced
|
|
494
|
+
metrics.cost_usd = prompt_cost + completion_cost
|
|
495
|
+
# harbor labels every priced step "litellm_estimate"; a step priced from a
|
|
496
|
+
# local override says so, as the session label does.
|
|
497
|
+
metrics.extra = {**extra, "cost_source": pricing.cost_source_label([step.model_name])}
|
|
498
|
+
total_cost += metrics.cost_usd
|
|
499
|
+
priced_any_step = True
|
|
500
|
+
|
|
501
|
+
return total_cost if priced_any_step and priced_all_steps else None
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
def _first_event_model(events: list[dict[str, Any]], *, include_sidechain: bool) -> str | None:
|
|
505
|
+
for event in events:
|
|
506
|
+
if not include_sidechain and event.get("isSidechain"):
|
|
507
|
+
continue
|
|
508
|
+
message = event.get("message")
|
|
509
|
+
if not isinstance(message, dict):
|
|
510
|
+
continue
|
|
511
|
+
model_name = message.get("model")
|
|
512
|
+
if isinstance(model_name, str) and model_name:
|
|
513
|
+
return model_name
|
|
514
|
+
return None
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
@dataclass(slots=True)
|
|
518
|
+
class _NormalizationState:
|
|
519
|
+
"""The locals ``_convert_events_to_trajectory`` shared across its event loop.
|
|
520
|
+
|
|
521
|
+
One object rather than seven positional arguments, so the per-event-type
|
|
522
|
+
helpers below read like the branches they were cut from.
|
|
523
|
+
"""
|
|
524
|
+
|
|
525
|
+
default_model_name: str | None
|
|
526
|
+
#: Per message id, the last usage seen (streaming updates it on each chunk).
|
|
527
|
+
last_usage_by_msg_id: dict[str, Any]
|
|
528
|
+
normalized_events: list[dict[str, Any]] = field(default_factory=list)
|
|
529
|
+
pending_calls: dict[str, dict[str, Any]] = field(default_factory=dict)
|
|
530
|
+
completed_call_ids: set[str] = field(default_factory=set)
|
|
531
|
+
seen_message_ids: set[str] = field(default_factory=set)
|
|
532
|
+
#: Maps an assistant ``message.id`` to the single agent_step it bundles, so
|
|
533
|
+
#: text / reasoning / every tool_use from one LLM inference land on one
|
|
534
|
+
#: ATIF step even if the session log splits them across events.
|
|
535
|
+
turn_by_msgid: dict[str, dict[str, Any]] = field(default_factory=dict)
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
def _collect_events(
|
|
539
|
+
main_records: Sequence[dict[str, Any]],
|
|
540
|
+
side_records: Mapping[str, Sequence[dict[str, Any]]] | None,
|
|
541
|
+
) -> list[dict[str, Any]]:
|
|
542
|
+
"""Concatenate in harbor's read order, de-duplicate by uuid, sort by timestamp."""
|
|
543
|
+
raw_events: list[dict[str, Any]] = list(main_records)
|
|
544
|
+
if side_records:
|
|
545
|
+
for name in sorted(side_records):
|
|
546
|
+
raw_events.extend(side_records[name])
|
|
547
|
+
|
|
548
|
+
seen_event_uuids: set[str] = set()
|
|
549
|
+
deduped_raw_events: list[dict[str, Any]] = []
|
|
550
|
+
for event in raw_events:
|
|
551
|
+
uuid = event.get("uuid")
|
|
552
|
+
if isinstance(uuid, str) and uuid:
|
|
553
|
+
if uuid in seen_event_uuids:
|
|
554
|
+
logger.debug("Skipping duplicate Claude Code session event {}", uuid)
|
|
555
|
+
continue
|
|
556
|
+
seen_event_uuids.add(uuid)
|
|
557
|
+
deduped_raw_events.append(event)
|
|
558
|
+
|
|
559
|
+
# Keep events in chronological order across the main chain and any
|
|
560
|
+
# subagent sidechains, so the first user step remains the instruction
|
|
561
|
+
# (downstream byte-identity checks rely on this) and step timestamps
|
|
562
|
+
# stay monotonic; sidechain steps carry `extra.is_sidechain` and
|
|
563
|
+
# `extra.agent_id` so consumers can distinguish concurrent subagents.
|
|
564
|
+
deduped_raw_events.sort(key=lambda e: e.get("timestamp", ""))
|
|
565
|
+
return deduped_raw_events
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
def _first_seen(events: list[dict[str, Any]], key: str) -> list[str]:
|
|
569
|
+
"""Every distinct non-empty string ``event[key]``, in the order events carry it."""
|
|
570
|
+
values = (event.get(key) for event in events)
|
|
571
|
+
return list(dict.fromkeys(value for value in values if isinstance(value, str) and value))
|
|
572
|
+
|
|
573
|
+
|
|
574
|
+
def _agent_extra(events: list[dict[str, Any]]) -> dict[str, Any] | None:
|
|
575
|
+
"""``agent.extra``: the session's working dirs, branches and subagent ids.
|
|
576
|
+
|
|
577
|
+
harbor collects each into a ``set`` and dumps it as a list, so its order
|
|
578
|
+
follows the process hash seed and two conversions of one session disagree.
|
|
579
|
+
Here each list keeps FIRST-SEEN order over ``events``, which are already in
|
|
580
|
+
timestamp order (:func:`_collect_events`): the first ``cwds`` entry is where
|
|
581
|
+
the session started, later ones are where it moved, and the same holds for
|
|
582
|
+
branch switches and for the order subagents first spoke. The parity oracle
|
|
583
|
+
forgives the reordering only when the two lists hold the same values.
|
|
584
|
+
"""
|
|
585
|
+
cwds = _first_seen(events, "cwd")
|
|
586
|
+
git_branches = _first_seen(events, "gitBranch")
|
|
587
|
+
agent_ids = _first_seen(events, "agentId")
|
|
588
|
+
|
|
589
|
+
agent_extra: dict[str, Any] = {}
|
|
590
|
+
if cwds:
|
|
591
|
+
agent_extra["cwds"] = cwds
|
|
592
|
+
if git_branches:
|
|
593
|
+
agent_extra["git_branches"] = git_branches
|
|
594
|
+
if agent_ids:
|
|
595
|
+
agent_extra["agent_ids"] = agent_ids
|
|
596
|
+
return agent_extra or None
|
|
597
|
+
|
|
598
|
+
|
|
599
|
+
def _event_identity_extra(event: dict[str, Any]) -> dict[str, Any]:
|
|
600
|
+
"""Keep each Claude session event's agent identity on its ATIF step.
|
|
601
|
+
|
|
602
|
+
harbor 0.24.0's ``ClaudeCode._event_identity_extra`` (harbor #3434): every
|
|
603
|
+
step a session event becomes, the agent turn, a user message and a tool
|
|
604
|
+
result alike, carries ``is_sidechain`` and, when the event names one, the
|
|
605
|
+
``agentId`` of the subagent that wrote it.
|
|
606
|
+
"""
|
|
607
|
+
extra: dict[str, Any] = {"is_sidechain": event.get("isSidechain", False)}
|
|
608
|
+
if agent_id := event.get("agentId"):
|
|
609
|
+
extra["agent_id"] = agent_id
|
|
610
|
+
return extra
|
|
611
|
+
|
|
612
|
+
|
|
613
|
+
def _last_usage_by_msg_id(events: list[dict[str, Any]]) -> dict[str, Any]:
|
|
614
|
+
last_usage_by_msg_id: dict[str, Any] = {}
|
|
615
|
+
for ev in events:
|
|
616
|
+
if ev.get("type") != "assistant":
|
|
617
|
+
continue
|
|
618
|
+
msg = ev.get("message")
|
|
619
|
+
if not isinstance(msg, dict):
|
|
620
|
+
continue
|
|
621
|
+
mid = msg.get("id")
|
|
622
|
+
usage = msg.get("usage")
|
|
623
|
+
if mid and usage is not None:
|
|
624
|
+
last_usage_by_msg_id[mid] = usage
|
|
625
|
+
return last_usage_by_msg_id
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def _normalize_assistant_event(
|
|
629
|
+
event: dict[str, Any], message: dict[str, Any], state: _NormalizationState
|
|
630
|
+
) -> None:
|
|
631
|
+
timestamp = event.get("timestamp")
|
|
632
|
+
text, reasoning, tool_blocks = _extract_text_reasoning_tool_uses(message.get("content"))
|
|
633
|
+
|
|
634
|
+
msg_id = message.get("id")
|
|
635
|
+
if msg_id and msg_id in state.seen_message_ids:
|
|
636
|
+
metrics = None
|
|
637
|
+
else:
|
|
638
|
+
# Claude Code accumulates usage, so the "real" usage is the last
|
|
639
|
+
# message in the chain. We still report this usage on the first
|
|
640
|
+
# shown in the trajectory.
|
|
641
|
+
usage = (
|
|
642
|
+
state.last_usage_by_msg_id.get(msg_id, message.get("usage"))
|
|
643
|
+
if msg_id
|
|
644
|
+
else message.get("usage")
|
|
645
|
+
)
|
|
646
|
+
metrics = _build_metrics(usage)
|
|
647
|
+
if msg_id:
|
|
648
|
+
state.seen_message_ids.add(msg_id)
|
|
649
|
+
extra: dict[str, Any] = {}
|
|
650
|
+
for key in ("stop_reason", "stop_sequence", "requestId"):
|
|
651
|
+
value = message.get(key)
|
|
652
|
+
if value is not None:
|
|
653
|
+
extra[key] = value
|
|
654
|
+
if event.get("id"):
|
|
655
|
+
extra["id"] = event["id"]
|
|
656
|
+
extra.update(_event_identity_extra(event))
|
|
657
|
+
if event.get("cwd"):
|
|
658
|
+
extra.setdefault("cwd", event["cwd"])
|
|
659
|
+
if event.get("userType") and event.get("userType") != "external":
|
|
660
|
+
extra["user_type"] = event["userType"]
|
|
661
|
+
|
|
662
|
+
model_name = message.get("model") or state.default_model_name
|
|
663
|
+
|
|
664
|
+
# Bundle one LLM inference (text, reasoning, and all tool_use calls share
|
|
665
|
+
# a `message.id`) into a single ATIF step, per RFC-0001 (`step` == one
|
|
666
|
+
# turn; `tool_calls` is multi-valued). Reuse the turn when the same
|
|
667
|
+
# `message.id` is split across several session-log events.
|
|
668
|
+
turn = state.turn_by_msgid.get(msg_id) if msg_id else None
|
|
669
|
+
if turn is None:
|
|
670
|
+
turn = {
|
|
671
|
+
"kind": "agent_step",
|
|
672
|
+
"timestamp": timestamp,
|
|
673
|
+
"role": message.get("role", "assistant"),
|
|
674
|
+
"text": "",
|
|
675
|
+
"reasoning": None,
|
|
676
|
+
"metrics": None,
|
|
677
|
+
"extra": extra or None,
|
|
678
|
+
"model_name": model_name,
|
|
679
|
+
"tool_calls": [],
|
|
680
|
+
}
|
|
681
|
+
state.normalized_events.append(turn)
|
|
682
|
+
if msg_id:
|
|
683
|
+
state.turn_by_msgid[msg_id] = turn
|
|
684
|
+
|
|
685
|
+
if text:
|
|
686
|
+
turn["text"] = f"{turn['text']}\n\n{text}".strip() if turn["text"] else text
|
|
687
|
+
if reasoning and message.get("role") == "assistant":
|
|
688
|
+
turn["reasoning"] = (
|
|
689
|
+
f"{turn['reasoning']}\n\n{reasoning}" if turn["reasoning"] else reasoning
|
|
690
|
+
)
|
|
691
|
+
if turn["metrics"] is None and metrics is not None:
|
|
692
|
+
turn["metrics"] = metrics
|
|
693
|
+
metrics = None
|
|
694
|
+
|
|
695
|
+
turn_calls = turn["tool_calls"]
|
|
696
|
+
if not isinstance(turn_calls, list):
|
|
697
|
+
turn_calls = []
|
|
698
|
+
turn["tool_calls"] = turn_calls
|
|
699
|
+
for tool_block in tool_blocks:
|
|
700
|
+
call_id = tool_block.get("id") or tool_block.get("tool_use_id")
|
|
701
|
+
if not call_id:
|
|
702
|
+
continue
|
|
703
|
+
# A call is keyed by call_id; skip a tool_use already seen (e.g. an
|
|
704
|
+
# event replayed after compaction) so it is not bundled twice.
|
|
705
|
+
if call_id in state.pending_calls or call_id in state.completed_call_ids:
|
|
706
|
+
continue
|
|
707
|
+
|
|
708
|
+
raw_arguments = tool_block.get("input")
|
|
709
|
+
arguments = raw_arguments if isinstance(raw_arguments, dict) else {"input": raw_arguments}
|
|
710
|
+
|
|
711
|
+
call_extra: dict[str, Any] = {}
|
|
712
|
+
if raw_arguments is not None:
|
|
713
|
+
call_extra["raw_arguments"] = raw_arguments
|
|
714
|
+
if tool_block.get("status") is not None:
|
|
715
|
+
call_extra["status"] = tool_block.get("status")
|
|
716
|
+
if tool_block.get("is_error") is not None:
|
|
717
|
+
call_extra["tool_use_is_error"] = tool_block.get("is_error")
|
|
718
|
+
if tool_block.get("name"):
|
|
719
|
+
call_extra.setdefault("tool_use_name", tool_block.get("name"))
|
|
720
|
+
|
|
721
|
+
tool_call_spec: dict[str, Any] = {
|
|
722
|
+
"call_id": call_id,
|
|
723
|
+
"tool_name": tool_block.get("name") or "",
|
|
724
|
+
"arguments": arguments or {},
|
|
725
|
+
"extra": call_extra or None,
|
|
726
|
+
"output": None,
|
|
727
|
+
"result_extra": None,
|
|
728
|
+
}
|
|
729
|
+
turn_calls.append(tool_call_spec)
|
|
730
|
+
state.pending_calls[call_id] = tool_call_spec
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
def _normalize_tool_result_block(
|
|
734
|
+
block: dict[str, Any], event: dict[str, Any], state: _NormalizationState
|
|
735
|
+
) -> None:
|
|
736
|
+
timestamp = event.get("timestamp")
|
|
737
|
+
call_id = block.get("tool_use_id")
|
|
738
|
+
formatted_output, metadata = _format_tool_result(block, event.get("toolUseResult"))
|
|
739
|
+
call_info = state.pending_calls.pop(call_id, None) if call_id else None
|
|
740
|
+
if call_info is not None:
|
|
741
|
+
# Matched a pending tool call: attach the result in place on its
|
|
742
|
+
# bundled turn (already appended to normalized_events as one
|
|
743
|
+
# agent_step).
|
|
744
|
+
result_extra: dict[str, Any] = {}
|
|
745
|
+
if metadata:
|
|
746
|
+
result_extra["tool_result_metadata"] = metadata
|
|
747
|
+
if block.get("is_error") is not None:
|
|
748
|
+
result_extra["tool_result_is_error"] = block.get("is_error")
|
|
749
|
+
call_info["output"] = formatted_output
|
|
750
|
+
call_info["result_extra"] = result_extra or None
|
|
751
|
+
if call_id:
|
|
752
|
+
state.completed_call_ids.add(call_id)
|
|
753
|
+
return
|
|
754
|
+
|
|
755
|
+
# Orphan tool_result with no matching tool_use in this window (e.g.
|
|
756
|
+
# replayed after compaction): keep the legacy single-call handling so its
|
|
757
|
+
# output is not lost. A duplicate of an already-completed call is
|
|
758
|
+
# dropped; a result with no tool name is skipped.
|
|
759
|
+
if call_id and call_id in state.completed_call_ids:
|
|
760
|
+
logger.debug("Skipping duplicate Claude Code tool result {}", call_id)
|
|
761
|
+
return
|
|
762
|
+
tool_name = block.get("name") or block.get("tool_name") or ""
|
|
763
|
+
if not tool_name:
|
|
764
|
+
logger.debug(
|
|
765
|
+
"Skipping orphan Claude Code tool result {} without tool name",
|
|
766
|
+
call_id or "<missing>",
|
|
767
|
+
)
|
|
768
|
+
return
|
|
769
|
+
call_info = {
|
|
770
|
+
"kind": "tool_call",
|
|
771
|
+
"timestamp": timestamp,
|
|
772
|
+
"call_id": call_id or "",
|
|
773
|
+
"tool_name": tool_name,
|
|
774
|
+
"arguments": {},
|
|
775
|
+
"raw_arguments": None,
|
|
776
|
+
"reasoning": None,
|
|
777
|
+
"status": None,
|
|
778
|
+
"message": None,
|
|
779
|
+
"extra": None,
|
|
780
|
+
"metrics": None,
|
|
781
|
+
"model_name": state.default_model_name,
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
extra_val = call_info.get("extra")
|
|
785
|
+
extra = extra_val if isinstance(extra_val, dict) else {}
|
|
786
|
+
extra.update(_event_identity_extra(event))
|
|
787
|
+
if metadata:
|
|
788
|
+
extra.setdefault("tool_result_metadata", metadata)
|
|
789
|
+
if block.get("is_error") is not None:
|
|
790
|
+
extra.setdefault("tool_result_is_error", block.get("is_error"))
|
|
791
|
+
|
|
792
|
+
call_info["extra"] = extra or None
|
|
793
|
+
call_info["output"] = formatted_output
|
|
794
|
+
call_info["metadata"] = metadata
|
|
795
|
+
call_info["timestamp"] = call_info.get("timestamp") or timestamp
|
|
796
|
+
call_info.setdefault("model_name", state.default_model_name)
|
|
797
|
+
|
|
798
|
+
state.normalized_events.append(call_info)
|
|
799
|
+
if call_id:
|
|
800
|
+
state.completed_call_ids.add(call_id)
|
|
801
|
+
|
|
802
|
+
|
|
803
|
+
def _normalize_user_event(
|
|
804
|
+
event: dict[str, Any], message: dict[str, Any], state: _NormalizationState
|
|
805
|
+
) -> None:
|
|
806
|
+
timestamp = event.get("timestamp")
|
|
807
|
+
content = message.get("content")
|
|
808
|
+
if isinstance(content, str):
|
|
809
|
+
# Preserve the raw bytes of the user message so downstream
|
|
810
|
+
# byte-identity checks (sha256 of the canonical instruction.md vs the
|
|
811
|
+
# first user step) hold; the strip is only the empty-skip filter.
|
|
812
|
+
text = content
|
|
813
|
+
if text.strip():
|
|
814
|
+
extra = _event_identity_extra(event)
|
|
815
|
+
state.normalized_events.append(
|
|
816
|
+
{
|
|
817
|
+
"kind": "message",
|
|
818
|
+
"timestamp": timestamp,
|
|
819
|
+
"role": "user",
|
|
820
|
+
"text": text,
|
|
821
|
+
"extra": extra,
|
|
822
|
+
}
|
|
823
|
+
)
|
|
824
|
+
return
|
|
825
|
+
|
|
826
|
+
if isinstance(content, list):
|
|
827
|
+
text_parts: list[str] = []
|
|
828
|
+
for block in content:
|
|
829
|
+
# Byte-faithful: a plain text content block contributes its
|
|
830
|
+
# inner string verbatim instead of a json-encoded dict, keeping
|
|
831
|
+
# trailing/internal whitespace intact.
|
|
832
|
+
if (
|
|
833
|
+
isinstance(block, dict)
|
|
834
|
+
and block.get("type") == "text"
|
|
835
|
+
and isinstance(block.get("text"), str)
|
|
836
|
+
):
|
|
837
|
+
text_parts.append(block["text"])
|
|
838
|
+
continue
|
|
839
|
+
|
|
840
|
+
if isinstance(block, dict) and block.get("type") == "tool_result":
|
|
841
|
+
_normalize_tool_result_block(block, event, state)
|
|
842
|
+
continue
|
|
843
|
+
|
|
844
|
+
# harbor repeats the text-block check here, after the tool_result
|
|
845
|
+
# branch; it is unreachable (the first check above already took
|
|
846
|
+
# every such block) and kept only so the upstream diff stays
|
|
847
|
+
# line-for-line.
|
|
848
|
+
if (
|
|
849
|
+
isinstance(block, dict)
|
|
850
|
+
and block.get("type") == "text"
|
|
851
|
+
and isinstance(block.get("text"), str)
|
|
852
|
+
):
|
|
853
|
+
text_parts.append(block["text"])
|
|
854
|
+
continue
|
|
855
|
+
|
|
856
|
+
text_parts.append(_stringify(block))
|
|
857
|
+
|
|
858
|
+
# No per-part strip, so byte spans inside each part round-trip; parts
|
|
859
|
+
# that are empty or whitespace-only are still filtered so the join
|
|
860
|
+
# does not materialise separators between nothing.
|
|
861
|
+
text_message = "\n\n".join(part for part in text_parts if part.strip())
|
|
862
|
+
if text_message:
|
|
863
|
+
state.normalized_events.append(
|
|
864
|
+
{
|
|
865
|
+
"kind": "message",
|
|
866
|
+
"timestamp": timestamp,
|
|
867
|
+
"role": "user",
|
|
868
|
+
"text": text_message,
|
|
869
|
+
"extra": _event_identity_extra(event),
|
|
870
|
+
}
|
|
871
|
+
)
|
|
872
|
+
return
|
|
873
|
+
|
|
874
|
+
if content not in (None, ""):
|
|
875
|
+
# Same byte-faithful rule: keep the stringified content's raw bytes
|
|
876
|
+
# and use strip only for the empty-skip filter.
|
|
877
|
+
text = _stringify(content)
|
|
878
|
+
if text.strip():
|
|
879
|
+
state.normalized_events.append(
|
|
880
|
+
{
|
|
881
|
+
"kind": "message",
|
|
882
|
+
"timestamp": timestamp,
|
|
883
|
+
"role": "user",
|
|
884
|
+
"text": text,
|
|
885
|
+
"extra": _event_identity_extra(event),
|
|
886
|
+
}
|
|
887
|
+
)
|
|
888
|
+
|
|
889
|
+
|
|
890
|
+
def _normalize_events(
|
|
891
|
+
events: list[dict[str, Any]], default_model_name: str | None
|
|
892
|
+
) -> list[dict[str, Any]]:
|
|
893
|
+
state = _NormalizationState(
|
|
894
|
+
default_model_name=default_model_name,
|
|
895
|
+
last_usage_by_msg_id=_last_usage_by_msg_id(events),
|
|
896
|
+
)
|
|
897
|
+
for event in events:
|
|
898
|
+
message = event.get("message")
|
|
899
|
+
if not isinstance(message, dict):
|
|
900
|
+
continue
|
|
901
|
+
|
|
902
|
+
event_type = event.get("type")
|
|
903
|
+
if event_type == "assistant":
|
|
904
|
+
_normalize_assistant_event(event, message, state)
|
|
905
|
+
elif event_type == "user":
|
|
906
|
+
_normalize_user_event(event, message, state)
|
|
907
|
+
|
|
908
|
+
# Leftover pending calls (a tool_use that never received a result) are
|
|
909
|
+
# already embedded in their bundled turn's `tool_calls`, so there is
|
|
910
|
+
# nothing to flush here; they render as a ToolCall with no observation.
|
|
911
|
+
return state.normalized_events
|
|
912
|
+
|
|
913
|
+
|
|
914
|
+
def _build_final_metrics(steps: list[Step]) -> FinalMetrics:
|
|
915
|
+
prompt_values = [
|
|
916
|
+
step.metrics.prompt_tokens
|
|
917
|
+
for step in steps
|
|
918
|
+
if step.metrics and step.metrics.prompt_tokens is not None
|
|
919
|
+
]
|
|
920
|
+
completion_values = [
|
|
921
|
+
step.metrics.completion_tokens
|
|
922
|
+
for step in steps
|
|
923
|
+
if step.metrics and step.metrics.completion_tokens is not None
|
|
924
|
+
]
|
|
925
|
+
cached_values = [
|
|
926
|
+
step.metrics.cached_tokens
|
|
927
|
+
for step in steps
|
|
928
|
+
if step.metrics and step.metrics.cached_tokens is not None
|
|
929
|
+
]
|
|
930
|
+
|
|
931
|
+
total_prompt_tokens = sum(prompt_values) if prompt_values else None
|
|
932
|
+
total_completion_tokens = sum(completion_values) if completion_values else None
|
|
933
|
+
total_cached_tokens = sum(cached_values) if cached_values else None
|
|
934
|
+
|
|
935
|
+
service_tiers: set[str] = set()
|
|
936
|
+
cache_creation_total, cache_read_total = 0, 0
|
|
937
|
+
cache_creation_seen, cache_read_seen = False, False
|
|
938
|
+
for step in steps:
|
|
939
|
+
if not step.metrics or not step.metrics.extra:
|
|
940
|
+
continue
|
|
941
|
+
extra = step.metrics.extra
|
|
942
|
+
tier = extra.get("service_tier")
|
|
943
|
+
if isinstance(tier, str):
|
|
944
|
+
service_tiers.add(tier)
|
|
945
|
+
cache_creation = extra.get("cache_creation_input_tokens")
|
|
946
|
+
if isinstance(cache_creation, int):
|
|
947
|
+
cache_creation_total += cache_creation
|
|
948
|
+
cache_creation_seen = True
|
|
949
|
+
cache_read = extra.get("cache_read_input_tokens")
|
|
950
|
+
if isinstance(cache_read, int):
|
|
951
|
+
cache_read_total += cache_read
|
|
952
|
+
cache_read_seen = True
|
|
953
|
+
|
|
954
|
+
final_extra: dict[str, Any] = {}
|
|
955
|
+
if service_tiers:
|
|
956
|
+
final_extra["service_tiers"] = sorted(service_tiers)
|
|
957
|
+
if cache_creation_seen:
|
|
958
|
+
final_extra["total_cache_creation_input_tokens"] = cache_creation_total
|
|
959
|
+
if cache_read_seen:
|
|
960
|
+
final_extra["total_cache_read_input_tokens"] = cache_read_total
|
|
961
|
+
|
|
962
|
+
# harbor consults ``_parse_total_cost_from_stream_json`` first; in our
|
|
963
|
+
# usage it always returned ``None`` (see the module docstring), so the
|
|
964
|
+
# estimate is the only cost source.
|
|
965
|
+
total_cost_usd = _estimate_step_costs(steps)
|
|
966
|
+
if total_cost_usd is not None:
|
|
967
|
+
final_extra["cost_source"] = pricing.cost_source_label(
|
|
968
|
+
step.model_name for step in steps if step.metrics is not None
|
|
969
|
+
)
|
|
970
|
+
|
|
971
|
+
return FinalMetrics(
|
|
972
|
+
total_prompt_tokens=total_prompt_tokens,
|
|
973
|
+
total_completion_tokens=total_completion_tokens,
|
|
974
|
+
total_cached_tokens=total_cached_tokens,
|
|
975
|
+
total_cost_usd=total_cost_usd,
|
|
976
|
+
total_steps=len(steps),
|
|
977
|
+
extra=final_extra or None,
|
|
978
|
+
)
|
|
979
|
+
|
|
980
|
+
|
|
981
|
+
def convert_claude_code_records(
|
|
982
|
+
main_records: Sequence[dict[str, Any]],
|
|
983
|
+
side_records: Mapping[str, Sequence[dict[str, Any]]] | None = None,
|
|
984
|
+
*,
|
|
985
|
+
fallback_session_id: str = "-unknown",
|
|
986
|
+
model_name: str | None = None,
|
|
987
|
+
) -> Trajectory | None:
|
|
988
|
+
"""Convert one Claude Code session's parsed records into an ATIF trajectory.
|
|
989
|
+
|
|
990
|
+
This is harbor's ``_convert_events_to_trajectory`` from the point where the
|
|
991
|
+
session files have been read.
|
|
992
|
+
|
|
993
|
+
Parameters
|
|
994
|
+
----------
|
|
995
|
+
main_records
|
|
996
|
+
Every JSON record of the main ``<session-id>.jsonl``, in file order.
|
|
997
|
+
side_records
|
|
998
|
+
Staged side-file NAME -> that file's records (see the module docstring
|
|
999
|
+
for the name shape). Read after the main records, in sorted-name
|
|
1000
|
+
order, exactly as harbor's ``sorted(rglob("subagents/*.jsonl"))``.
|
|
1001
|
+
fallback_session_id
|
|
1002
|
+
harbor falls back to the session directory's name when no record
|
|
1003
|
+
carries a ``sessionId``; the old adapter staged under
|
|
1004
|
+
``<parent-dir-name or "-unknown">``, so callers pass that.
|
|
1005
|
+
model_name
|
|
1006
|
+
harbor's ``self.model_name``: the run-configured model, used only when
|
|
1007
|
+
no record names one. Our adapter never set it.
|
|
1008
|
+
|
|
1009
|
+
Returns
|
|
1010
|
+
-------
|
|
1011
|
+
Trajectory | None
|
|
1012
|
+
``None`` exactly when harbor returns ``None``: no records at all, or no
|
|
1013
|
+
record produced a step.
|
|
1014
|
+
"""
|
|
1015
|
+
# harbor builds agent.extra's lists from SETS, so their order follows the
|
|
1016
|
+
# process hash seed. Ours keep first-seen order (``_agent_extra``), a
|
|
1017
|
+
# deliberate divergence the oracle forgives only for a reordering.
|
|
1018
|
+
events = _collect_events(main_records, side_records)
|
|
1019
|
+
if not events:
|
|
1020
|
+
return None
|
|
1021
|
+
|
|
1022
|
+
session_id: str = fallback_session_id
|
|
1023
|
+
for event in events:
|
|
1024
|
+
sid = event.get("sessionId")
|
|
1025
|
+
if isinstance(sid, str):
|
|
1026
|
+
session_id = sid
|
|
1027
|
+
break
|
|
1028
|
+
|
|
1029
|
+
agent_version: str = "unknown"
|
|
1030
|
+
for event in events:
|
|
1031
|
+
ver = event.get("version")
|
|
1032
|
+
if isinstance(ver, str) and ver:
|
|
1033
|
+
agent_version = ver
|
|
1034
|
+
break
|
|
1035
|
+
|
|
1036
|
+
agent_extra = _agent_extra(events)
|
|
1037
|
+
|
|
1038
|
+
# Prefer the main chain's model so a subagent running a different model
|
|
1039
|
+
# (e.g. a small/fast model) cannot become the trajectory's
|
|
1040
|
+
# `agent.model_name` or the fallback for model-less steps.
|
|
1041
|
+
default_model_name = (
|
|
1042
|
+
_first_event_model(events, include_sidechain=False)
|
|
1043
|
+
or _first_event_model(events, include_sidechain=True)
|
|
1044
|
+
or model_name
|
|
1045
|
+
)
|
|
1046
|
+
|
|
1047
|
+
normalized_events = _normalize_events(events, default_model_name)
|
|
1048
|
+
|
|
1049
|
+
steps: list[Step] = []
|
|
1050
|
+
for norm_event in normalized_events:
|
|
1051
|
+
try:
|
|
1052
|
+
step = _convert_event_to_step(norm_event, len(steps) + 1, model_name)
|
|
1053
|
+
except ValueError as exc:
|
|
1054
|
+
logger.debug("Skipping event during step conversion: {}", exc)
|
|
1055
|
+
continue
|
|
1056
|
+
|
|
1057
|
+
if step.source == "agent" and not step.model_name and default_model_name:
|
|
1058
|
+
step.model_name = default_model_name
|
|
1059
|
+
|
|
1060
|
+
steps.append(step)
|
|
1061
|
+
|
|
1062
|
+
if not steps:
|
|
1063
|
+
logger.debug("No valid steps produced from Claude Code session")
|
|
1064
|
+
return None
|
|
1065
|
+
|
|
1066
|
+
return Trajectory(
|
|
1067
|
+
schema_version=SCHEMA_VERSION,
|
|
1068
|
+
session_id=session_id,
|
|
1069
|
+
agent=Agent(
|
|
1070
|
+
name=AgentSource.CLAUDE_CODE.value,
|
|
1071
|
+
version=agent_version,
|
|
1072
|
+
model_name=default_model_name,
|
|
1073
|
+
extra=agent_extra,
|
|
1074
|
+
),
|
|
1075
|
+
steps=steps,
|
|
1076
|
+
final_metrics=_build_final_metrics(steps),
|
|
1077
|
+
)
|