atif-sql 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- atif_analytics/__init__.py +9 -0
- atif_analytics/application/__init__.py +3 -0
- atif_analytics/application/analyze.py +158 -0
- atif_analytics/application/prompts.py +750 -0
- atif_analytics/application/use_cases/__init__.py +3 -0
- atif_analytics/application/use_cases/_shared.py +264 -0
- atif_analytics/application/use_cases/classify.py +430 -0
- atif_analytics/application/use_cases/conflicts.py +448 -0
- atif_analytics/application/use_cases/friction.py +634 -0
- atif_analytics/application/use_cases/perceived.py +451 -0
- atif_analytics/domain/__init__.py +3 -0
- atif_analytics/domain/authorship.py +166 -0
- atif_analytics/domain/config.py +30 -0
- atif_analytics/domain/costs.py +50 -0
- atif_analytics/domain/friction.py +117 -0
- atif_analytics/domain/layout.py +94 -0
- atif_analytics/domain/models.py +385 -0
- atif_analytics/domain/ports.py +100 -0
- atif_analytics/domain/transcript.py +359 -0
- atif_analytics/infrastructure/__init__.py +3 -0
- atif_analytics/infrastructure/corpus_reader.py +546 -0
- atif_analytics/infrastructure/parquet_cache.py +186 -0
- atif_analytics/infrastructure/settings.py +114 -0
- atif_analytics/infrastructure/sqlite_state/__init__.py +3 -0
- atif_analytics/infrastructure/sqlite_state/checkpointer.py +326 -0
- atif_analytics/infrastructure/sqlite_state/retry_queue.py +296 -0
- atif_analytics/py.typed +0 -0
- atif_cli/__init__.py +3 -0
- atif_cli/__main__.py +8 -0
- atif_cli/app.py +2459 -0
- atif_cli/converter_adapter.py +151 -0
- atif_cli/corpus.py +409 -0
- atif_cli/cron.py +311 -0
- atif_cli/duck_errors.py +97 -0
- atif_cli/embed_lake.py +101 -0
- atif_cli/errors.py +89 -0
- atif_cli/lake.py +373 -0
- atif_cli/lake_sessions.py +270 -0
- atif_cli/output.py +297 -0
- atif_cli/py.typed +0 -0
- atif_converter/__init__.py +3 -0
- atif_converter/application/__init__.py +3 -0
- atif_converter/application/convert_and_audit.py +294 -0
- atif_converter/application/convert_codex.py +210 -0
- atif_converter/domain/__init__.py +9 -0
- atif_converter/domain/agents.py +40 -0
- atif_converter/domain/atif/__init__.py +55 -0
- atif_converter/domain/atif/agent.py +37 -0
- atif_converter/domain/atif/content.py +161 -0
- atif_converter/domain/atif/final_metrics.py +45 -0
- atif_converter/domain/atif/metrics.py +49 -0
- atif_converter/domain/atif/observation.py +21 -0
- atif_converter/domain/atif/observation_result.py +47 -0
- atif_converter/domain/atif/step.py +144 -0
- atif_converter/domain/atif/subagent_trajectory_ref.py +90 -0
- atif_converter/domain/atif/tool_call.py +36 -0
- atif_converter/domain/atif/trajectory.py +211 -0
- atif_converter/domain/atif/trajectory_validator.py +299 -0
- atif_converter/domain/blobs.py +510 -0
- atif_converter/domain/claude_code_conversion.py +1077 -0
- atif_converter/domain/codex_conversion.py +1036 -0
- atif_converter/domain/codex_edges.py +192 -0
- atif_converter/domain/codex_enrichment.py +419 -0
- atif_converter/domain/codex_fidelity.py +124 -0
- atif_converter/domain/codex_nested_calls.py +165 -0
- atif_converter/domain/edges.py +113 -0
- atif_converter/domain/enrichment.py +412 -0
- atif_converter/domain/errors.py +60 -0
- atif_converter/domain/fidelity.py +171 -0
- atif_converter/domain/model_prices.json +8538 -0
- atif_converter/domain/pricing.py +790 -0
- atif_converter/domain/result_signals.py +774 -0
- atif_converter/domain/schema_version.py +90 -0
- atif_converter/domain/session_events.py +408 -0
- atif_converter/infrastructure/__init__.py +3 -0
- atif_converter/infrastructure/census.py +89 -0
- atif_converter/infrastructure/claude_code_converter.py +127 -0
- atif_converter/infrastructure/codex_adapter.py +132 -0
- atif_converter/infrastructure/codex_census.py +100 -0
- atif_converter/infrastructure/codex_converter.py +82 -0
- atif_converter/infrastructure/harbor_adapter.py +187 -0
- atif_converter/infrastructure/raw_records.py +426 -0
- atif_converter/infrastructure/source_archive.py +154 -0
- atif_converter/py.typed +0 -0
- atif_corpus/__init__.py +13 -0
- atif_corpus/application/__init__.py +7 -0
- atif_corpus/application/materialize.py +1826 -0
- atif_corpus/domain/__init__.py +9 -0
- atif_corpus/domain/agents.py +33 -0
- atif_corpus/domain/generation.py +34 -0
- atif_corpus/domain/layout.py +168 -0
- atif_corpus/domain/ports.py +175 -0
- atif_corpus/domain/session_id.py +68 -0
- atif_corpus/domain/sessions.py +228 -0
- atif_corpus/domain/slug.py +72 -0
- atif_corpus/domain/source_layout.py +143 -0
- atif_corpus/domain/watermark.py +76 -0
- atif_corpus/infrastructure/__init__.py +7 -0
- atif_corpus/infrastructure/atomic.py +266 -0
- atif_corpus/infrastructure/compress_artifacts.py +161 -0
- atif_corpus/infrastructure/fake_converter.py +174 -0
- atif_corpus/infrastructure/scanner.py +316 -0
- atif_corpus/infrastructure/settings.py +130 -0
- atif_corpus/infrastructure/source_archive.py +202 -0
- atif_corpus/py.typed +0 -0
- atif_duck/__init__.py +43 -0
- atif_duck/domain/__init__.py +32 -0
- atif_duck/domain/artifacts.py +66 -0
- atif_duck/domain/authorship.py +165 -0
- atif_duck/domain/catalog.py +626 -0
- atif_duck/domain/columnar.py +149 -0
- atif_duck/domain/embedding_guard.py +88 -0
- atif_duck/domain/examples.py +265 -0
- atif_duck/domain/lake.py +427 -0
- atif_duck/domain/lake_sessions.py +97 -0
- atif_duck/domain/raw_readers.py +140 -0
- atif_duck/domain/session_id.py +69 -0
- atif_duck/domain/sql_literal.py +50 -0
- atif_duck/infrastructure/__init__.py +34 -0
- atif_duck/infrastructure/analytics.py +469 -0
- atif_duck/infrastructure/authorship.py +139 -0
- atif_duck/infrastructure/columnar.py +641 -0
- atif_duck/infrastructure/lake.py +1383 -0
- atif_duck/infrastructure/lake_sessions.py +194 -0
- atif_duck/infrastructure/lake_settings.py +63 -0
- atif_duck/infrastructure/lake_steps.py +225 -0
- atif_duck/infrastructure/projections.py +221 -0
- atif_duck/infrastructure/registry.py +2194 -0
- atif_duck/infrastructure/stored_artifacts.py +115 -0
- atif_duck/py.typed +0 -0
- atif_embed/__init__.py +12 -0
- atif_embed/application/__init__.py +3 -0
- atif_embed/application/embed.py +293 -0
- atif_embed/application/prune.py +80 -0
- atif_embed/domain/__init__.py +3 -0
- atif_embed/domain/discovery.py +70 -0
- atif_embed/domain/embedding_guard.py +83 -0
- atif_embed/domain/errors.py +94 -0
- atif_embed/domain/ports.py +223 -0
- atif_embed/domain/sql_literal.py +49 -0
- atif_embed/domain/text_stamp.py +71 -0
- atif_embed/infrastructure/__init__.py +3 -0
- atif_embed/infrastructure/cohere_bedrock.py +433 -0
- atif_embed/infrastructure/corpus_text_rows.py +273 -0
- atif_embed/infrastructure/gemma_local.py +355 -0
- atif_embed/infrastructure/lake_text_rows.py +203 -0
- atif_embed/infrastructure/lance_store.py +440 -0
- atif_embed/infrastructure/providers.py +30 -0
- atif_embed/infrastructure/settings.py +154 -0
- atif_embed/py.typed +0 -0
- atif_models/__init__.py +14 -0
- atif_models/domain/__init__.py +9 -0
- atif_models/domain/ports.py +143 -0
- atif_models/domain/registry.py +151 -0
- atif_models/domain/schema.py +109 -0
- atif_models/infrastructure/__init__.py +8 -0
- atif_models/infrastructure/openai_bedrock.py +314 -0
- atif_models/infrastructure/settings.py +85 -0
- atif_models/py.typed +0 -0
- atif_sql-0.2.0.dist-info/METADATA +272 -0
- atif_sql-0.2.0.dist-info/RECORD +164 -0
- atif_sql-0.2.0.dist-info/WHEEL +4 -0
- atif_sql-0.2.0.dist-info/entry_points.txt +2 -0
- atif_sql-0.2.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
|
|
3
|
+
"""Use case: convert one Codex rollout AND account for what conversion lost.
|
|
4
|
+
|
|
5
|
+
The Codex counterpart of
|
|
6
|
+
:mod:`atif_converter.application.convert_and_audit`, composed the same way and
|
|
7
|
+
under the same discipline: the rollout is read once, fingerprinted and parsed
|
|
8
|
+
in the same pass, the converter and the audit consume that one list of
|
|
9
|
+
records, and the fingerprints are re-checked once the artifacts are built, so
|
|
10
|
+
the census, the trajectory and ``edges.jsonl`` all describe the same bytes of a
|
|
11
|
+
rollout that Codex may still be appending to.
|
|
12
|
+
|
|
13
|
+
What differs is only what a rollout IS. One file, no side-file tree, a record
|
|
14
|
+
taxonomy of its own, and a set of gaps of its own — so the census, the edges
|
|
15
|
+
and the enrichment come from the Codex modules while the snapshot machinery,
|
|
16
|
+
the error taxonomy and the returned ``(ConversionResult, LossReport)`` pair are
|
|
17
|
+
shared with the Claude Code path.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
from loguru import logger
|
|
25
|
+
|
|
26
|
+
from atif_converter.domain.blobs import BlobCollector, extract_codex_blobs
|
|
27
|
+
from atif_converter.domain.codex_edges import build_codex_edges, codex_edges_jsonl_lines
|
|
28
|
+
from atif_converter.domain.codex_enrichment import enrich_codex_trajectory
|
|
29
|
+
from atif_converter.domain.codex_fidelity import (
|
|
30
|
+
CODEX_STRUCTURAL_GAPS,
|
|
31
|
+
CodexFidelityGap,
|
|
32
|
+
CodexRecordType,
|
|
33
|
+
)
|
|
34
|
+
from atif_converter.domain.errors import SourceMutatedDuringConversion
|
|
35
|
+
from atif_converter.domain.fidelity import AnyRecordType, LossReport
|
|
36
|
+
from atif_converter.domain.result_signals import annotate_codex_trajectory
|
|
37
|
+
from atif_converter.domain.session_events import (
|
|
38
|
+
codex_session_events,
|
|
39
|
+
session_events_jsonl_lines,
|
|
40
|
+
)
|
|
41
|
+
from atif_converter.infrastructure.codex_adapter import convert_loaded_codex_session
|
|
42
|
+
from atif_converter.infrastructure.codex_census import (
|
|
43
|
+
CodexSessionCensus,
|
|
44
|
+
codex_census_from_records,
|
|
45
|
+
)
|
|
46
|
+
from atif_converter.infrastructure.harbor_adapter import (
|
|
47
|
+
ConversionResult,
|
|
48
|
+
read_session,
|
|
49
|
+
validate_trajectory,
|
|
50
|
+
)
|
|
51
|
+
from atif_converter.infrastructure.raw_records import SessionSnapshot, mutated_files
|
|
52
|
+
from atif_converter.infrastructure.source_archive import SourceArchiveWriter
|
|
53
|
+
|
|
54
|
+
#: Response-item payload types whose presence in a rollout means a specific
|
|
55
|
+
#: content-shaped gap was actually exercised, rather than merely being possible.
|
|
56
|
+
_ITEM_TYPE_GAPS: dict[str, CodexFidelityGap] = {
|
|
57
|
+
"reasoning": CodexFidelityGap.REASONING_ENCRYPTED_DROPPED,
|
|
58
|
+
"tool_search_call": CodexFidelityGap.TOOL_SEARCH_CALLS_DROPPED,
|
|
59
|
+
"tool_search_output": CodexFidelityGap.TOOL_SEARCH_CALLS_DROPPED,
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _loss_report(
|
|
64
|
+
census: CodexSessionCensus, *, developer_messages: int, captured: int = 0
|
|
65
|
+
) -> LossReport:
|
|
66
|
+
"""Loss accounting for one rollout: total records vs items that convert.
|
|
67
|
+
|
|
68
|
+
``records_converted`` counts the CONVERTIBLE RESPONSE ITEMS, not every
|
|
69
|
+
response item and not every record: ``event_msg`` and ``turn_context``
|
|
70
|
+
records are read for token counts and turn ids while producing no step of
|
|
71
|
+
their own, so counting them as converted would claim the trajectory
|
|
72
|
+
carries content it does not.
|
|
73
|
+
"""
|
|
74
|
+
total = sum(census.record_counts.values())
|
|
75
|
+
converted = census.convertible_items
|
|
76
|
+
dropped = total - converted - captured
|
|
77
|
+
|
|
78
|
+
gaps = set(CODEX_STRUCTURAL_GAPS)
|
|
79
|
+
if dropped:
|
|
80
|
+
gaps.add(CodexFidelityGap.NON_ITEM_RECORDS_DROPPED)
|
|
81
|
+
if census.record_counts.get(CodexRecordType.COMPACTED):
|
|
82
|
+
gaps.add(CodexFidelityGap.COMPACTION_UNHANDLED)
|
|
83
|
+
if developer_messages:
|
|
84
|
+
gaps.add(CodexFidelityGap.DEVELOPER_ROLE_FLATTENED_TO_SYSTEM)
|
|
85
|
+
for item_type, gap in _ITEM_TYPE_GAPS.items():
|
|
86
|
+
if census.item_type_counts.get(item_type):
|
|
87
|
+
gaps.add(gap)
|
|
88
|
+
|
|
89
|
+
# Named rather than inlined: LossReport is shared by both agents, so its
|
|
90
|
+
# key type is the UNION of the two record taxonomies and a same-typed dict
|
|
91
|
+
# of one agent's members does not match the constructor overload without
|
|
92
|
+
# being widened here.
|
|
93
|
+
record_counts: dict[AnyRecordType, int] = {**census.record_counts}
|
|
94
|
+
return LossReport(
|
|
95
|
+
record_counts=record_counts,
|
|
96
|
+
records_converted=converted,
|
|
97
|
+
records_captured=captured,
|
|
98
|
+
records_dropped=dropped,
|
|
99
|
+
gaps_observed=frozenset(gaps),
|
|
100
|
+
# A Codex sub-agent writes its OWN rollout under its own session id, so
|
|
101
|
+
# a rollout never carries a subagent side-file. Zero here is a fact
|
|
102
|
+
# about the format, not an unmeasured field.
|
|
103
|
+
subagent_files_found=0,
|
|
104
|
+
subagent_files_convertible=0,
|
|
105
|
+
workflow_subagent_files_found=0,
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _developer_message_count(records: list[tuple[dict[str, object], str]]) -> int:
|
|
110
|
+
"""How many ``developer``-role messages the rollout carried."""
|
|
111
|
+
count = 0
|
|
112
|
+
for record, _source_file in records:
|
|
113
|
+
if record.get("type") != "response_item":
|
|
114
|
+
continue
|
|
115
|
+
payload = record.get("payload")
|
|
116
|
+
if not isinstance(payload, dict):
|
|
117
|
+
continue
|
|
118
|
+
if payload.get("type") == "message" and payload.get("role") == "developer":
|
|
119
|
+
count += 1
|
|
120
|
+
return count
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _refuse_if_mutated(
|
|
124
|
+
snapshot: SessionSnapshot, archive: SourceArchiveWriter | None = None
|
|
125
|
+
) -> None:
|
|
126
|
+
mutated = mutated_files(snapshot, archive=archive)
|
|
127
|
+
if not mutated:
|
|
128
|
+
if archive is not None:
|
|
129
|
+
# The verifying read already archived the transcripts AND the
|
|
130
|
+
# parsed sidecars; the sweep takes only files nothing parsed.
|
|
131
|
+
archive.archive_side_files(
|
|
132
|
+
snapshot.session_jsonl,
|
|
133
|
+
already=(*snapshot.files, *snapshot.sidecar_fingerprints),
|
|
134
|
+
)
|
|
135
|
+
return
|
|
136
|
+
logger.warning(
|
|
137
|
+
"convert_codex: {} source file(s) changed while converting {}; refusing",
|
|
138
|
+
len(mutated),
|
|
139
|
+
snapshot.session_jsonl,
|
|
140
|
+
)
|
|
141
|
+
raise SourceMutatedDuringConversion(snapshot.session_jsonl, mutated)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def convert_codex_and_audit(
|
|
145
|
+
rollout_jsonl: Path,
|
|
146
|
+
*,
|
|
147
|
+
archive: SourceArchiveWriter | None = None,
|
|
148
|
+
) -> tuple[ConversionResult, LossReport]:
|
|
149
|
+
"""Convert ``rollout_jsonl`` to ATIF and produce its loss accounting.
|
|
150
|
+
|
|
151
|
+
The returned :class:`ConversionResult` carries the ENRICHED trajectory
|
|
152
|
+
(``source_uuids`` on steps, ``cache_creation_total`` and the compaction
|
|
153
|
+
count on the trajectory — see
|
|
154
|
+
:func:`~atif_converter.domain.codex_enrichment.enrich_codex_trajectory`)
|
|
155
|
+
plus the ready-to-write ``edges.jsonl`` lines derived from the RAW records.
|
|
156
|
+
Validation is re-run post-enrichment.
|
|
157
|
+
|
|
158
|
+
With ``archive``, the verifying re-read also streams every source file
|
|
159
|
+
into that writer (see
|
|
160
|
+
:mod:`atif_converter.infrastructure.source_archive`): the archive costs
|
|
161
|
+
no extra open of a transcript and holds exactly the bytes that were
|
|
162
|
+
parsed. It is complete only when this function returns; on any raise the
|
|
163
|
+
caller must discard it.
|
|
164
|
+
|
|
165
|
+
Raises
|
|
166
|
+
------
|
|
167
|
+
InvalidSessionInput: ``rollout_jsonl`` is not an existing ``.jsonl`` file.
|
|
168
|
+
SourceMutatedDuringConversion: the rollout changed between the
|
|
169
|
+
read and the end of the audit, so the artifacts would not
|
|
170
|
+
describe the rollout as it stands.
|
|
171
|
+
"""
|
|
172
|
+
loaded = read_session(rollout_jsonl)
|
|
173
|
+
snapshot = loaded.snapshot
|
|
174
|
+
# Attachments out BEFORE conversion, named by the same record ids the
|
|
175
|
+
# enrichment pass writes into ``source_uuids`` (an edge's uuid depends on
|
|
176
|
+
# the record's ids and position, never on the content being rewritten).
|
|
177
|
+
records = loaded.record_pairs()
|
|
178
|
+
record_keys = [str(edge["uuid"]) for edge in build_codex_edges(records)]
|
|
179
|
+
collector = BlobCollector()
|
|
180
|
+
blob_index = extract_codex_blobs(
|
|
181
|
+
[record for record, _src in records], collector, record_keys=record_keys
|
|
182
|
+
)
|
|
183
|
+
result = convert_loaded_codex_session(loaded)
|
|
184
|
+
|
|
185
|
+
census = codex_census_from_records(rollout_jsonl, records)
|
|
186
|
+
events = codex_session_events(records)
|
|
187
|
+
report = _loss_report(
|
|
188
|
+
census,
|
|
189
|
+
developer_messages=_developer_message_count(records),
|
|
190
|
+
captured=len(events),
|
|
191
|
+
)
|
|
192
|
+
edges_lines = tuple(codex_edges_jsonl_lines(records))
|
|
193
|
+
events_lines = tuple(session_events_jsonl_lines(events))
|
|
194
|
+
# The harbor trajectory has no other consumer, so enrich in place rather
|
|
195
|
+
# than deep-copying a whole rollout's worth of steps.
|
|
196
|
+
enriched = enrich_codex_trajectory(result.trajectory, records, copy_input=False)
|
|
197
|
+
annotate_codex_trajectory(enriched, [record for record, _src in records], blob_index)
|
|
198
|
+
del records, loaded, events
|
|
199
|
+
_refuse_if_mutated(snapshot, archive)
|
|
200
|
+
|
|
201
|
+
return (
|
|
202
|
+
ConversionResult(
|
|
203
|
+
trajectory=enriched,
|
|
204
|
+
validation_errors=validate_trajectory(enriched),
|
|
205
|
+
edges_lines=edges_lines,
|
|
206
|
+
events_lines=events_lines,
|
|
207
|
+
blobs=collector.blobs,
|
|
208
|
+
),
|
|
209
|
+
report,
|
|
210
|
+
)
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
|
|
3
|
+
"""Pure domain layer: fidelity policy types, the error taxonomy, the two conversions.
|
|
4
|
+
|
|
5
|
+
The innermost layer of the converter hexagon. It holds the ATIF data classes
|
|
6
|
+
(the output contract, vendored from harbor under :mod:`atif_converter.domain.atif`)
|
|
7
|
+
and does no file I/O beyond :mod:`atif_converter.domain.pricing` reading the
|
|
8
|
+
vendored price table beside it. Nothing here imports harbor or litellm.
|
|
9
|
+
"""
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
|
|
3
|
+
"""Which agent produced a transcript — the one enum every layer keys off.
|
|
4
|
+
|
|
5
|
+
An ATIF corpus can hold trajectories from more than one coding agent, and the
|
|
6
|
+
two this workspace converts write nothing alike: Claude Code appends one JSONL
|
|
7
|
+
per session under ``<config>/projects/<project>/``, Codex CLI writes one
|
|
8
|
+
rollout per session under ``$CODEX_HOME/sessions/<YYYY>/<MM>/<DD>/``. Discovery,
|
|
9
|
+
the record taxonomy, the fidelity gaps and the harbor adapter all differ, so
|
|
10
|
+
the agent is a first-class value carried from the CLI flag down to the census
|
|
11
|
+
rather than inferred from a path shape at each layer.
|
|
12
|
+
|
|
13
|
+
Pure value object: no harbor import, no I/O. The string values are WIRE
|
|
14
|
+
CONTRACT — they are the ``--agent`` flag's accepted spellings, the
|
|
15
|
+
``agent.name`` harbor stamps into a trajectory, and the ``agent`` key in
|
|
16
|
+
``meta.json``.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from enum import StrEnum
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class AgentSource(StrEnum):
|
|
25
|
+
"""A coding agent whose transcripts this workspace can convert.
|
|
26
|
+
|
|
27
|
+
The value doubles as harbor's ``Trajectory.agent.name`` for that agent
|
|
28
|
+
(verified 2026-09-11 against harbor 0.22.0: ``ClaudeCode`` stamps
|
|
29
|
+
``claude-code`` and ``Codex`` stamps ``codex``), which is what lets
|
|
30
|
+
atif-duck's ``sessions.agent`` column come straight from the trajectory
|
|
31
|
+
instead of from a second provenance file.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
CLAUDE_CODE = "claude-code"
|
|
35
|
+
CODEX = "codex"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
#: The agent assumed when a caller names none — the interactive corpus nearly
|
|
39
|
+
#: every user has, and the only agent this workspace supported before 2026-09-11.
|
|
40
|
+
DEFAULT_AGENT: AgentSource = AgentSource.CLAUDE_CODE
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
# Vendored from harbor 0.24.0, src/harbor/models/trajectories/__init__.py
|
|
3
|
+
# (Apache-2.0, Copyright the Harbor authors), with the provenance constants
|
|
4
|
+
# below added.
|
|
5
|
+
|
|
6
|
+
"""Pydantic models for Agent Trajectory Interchange Format (ATIF).
|
|
7
|
+
|
|
8
|
+
The ATIF data classes and the trajectory validator, vendored from harbor
|
|
9
|
+
(https://github.com/laude-institute/harbor) at :data:`UPSTREAM_VERSION` under
|
|
10
|
+
its Apache-2.0 license, so atif-sql converts and validates trajectories
|
|
11
|
+
without installing harbor and the dependency tree harbor's agent runtime
|
|
12
|
+
carries. Every sibling module is upstream's file byte for byte apart from its
|
|
13
|
+
header and the import path (``harbor.models.trajectories`` reads
|
|
14
|
+
``atif_converter.domain.atif`` here); this ``__init__`` adds only the two
|
|
15
|
+
constants.
|
|
16
|
+
|
|
17
|
+
harbor stays a dev dependency, and the converter's tests hold this copy to it:
|
|
18
|
+
``tests/test_vendored_atif.py`` compares each file to the installed upstream
|
|
19
|
+
source and round-trips the golden trajectories through both sets of models and
|
|
20
|
+
both validators. A harbor bump that changes the models fails there, and the fix
|
|
21
|
+
is to re-vendor the changed files and move :data:`UPSTREAM_VERSION`.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from atif_converter.domain.atif.agent import Agent
|
|
25
|
+
from atif_converter.domain.atif.content import AudioSource, ContentPart, ImageSource
|
|
26
|
+
from atif_converter.domain.atif.final_metrics import FinalMetrics
|
|
27
|
+
from atif_converter.domain.atif.metrics import Metrics
|
|
28
|
+
from atif_converter.domain.atif.observation import Observation
|
|
29
|
+
from atif_converter.domain.atif.observation_result import ObservationResult
|
|
30
|
+
from atif_converter.domain.atif.step import Step
|
|
31
|
+
from atif_converter.domain.atif.subagent_trajectory_ref import SubagentTrajectoryRef
|
|
32
|
+
from atif_converter.domain.atif.tool_call import ToolCall
|
|
33
|
+
from atif_converter.domain.atif.trajectory import Trajectory
|
|
34
|
+
|
|
35
|
+
#: The distribution these files were vendored from.
|
|
36
|
+
UPSTREAM_DISTRIBUTION = "harbor"
|
|
37
|
+
#: The upstream release the files match; ``meta.json`` records it as ``harbor_version``.
|
|
38
|
+
UPSTREAM_VERSION = "0.24.0"
|
|
39
|
+
|
|
40
|
+
__all__ = [
|
|
41
|
+
"UPSTREAM_DISTRIBUTION",
|
|
42
|
+
"UPSTREAM_VERSION",
|
|
43
|
+
"Agent",
|
|
44
|
+
"AudioSource",
|
|
45
|
+
"ContentPart",
|
|
46
|
+
"FinalMetrics",
|
|
47
|
+
"ImageSource",
|
|
48
|
+
"Metrics",
|
|
49
|
+
"Observation",
|
|
50
|
+
"ObservationResult",
|
|
51
|
+
"Step",
|
|
52
|
+
"SubagentTrajectoryRef",
|
|
53
|
+
"ToolCall",
|
|
54
|
+
"Trajectory",
|
|
55
|
+
]
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
# Vendored from harbor 0.24.0, src/harbor/models/trajectories/agent.py
|
|
3
|
+
# (Apache-2.0, Copyright the Harbor authors). Byte for byte upstream apart
|
|
4
|
+
# from these header lines and the import path, which reads
|
|
5
|
+
# atif_converter.domain.atif where upstream reads harbor.models.trajectories.
|
|
6
|
+
"""Agent configuration model for ATIF trajectories."""
|
|
7
|
+
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from pydantic import BaseModel, Field
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Agent(BaseModel):
|
|
14
|
+
"""Agent configuration."""
|
|
15
|
+
|
|
16
|
+
name: str = Field(
|
|
17
|
+
default=...,
|
|
18
|
+
description="The name of the agent system",
|
|
19
|
+
)
|
|
20
|
+
version: str = Field(
|
|
21
|
+
default=...,
|
|
22
|
+
description="The version identifier of the agent system",
|
|
23
|
+
)
|
|
24
|
+
model_name: str | None = Field(
|
|
25
|
+
default=None,
|
|
26
|
+
description="Default LLM model used for this trajectory",
|
|
27
|
+
)
|
|
28
|
+
tool_definitions: list[dict[str, Any]] | None = Field(
|
|
29
|
+
default=None,
|
|
30
|
+
description="Array of tool/function definitions available to the agent. Each element follows OpenAI's function calling schema.",
|
|
31
|
+
)
|
|
32
|
+
extra: dict[str, Any] | None = Field(
|
|
33
|
+
default=None,
|
|
34
|
+
description="Custom agent configuration details",
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
model_config = {"extra": "forbid"}
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
# Vendored from harbor 0.24.0, src/harbor/models/trajectories/content.py
|
|
3
|
+
# (Apache-2.0, Copyright the Harbor authors). Byte for byte upstream apart
|
|
4
|
+
# from these header lines and the import path, which reads
|
|
5
|
+
# atif_converter.domain.atif where upstream reads harbor.models.trajectories.
|
|
6
|
+
"""Content models for multimodal ATIF trajectories.
|
|
7
|
+
|
|
8
|
+
Added in ATIF-v1.6 to support multimodal content (images) in trajectories.
|
|
9
|
+
Extended in ATIF-v1.8 with audio.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from typing import Any, Literal
|
|
13
|
+
|
|
14
|
+
from pydantic import BaseModel, Field, field_validator, model_validator
|
|
15
|
+
|
|
16
|
+
# Canonical audio MIME types, chosen to cover what multimodal and transcription
|
|
17
|
+
# APIs actually accept: Gemini audio understanding (wav, mp3, aiff, aac, ogg,
|
|
18
|
+
# flac), OpenAI transcription (flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, webm),
|
|
19
|
+
# and `audio/webm`, which is what a browser's MediaRecorder produces and so what
|
|
20
|
+
# human-recorded task data usually arrives as.
|
|
21
|
+
AudioMediaType = Literal[
|
|
22
|
+
"audio/wav",
|
|
23
|
+
"audio/mpeg",
|
|
24
|
+
"audio/mp4",
|
|
25
|
+
"audio/aac",
|
|
26
|
+
"audio/ogg",
|
|
27
|
+
"audio/flac",
|
|
28
|
+
"audio/webm",
|
|
29
|
+
"audio/aiff",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
# Aliases producers emit in place of the registered MIME type, normalized on the
|
|
33
|
+
# way in so consumers only ever see one spelling per format. `audio/mp3` is the
|
|
34
|
+
# important one: it is not a registered type, but the Gemini API documents and
|
|
35
|
+
# emits it, so a producer copying a provider response verbatim would otherwise
|
|
36
|
+
# fail validation.
|
|
37
|
+
_AUDIO_MEDIA_TYPE_ALIASES = {
|
|
38
|
+
"audio/mp3": "audio/mpeg",
|
|
39
|
+
"audio/mpga": "audio/mpeg",
|
|
40
|
+
"audio/x-mpeg": "audio/mpeg",
|
|
41
|
+
"audio/x-wav": "audio/wav",
|
|
42
|
+
"audio/wave": "audio/wav",
|
|
43
|
+
"audio/vnd.wave": "audio/wav",
|
|
44
|
+
"audio/x-m4a": "audio/mp4",
|
|
45
|
+
"audio/m4a": "audio/mp4",
|
|
46
|
+
"audio/x-aac": "audio/aac",
|
|
47
|
+
"audio/x-flac": "audio/flac",
|
|
48
|
+
"audio/x-aiff": "audio/aiff",
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class ImageSource(BaseModel):
|
|
53
|
+
"""Image source specification for images stored as files or at remote URLs."""
|
|
54
|
+
|
|
55
|
+
media_type: Literal["image/jpeg", "image/png", "image/gif", "image/webp"] = Field(
|
|
56
|
+
default=...,
|
|
57
|
+
description="MIME type of the image",
|
|
58
|
+
)
|
|
59
|
+
path: str = Field(
|
|
60
|
+
default=...,
|
|
61
|
+
description=(
|
|
62
|
+
"Location of the image. Can be a relative or absolute file path, or a URL."
|
|
63
|
+
),
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
model_config = {"extra": "forbid"}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class AudioSource(BaseModel):
|
|
70
|
+
"""Audio source specification for audio stored as files or at remote URLs.
|
|
71
|
+
|
|
72
|
+
Added in ATIF-v1.8. Mirrors ``ImageSource`` so that consumers can treat any
|
|
73
|
+
referenced media the same way, with one addition: ``duration_sec``. Duration
|
|
74
|
+
is worth recording because several providers bill audio per second and
|
|
75
|
+
dataset statistics need it, and unlike an image's dimensions it cannot be
|
|
76
|
+
recovered without decoding the file.
|
|
77
|
+
"""
|
|
78
|
+
|
|
79
|
+
media_type: AudioMediaType = Field(
|
|
80
|
+
default=...,
|
|
81
|
+
description="MIME type of the audio",
|
|
82
|
+
)
|
|
83
|
+
path: str = Field(
|
|
84
|
+
default=...,
|
|
85
|
+
description=(
|
|
86
|
+
"Location of the audio. Can be a relative or absolute file path, or a URL."
|
|
87
|
+
),
|
|
88
|
+
)
|
|
89
|
+
duration_sec: float | None = Field(
|
|
90
|
+
default=None,
|
|
91
|
+
ge=0,
|
|
92
|
+
description=(
|
|
93
|
+
"Optional duration of the audio in seconds. Useful for cost "
|
|
94
|
+
"accounting and dataset statistics; omit when unknown."
|
|
95
|
+
),
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
model_config = {"extra": "forbid"}
|
|
99
|
+
|
|
100
|
+
@field_validator("media_type", mode="before")
|
|
101
|
+
@classmethod
|
|
102
|
+
def normalize_media_type(cls, value: Any) -> Any:
|
|
103
|
+
"""Accept common non-registered spellings, storing the canonical one."""
|
|
104
|
+
if isinstance(value, str):
|
|
105
|
+
normalized = value.strip().lower()
|
|
106
|
+
return _AUDIO_MEDIA_TYPE_ALIASES.get(normalized, normalized)
|
|
107
|
+
return value
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class ContentPart(BaseModel):
|
|
111
|
+
"""A single content part within a multimodal message.
|
|
112
|
+
|
|
113
|
+
Used when a message or observation contains mixed content types (text,
|
|
114
|
+
images, or audio). For text-only content, a plain string can still be used
|
|
115
|
+
instead of a ContentPart array.
|
|
116
|
+
"""
|
|
117
|
+
|
|
118
|
+
type: Literal["text", "image", "audio"] = Field(
|
|
119
|
+
default=...,
|
|
120
|
+
description="The type of content",
|
|
121
|
+
)
|
|
122
|
+
text: str | None = Field(
|
|
123
|
+
default=None,
|
|
124
|
+
description="Text content. Required when type='text'.",
|
|
125
|
+
)
|
|
126
|
+
source: ImageSource | AudioSource | None = Field(
|
|
127
|
+
default=None,
|
|
128
|
+
description=(
|
|
129
|
+
"Media source (file reference). Required when type='image' or "
|
|
130
|
+
"type='audio', and must match that type."
|
|
131
|
+
),
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
model_config = {"extra": "forbid"}
|
|
135
|
+
|
|
136
|
+
@model_validator(mode="after")
|
|
137
|
+
def validate_content_type(self) -> "ContentPart":
|
|
138
|
+
"""Validate that the correct fields are present for each content type."""
|
|
139
|
+
if self.type == "text":
|
|
140
|
+
if self.text is None:
|
|
141
|
+
raise ValueError("'text' field is required when type='text'")
|
|
142
|
+
if self.source is not None:
|
|
143
|
+
raise ValueError("'source' field is not allowed when type='text'")
|
|
144
|
+
return self
|
|
145
|
+
|
|
146
|
+
# Media parts: a source is required, text is not allowed, and the source
|
|
147
|
+
# must be of the kind the `type` declares. Without the last check a
|
|
148
|
+
# `source` union would happily accept an ImageSource under
|
|
149
|
+
# type='audio' (and vice versa), silently mislabelling the content.
|
|
150
|
+
if self.source is None:
|
|
151
|
+
raise ValueError(f"'source' field is required when type='{self.type}'")
|
|
152
|
+
if self.text is not None:
|
|
153
|
+
raise ValueError(f"'text' field is not allowed when type='{self.type}'")
|
|
154
|
+
|
|
155
|
+
expected = ImageSource if self.type == "image" else AudioSource
|
|
156
|
+
if not isinstance(self.source, expected):
|
|
157
|
+
raise ValueError(
|
|
158
|
+
f"type='{self.type}' requires a {expected.__name__}, but the "
|
|
159
|
+
f"source's media_type is {self.source.media_type!r}"
|
|
160
|
+
)
|
|
161
|
+
return self
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
# Vendored from harbor 0.24.0, src/harbor/models/trajectories/final_metrics.py
|
|
3
|
+
# (Apache-2.0, Copyright the Harbor authors). Byte for byte upstream apart
|
|
4
|
+
# from these header lines and the import path, which reads
|
|
5
|
+
# atif_converter.domain.atif where upstream reads harbor.models.trajectories.
|
|
6
|
+
"""Final metrics model for ATIF trajectories."""
|
|
7
|
+
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from pydantic import BaseModel, Field
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class FinalMetrics(BaseModel):
|
|
14
|
+
"""Aggregate statistics for the entire trajectory."""
|
|
15
|
+
|
|
16
|
+
total_prompt_tokens: int | None = Field(
|
|
17
|
+
default=None,
|
|
18
|
+
description="Sum of all prompt tokens across all steps, including cached tokens",
|
|
19
|
+
)
|
|
20
|
+
total_completion_tokens: int | None = Field(
|
|
21
|
+
default=None,
|
|
22
|
+
description="Sum of all completion tokens across all steps",
|
|
23
|
+
)
|
|
24
|
+
total_cached_tokens: int | None = Field(
|
|
25
|
+
default=None,
|
|
26
|
+
description="Sum of all cached tokens across all steps",
|
|
27
|
+
)
|
|
28
|
+
total_cost_usd: float | None = Field(
|
|
29
|
+
default=None,
|
|
30
|
+
description="Total real monetary cost for the entire trajectory, including cost for subagents, if any",
|
|
31
|
+
)
|
|
32
|
+
total_steps: int | None = Field(
|
|
33
|
+
default=None,
|
|
34
|
+
ge=0,
|
|
35
|
+
description=(
|
|
36
|
+
"Total number of steps. If not equivalent to the number of steps in the "
|
|
37
|
+
"trajectory, must be documented in the root-level notes field."
|
|
38
|
+
),
|
|
39
|
+
)
|
|
40
|
+
extra: dict[str, Any] | None = Field(
|
|
41
|
+
default=None,
|
|
42
|
+
description="Custom aggregate metrics",
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
model_config = {"extra": "forbid"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
# Vendored from harbor 0.24.0, src/harbor/models/trajectories/metrics.py
|
|
3
|
+
# (Apache-2.0, Copyright the Harbor authors). Byte for byte upstream apart
|
|
4
|
+
# from these header lines and the import path, which reads
|
|
5
|
+
# atif_converter.domain.atif where upstream reads harbor.models.trajectories.
|
|
6
|
+
"""Metrics model for ATIF trajectories."""
|
|
7
|
+
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from pydantic import BaseModel, Field
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Metrics(BaseModel):
|
|
14
|
+
"""LLM operational and confidence data."""
|
|
15
|
+
|
|
16
|
+
prompt_tokens: int | None = Field(
|
|
17
|
+
default=None,
|
|
18
|
+
description="Total input tokens including cached and non-cached",
|
|
19
|
+
)
|
|
20
|
+
completion_tokens: int | None = Field(
|
|
21
|
+
default=None,
|
|
22
|
+
description="Total tokens generated by the LLM response",
|
|
23
|
+
)
|
|
24
|
+
cached_tokens: int | None = Field(
|
|
25
|
+
default=None,
|
|
26
|
+
description="Subset of prompt_tokens that were cache hits",
|
|
27
|
+
)
|
|
28
|
+
cost_usd: float | None = Field(
|
|
29
|
+
default=None,
|
|
30
|
+
description="Monetary cost of the API call in USD",
|
|
31
|
+
)
|
|
32
|
+
prompt_token_ids: list[int] | None = Field(
|
|
33
|
+
default=None,
|
|
34
|
+
description="Token IDs for prompt (input) tokens sent to the LLM, including chat history (if applicable)",
|
|
35
|
+
)
|
|
36
|
+
completion_token_ids: list[int] | None = Field(
|
|
37
|
+
default=None,
|
|
38
|
+
description="Token IDs for completion (response) tokens, enabling RL training without retokenization drift",
|
|
39
|
+
)
|
|
40
|
+
logprobs: list[float] | None = Field(
|
|
41
|
+
default=None,
|
|
42
|
+
description="Log probability assigned to each generated token",
|
|
43
|
+
)
|
|
44
|
+
extra: dict[str, Any] | None = Field(
|
|
45
|
+
default=None,
|
|
46
|
+
description="Other metrics",
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
model_config = {"extra": "forbid"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
# Vendored from harbor 0.24.0, src/harbor/models/trajectories/observation.py
|
|
3
|
+
# (Apache-2.0, Copyright the Harbor authors). Byte for byte upstream apart
|
|
4
|
+
# from these header lines and the import path, which reads
|
|
5
|
+
# atif_converter.domain.atif where upstream reads harbor.models.trajectories.
|
|
6
|
+
"""Observation model for ATIF trajectories."""
|
|
7
|
+
|
|
8
|
+
from pydantic import BaseModel, Field
|
|
9
|
+
|
|
10
|
+
from atif_converter.domain.atif.observation_result import ObservationResult
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Observation(BaseModel):
|
|
14
|
+
"""Environment feedback/result after actions or system events."""
|
|
15
|
+
|
|
16
|
+
results: list[ObservationResult] = Field(
|
|
17
|
+
default=...,
|
|
18
|
+
description="Array of result objects from tool calls or actions",
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
model_config = {"extra": "forbid"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
# Vendored from harbor 0.24.0, src/harbor/models/trajectories/observation_result.py
|
|
3
|
+
# (Apache-2.0, Copyright the Harbor authors). Byte for byte upstream apart
|
|
4
|
+
# from these header lines and the import path, which reads
|
|
5
|
+
# atif_converter.domain.atif where upstream reads harbor.models.trajectories.
|
|
6
|
+
"""Observation result model for ATIF trajectories."""
|
|
7
|
+
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from pydantic import BaseModel, Field
|
|
11
|
+
|
|
12
|
+
from atif_converter.domain.atif.content import ContentPart
|
|
13
|
+
from atif_converter.domain.atif.subagent_trajectory_ref import SubagentTrajectoryRef
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class ObservationResult(BaseModel):
|
|
17
|
+
"""A single result within an observation."""
|
|
18
|
+
|
|
19
|
+
source_call_id: str | None = Field(
|
|
20
|
+
default=None,
|
|
21
|
+
description=(
|
|
22
|
+
"The `tool_call_id` from the _tool_calls_ array in _StepObject_ that this "
|
|
23
|
+
"result corresponds to. If null or omitted, the result comes from an "
|
|
24
|
+
"action that doesn't use the standard tool calling format (e.g., agent "
|
|
25
|
+
"actions without tool calls or system-initiated operations)."
|
|
26
|
+
),
|
|
27
|
+
)
|
|
28
|
+
content: str | list[ContentPart] | None = Field(
|
|
29
|
+
default=None,
|
|
30
|
+
description=(
|
|
31
|
+
"The output or result from the tool execution. String for text-only "
|
|
32
|
+
"content, or array of ContentPart for multimodal content (added in ATIF-v1.6)."
|
|
33
|
+
),
|
|
34
|
+
)
|
|
35
|
+
subagent_trajectory_ref: list[SubagentTrajectoryRef] | None = Field(
|
|
36
|
+
default=None,
|
|
37
|
+
description="Array of references to delegated subagent trajectories",
|
|
38
|
+
)
|
|
39
|
+
extra: dict[str, Any] | None = Field(
|
|
40
|
+
default=None,
|
|
41
|
+
description=(
|
|
42
|
+
"Custom observation-result-level metadata (e.g., confidence score, "
|
|
43
|
+
"retrieval score, source document ID). Added in ATIF-v1.7."
|
|
44
|
+
),
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
model_config = {"extra": "forbid"}
|