atif-sql 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- atif_analytics/__init__.py +9 -0
- atif_analytics/application/__init__.py +3 -0
- atif_analytics/application/analyze.py +158 -0
- atif_analytics/application/prompts.py +750 -0
- atif_analytics/application/use_cases/__init__.py +3 -0
- atif_analytics/application/use_cases/_shared.py +264 -0
- atif_analytics/application/use_cases/classify.py +430 -0
- atif_analytics/application/use_cases/conflicts.py +448 -0
- atif_analytics/application/use_cases/friction.py +634 -0
- atif_analytics/application/use_cases/perceived.py +451 -0
- atif_analytics/domain/__init__.py +3 -0
- atif_analytics/domain/authorship.py +166 -0
- atif_analytics/domain/config.py +30 -0
- atif_analytics/domain/costs.py +50 -0
- atif_analytics/domain/friction.py +117 -0
- atif_analytics/domain/layout.py +94 -0
- atif_analytics/domain/models.py +385 -0
- atif_analytics/domain/ports.py +100 -0
- atif_analytics/domain/transcript.py +359 -0
- atif_analytics/infrastructure/__init__.py +3 -0
- atif_analytics/infrastructure/corpus_reader.py +546 -0
- atif_analytics/infrastructure/parquet_cache.py +186 -0
- atif_analytics/infrastructure/settings.py +114 -0
- atif_analytics/infrastructure/sqlite_state/__init__.py +3 -0
- atif_analytics/infrastructure/sqlite_state/checkpointer.py +326 -0
- atif_analytics/infrastructure/sqlite_state/retry_queue.py +296 -0
- atif_analytics/py.typed +0 -0
- atif_cli/__init__.py +3 -0
- atif_cli/__main__.py +8 -0
- atif_cli/app.py +2459 -0
- atif_cli/converter_adapter.py +151 -0
- atif_cli/corpus.py +409 -0
- atif_cli/cron.py +311 -0
- atif_cli/duck_errors.py +97 -0
- atif_cli/embed_lake.py +101 -0
- atif_cli/errors.py +89 -0
- atif_cli/lake.py +373 -0
- atif_cli/lake_sessions.py +270 -0
- atif_cli/output.py +297 -0
- atif_cli/py.typed +0 -0
- atif_converter/__init__.py +3 -0
- atif_converter/application/__init__.py +3 -0
- atif_converter/application/convert_and_audit.py +294 -0
- atif_converter/application/convert_codex.py +210 -0
- atif_converter/domain/__init__.py +9 -0
- atif_converter/domain/agents.py +40 -0
- atif_converter/domain/atif/__init__.py +55 -0
- atif_converter/domain/atif/agent.py +37 -0
- atif_converter/domain/atif/content.py +161 -0
- atif_converter/domain/atif/final_metrics.py +45 -0
- atif_converter/domain/atif/metrics.py +49 -0
- atif_converter/domain/atif/observation.py +21 -0
- atif_converter/domain/atif/observation_result.py +47 -0
- atif_converter/domain/atif/step.py +144 -0
- atif_converter/domain/atif/subagent_trajectory_ref.py +90 -0
- atif_converter/domain/atif/tool_call.py +36 -0
- atif_converter/domain/atif/trajectory.py +211 -0
- atif_converter/domain/atif/trajectory_validator.py +299 -0
- atif_converter/domain/blobs.py +510 -0
- atif_converter/domain/claude_code_conversion.py +1077 -0
- atif_converter/domain/codex_conversion.py +1036 -0
- atif_converter/domain/codex_edges.py +192 -0
- atif_converter/domain/codex_enrichment.py +419 -0
- atif_converter/domain/codex_fidelity.py +124 -0
- atif_converter/domain/codex_nested_calls.py +165 -0
- atif_converter/domain/edges.py +113 -0
- atif_converter/domain/enrichment.py +412 -0
- atif_converter/domain/errors.py +60 -0
- atif_converter/domain/fidelity.py +171 -0
- atif_converter/domain/model_prices.json +8538 -0
- atif_converter/domain/pricing.py +790 -0
- atif_converter/domain/result_signals.py +774 -0
- atif_converter/domain/schema_version.py +90 -0
- atif_converter/domain/session_events.py +408 -0
- atif_converter/infrastructure/__init__.py +3 -0
- atif_converter/infrastructure/census.py +89 -0
- atif_converter/infrastructure/claude_code_converter.py +127 -0
- atif_converter/infrastructure/codex_adapter.py +132 -0
- atif_converter/infrastructure/codex_census.py +100 -0
- atif_converter/infrastructure/codex_converter.py +82 -0
- atif_converter/infrastructure/harbor_adapter.py +187 -0
- atif_converter/infrastructure/raw_records.py +426 -0
- atif_converter/infrastructure/source_archive.py +154 -0
- atif_converter/py.typed +0 -0
- atif_corpus/__init__.py +13 -0
- atif_corpus/application/__init__.py +7 -0
- atif_corpus/application/materialize.py +1826 -0
- atif_corpus/domain/__init__.py +9 -0
- atif_corpus/domain/agents.py +33 -0
- atif_corpus/domain/generation.py +34 -0
- atif_corpus/domain/layout.py +168 -0
- atif_corpus/domain/ports.py +175 -0
- atif_corpus/domain/session_id.py +68 -0
- atif_corpus/domain/sessions.py +228 -0
- atif_corpus/domain/slug.py +72 -0
- atif_corpus/domain/source_layout.py +143 -0
- atif_corpus/domain/watermark.py +76 -0
- atif_corpus/infrastructure/__init__.py +7 -0
- atif_corpus/infrastructure/atomic.py +266 -0
- atif_corpus/infrastructure/compress_artifacts.py +161 -0
- atif_corpus/infrastructure/fake_converter.py +174 -0
- atif_corpus/infrastructure/scanner.py +316 -0
- atif_corpus/infrastructure/settings.py +130 -0
- atif_corpus/infrastructure/source_archive.py +202 -0
- atif_corpus/py.typed +0 -0
- atif_duck/__init__.py +43 -0
- atif_duck/domain/__init__.py +32 -0
- atif_duck/domain/artifacts.py +66 -0
- atif_duck/domain/authorship.py +165 -0
- atif_duck/domain/catalog.py +626 -0
- atif_duck/domain/columnar.py +149 -0
- atif_duck/domain/embedding_guard.py +88 -0
- atif_duck/domain/examples.py +265 -0
- atif_duck/domain/lake.py +427 -0
- atif_duck/domain/lake_sessions.py +97 -0
- atif_duck/domain/raw_readers.py +140 -0
- atif_duck/domain/session_id.py +69 -0
- atif_duck/domain/sql_literal.py +50 -0
- atif_duck/infrastructure/__init__.py +34 -0
- atif_duck/infrastructure/analytics.py +469 -0
- atif_duck/infrastructure/authorship.py +139 -0
- atif_duck/infrastructure/columnar.py +641 -0
- atif_duck/infrastructure/lake.py +1383 -0
- atif_duck/infrastructure/lake_sessions.py +194 -0
- atif_duck/infrastructure/lake_settings.py +63 -0
- atif_duck/infrastructure/lake_steps.py +225 -0
- atif_duck/infrastructure/projections.py +221 -0
- atif_duck/infrastructure/registry.py +2194 -0
- atif_duck/infrastructure/stored_artifacts.py +115 -0
- atif_duck/py.typed +0 -0
- atif_embed/__init__.py +12 -0
- atif_embed/application/__init__.py +3 -0
- atif_embed/application/embed.py +293 -0
- atif_embed/application/prune.py +80 -0
- atif_embed/domain/__init__.py +3 -0
- atif_embed/domain/discovery.py +70 -0
- atif_embed/domain/embedding_guard.py +83 -0
- atif_embed/domain/errors.py +94 -0
- atif_embed/domain/ports.py +223 -0
- atif_embed/domain/sql_literal.py +49 -0
- atif_embed/domain/text_stamp.py +71 -0
- atif_embed/infrastructure/__init__.py +3 -0
- atif_embed/infrastructure/cohere_bedrock.py +433 -0
- atif_embed/infrastructure/corpus_text_rows.py +273 -0
- atif_embed/infrastructure/gemma_local.py +355 -0
- atif_embed/infrastructure/lake_text_rows.py +203 -0
- atif_embed/infrastructure/lance_store.py +440 -0
- atif_embed/infrastructure/providers.py +30 -0
- atif_embed/infrastructure/settings.py +154 -0
- atif_embed/py.typed +0 -0
- atif_models/__init__.py +14 -0
- atif_models/domain/__init__.py +9 -0
- atif_models/domain/ports.py +143 -0
- atif_models/domain/registry.py +151 -0
- atif_models/domain/schema.py +109 -0
- atif_models/infrastructure/__init__.py +8 -0
- atif_models/infrastructure/openai_bedrock.py +314 -0
- atif_models/infrastructure/settings.py +85 -0
- atif_models/py.typed +0 -0
- atif_sql-0.2.0.dist-info/METADATA +272 -0
- atif_sql-0.2.0.dist-info/RECORD +164 -0
- atif_sql-0.2.0.dist-info/WHEEL +4 -0
- atif_sql-0.2.0.dist-info/entry_points.txt +2 -0
- atif_sql-0.2.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
|
|
3
|
+
"""The converter-owned version of what a conversion produces.
|
|
4
|
+
|
|
5
|
+
The package version (``converter_version`` in ``meta.json``) moves with every
|
|
6
|
+
release, including ones that change nothing a corpus holds. This number moves
|
|
7
|
+
only when the artifacts a conversion writes change shape or meaning, so a
|
|
8
|
+
materialize pass can re-convert exactly the sessions an older converter wrote:
|
|
9
|
+
materialize stamps it into every ``meta.json`` as ``converter_schema`` and
|
|
10
|
+
treats a session recording a different value as stale, whether or not its
|
|
11
|
+
source moved. A source-removed session re-converts from its raw source archive.
|
|
12
|
+
|
|
13
|
+
THE RULE: bump this in the same commit as any change that can alter a byte of
|
|
14
|
+
any artifact for any input. A refactor that provably changes no output does not
|
|
15
|
+
bump it. ``packages/atif-converter/tests/test_converter_schema_version.py`` pins
|
|
16
|
+
a digest of the converter's code (docstrings and comments excluded) beside this
|
|
17
|
+
value, so any converter code change fails that test until someone decides:
|
|
18
|
+
bump if output can change, then re-pin the digest either way.
|
|
19
|
+
|
|
20
|
+
History:
|
|
21
|
+
|
|
22
|
+
* 1 — every converter before this constant existed (implicit).
|
|
23
|
+
* 2 — ``session_events.jsonl`` (hooks, injected context, API errors,
|
|
24
|
+
compaction boundaries, cost-state, mode changes); ``records_captured`` in
|
|
25
|
+
``loss_report.json`` and the structural gaps reported only when enrichment
|
|
26
|
+
didn't repair them; an unpriced model yields ``total_cost_usd`` NULL instead
|
|
27
|
+
of $0; ``final_metrics.extra.reported_cost_usd`` from Claude Code's
|
|
28
|
+
``cost-state``; local price overrides for claude-opus-5-5 / claude-fable-5-1.
|
|
29
|
+
In the same version: inline base64 attachments replaced by blob placeholders
|
|
30
|
+
(and the bytes handed out for the corpus blob store); typed ``is_error`` /
|
|
31
|
+
``exit_code`` / ``interrupted`` / ``images`` on observation results;
|
|
32
|
+
``agent_id`` read from ``agentId`` and filled on every sidechain step;
|
|
33
|
+
``trajectory.extra.subagents`` from the ``agent-*.meta.json`` sidecars.
|
|
34
|
+
Both change sets ship together and no corpus was ever stamped with only one
|
|
35
|
+
of them, so one bump separates every version-1 session from every version-2
|
|
36
|
+
one.
|
|
37
|
+
* 3 — harbor 0.23.0: every priced Claude Code step carries
|
|
38
|
+
``metrics.cost_usd`` and ``metrics.extra.cost_source`` (an unpriced step
|
|
39
|
+
carries neither), and ``meta.json`` records ``harbor_version`` 0.23.0. Prices
|
|
40
|
+
from litellm v1.102.0, which also prices claude-fable-5-1, so its local
|
|
41
|
+
override retired and its sessions are labeled ``litellm_estimate``.
|
|
42
|
+
* 4 — Codex ``is_error`` / ``exit_code`` for code-mode ``exec`` scripts, from
|
|
43
|
+
the script's own header and the items its nested commands, MCP calls and
|
|
44
|
+
patches complete; ``is_error`` from the ``status`` of ``FileChange`` and
|
|
45
|
+
``CollabAgentToolCall`` items.
|
|
46
|
+
* 5 — Claude Code ``agent.extra`` lists (``cwds``, ``git_branches``,
|
|
47
|
+
``agent_ids``) in first-seen order instead of set order, so a session
|
|
48
|
+
converts to the same bytes in every process.
|
|
49
|
+
* 6 — prices from litellm v1.103.2, whose arithmetic bills cache writes and
|
|
50
|
+
cache reads at the input rate for a model that publishes no cache-creation
|
|
51
|
+
or cache-read rate (every OpenAI text model's cache writes, and cached
|
|
52
|
+
tokens of the 46 entries with no cache-read rate), where v1.102.0 billed
|
|
53
|
+
them $0. The table also adds flex rates for gpt-5.1 and gpt-5.2 and moves
|
|
54
|
+
o4-mini's flex cache-read rate. ``total_cost_usd`` and ``metrics.cost_usd``
|
|
55
|
+
change for every such session.
|
|
56
|
+
* 7 — harbor 0.24.0: ``meta.json`` records ``harbor_version`` 0.24.0. The
|
|
57
|
+
Claude Code port itself puts each event's ``agentId`` on every step the
|
|
58
|
+
event becomes (prompts, tool results and turns, which enrichment filled
|
|
59
|
+
before) and writes ``is_sidechain`` and ``agent_id`` right after ``id`` in
|
|
60
|
+
``extra``, harbor's key order; a snake_case ``agent_id`` key names no
|
|
61
|
+
subagent (harbor #3434). A Codex tool output that is a list of content blocks or
|
|
62
|
+
items is its text parts joined by newlines, with an inline image as its blob
|
|
63
|
+
placeholder, where it was Python's ``str()`` of the list; a non-string
|
|
64
|
+
``output`` inside an output object is its JSON text; a JSON scalar output
|
|
65
|
+
keeps its raw text (harbor #3467). A Codex ``web_search_call`` carries its
|
|
66
|
+
item id, or ``<api call>_web_search_<n>``, where its id was empty (harbor
|
|
67
|
+
#2972).
|
|
68
|
+
* 8 — every Codex tool output image the port writes a blob placeholder for is
|
|
69
|
+
lifted by the pre-pass too: an MCP ``image`` block with inline base64 (by
|
|
70
|
+
``mimeType`` or ``mime_type``), and an ``input_image`` whose data URL
|
|
71
|
+
carries parameters or whose base64 is wrapped. Its bytes reach the blob
|
|
72
|
+
store, its result carries ``extra.images``, and a tool output list the port
|
|
73
|
+
dumps as JSON (one element of an unknown type) holds the placeholder item
|
|
74
|
+
where it held the base64. Placeholder text is unchanged.
|
|
75
|
+
* 9 — a Codex code-mode ``exec`` script's nested MCP calls and commands (the
|
|
76
|
+
``McpToolCall`` / ``CommandExecution`` items attributed to it) are tool
|
|
77
|
+
calls of their own on its step, after the ``exec`` call, with
|
|
78
|
+
``extra.nested_in``, ``via`` and ``duration_ms``, and one result each
|
|
79
|
+
carrying its content and its own ``is_error`` / ``exit_code`` (and
|
|
80
|
+
``images``: the pre-pass lifts images out of those items' MCP results too).
|
|
81
|
+
The ``exec`` result's own fields are unchanged.
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
from __future__ import annotations
|
|
85
|
+
|
|
86
|
+
from typing import Final
|
|
87
|
+
|
|
88
|
+
CONVERTER_SCHEMA_VERSION: Final = 9
|
|
89
|
+
|
|
90
|
+
__all__ = ["CONVERTER_SCHEMA_VERSION"]
|
|
@@ -0,0 +1,408 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
|
|
3
|
+
"""session_events.jsonl: the non-message records worth keeping, one typed row each.
|
|
4
|
+
|
|
5
|
+
Pure functions over already-parsed ``(record, source_file)`` pairs; no harbor,
|
|
6
|
+
no I/O. The step converter keeps user and assistant records only
|
|
7
|
+
(:attr:`~atif_converter.domain.fidelity.FidelityGap.NON_MESSAGE_RECORDS_DROPPED`),
|
|
8
|
+
which loses records that say what happened AROUND the turns: hooks that ran,
|
|
9
|
+
blocked or were cancelled, context a hook or a queued command injected into
|
|
10
|
+
the model's view, API errors, compaction boundaries, refusal fallbacks,
|
|
11
|
+
Claude Code's own running cost, and mode changes. This module picks those out
|
|
12
|
+
and writes each one as a row. They're deliberately NOT ATIF steps: the steps
|
|
13
|
+
view and everything built on it stay exactly as they were.
|
|
14
|
+
|
|
15
|
+
ROW SHAPE (:data:`SESSION_EVENT_FIELDS`, key order stable)::
|
|
16
|
+
|
|
17
|
+
{
|
|
18
|
+
seq,
|
|
19
|
+
ts,
|
|
20
|
+
event_type,
|
|
21
|
+
subtype,
|
|
22
|
+
uuid,
|
|
23
|
+
parent_uuid,
|
|
24
|
+
tool_use_id,
|
|
25
|
+
is_sidechain,
|
|
26
|
+
source_file,
|
|
27
|
+
payload,
|
|
28
|
+
payload_bytes,
|
|
29
|
+
payload_truncated,
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
``event_type`` is the raw record ``type`` (``system``, ``attachment``,
|
|
33
|
+
``cost-state``, ``mode``, ``permission-mode``; for Codex ``compacted`` or
|
|
34
|
+
``event_msg``) and ``subtype`` names the kind inside it (``system.subtype``,
|
|
35
|
+
``attachment.type``, a Codex ``payload.type``; ``None`` where the record type
|
|
36
|
+
is the whole story). ``seq`` is the row's position in the session's record
|
|
37
|
+
order (main file first, then side files, as read), because ``cost-state`` and
|
|
38
|
+
``mode`` records carry no timestamp. ``parent_uuid`` is the record's
|
|
39
|
+
``parentUuid``: for an attachment it's the record the attachment follows,
|
|
40
|
+
which is how a row joins back to ``steps.source_uuids``.
|
|
41
|
+
|
|
42
|
+
BOUNDED PAYLOAD. ``payload`` is the record minus its envelope (the columns
|
|
43
|
+
above plus per-record noise such as ``cwd`` and ``version``). Strings longer
|
|
44
|
+
than :data:`PAYLOAD_STRING_MAX_CHARS` and lists longer than
|
|
45
|
+
:data:`PAYLOAD_LIST_MAX_ITEMS` are cut, and a payload still over
|
|
46
|
+
:data:`PAYLOAD_MAX_BYTES` becomes ``{"truncated_preview": <first bytes>}``.
|
|
47
|
+
``payload_bytes`` is always the ORIGINAL payload's UTF-8 JSON length and
|
|
48
|
+
``payload_truncated`` says whether anything was cut, so a reader can tell a
|
|
49
|
+
short payload from a shortened one.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
from __future__ import annotations
|
|
53
|
+
|
|
54
|
+
import json
|
|
55
|
+
from collections.abc import Iterable
|
|
56
|
+
from typing import Any, Final
|
|
57
|
+
|
|
58
|
+
from atif_converter.domain.codex_edges import record_id
|
|
59
|
+
|
|
60
|
+
#: Stable key order for one session_events.jsonl line.
|
|
61
|
+
SESSION_EVENT_FIELDS: Final = (
|
|
62
|
+
"seq",
|
|
63
|
+
"ts",
|
|
64
|
+
"event_type",
|
|
65
|
+
"subtype",
|
|
66
|
+
"uuid",
|
|
67
|
+
"parent_uuid",
|
|
68
|
+
"tool_use_id",
|
|
69
|
+
"is_sidechain",
|
|
70
|
+
"source_file",
|
|
71
|
+
"payload",
|
|
72
|
+
"payload_bytes",
|
|
73
|
+
"payload_truncated",
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
#: Hard ceiling for one serialized payload, in UTF-8 bytes.
|
|
77
|
+
PAYLOAD_MAX_BYTES: Final = 8192
|
|
78
|
+
#: Any string leaf longer than this is cut to it.
|
|
79
|
+
PAYLOAD_STRING_MAX_CHARS: Final = 2048
|
|
80
|
+
#: Any list longer than this keeps its first items only.
|
|
81
|
+
PAYLOAD_LIST_MAX_ITEMS: Final = 32
|
|
82
|
+
#: Nesting below this depth is replaced by its JSON text, then cut as a string.
|
|
83
|
+
_PAYLOAD_MAX_DEPTH: Final = 8
|
|
84
|
+
_ELLIPSIS: Final = "…"
|
|
85
|
+
|
|
86
|
+
# --------------------------------------------------------------------------- Claude Code
|
|
87
|
+
|
|
88
|
+
#: ``system`` records kept, by ``subtype``.
|
|
89
|
+
CLAUDE_SYSTEM_SUBTYPES: Final = frozenset(
|
|
90
|
+
{"stop_hook_summary", "api_error", "compact_boundary", "model_refusal_fallback"}
|
|
91
|
+
)
|
|
92
|
+
#: ``attachment`` records kept, by ``attachment.type``, besides every ``hook_*`` type.
|
|
93
|
+
CLAUDE_ATTACHMENT_TYPES: Final = frozenset({"queued_command"})
|
|
94
|
+
_CLAUDE_HOOK_ATTACHMENT_PREFIX: Final = "hook_"
|
|
95
|
+
#: Record types kept whole (no subtype).
|
|
96
|
+
CLAUDE_WHOLE_RECORD_TYPES: Final = frozenset({"cost-state", "mode", "permission-mode"})
|
|
97
|
+
|
|
98
|
+
#: Record keys that are columns or per-record noise, never payload.
|
|
99
|
+
_CLAUDE_ENVELOPE: Final = frozenset(
|
|
100
|
+
{
|
|
101
|
+
"type",
|
|
102
|
+
"subtype",
|
|
103
|
+
"uuid",
|
|
104
|
+
"parentUuid",
|
|
105
|
+
"timestamp",
|
|
106
|
+
"isSidechain",
|
|
107
|
+
"toolUseID",
|
|
108
|
+
"sessionId",
|
|
109
|
+
"userType",
|
|
110
|
+
"entrypoint",
|
|
111
|
+
"cwd",
|
|
112
|
+
"version",
|
|
113
|
+
"gitBranch",
|
|
114
|
+
"slug",
|
|
115
|
+
"isMeta",
|
|
116
|
+
"attachment",
|
|
117
|
+
"rendered",
|
|
118
|
+
}
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
# --------------------------------------------------------------------------- Codex
|
|
122
|
+
|
|
123
|
+
#: Top-level Codex record types kept whole.
|
|
124
|
+
CODEX_WHOLE_RECORD_TYPES: Final = frozenset({"compacted"})
|
|
125
|
+
#: ``event_msg`` records kept, by ``payload.type``.
|
|
126
|
+
CODEX_EVENT_MSG_TYPES: Final = frozenset(
|
|
127
|
+
{"turn_aborted", "error", "stream_error", "context_compacted", "warning"}
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _bounded_container(value: dict[str, Any] | list[Any], depth: int) -> tuple[Any, bool]:
|
|
132
|
+
if depth >= _PAYLOAD_MAX_DEPTH:
|
|
133
|
+
text = json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
|
134
|
+
return _bounded(text, depth)[0], True
|
|
135
|
+
if isinstance(value, dict):
|
|
136
|
+
cut = False
|
|
137
|
+
out: dict[str, Any] = {}
|
|
138
|
+
for key, item in value.items():
|
|
139
|
+
out[str(key)], item_cut = _bounded(item, depth + 1)
|
|
140
|
+
cut = cut or item_cut
|
|
141
|
+
return out, cut
|
|
142
|
+
cut = len(value) > PAYLOAD_LIST_MAX_ITEMS
|
|
143
|
+
items: list[Any] = []
|
|
144
|
+
for item in value[:PAYLOAD_LIST_MAX_ITEMS]:
|
|
145
|
+
bounded_item, item_cut = _bounded(item, depth + 1)
|
|
146
|
+
items.append(bounded_item)
|
|
147
|
+
cut = cut or item_cut
|
|
148
|
+
return items, cut
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _bounded(value: Any, depth: int = 0) -> tuple[Any, bool]:
|
|
152
|
+
"""``value`` with long strings and lists cut; the flag says whether anything was."""
|
|
153
|
+
if isinstance(value, str):
|
|
154
|
+
if len(value) > PAYLOAD_STRING_MAX_CHARS:
|
|
155
|
+
return value[:PAYLOAD_STRING_MAX_CHARS] + _ELLIPSIS, True
|
|
156
|
+
return value, False
|
|
157
|
+
if isinstance(value, dict | list):
|
|
158
|
+
return _bounded_container(value, depth)
|
|
159
|
+
return value, False
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def bound_payload(payload: dict[str, Any]) -> tuple[dict[str, Any], int, bool]:
|
|
163
|
+
"""``(bounded payload, original UTF-8 JSON length, truncated?)`` for one event."""
|
|
164
|
+
original = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
|
|
165
|
+
original_bytes = len(original.encode("utf-8"))
|
|
166
|
+
if original_bytes <= PAYLOAD_MAX_BYTES:
|
|
167
|
+
return payload, original_bytes, False
|
|
168
|
+
bounded, _cut = _bounded(payload)
|
|
169
|
+
text = json.dumps(bounded, ensure_ascii=False, separators=(",", ":"))
|
|
170
|
+
if len(text.encode("utf-8")) <= PAYLOAD_MAX_BYTES:
|
|
171
|
+
return bounded, original_bytes, True
|
|
172
|
+
preview = original.encode("utf-8")[:PAYLOAD_MAX_BYTES].decode("utf-8", errors="ignore")
|
|
173
|
+
return {"truncated_preview": preview}, original_bytes, True
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _str_or_none(value: Any) -> str | None:
|
|
177
|
+
return value if isinstance(value, str) and value else None
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _claude_subtype(record: dict[str, Any], record_type: str) -> str | None:
|
|
181
|
+
"""The kept subtype of a ``system`` / ``attachment`` record, else ``None``."""
|
|
182
|
+
if record_type == "system":
|
|
183
|
+
subtype = record.get("subtype")
|
|
184
|
+
return subtype if isinstance(subtype, str) and subtype in CLAUDE_SYSTEM_SUBTYPES else None
|
|
185
|
+
if record_type == "attachment":
|
|
186
|
+
attachment = record.get("attachment")
|
|
187
|
+
kind = attachment.get("type") if isinstance(attachment, dict) else None
|
|
188
|
+
if isinstance(kind, str) and (
|
|
189
|
+
kind.startswith(_CLAUDE_HOOK_ATTACHMENT_PREFIX) or kind in CLAUDE_ATTACHMENT_TYPES
|
|
190
|
+
):
|
|
191
|
+
return kind
|
|
192
|
+
return None
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _claude_kind(record: dict[str, Any]) -> tuple[str, str | None] | None:
|
|
196
|
+
"""``(event_type, subtype)`` when a Claude Code record is kept, else ``None``."""
|
|
197
|
+
record_type = record.get("type")
|
|
198
|
+
if not isinstance(record_type, str):
|
|
199
|
+
return None
|
|
200
|
+
if record_type in CLAUDE_WHOLE_RECORD_TYPES:
|
|
201
|
+
return record_type, None
|
|
202
|
+
subtype = _claude_subtype(record, record_type)
|
|
203
|
+
return None if subtype is None else (record_type, subtype)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def is_claude_session_event(record: dict[str, Any]) -> bool:
|
|
207
|
+
"""Whether a Claude Code record becomes a session_events row."""
|
|
208
|
+
return _claude_kind(record) is not None
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _claude_payload(record: dict[str, Any]) -> dict[str, Any]:
|
|
212
|
+
payload = {key: value for key, value in record.items() if key not in _CLAUDE_ENVELOPE}
|
|
213
|
+
attachment = record.get("attachment")
|
|
214
|
+
if isinstance(attachment, dict):
|
|
215
|
+
payload.update(
|
|
216
|
+
{key: value for key, value in attachment.items() if key not in {"type", "toolUseID"}}
|
|
217
|
+
)
|
|
218
|
+
return payload
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _claude_tool_use_id(record: dict[str, Any]) -> str | None:
|
|
222
|
+
own = _str_or_none(record.get("toolUseID"))
|
|
223
|
+
if own is not None:
|
|
224
|
+
return own
|
|
225
|
+
attachment = record.get("attachment")
|
|
226
|
+
return _str_or_none(attachment.get("toolUseID")) if isinstance(attachment, dict) else None
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _row(
|
|
230
|
+
seq: int,
|
|
231
|
+
*,
|
|
232
|
+
ts: Any,
|
|
233
|
+
kind: tuple[str, str | None],
|
|
234
|
+
uuid: str | None,
|
|
235
|
+
parent_uuid: str | None,
|
|
236
|
+
tool_use_id: str | None,
|
|
237
|
+
is_sidechain: bool,
|
|
238
|
+
source_file: str,
|
|
239
|
+
payload: dict[str, Any],
|
|
240
|
+
) -> dict[str, Any]:
|
|
241
|
+
bounded, original_bytes, truncated = bound_payload(payload)
|
|
242
|
+
return {
|
|
243
|
+
"seq": seq,
|
|
244
|
+
"ts": _str_or_none(ts),
|
|
245
|
+
"event_type": kind[0],
|
|
246
|
+
"subtype": kind[1],
|
|
247
|
+
"uuid": uuid,
|
|
248
|
+
"parent_uuid": parent_uuid,
|
|
249
|
+
"tool_use_id": tool_use_id,
|
|
250
|
+
"is_sidechain": is_sidechain,
|
|
251
|
+
"source_file": source_file,
|
|
252
|
+
"payload": bounded,
|
|
253
|
+
"payload_bytes": original_bytes,
|
|
254
|
+
"payload_truncated": truncated,
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def claude_session_events(records: Iterable[tuple[dict[str, Any], str]]) -> list[dict[str, Any]]:
|
|
259
|
+
"""The kept Claude Code records as rows, in the order the pairs arrive."""
|
|
260
|
+
rows: list[dict[str, Any]] = []
|
|
261
|
+
for record, source_file in records:
|
|
262
|
+
kind = _claude_kind(record)
|
|
263
|
+
if kind is None:
|
|
264
|
+
continue
|
|
265
|
+
rows.append(
|
|
266
|
+
_row(
|
|
267
|
+
len(rows),
|
|
268
|
+
ts=record.get("timestamp"),
|
|
269
|
+
kind=kind,
|
|
270
|
+
uuid=_str_or_none(record.get("uuid")),
|
|
271
|
+
parent_uuid=_str_or_none(record.get("parentUuid")),
|
|
272
|
+
tool_use_id=_claude_tool_use_id(record),
|
|
273
|
+
is_sidechain=bool(record.get("isSidechain", False)),
|
|
274
|
+
source_file=source_file,
|
|
275
|
+
payload=_claude_payload(record),
|
|
276
|
+
)
|
|
277
|
+
)
|
|
278
|
+
return rows
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _codex_kind(record: dict[str, Any]) -> tuple[str, str | None] | None:
|
|
282
|
+
record_type = record.get("type")
|
|
283
|
+
if not isinstance(record_type, str):
|
|
284
|
+
return None
|
|
285
|
+
if record_type in CODEX_WHOLE_RECORD_TYPES:
|
|
286
|
+
return record_type, None
|
|
287
|
+
payload = record.get("payload")
|
|
288
|
+
if record_type == "event_msg" and isinstance(payload, dict):
|
|
289
|
+
kind = payload.get("type")
|
|
290
|
+
if isinstance(kind, str) and kind in CODEX_EVENT_MSG_TYPES:
|
|
291
|
+
return record_type, kind
|
|
292
|
+
return None
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def is_codex_session_event(record: dict[str, Any]) -> bool:
|
|
296
|
+
"""Whether a Codex rollout record becomes a session_events row."""
|
|
297
|
+
return _codex_kind(record) is not None
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def codex_session_events(records: Iterable[tuple[dict[str, Any], str]]) -> list[dict[str, Any]]:
|
|
301
|
+
"""The kept Codex records as rows, in file order.
|
|
302
|
+
|
|
303
|
+
``uuid`` is the same id ``edges.jsonl`` gives the record
|
|
304
|
+
(:func:`~atif_converter.domain.codex_edges.record_id`, line-numbered per
|
|
305
|
+
source file), so a row joins ``messages`` on it. A rollout has no parent
|
|
306
|
+
pointer and no sidechain, so those columns are ``None`` / ``False``.
|
|
307
|
+
"""
|
|
308
|
+
rows: list[dict[str, Any]] = []
|
|
309
|
+
line_numbers: dict[str, int] = {}
|
|
310
|
+
for record, source_file in records:
|
|
311
|
+
line_numbers[source_file] = line_numbers.get(source_file, 0) + 1
|
|
312
|
+
kind = _codex_kind(record)
|
|
313
|
+
if kind is None:
|
|
314
|
+
continue
|
|
315
|
+
payload = record.get("payload")
|
|
316
|
+
body = (
|
|
317
|
+
{key: value for key, value in payload.items() if key != "type"}
|
|
318
|
+
if isinstance(payload, dict)
|
|
319
|
+
else {}
|
|
320
|
+
)
|
|
321
|
+
rows.append(
|
|
322
|
+
_row(
|
|
323
|
+
len(rows),
|
|
324
|
+
ts=record.get("timestamp"),
|
|
325
|
+
kind=kind,
|
|
326
|
+
uuid=record_id(record, line_numbers[source_file]),
|
|
327
|
+
parent_uuid=None,
|
|
328
|
+
tool_use_id=None,
|
|
329
|
+
is_sidechain=False,
|
|
330
|
+
source_file=source_file,
|
|
331
|
+
payload=body,
|
|
332
|
+
)
|
|
333
|
+
)
|
|
334
|
+
return rows
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
#: ``final_metrics.extra`` keys the reported cost lands under.
|
|
338
|
+
REPORTED_COST_KEY: Final = "reported_cost_usd"
|
|
339
|
+
REPORTED_COST_SOURCE_KEY: Final = "reported_cost_source"
|
|
340
|
+
REPORTED_UNKNOWN_MODEL_COST_KEY: Final = "reported_cost_has_unknown_model_cost"
|
|
341
|
+
#: ``reported_cost_source`` for a figure read off a Claude Code ``cost-state`` record.
|
|
342
|
+
REPORTED_COST_SOURCE_COST_STATE: Final = "claude_code_cost_state"
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def claude_reported_cost(records: Iterable[tuple[dict[str, Any], str]]) -> dict[str, Any]:
|
|
346
|
+
"""What Claude Code itself says the session cost, for ``final_metrics.extra``.
|
|
347
|
+
|
|
348
|
+
Claude Code appends a ``cost-state`` record as the session runs; its
|
|
349
|
+
``totalCostUSD`` is cumulative across resumes (measured 2026-09-27: the
|
|
350
|
+
value only grows and ``startTime`` stays fixed across a resume), so the
|
|
351
|
+
LAST one in record order is the session's figure. Returns ``{}`` when no
|
|
352
|
+
record carries a numeric ``totalCostUSD``. ``hasUnknownModelCost`` is kept
|
|
353
|
+
beside it because Claude Code sets it when it met a model it couldn't
|
|
354
|
+
price, which makes its own total a lower bound.
|
|
355
|
+
"""
|
|
356
|
+
last: dict[str, Any] | None = None
|
|
357
|
+
for record, _source_file in records:
|
|
358
|
+
if record.get("type") != "cost-state" or record.get("isSidechain"):
|
|
359
|
+
continue
|
|
360
|
+
total = record.get("totalCostUSD")
|
|
361
|
+
if isinstance(total, int | float) and not isinstance(total, bool):
|
|
362
|
+
last = record
|
|
363
|
+
if last is None:
|
|
364
|
+
return {}
|
|
365
|
+
reported: dict[str, Any] = {
|
|
366
|
+
REPORTED_COST_KEY: float(last["totalCostUSD"]),
|
|
367
|
+
REPORTED_COST_SOURCE_KEY: REPORTED_COST_SOURCE_COST_STATE,
|
|
368
|
+
}
|
|
369
|
+
unknown = last.get("hasUnknownModelCost")
|
|
370
|
+
if isinstance(unknown, bool):
|
|
371
|
+
reported[REPORTED_UNKNOWN_MODEL_COST_KEY] = unknown
|
|
372
|
+
return reported
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def session_event_to_jsonl_line(row: dict[str, Any]) -> str:
|
|
376
|
+
"""Serialize one row as a compact, key-order-stable JSONL line."""
|
|
377
|
+
ordered = {key: row.get(key) for key in SESSION_EVENT_FIELDS}
|
|
378
|
+
return json.dumps(ordered, ensure_ascii=False, separators=(",", ":"))
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def session_events_jsonl_lines(rows: Iterable[dict[str, Any]]) -> list[str]:
|
|
382
|
+
"""The ready-to-write session_events.jsonl body."""
|
|
383
|
+
return [session_event_to_jsonl_line(row) for row in rows]
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
__all__ = [
|
|
387
|
+
"CLAUDE_ATTACHMENT_TYPES",
|
|
388
|
+
"CLAUDE_SYSTEM_SUBTYPES",
|
|
389
|
+
"CLAUDE_WHOLE_RECORD_TYPES",
|
|
390
|
+
"CODEX_EVENT_MSG_TYPES",
|
|
391
|
+
"CODEX_WHOLE_RECORD_TYPES",
|
|
392
|
+
"PAYLOAD_LIST_MAX_ITEMS",
|
|
393
|
+
"PAYLOAD_MAX_BYTES",
|
|
394
|
+
"PAYLOAD_STRING_MAX_CHARS",
|
|
395
|
+
"REPORTED_COST_KEY",
|
|
396
|
+
"REPORTED_COST_SOURCE_COST_STATE",
|
|
397
|
+
"REPORTED_COST_SOURCE_KEY",
|
|
398
|
+
"REPORTED_UNKNOWN_MODEL_COST_KEY",
|
|
399
|
+
"SESSION_EVENT_FIELDS",
|
|
400
|
+
"bound_payload",
|
|
401
|
+
"claude_reported_cost",
|
|
402
|
+
"claude_session_events",
|
|
403
|
+
"codex_session_events",
|
|
404
|
+
"is_claude_session_event",
|
|
405
|
+
"is_codex_session_event",
|
|
406
|
+
"session_event_to_jsonl_line",
|
|
407
|
+
"session_events_jsonl_lines",
|
|
408
|
+
]
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
|
|
3
|
+
"""Raw-side census of a Claude Code session: the input half of a LossReport.
|
|
4
|
+
|
|
5
|
+
Counts come from records already parsed out of a
|
|
6
|
+
:class:`~atif_converter.infrastructure.raw_records.SessionSnapshot`, and the
|
|
7
|
+
side-file classification is derived from that same snapshot's file list — no
|
|
8
|
+
second parse, no disk access, no harbor, no env, no network. Both halves read
|
|
9
|
+
from the one snapshot, so the census, the trajectory, and edges.jsonl
|
|
10
|
+
describe the same bytes of a session that may still be growing.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from collections.abc import Iterable
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from atif_converter.domain.fidelity import RecordType
|
|
21
|
+
from atif_converter.infrastructure.raw_records import SessionSnapshot
|
|
22
|
+
|
|
23
|
+
_KNOWN_TYPES: dict[str, RecordType] = {
|
|
24
|
+
rt.value: rt for rt in RecordType if rt is not RecordType.OTHER
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
#: harbor's ``rglob("subagents/*.jsonl")`` discovers a side-file only when this
|
|
28
|
+
#: is the name of its immediate parent directory.
|
|
29
|
+
_HARBOR_SUBAGENT_DIR = "subagents"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True, slots=True)
|
|
33
|
+
class SessionCensus:
|
|
34
|
+
"""Raw record counts for one session (main file + every side-file).
|
|
35
|
+
|
|
36
|
+
``record_counts`` covers EVERY record of EVERY discovered ``*.jsonl``, so
|
|
37
|
+
it agrees line-for-line with edges.jsonl. ``subagent_files`` are the
|
|
38
|
+
side-files harbor 0.22.0's ``rglob("subagents/*.jsonl")`` can discover;
|
|
39
|
+
``workflow_subagent_files`` are the ones it cannot — those nested deeper
|
|
40
|
+
(``subagents/workflows/wf_*/agent-*.jsonl``) plus any other ``*.jsonl``
|
|
41
|
+
elsewhere in the side directory — fidelity gap 1.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
session_jsonl: Path
|
|
45
|
+
record_counts: dict[RecordType, int] = field(default_factory=dict)
|
|
46
|
+
subagent_files: tuple[Path, ...] = ()
|
|
47
|
+
workflow_subagent_files: tuple[Path, ...] = ()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _record_type(record: dict[str, Any]) -> RecordType:
|
|
51
|
+
raw_type = record.get("type")
|
|
52
|
+
if isinstance(raw_type, str):
|
|
53
|
+
return _KNOWN_TYPES.get(raw_type, RecordType.OTHER)
|
|
54
|
+
return RecordType.OTHER
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def census_from_snapshot(
|
|
58
|
+
snapshot: SessionSnapshot,
|
|
59
|
+
records: Iterable[tuple[dict[str, Any], str]],
|
|
60
|
+
) -> SessionCensus:
|
|
61
|
+
"""Census one snapshot: record-type counts plus side-file classification.
|
|
62
|
+
|
|
63
|
+
``records`` are that snapshot's parsed pairs from
|
|
64
|
+
:func:`~atif_converter.infrastructure.raw_records.read_snapshot_records`.
|
|
65
|
+
Only each record's ``type`` is read, so a generator whose records are
|
|
66
|
+
discarded as they are consumed counts the same as a list.
|
|
67
|
+
"""
|
|
68
|
+
counts: dict[RecordType, int] = {}
|
|
69
|
+
for record, _source_file in records:
|
|
70
|
+
record_type = _record_type(record)
|
|
71
|
+
counts[record_type] = counts.get(record_type, 0) + 1
|
|
72
|
+
|
|
73
|
+
session_jsonl = snapshot.session_jsonl
|
|
74
|
+
subagent_files: list[Path] = []
|
|
75
|
+
workflow_files: list[Path] = []
|
|
76
|
+
for candidate in snapshot.files:
|
|
77
|
+
if candidate == session_jsonl:
|
|
78
|
+
continue
|
|
79
|
+
if candidate.parent.name == _HARBOR_SUBAGENT_DIR:
|
|
80
|
+
subagent_files.append(candidate)
|
|
81
|
+
else:
|
|
82
|
+
workflow_files.append(candidate)
|
|
83
|
+
|
|
84
|
+
return SessionCensus(
|
|
85
|
+
session_jsonl=session_jsonl,
|
|
86
|
+
record_counts=counts,
|
|
87
|
+
subagent_files=tuple(subagent_files),
|
|
88
|
+
workflow_subagent_files=tuple(workflow_files),
|
|
89
|
+
)
|