atif-sql 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. atif_analytics/__init__.py +9 -0
  2. atif_analytics/application/__init__.py +3 -0
  3. atif_analytics/application/analyze.py +158 -0
  4. atif_analytics/application/prompts.py +750 -0
  5. atif_analytics/application/use_cases/__init__.py +3 -0
  6. atif_analytics/application/use_cases/_shared.py +264 -0
  7. atif_analytics/application/use_cases/classify.py +430 -0
  8. atif_analytics/application/use_cases/conflicts.py +448 -0
  9. atif_analytics/application/use_cases/friction.py +634 -0
  10. atif_analytics/application/use_cases/perceived.py +451 -0
  11. atif_analytics/domain/__init__.py +3 -0
  12. atif_analytics/domain/authorship.py +166 -0
  13. atif_analytics/domain/config.py +30 -0
  14. atif_analytics/domain/costs.py +50 -0
  15. atif_analytics/domain/friction.py +117 -0
  16. atif_analytics/domain/layout.py +94 -0
  17. atif_analytics/domain/models.py +385 -0
  18. atif_analytics/domain/ports.py +100 -0
  19. atif_analytics/domain/transcript.py +359 -0
  20. atif_analytics/infrastructure/__init__.py +3 -0
  21. atif_analytics/infrastructure/corpus_reader.py +546 -0
  22. atif_analytics/infrastructure/parquet_cache.py +186 -0
  23. atif_analytics/infrastructure/settings.py +114 -0
  24. atif_analytics/infrastructure/sqlite_state/__init__.py +3 -0
  25. atif_analytics/infrastructure/sqlite_state/checkpointer.py +326 -0
  26. atif_analytics/infrastructure/sqlite_state/retry_queue.py +296 -0
  27. atif_analytics/py.typed +0 -0
  28. atif_cli/__init__.py +3 -0
  29. atif_cli/__main__.py +8 -0
  30. atif_cli/app.py +2459 -0
  31. atif_cli/converter_adapter.py +151 -0
  32. atif_cli/corpus.py +409 -0
  33. atif_cli/cron.py +311 -0
  34. atif_cli/duck_errors.py +97 -0
  35. atif_cli/embed_lake.py +101 -0
  36. atif_cli/errors.py +89 -0
  37. atif_cli/lake.py +373 -0
  38. atif_cli/lake_sessions.py +270 -0
  39. atif_cli/output.py +297 -0
  40. atif_cli/py.typed +0 -0
  41. atif_converter/__init__.py +3 -0
  42. atif_converter/application/__init__.py +3 -0
  43. atif_converter/application/convert_and_audit.py +294 -0
  44. atif_converter/application/convert_codex.py +210 -0
  45. atif_converter/domain/__init__.py +9 -0
  46. atif_converter/domain/agents.py +40 -0
  47. atif_converter/domain/atif/__init__.py +55 -0
  48. atif_converter/domain/atif/agent.py +37 -0
  49. atif_converter/domain/atif/content.py +161 -0
  50. atif_converter/domain/atif/final_metrics.py +45 -0
  51. atif_converter/domain/atif/metrics.py +49 -0
  52. atif_converter/domain/atif/observation.py +21 -0
  53. atif_converter/domain/atif/observation_result.py +47 -0
  54. atif_converter/domain/atif/step.py +144 -0
  55. atif_converter/domain/atif/subagent_trajectory_ref.py +90 -0
  56. atif_converter/domain/atif/tool_call.py +36 -0
  57. atif_converter/domain/atif/trajectory.py +211 -0
  58. atif_converter/domain/atif/trajectory_validator.py +299 -0
  59. atif_converter/domain/blobs.py +510 -0
  60. atif_converter/domain/claude_code_conversion.py +1077 -0
  61. atif_converter/domain/codex_conversion.py +1036 -0
  62. atif_converter/domain/codex_edges.py +192 -0
  63. atif_converter/domain/codex_enrichment.py +419 -0
  64. atif_converter/domain/codex_fidelity.py +124 -0
  65. atif_converter/domain/codex_nested_calls.py +165 -0
  66. atif_converter/domain/edges.py +113 -0
  67. atif_converter/domain/enrichment.py +412 -0
  68. atif_converter/domain/errors.py +60 -0
  69. atif_converter/domain/fidelity.py +171 -0
  70. atif_converter/domain/model_prices.json +8538 -0
  71. atif_converter/domain/pricing.py +790 -0
  72. atif_converter/domain/result_signals.py +774 -0
  73. atif_converter/domain/schema_version.py +90 -0
  74. atif_converter/domain/session_events.py +408 -0
  75. atif_converter/infrastructure/__init__.py +3 -0
  76. atif_converter/infrastructure/census.py +89 -0
  77. atif_converter/infrastructure/claude_code_converter.py +127 -0
  78. atif_converter/infrastructure/codex_adapter.py +132 -0
  79. atif_converter/infrastructure/codex_census.py +100 -0
  80. atif_converter/infrastructure/codex_converter.py +82 -0
  81. atif_converter/infrastructure/harbor_adapter.py +187 -0
  82. atif_converter/infrastructure/raw_records.py +426 -0
  83. atif_converter/infrastructure/source_archive.py +154 -0
  84. atif_converter/py.typed +0 -0
  85. atif_corpus/__init__.py +13 -0
  86. atif_corpus/application/__init__.py +7 -0
  87. atif_corpus/application/materialize.py +1826 -0
  88. atif_corpus/domain/__init__.py +9 -0
  89. atif_corpus/domain/agents.py +33 -0
  90. atif_corpus/domain/generation.py +34 -0
  91. atif_corpus/domain/layout.py +168 -0
  92. atif_corpus/domain/ports.py +175 -0
  93. atif_corpus/domain/session_id.py +68 -0
  94. atif_corpus/domain/sessions.py +228 -0
  95. atif_corpus/domain/slug.py +72 -0
  96. atif_corpus/domain/source_layout.py +143 -0
  97. atif_corpus/domain/watermark.py +76 -0
  98. atif_corpus/infrastructure/__init__.py +7 -0
  99. atif_corpus/infrastructure/atomic.py +266 -0
  100. atif_corpus/infrastructure/compress_artifacts.py +161 -0
  101. atif_corpus/infrastructure/fake_converter.py +174 -0
  102. atif_corpus/infrastructure/scanner.py +316 -0
  103. atif_corpus/infrastructure/settings.py +130 -0
  104. atif_corpus/infrastructure/source_archive.py +202 -0
  105. atif_corpus/py.typed +0 -0
  106. atif_duck/__init__.py +43 -0
  107. atif_duck/domain/__init__.py +32 -0
  108. atif_duck/domain/artifacts.py +66 -0
  109. atif_duck/domain/authorship.py +165 -0
  110. atif_duck/domain/catalog.py +626 -0
  111. atif_duck/domain/columnar.py +149 -0
  112. atif_duck/domain/embedding_guard.py +88 -0
  113. atif_duck/domain/examples.py +265 -0
  114. atif_duck/domain/lake.py +427 -0
  115. atif_duck/domain/lake_sessions.py +97 -0
  116. atif_duck/domain/raw_readers.py +140 -0
  117. atif_duck/domain/session_id.py +69 -0
  118. atif_duck/domain/sql_literal.py +50 -0
  119. atif_duck/infrastructure/__init__.py +34 -0
  120. atif_duck/infrastructure/analytics.py +469 -0
  121. atif_duck/infrastructure/authorship.py +139 -0
  122. atif_duck/infrastructure/columnar.py +641 -0
  123. atif_duck/infrastructure/lake.py +1383 -0
  124. atif_duck/infrastructure/lake_sessions.py +194 -0
  125. atif_duck/infrastructure/lake_settings.py +63 -0
  126. atif_duck/infrastructure/lake_steps.py +225 -0
  127. atif_duck/infrastructure/projections.py +221 -0
  128. atif_duck/infrastructure/registry.py +2194 -0
  129. atif_duck/infrastructure/stored_artifacts.py +115 -0
  130. atif_duck/py.typed +0 -0
  131. atif_embed/__init__.py +12 -0
  132. atif_embed/application/__init__.py +3 -0
  133. atif_embed/application/embed.py +293 -0
  134. atif_embed/application/prune.py +80 -0
  135. atif_embed/domain/__init__.py +3 -0
  136. atif_embed/domain/discovery.py +70 -0
  137. atif_embed/domain/embedding_guard.py +83 -0
  138. atif_embed/domain/errors.py +94 -0
  139. atif_embed/domain/ports.py +223 -0
  140. atif_embed/domain/sql_literal.py +49 -0
  141. atif_embed/domain/text_stamp.py +71 -0
  142. atif_embed/infrastructure/__init__.py +3 -0
  143. atif_embed/infrastructure/cohere_bedrock.py +433 -0
  144. atif_embed/infrastructure/corpus_text_rows.py +273 -0
  145. atif_embed/infrastructure/gemma_local.py +355 -0
  146. atif_embed/infrastructure/lake_text_rows.py +203 -0
  147. atif_embed/infrastructure/lance_store.py +440 -0
  148. atif_embed/infrastructure/providers.py +30 -0
  149. atif_embed/infrastructure/settings.py +154 -0
  150. atif_embed/py.typed +0 -0
  151. atif_models/__init__.py +14 -0
  152. atif_models/domain/__init__.py +9 -0
  153. atif_models/domain/ports.py +143 -0
  154. atif_models/domain/registry.py +151 -0
  155. atif_models/domain/schema.py +109 -0
  156. atif_models/infrastructure/__init__.py +8 -0
  157. atif_models/infrastructure/openai_bedrock.py +314 -0
  158. atif_models/infrastructure/settings.py +85 -0
  159. atif_models/py.typed +0 -0
  160. atif_sql-0.2.0.dist-info/METADATA +272 -0
  161. atif_sql-0.2.0.dist-info/RECORD +164 -0
  162. atif_sql-0.2.0.dist-info/WHEEL +4 -0
  163. atif_sql-0.2.0.dist-info/entry_points.txt +2 -0
  164. atif_sql-0.2.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,90 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+
3
+ """The converter-owned version of what a conversion produces.
4
+
5
+ The package version (``converter_version`` in ``meta.json``) moves with every
6
+ release, including ones that change nothing a corpus holds. This number moves
7
+ only when the artifacts a conversion writes change shape or meaning, so a
8
+ materialize pass can re-convert exactly the sessions an older converter wrote:
9
+ materialize stamps it into every ``meta.json`` as ``converter_schema`` and
10
+ treats a session recording a different value as stale, whether or not its
11
+ source moved. A source-removed session re-converts from its raw source archive.
12
+
13
+ THE RULE: bump this in the same commit as any change that can alter a byte of
14
+ any artifact for any input. A refactor that provably changes no output does not
15
+ bump it. ``packages/atif-converter/tests/test_converter_schema_version.py`` pins
16
+ a digest of the converter's code (docstrings and comments excluded) beside this
17
+ value, so any converter code change fails that test until someone decides:
18
+ bump if output can change, then re-pin the digest either way.
19
+
20
+ History:
21
+
22
+ * 1 — every converter before this constant existed (implicit).
23
+ * 2 — ``session_events.jsonl`` (hooks, injected context, API errors,
24
+ compaction boundaries, cost-state, mode changes); ``records_captured`` in
25
+ ``loss_report.json`` and the structural gaps reported only when enrichment
26
+ didn't repair them; an unpriced model yields ``total_cost_usd`` NULL instead
27
+ of $0; ``final_metrics.extra.reported_cost_usd`` from Claude Code's
28
+ ``cost-state``; local price overrides for claude-opus-5-5 / claude-fable-5-1.
29
+ In the same version: inline base64 attachments replaced by blob placeholders
30
+ (and the bytes handed out for the corpus blob store); typed ``is_error`` /
31
+ ``exit_code`` / ``interrupted`` / ``images`` on observation results;
32
+ ``agent_id`` read from ``agentId`` and filled on every sidechain step;
33
+ ``trajectory.extra.subagents`` from the ``agent-*.meta.json`` sidecars.
34
+ Both change sets ship together and no corpus was ever stamped with only one
35
+ of them, so one bump separates every version-1 session from every version-2
36
+ one.
37
+ * 3 — harbor 0.23.0: every priced Claude Code step carries
38
+ ``metrics.cost_usd`` and ``metrics.extra.cost_source`` (an unpriced step
39
+ carries neither), and ``meta.json`` records ``harbor_version`` 0.23.0. Prices
40
+ from litellm v1.102.0, which also prices claude-fable-5-1, so its local
41
+ override retired and its sessions are labeled ``litellm_estimate``.
42
+ * 4 — Codex ``is_error`` / ``exit_code`` for code-mode ``exec`` scripts, from
43
+ the script's own header and the items its nested commands, MCP calls and
44
+ patches complete; ``is_error`` from the ``status`` of ``FileChange`` and
45
+ ``CollabAgentToolCall`` items.
46
+ * 5 — Claude Code ``agent.extra`` lists (``cwds``, ``git_branches``,
47
+ ``agent_ids``) in first-seen order instead of set order, so a session
48
+ converts to the same bytes in every process.
49
+ * 6 — prices from litellm v1.103.2, whose arithmetic bills cache writes and
50
+ cache reads at the input rate for a model that publishes no cache-creation
51
+ or cache-read rate (every OpenAI text model's cache writes, and cached
52
+ tokens of the 46 entries with no cache-read rate), where v1.102.0 billed
53
+ them $0. The table also adds flex rates for gpt-5.1 and gpt-5.2 and moves
54
+ o4-mini's flex cache-read rate. ``total_cost_usd`` and ``metrics.cost_usd``
55
+ change for every such session.
56
+ * 7 — harbor 0.24.0: ``meta.json`` records ``harbor_version`` 0.24.0. The
57
+ Claude Code port itself puts each event's ``agentId`` on every step the
58
+ event becomes (prompts, tool results and turns, which enrichment filled
59
+ before) and writes ``is_sidechain`` and ``agent_id`` right after ``id`` in
60
+ ``extra``, harbor's key order; a snake_case ``agent_id`` key names no
61
+ subagent (harbor #3434). A Codex tool output that is a list of content blocks or
62
+ items is its text parts joined by newlines, with an inline image as its blob
63
+ placeholder, where it was Python's ``str()`` of the list; a non-string
64
+ ``output`` inside an output object is its JSON text; a JSON scalar output
65
+ keeps its raw text (harbor #3467). A Codex ``web_search_call`` carries its
66
+ item id, or ``<api call>_web_search_<n>``, where its id was empty (harbor
67
+ #2972).
68
+ * 8 — every Codex tool output image the port writes a blob placeholder for is
69
+ lifted by the pre-pass too: an MCP ``image`` block with inline base64 (by
70
+ ``mimeType`` or ``mime_type``), and an ``input_image`` whose data URL
71
+ carries parameters or whose base64 is wrapped. Its bytes reach the blob
72
+ store, its result carries ``extra.images``, and a tool output list the port
73
+ dumps as JSON (one element of an unknown type) holds the placeholder item
74
+ where it held the base64. Placeholder text is unchanged.
75
+ * 9 — a Codex code-mode ``exec`` script's nested MCP calls and commands (the
76
+ ``McpToolCall`` / ``CommandExecution`` items attributed to it) are tool
77
+ calls of their own on its step, after the ``exec`` call, with
78
+ ``extra.nested_in``, ``via`` and ``duration_ms``, and one result each
79
+ carrying its content and its own ``is_error`` / ``exit_code`` (and
80
+ ``images``: the pre-pass lifts images out of those items' MCP results too).
81
+ The ``exec`` result's own fields are unchanged.
82
+ """
83
+
84
+ from __future__ import annotations
85
+
86
+ from typing import Final
87
+
88
+ CONVERTER_SCHEMA_VERSION: Final = 9
89
+
90
+ __all__ = ["CONVERTER_SCHEMA_VERSION"]
@@ -0,0 +1,408 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+
3
+ """session_events.jsonl: the non-message records worth keeping, one typed row each.
4
+
5
+ Pure functions over already-parsed ``(record, source_file)`` pairs; no harbor,
6
+ no I/O. The step converter keeps user and assistant records only
7
+ (:attr:`~atif_converter.domain.fidelity.FidelityGap.NON_MESSAGE_RECORDS_DROPPED`),
8
+ which loses records that say what happened AROUND the turns: hooks that ran,
9
+ blocked or were cancelled, context a hook or a queued command injected into
10
+ the model's view, API errors, compaction boundaries, refusal fallbacks,
11
+ Claude Code's own running cost, and mode changes. This module picks those out
12
+ and writes each one as a row. They're deliberately NOT ATIF steps: the steps
13
+ view and everything built on it stay exactly as they were.
14
+
15
+ ROW SHAPE (:data:`SESSION_EVENT_FIELDS`, key order stable)::
16
+
17
+ {
18
+ seq,
19
+ ts,
20
+ event_type,
21
+ subtype,
22
+ uuid,
23
+ parent_uuid,
24
+ tool_use_id,
25
+ is_sidechain,
26
+ source_file,
27
+ payload,
28
+ payload_bytes,
29
+ payload_truncated,
30
+ }
31
+
32
+ ``event_type`` is the raw record ``type`` (``system``, ``attachment``,
33
+ ``cost-state``, ``mode``, ``permission-mode``; for Codex ``compacted`` or
34
+ ``event_msg``) and ``subtype`` names the kind inside it (``system.subtype``,
35
+ ``attachment.type``, a Codex ``payload.type``; ``None`` where the record type
36
+ is the whole story). ``seq`` is the row's position in the session's record
37
+ order (main file first, then side files, as read), because ``cost-state`` and
38
+ ``mode`` records carry no timestamp. ``parent_uuid`` is the record's
39
+ ``parentUuid``: for an attachment it's the record the attachment follows,
40
+ which is how a row joins back to ``steps.source_uuids``.
41
+
42
+ BOUNDED PAYLOAD. ``payload`` is the record minus its envelope (the columns
43
+ above plus per-record noise such as ``cwd`` and ``version``). Strings longer
44
+ than :data:`PAYLOAD_STRING_MAX_CHARS` and lists longer than
45
+ :data:`PAYLOAD_LIST_MAX_ITEMS` are cut, and a payload still over
46
+ :data:`PAYLOAD_MAX_BYTES` becomes ``{"truncated_preview": <first bytes>}``.
47
+ ``payload_bytes`` is always the ORIGINAL payload's UTF-8 JSON length and
48
+ ``payload_truncated`` says whether anything was cut, so a reader can tell a
49
+ short payload from a shortened one.
50
+ """
51
+
52
+ from __future__ import annotations
53
+
54
+ import json
55
+ from collections.abc import Iterable
56
+ from typing import Any, Final
57
+
58
+ from atif_converter.domain.codex_edges import record_id
59
+
60
+ #: Stable key order for one session_events.jsonl line.
61
+ SESSION_EVENT_FIELDS: Final = (
62
+ "seq",
63
+ "ts",
64
+ "event_type",
65
+ "subtype",
66
+ "uuid",
67
+ "parent_uuid",
68
+ "tool_use_id",
69
+ "is_sidechain",
70
+ "source_file",
71
+ "payload",
72
+ "payload_bytes",
73
+ "payload_truncated",
74
+ )
75
+
76
+ #: Hard ceiling for one serialized payload, in UTF-8 bytes.
77
+ PAYLOAD_MAX_BYTES: Final = 8192
78
+ #: Any string leaf longer than this is cut to it.
79
+ PAYLOAD_STRING_MAX_CHARS: Final = 2048
80
+ #: Any list longer than this keeps its first items only.
81
+ PAYLOAD_LIST_MAX_ITEMS: Final = 32
82
+ #: Nesting below this depth is replaced by its JSON text, then cut as a string.
83
+ _PAYLOAD_MAX_DEPTH: Final = 8
84
+ _ELLIPSIS: Final = "…"
85
+
86
+ # --------------------------------------------------------------------------- Claude Code
87
+
88
+ #: ``system`` records kept, by ``subtype``.
89
+ CLAUDE_SYSTEM_SUBTYPES: Final = frozenset(
90
+ {"stop_hook_summary", "api_error", "compact_boundary", "model_refusal_fallback"}
91
+ )
92
+ #: ``attachment`` records kept, by ``attachment.type``, besides every ``hook_*`` type.
93
+ CLAUDE_ATTACHMENT_TYPES: Final = frozenset({"queued_command"})
94
+ _CLAUDE_HOOK_ATTACHMENT_PREFIX: Final = "hook_"
95
+ #: Record types kept whole (no subtype).
96
+ CLAUDE_WHOLE_RECORD_TYPES: Final = frozenset({"cost-state", "mode", "permission-mode"})
97
+
98
+ #: Record keys that are columns or per-record noise, never payload.
99
+ _CLAUDE_ENVELOPE: Final = frozenset(
100
+ {
101
+ "type",
102
+ "subtype",
103
+ "uuid",
104
+ "parentUuid",
105
+ "timestamp",
106
+ "isSidechain",
107
+ "toolUseID",
108
+ "sessionId",
109
+ "userType",
110
+ "entrypoint",
111
+ "cwd",
112
+ "version",
113
+ "gitBranch",
114
+ "slug",
115
+ "isMeta",
116
+ "attachment",
117
+ "rendered",
118
+ }
119
+ )
120
+
121
+ # --------------------------------------------------------------------------- Codex
122
+
123
+ #: Top-level Codex record types kept whole.
124
+ CODEX_WHOLE_RECORD_TYPES: Final = frozenset({"compacted"})
125
+ #: ``event_msg`` records kept, by ``payload.type``.
126
+ CODEX_EVENT_MSG_TYPES: Final = frozenset(
127
+ {"turn_aborted", "error", "stream_error", "context_compacted", "warning"}
128
+ )
129
+
130
+
131
+ def _bounded_container(value: dict[str, Any] | list[Any], depth: int) -> tuple[Any, bool]:
132
+ if depth >= _PAYLOAD_MAX_DEPTH:
133
+ text = json.dumps(value, ensure_ascii=False, separators=(",", ":"))
134
+ return _bounded(text, depth)[0], True
135
+ if isinstance(value, dict):
136
+ cut = False
137
+ out: dict[str, Any] = {}
138
+ for key, item in value.items():
139
+ out[str(key)], item_cut = _bounded(item, depth + 1)
140
+ cut = cut or item_cut
141
+ return out, cut
142
+ cut = len(value) > PAYLOAD_LIST_MAX_ITEMS
143
+ items: list[Any] = []
144
+ for item in value[:PAYLOAD_LIST_MAX_ITEMS]:
145
+ bounded_item, item_cut = _bounded(item, depth + 1)
146
+ items.append(bounded_item)
147
+ cut = cut or item_cut
148
+ return items, cut
149
+
150
+
151
+ def _bounded(value: Any, depth: int = 0) -> tuple[Any, bool]:
152
+ """``value`` with long strings and lists cut; the flag says whether anything was."""
153
+ if isinstance(value, str):
154
+ if len(value) > PAYLOAD_STRING_MAX_CHARS:
155
+ return value[:PAYLOAD_STRING_MAX_CHARS] + _ELLIPSIS, True
156
+ return value, False
157
+ if isinstance(value, dict | list):
158
+ return _bounded_container(value, depth)
159
+ return value, False
160
+
161
+
162
+ def bound_payload(payload: dict[str, Any]) -> tuple[dict[str, Any], int, bool]:
163
+ """``(bounded payload, original UTF-8 JSON length, truncated?)`` for one event."""
164
+ original = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
165
+ original_bytes = len(original.encode("utf-8"))
166
+ if original_bytes <= PAYLOAD_MAX_BYTES:
167
+ return payload, original_bytes, False
168
+ bounded, _cut = _bounded(payload)
169
+ text = json.dumps(bounded, ensure_ascii=False, separators=(",", ":"))
170
+ if len(text.encode("utf-8")) <= PAYLOAD_MAX_BYTES:
171
+ return bounded, original_bytes, True
172
+ preview = original.encode("utf-8")[:PAYLOAD_MAX_BYTES].decode("utf-8", errors="ignore")
173
+ return {"truncated_preview": preview}, original_bytes, True
174
+
175
+
176
+ def _str_or_none(value: Any) -> str | None:
177
+ return value if isinstance(value, str) and value else None
178
+
179
+
180
+ def _claude_subtype(record: dict[str, Any], record_type: str) -> str | None:
181
+ """The kept subtype of a ``system`` / ``attachment`` record, else ``None``."""
182
+ if record_type == "system":
183
+ subtype = record.get("subtype")
184
+ return subtype if isinstance(subtype, str) and subtype in CLAUDE_SYSTEM_SUBTYPES else None
185
+ if record_type == "attachment":
186
+ attachment = record.get("attachment")
187
+ kind = attachment.get("type") if isinstance(attachment, dict) else None
188
+ if isinstance(kind, str) and (
189
+ kind.startswith(_CLAUDE_HOOK_ATTACHMENT_PREFIX) or kind in CLAUDE_ATTACHMENT_TYPES
190
+ ):
191
+ return kind
192
+ return None
193
+
194
+
195
+ def _claude_kind(record: dict[str, Any]) -> tuple[str, str | None] | None:
196
+ """``(event_type, subtype)`` when a Claude Code record is kept, else ``None``."""
197
+ record_type = record.get("type")
198
+ if not isinstance(record_type, str):
199
+ return None
200
+ if record_type in CLAUDE_WHOLE_RECORD_TYPES:
201
+ return record_type, None
202
+ subtype = _claude_subtype(record, record_type)
203
+ return None if subtype is None else (record_type, subtype)
204
+
205
+
206
+ def is_claude_session_event(record: dict[str, Any]) -> bool:
207
+ """Whether a Claude Code record becomes a session_events row."""
208
+ return _claude_kind(record) is not None
209
+
210
+
211
+ def _claude_payload(record: dict[str, Any]) -> dict[str, Any]:
212
+ payload = {key: value for key, value in record.items() if key not in _CLAUDE_ENVELOPE}
213
+ attachment = record.get("attachment")
214
+ if isinstance(attachment, dict):
215
+ payload.update(
216
+ {key: value for key, value in attachment.items() if key not in {"type", "toolUseID"}}
217
+ )
218
+ return payload
219
+
220
+
221
+ def _claude_tool_use_id(record: dict[str, Any]) -> str | None:
222
+ own = _str_or_none(record.get("toolUseID"))
223
+ if own is not None:
224
+ return own
225
+ attachment = record.get("attachment")
226
+ return _str_or_none(attachment.get("toolUseID")) if isinstance(attachment, dict) else None
227
+
228
+
229
+ def _row(
230
+ seq: int,
231
+ *,
232
+ ts: Any,
233
+ kind: tuple[str, str | None],
234
+ uuid: str | None,
235
+ parent_uuid: str | None,
236
+ tool_use_id: str | None,
237
+ is_sidechain: bool,
238
+ source_file: str,
239
+ payload: dict[str, Any],
240
+ ) -> dict[str, Any]:
241
+ bounded, original_bytes, truncated = bound_payload(payload)
242
+ return {
243
+ "seq": seq,
244
+ "ts": _str_or_none(ts),
245
+ "event_type": kind[0],
246
+ "subtype": kind[1],
247
+ "uuid": uuid,
248
+ "parent_uuid": parent_uuid,
249
+ "tool_use_id": tool_use_id,
250
+ "is_sidechain": is_sidechain,
251
+ "source_file": source_file,
252
+ "payload": bounded,
253
+ "payload_bytes": original_bytes,
254
+ "payload_truncated": truncated,
255
+ }
256
+
257
+
258
+ def claude_session_events(records: Iterable[tuple[dict[str, Any], str]]) -> list[dict[str, Any]]:
259
+ """The kept Claude Code records as rows, in the order the pairs arrive."""
260
+ rows: list[dict[str, Any]] = []
261
+ for record, source_file in records:
262
+ kind = _claude_kind(record)
263
+ if kind is None:
264
+ continue
265
+ rows.append(
266
+ _row(
267
+ len(rows),
268
+ ts=record.get("timestamp"),
269
+ kind=kind,
270
+ uuid=_str_or_none(record.get("uuid")),
271
+ parent_uuid=_str_or_none(record.get("parentUuid")),
272
+ tool_use_id=_claude_tool_use_id(record),
273
+ is_sidechain=bool(record.get("isSidechain", False)),
274
+ source_file=source_file,
275
+ payload=_claude_payload(record),
276
+ )
277
+ )
278
+ return rows
279
+
280
+
281
+ def _codex_kind(record: dict[str, Any]) -> tuple[str, str | None] | None:
282
+ record_type = record.get("type")
283
+ if not isinstance(record_type, str):
284
+ return None
285
+ if record_type in CODEX_WHOLE_RECORD_TYPES:
286
+ return record_type, None
287
+ payload = record.get("payload")
288
+ if record_type == "event_msg" and isinstance(payload, dict):
289
+ kind = payload.get("type")
290
+ if isinstance(kind, str) and kind in CODEX_EVENT_MSG_TYPES:
291
+ return record_type, kind
292
+ return None
293
+
294
+
295
+ def is_codex_session_event(record: dict[str, Any]) -> bool:
296
+ """Whether a Codex rollout record becomes a session_events row."""
297
+ return _codex_kind(record) is not None
298
+
299
+
300
+ def codex_session_events(records: Iterable[tuple[dict[str, Any], str]]) -> list[dict[str, Any]]:
301
+ """The kept Codex records as rows, in file order.
302
+
303
+ ``uuid`` is the same id ``edges.jsonl`` gives the record
304
+ (:func:`~atif_converter.domain.codex_edges.record_id`, line-numbered per
305
+ source file), so a row joins ``messages`` on it. A rollout has no parent
306
+ pointer and no sidechain, so those columns are ``None`` / ``False``.
307
+ """
308
+ rows: list[dict[str, Any]] = []
309
+ line_numbers: dict[str, int] = {}
310
+ for record, source_file in records:
311
+ line_numbers[source_file] = line_numbers.get(source_file, 0) + 1
312
+ kind = _codex_kind(record)
313
+ if kind is None:
314
+ continue
315
+ payload = record.get("payload")
316
+ body = (
317
+ {key: value for key, value in payload.items() if key != "type"}
318
+ if isinstance(payload, dict)
319
+ else {}
320
+ )
321
+ rows.append(
322
+ _row(
323
+ len(rows),
324
+ ts=record.get("timestamp"),
325
+ kind=kind,
326
+ uuid=record_id(record, line_numbers[source_file]),
327
+ parent_uuid=None,
328
+ tool_use_id=None,
329
+ is_sidechain=False,
330
+ source_file=source_file,
331
+ payload=body,
332
+ )
333
+ )
334
+ return rows
335
+
336
+
337
+ #: ``final_metrics.extra`` keys the reported cost lands under.
338
+ REPORTED_COST_KEY: Final = "reported_cost_usd"
339
+ REPORTED_COST_SOURCE_KEY: Final = "reported_cost_source"
340
+ REPORTED_UNKNOWN_MODEL_COST_KEY: Final = "reported_cost_has_unknown_model_cost"
341
+ #: ``reported_cost_source`` for a figure read off a Claude Code ``cost-state`` record.
342
+ REPORTED_COST_SOURCE_COST_STATE: Final = "claude_code_cost_state"
343
+
344
+
345
+ def claude_reported_cost(records: Iterable[tuple[dict[str, Any], str]]) -> dict[str, Any]:
346
+ """What Claude Code itself says the session cost, for ``final_metrics.extra``.
347
+
348
+ Claude Code appends a ``cost-state`` record as the session runs; its
349
+ ``totalCostUSD`` is cumulative across resumes (measured 2026-09-27: the
350
+ value only grows and ``startTime`` stays fixed across a resume), so the
351
+ LAST one in record order is the session's figure. Returns ``{}`` when no
352
+ record carries a numeric ``totalCostUSD``. ``hasUnknownModelCost`` is kept
353
+ beside it because Claude Code sets it when it met a model it couldn't
354
+ price, which makes its own total a lower bound.
355
+ """
356
+ last: dict[str, Any] | None = None
357
+ for record, _source_file in records:
358
+ if record.get("type") != "cost-state" or record.get("isSidechain"):
359
+ continue
360
+ total = record.get("totalCostUSD")
361
+ if isinstance(total, int | float) and not isinstance(total, bool):
362
+ last = record
363
+ if last is None:
364
+ return {}
365
+ reported: dict[str, Any] = {
366
+ REPORTED_COST_KEY: float(last["totalCostUSD"]),
367
+ REPORTED_COST_SOURCE_KEY: REPORTED_COST_SOURCE_COST_STATE,
368
+ }
369
+ unknown = last.get("hasUnknownModelCost")
370
+ if isinstance(unknown, bool):
371
+ reported[REPORTED_UNKNOWN_MODEL_COST_KEY] = unknown
372
+ return reported
373
+
374
+
375
+ def session_event_to_jsonl_line(row: dict[str, Any]) -> str:
376
+ """Serialize one row as a compact, key-order-stable JSONL line."""
377
+ ordered = {key: row.get(key) for key in SESSION_EVENT_FIELDS}
378
+ return json.dumps(ordered, ensure_ascii=False, separators=(",", ":"))
379
+
380
+
381
+ def session_events_jsonl_lines(rows: Iterable[dict[str, Any]]) -> list[str]:
382
+ """The ready-to-write session_events.jsonl body."""
383
+ return [session_event_to_jsonl_line(row) for row in rows]
384
+
385
+
386
+ __all__ = [
387
+ "CLAUDE_ATTACHMENT_TYPES",
388
+ "CLAUDE_SYSTEM_SUBTYPES",
389
+ "CLAUDE_WHOLE_RECORD_TYPES",
390
+ "CODEX_EVENT_MSG_TYPES",
391
+ "CODEX_WHOLE_RECORD_TYPES",
392
+ "PAYLOAD_LIST_MAX_ITEMS",
393
+ "PAYLOAD_MAX_BYTES",
394
+ "PAYLOAD_STRING_MAX_CHARS",
395
+ "REPORTED_COST_KEY",
396
+ "REPORTED_COST_SOURCE_COST_STATE",
397
+ "REPORTED_COST_SOURCE_KEY",
398
+ "REPORTED_UNKNOWN_MODEL_COST_KEY",
399
+ "SESSION_EVENT_FIELDS",
400
+ "bound_payload",
401
+ "claude_reported_cost",
402
+ "claude_session_events",
403
+ "codex_session_events",
404
+ "is_claude_session_event",
405
+ "is_codex_session_event",
406
+ "session_event_to_jsonl_line",
407
+ "session_events_jsonl_lines",
408
+ ]
@@ -0,0 +1,3 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+
3
+ """Infrastructure layer: harbor adapter and raw-JSONL readers."""
@@ -0,0 +1,89 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+
3
+ """Raw-side census of a Claude Code session: the input half of a LossReport.
4
+
5
+ Counts come from records already parsed out of a
6
+ :class:`~atif_converter.infrastructure.raw_records.SessionSnapshot`, and the
7
+ side-file classification is derived from that same snapshot's file list — no
8
+ second parse, no disk access, no harbor, no env, no network. Both halves read
9
+ from the one snapshot, so the census, the trajectory, and edges.jsonl
10
+ describe the same bytes of a session that may still be growing.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ from collections.abc import Iterable
16
+ from dataclasses import dataclass, field
17
+ from pathlib import Path
18
+ from typing import Any
19
+
20
+ from atif_converter.domain.fidelity import RecordType
21
+ from atif_converter.infrastructure.raw_records import SessionSnapshot
22
+
23
+ _KNOWN_TYPES: dict[str, RecordType] = {
24
+ rt.value: rt for rt in RecordType if rt is not RecordType.OTHER
25
+ }
26
+
27
+ #: harbor's ``rglob("subagents/*.jsonl")`` discovers a side-file only when this
28
+ #: is the name of its immediate parent directory.
29
+ _HARBOR_SUBAGENT_DIR = "subagents"
30
+
31
+
32
+ @dataclass(frozen=True, slots=True)
33
+ class SessionCensus:
34
+ """Raw record counts for one session (main file + every side-file).
35
+
36
+ ``record_counts`` covers EVERY record of EVERY discovered ``*.jsonl``, so
37
+ it agrees line-for-line with edges.jsonl. ``subagent_files`` are the
38
+ side-files harbor 0.22.0's ``rglob("subagents/*.jsonl")`` can discover;
39
+ ``workflow_subagent_files`` are the ones it cannot — those nested deeper
40
+ (``subagents/workflows/wf_*/agent-*.jsonl``) plus any other ``*.jsonl``
41
+ elsewhere in the side directory — fidelity gap 1.
42
+ """
43
+
44
+ session_jsonl: Path
45
+ record_counts: dict[RecordType, int] = field(default_factory=dict)
46
+ subagent_files: tuple[Path, ...] = ()
47
+ workflow_subagent_files: tuple[Path, ...] = ()
48
+
49
+
50
+ def _record_type(record: dict[str, Any]) -> RecordType:
51
+ raw_type = record.get("type")
52
+ if isinstance(raw_type, str):
53
+ return _KNOWN_TYPES.get(raw_type, RecordType.OTHER)
54
+ return RecordType.OTHER
55
+
56
+
57
+ def census_from_snapshot(
58
+ snapshot: SessionSnapshot,
59
+ records: Iterable[tuple[dict[str, Any], str]],
60
+ ) -> SessionCensus:
61
+ """Census one snapshot: record-type counts plus side-file classification.
62
+
63
+ ``records`` are that snapshot's parsed pairs from
64
+ :func:`~atif_converter.infrastructure.raw_records.read_snapshot_records`.
65
+ Only each record's ``type`` is read, so a generator whose records are
66
+ discarded as they are consumed counts the same as a list.
67
+ """
68
+ counts: dict[RecordType, int] = {}
69
+ for record, _source_file in records:
70
+ record_type = _record_type(record)
71
+ counts[record_type] = counts.get(record_type, 0) + 1
72
+
73
+ session_jsonl = snapshot.session_jsonl
74
+ subagent_files: list[Path] = []
75
+ workflow_files: list[Path] = []
76
+ for candidate in snapshot.files:
77
+ if candidate == session_jsonl:
78
+ continue
79
+ if candidate.parent.name == _HARBOR_SUBAGENT_DIR:
80
+ subagent_files.append(candidate)
81
+ else:
82
+ workflow_files.append(candidate)
83
+
84
+ return SessionCensus(
85
+ session_jsonl=session_jsonl,
86
+ record_counts=counts,
87
+ subagent_files=tuple(subagent_files),
88
+ workflow_subagent_files=tuple(workflow_files),
89
+ )