atif-sql 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- atif_analytics/__init__.py +9 -0
- atif_analytics/application/__init__.py +3 -0
- atif_analytics/application/analyze.py +158 -0
- atif_analytics/application/prompts.py +750 -0
- atif_analytics/application/use_cases/__init__.py +3 -0
- atif_analytics/application/use_cases/_shared.py +264 -0
- atif_analytics/application/use_cases/classify.py +430 -0
- atif_analytics/application/use_cases/conflicts.py +448 -0
- atif_analytics/application/use_cases/friction.py +634 -0
- atif_analytics/application/use_cases/perceived.py +451 -0
- atif_analytics/domain/__init__.py +3 -0
- atif_analytics/domain/authorship.py +166 -0
- atif_analytics/domain/config.py +30 -0
- atif_analytics/domain/costs.py +50 -0
- atif_analytics/domain/friction.py +117 -0
- atif_analytics/domain/layout.py +94 -0
- atif_analytics/domain/models.py +385 -0
- atif_analytics/domain/ports.py +100 -0
- atif_analytics/domain/transcript.py +359 -0
- atif_analytics/infrastructure/__init__.py +3 -0
- atif_analytics/infrastructure/corpus_reader.py +546 -0
- atif_analytics/infrastructure/parquet_cache.py +186 -0
- atif_analytics/infrastructure/settings.py +114 -0
- atif_analytics/infrastructure/sqlite_state/__init__.py +3 -0
- atif_analytics/infrastructure/sqlite_state/checkpointer.py +326 -0
- atif_analytics/infrastructure/sqlite_state/retry_queue.py +296 -0
- atif_analytics/py.typed +0 -0
- atif_cli/__init__.py +3 -0
- atif_cli/__main__.py +8 -0
- atif_cli/app.py +2459 -0
- atif_cli/converter_adapter.py +151 -0
- atif_cli/corpus.py +409 -0
- atif_cli/cron.py +311 -0
- atif_cli/duck_errors.py +97 -0
- atif_cli/embed_lake.py +101 -0
- atif_cli/errors.py +89 -0
- atif_cli/lake.py +373 -0
- atif_cli/lake_sessions.py +270 -0
- atif_cli/output.py +297 -0
- atif_cli/py.typed +0 -0
- atif_converter/__init__.py +3 -0
- atif_converter/application/__init__.py +3 -0
- atif_converter/application/convert_and_audit.py +294 -0
- atif_converter/application/convert_codex.py +210 -0
- atif_converter/domain/__init__.py +9 -0
- atif_converter/domain/agents.py +40 -0
- atif_converter/domain/atif/__init__.py +55 -0
- atif_converter/domain/atif/agent.py +37 -0
- atif_converter/domain/atif/content.py +161 -0
- atif_converter/domain/atif/final_metrics.py +45 -0
- atif_converter/domain/atif/metrics.py +49 -0
- atif_converter/domain/atif/observation.py +21 -0
- atif_converter/domain/atif/observation_result.py +47 -0
- atif_converter/domain/atif/step.py +144 -0
- atif_converter/domain/atif/subagent_trajectory_ref.py +90 -0
- atif_converter/domain/atif/tool_call.py +36 -0
- atif_converter/domain/atif/trajectory.py +211 -0
- atif_converter/domain/atif/trajectory_validator.py +299 -0
- atif_converter/domain/blobs.py +510 -0
- atif_converter/domain/claude_code_conversion.py +1077 -0
- atif_converter/domain/codex_conversion.py +1036 -0
- atif_converter/domain/codex_edges.py +192 -0
- atif_converter/domain/codex_enrichment.py +419 -0
- atif_converter/domain/codex_fidelity.py +124 -0
- atif_converter/domain/codex_nested_calls.py +165 -0
- atif_converter/domain/edges.py +113 -0
- atif_converter/domain/enrichment.py +412 -0
- atif_converter/domain/errors.py +60 -0
- atif_converter/domain/fidelity.py +171 -0
- atif_converter/domain/model_prices.json +8538 -0
- atif_converter/domain/pricing.py +790 -0
- atif_converter/domain/result_signals.py +774 -0
- atif_converter/domain/schema_version.py +90 -0
- atif_converter/domain/session_events.py +408 -0
- atif_converter/infrastructure/__init__.py +3 -0
- atif_converter/infrastructure/census.py +89 -0
- atif_converter/infrastructure/claude_code_converter.py +127 -0
- atif_converter/infrastructure/codex_adapter.py +132 -0
- atif_converter/infrastructure/codex_census.py +100 -0
- atif_converter/infrastructure/codex_converter.py +82 -0
- atif_converter/infrastructure/harbor_adapter.py +187 -0
- atif_converter/infrastructure/raw_records.py +426 -0
- atif_converter/infrastructure/source_archive.py +154 -0
- atif_converter/py.typed +0 -0
- atif_corpus/__init__.py +13 -0
- atif_corpus/application/__init__.py +7 -0
- atif_corpus/application/materialize.py +1826 -0
- atif_corpus/domain/__init__.py +9 -0
- atif_corpus/domain/agents.py +33 -0
- atif_corpus/domain/generation.py +34 -0
- atif_corpus/domain/layout.py +168 -0
- atif_corpus/domain/ports.py +175 -0
- atif_corpus/domain/session_id.py +68 -0
- atif_corpus/domain/sessions.py +228 -0
- atif_corpus/domain/slug.py +72 -0
- atif_corpus/domain/source_layout.py +143 -0
- atif_corpus/domain/watermark.py +76 -0
- atif_corpus/infrastructure/__init__.py +7 -0
- atif_corpus/infrastructure/atomic.py +266 -0
- atif_corpus/infrastructure/compress_artifacts.py +161 -0
- atif_corpus/infrastructure/fake_converter.py +174 -0
- atif_corpus/infrastructure/scanner.py +316 -0
- atif_corpus/infrastructure/settings.py +130 -0
- atif_corpus/infrastructure/source_archive.py +202 -0
- atif_corpus/py.typed +0 -0
- atif_duck/__init__.py +43 -0
- atif_duck/domain/__init__.py +32 -0
- atif_duck/domain/artifacts.py +66 -0
- atif_duck/domain/authorship.py +165 -0
- atif_duck/domain/catalog.py +626 -0
- atif_duck/domain/columnar.py +149 -0
- atif_duck/domain/embedding_guard.py +88 -0
- atif_duck/domain/examples.py +265 -0
- atif_duck/domain/lake.py +427 -0
- atif_duck/domain/lake_sessions.py +97 -0
- atif_duck/domain/raw_readers.py +140 -0
- atif_duck/domain/session_id.py +69 -0
- atif_duck/domain/sql_literal.py +50 -0
- atif_duck/infrastructure/__init__.py +34 -0
- atif_duck/infrastructure/analytics.py +469 -0
- atif_duck/infrastructure/authorship.py +139 -0
- atif_duck/infrastructure/columnar.py +641 -0
- atif_duck/infrastructure/lake.py +1383 -0
- atif_duck/infrastructure/lake_sessions.py +194 -0
- atif_duck/infrastructure/lake_settings.py +63 -0
- atif_duck/infrastructure/lake_steps.py +225 -0
- atif_duck/infrastructure/projections.py +221 -0
- atif_duck/infrastructure/registry.py +2194 -0
- atif_duck/infrastructure/stored_artifacts.py +115 -0
- atif_duck/py.typed +0 -0
- atif_embed/__init__.py +12 -0
- atif_embed/application/__init__.py +3 -0
- atif_embed/application/embed.py +293 -0
- atif_embed/application/prune.py +80 -0
- atif_embed/domain/__init__.py +3 -0
- atif_embed/domain/discovery.py +70 -0
- atif_embed/domain/embedding_guard.py +83 -0
- atif_embed/domain/errors.py +94 -0
- atif_embed/domain/ports.py +223 -0
- atif_embed/domain/sql_literal.py +49 -0
- atif_embed/domain/text_stamp.py +71 -0
- atif_embed/infrastructure/__init__.py +3 -0
- atif_embed/infrastructure/cohere_bedrock.py +433 -0
- atif_embed/infrastructure/corpus_text_rows.py +273 -0
- atif_embed/infrastructure/gemma_local.py +355 -0
- atif_embed/infrastructure/lake_text_rows.py +203 -0
- atif_embed/infrastructure/lance_store.py +440 -0
- atif_embed/infrastructure/providers.py +30 -0
- atif_embed/infrastructure/settings.py +154 -0
- atif_embed/py.typed +0 -0
- atif_models/__init__.py +14 -0
- atif_models/domain/__init__.py +9 -0
- atif_models/domain/ports.py +143 -0
- atif_models/domain/registry.py +151 -0
- atif_models/domain/schema.py +109 -0
- atif_models/infrastructure/__init__.py +8 -0
- atif_models/infrastructure/openai_bedrock.py +314 -0
- atif_models/infrastructure/settings.py +85 -0
- atif_models/py.typed +0 -0
- atif_sql-0.2.0.dist-info/METADATA +272 -0
- atif_sql-0.2.0.dist-info/RECORD +164 -0
- atif_sql-0.2.0.dist-info/WHEEL +4 -0
- atif_sql-0.2.0.dist-info/entry_points.txt +2 -0
- atif_sql-0.2.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
# Ported from harbor 0.23.0, src/harbor/agents/installed/claude_code.py
|
|
3
|
+
# (Apache-2.0, Copyright the Harbor authors), onto the ATIF models vendored in
|
|
4
|
+
# atif_converter.domain.atif, so atif-converter doesn't depend on harbor.
|
|
5
|
+
|
|
6
|
+
"""Read one Claude Code session from disk and convert it, with no staging.
|
|
7
|
+
|
|
8
|
+
The file half of harbor's ``_convert_events_to_trajectory``: the JSONL reading
|
|
9
|
+
loop is harbor's (blank lines skipped, a malformed line logged and skipped,
|
|
10
|
+
the rest parsed), and side-file discovery is what the old adapter's staging
|
|
11
|
+
did before harbor ever saw the session — ``<parent>/<stem>/`` walked with
|
|
12
|
+
``rglob("*.jsonl")``, a leading ``subagents`` path part dropped, the remaining
|
|
13
|
+
parts joined with ``__``. That joined name is the key the domain function
|
|
14
|
+
sorts on, so a workflow-nested ``subagents/workflows/wf_1/agent-a.jsonl`` is
|
|
15
|
+
read in the same position it was when staged flat as
|
|
16
|
+
``subagents/workflows__wf_1__agent-a.jsonl``. When two side-files flatten to
|
|
17
|
+
the same name the first in ``rglob`` order wins, which is what the adapter's
|
|
18
|
+
``if not staged.exists()`` guard did.
|
|
19
|
+
|
|
20
|
+
No temp dir, no symlinks, no ``harbor.agents`` import. ``*.meta.json``
|
|
21
|
+
sidecars are excluded by the ``*.jsonl`` suffix filter, as before.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
from typing import Any
|
|
28
|
+
|
|
29
|
+
from atif_converter.domain.atif import Trajectory
|
|
30
|
+
from atif_converter.domain.claude_code_conversion import convert_claude_code_records
|
|
31
|
+
from atif_converter.infrastructure.raw_records import LoadedSession, parse_jsonl_records
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def read_session_records(path: Path) -> list[dict[str, Any]]:
|
|
35
|
+
"""Parse one JSONL transcript the way harbor's reading loop does.
|
|
36
|
+
|
|
37
|
+
Blank lines are skipped; a line that is not JSON is logged at debug and
|
|
38
|
+
skipped rather than failing the session. A line that IS JSON but not an
|
|
39
|
+
object is kept as-is, as harbor keeps it (it fails later, in the same
|
|
40
|
+
place harbor fails).
|
|
41
|
+
"""
|
|
42
|
+
with path.open(encoding="utf-8") as handle:
|
|
43
|
+
return parse_jsonl_records(handle, path)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def staged_side_name(side_file: Path, side_dir: Path) -> str:
|
|
47
|
+
"""The flat name harbor saw a side-file under: path parts joined with ``__``.
|
|
48
|
+
|
|
49
|
+
A leading ``subagents`` part is dropped, so
|
|
50
|
+
``subagents/workflows/wf_1/agent-a.jsonl`` becomes
|
|
51
|
+
``workflows__wf_1__agent-a.jsonl``. The domain converter sorts on this name.
|
|
52
|
+
"""
|
|
53
|
+
rel_parts = side_file.relative_to(side_dir).parts
|
|
54
|
+
if rel_parts and rel_parts[0] == "subagents":
|
|
55
|
+
rel_parts = rel_parts[1:]
|
|
56
|
+
return "__".join(rel_parts)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def discover_side_files(session_jsonl: Path, *, include_subagents: bool = True) -> dict[str, Path]:
|
|
60
|
+
"""Staged side-file name -> side-file path, for one session's ``<stem>/`` dir.
|
|
61
|
+
|
|
62
|
+
Empty when ``include_subagents`` is false or the session has no side dir.
|
|
63
|
+
"""
|
|
64
|
+
side_files: dict[str, Path] = {}
|
|
65
|
+
side_dir = session_jsonl.parent / session_jsonl.stem
|
|
66
|
+
if include_subagents and side_dir.is_dir():
|
|
67
|
+
for side_file in sorted(side_dir.rglob("*.jsonl")):
|
|
68
|
+
side_files.setdefault(staged_side_name(side_file, side_dir), side_file)
|
|
69
|
+
return side_files
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def convert_loaded_claude_code_session(
|
|
73
|
+
loaded: LoadedSession, *, include_subagents: bool = True
|
|
74
|
+
) -> Trajectory | None:
|
|
75
|
+
"""Convert an already-read session: the records of :func:`~atif_converter.infrastructure.raw_records.load_session`.
|
|
76
|
+
|
|
77
|
+
Same result as :func:`convert_claude_code_session` on the same bytes, with
|
|
78
|
+
no second read: the main transcript is the loaded main file and the
|
|
79
|
+
side-files are the loaded side-files keyed by their staged name, first in
|
|
80
|
+
sorted order winning on a name collision exactly as
|
|
81
|
+
:func:`discover_side_files` resolves it. ``include_subagents`` off converts
|
|
82
|
+
the main transcript alone, as before.
|
|
83
|
+
"""
|
|
84
|
+
session_jsonl = loaded.snapshot.session_jsonl
|
|
85
|
+
side_dir = session_jsonl.parent / session_jsonl.stem
|
|
86
|
+
side_records: dict[str, list[Any]] = {}
|
|
87
|
+
if include_subagents:
|
|
88
|
+
for path, records in loaded.records_by_file.items():
|
|
89
|
+
if path != session_jsonl:
|
|
90
|
+
side_records.setdefault(staged_side_name(path, side_dir), records)
|
|
91
|
+
return convert_claude_code_records(
|
|
92
|
+
loaded.main_records,
|
|
93
|
+
side_records,
|
|
94
|
+
fallback_session_id=session_jsonl.parent.name or "-unknown",
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def convert_claude_code_session(
|
|
99
|
+
session_jsonl: Path, *, include_subagents: bool = True
|
|
100
|
+
) -> Trajectory | None:
|
|
101
|
+
"""Convert one Claude Code session JSONL (plus side-files) into an ATIF trajectory.
|
|
102
|
+
|
|
103
|
+
Parameters
|
|
104
|
+
----------
|
|
105
|
+
session_jsonl
|
|
106
|
+
The main ``<session-id>.jsonl``. Read errors propagate as ``OSError``.
|
|
107
|
+
include_subagents
|
|
108
|
+
Whether to read the ``<stem>/`` side-files; off, only the main
|
|
109
|
+
transcript is converted.
|
|
110
|
+
|
|
111
|
+
Returns
|
|
112
|
+
-------
|
|
113
|
+
Trajectory | None
|
|
114
|
+
``None`` when the session yields no steps, as harbor returns ``None``.
|
|
115
|
+
"""
|
|
116
|
+
main_records = read_session_records(session_jsonl)
|
|
117
|
+
side_records = {
|
|
118
|
+
name: read_session_records(path)
|
|
119
|
+
for name, path in discover_side_files(
|
|
120
|
+
session_jsonl, include_subagents=include_subagents
|
|
121
|
+
).items()
|
|
122
|
+
}
|
|
123
|
+
return convert_claude_code_records(
|
|
124
|
+
main_records,
|
|
125
|
+
side_records,
|
|
126
|
+
fallback_session_id=session_jsonl.parent.name or "-unknown",
|
|
127
|
+
)
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
|
|
3
|
+
"""The Codex conversion seam: our converter in, a validated ATIF dict out.
|
|
4
|
+
|
|
5
|
+
The raw rollout -> ``Trajectory`` conversion is ours, in
|
|
6
|
+
:mod:`atif_converter.domain.codex_conversion` via
|
|
7
|
+
:mod:`atif_converter.infrastructure.codex_converter`, ported from harbor 0.22.0
|
|
8
|
+
and held to parity with it by the oracle in this package's tests. harbor itself
|
|
9
|
+
supplies only the public data classes and the validator. One rollout in, one
|
|
10
|
+
trajectory out, by construction: the converter reads the file it is given, so
|
|
11
|
+
the directory-globbing hazard the old wrapper staged around no longer exists.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import TYPE_CHECKING, Any
|
|
18
|
+
|
|
19
|
+
from loguru import logger
|
|
20
|
+
|
|
21
|
+
from atif_converter.domain.errors import (
|
|
22
|
+
ConversionError,
|
|
23
|
+
EmptySessionError,
|
|
24
|
+
InvalidSessionInput,
|
|
25
|
+
)
|
|
26
|
+
from atif_converter.infrastructure.harbor_adapter import (
|
|
27
|
+
ConversionResult,
|
|
28
|
+
validate_trajectory,
|
|
29
|
+
)
|
|
30
|
+
from atif_converter.infrastructure.raw_records import LoadedSession
|
|
31
|
+
|
|
32
|
+
if TYPE_CHECKING:
|
|
33
|
+
from atif_converter.domain.atif import Trajectory
|
|
34
|
+
|
|
35
|
+
#: A Codex rollout's filename shape. harbor's own
|
|
36
|
+
#: ``Codex._ROLLOUT_FILENAME_RE`` requires the same prefix for its upload path,
|
|
37
|
+
#: and the session id is the filename's trailing uuid — so a file that does not
|
|
38
|
+
#: match this is not a rollout and must not be guessed at.
|
|
39
|
+
_ROLLOUT_PREFIX = "rollout-"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def codex_session_id(rollout_jsonl: Path) -> str:
|
|
43
|
+
"""The session id a rollout filename carries.
|
|
44
|
+
|
|
45
|
+
``rollout-<ISO-ish timestamp>-<uuid>.jsonl`` — the uuid is the last five
|
|
46
|
+
hyphen-separated groups of the stem, and it equals the ``id`` inside the
|
|
47
|
+
file's ``session_meta`` record (verified 2026-09-11 over the local
|
|
48
|
+
rollouts). Read from the NAME rather than the contents because discovery
|
|
49
|
+
needs a session id before anything parses the file.
|
|
50
|
+
|
|
51
|
+
Raises
|
|
52
|
+
------
|
|
53
|
+
InvalidSessionInput: the name is not a rollout filename.
|
|
54
|
+
"""
|
|
55
|
+
stem = rollout_jsonl.stem
|
|
56
|
+
if not stem.startswith(_ROLLOUT_PREFIX):
|
|
57
|
+
msg = f"not a Codex rollout filename (expected rollout-<ts>-<uuid>.jsonl): {rollout_jsonl}"
|
|
58
|
+
raise InvalidSessionInput(msg)
|
|
59
|
+
parts = stem.split("-")
|
|
60
|
+
uuid_groups = 5
|
|
61
|
+
if len(parts) < uuid_groups + 1:
|
|
62
|
+
msg = f"Codex rollout filename carries no session uuid: {rollout_jsonl}"
|
|
63
|
+
raise InvalidSessionInput(msg)
|
|
64
|
+
return "-".join(parts[-uuid_groups:])
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def convert_codex_session(rollout_jsonl: Path) -> ConversionResult:
|
|
68
|
+
"""Convert one Codex rollout JSONL into a validated ATIF trajectory.
|
|
69
|
+
|
|
70
|
+
Raises
|
|
71
|
+
------
|
|
72
|
+
InvalidSessionInput: not an existing ``.jsonl`` file.
|
|
73
|
+
EmptySessionError: the converter found no convertible events.
|
|
74
|
+
ConversionError: any unexpected failure inside the converter.
|
|
75
|
+
"""
|
|
76
|
+
from atif_converter.infrastructure.codex_converter import convert_codex_rollout
|
|
77
|
+
|
|
78
|
+
if rollout_jsonl.suffix != ".jsonl" or not rollout_jsonl.is_file():
|
|
79
|
+
msg = f"not a rollout JSONL file: {rollout_jsonl}"
|
|
80
|
+
raise InvalidSessionInput(msg)
|
|
81
|
+
|
|
82
|
+
try:
|
|
83
|
+
trajectory = convert_codex_rollout(rollout_jsonl)
|
|
84
|
+
except (
|
|
85
|
+
Exception
|
|
86
|
+
) as exc: # the converter is a port of untyped upstream code; classify at the seam
|
|
87
|
+
msg = f"Codex conversion failed for {rollout_jsonl}"
|
|
88
|
+
raise ConversionError(msg) from exc
|
|
89
|
+
|
|
90
|
+
if trajectory is None:
|
|
91
|
+
msg = f"no convertible events in {rollout_jsonl}"
|
|
92
|
+
raise EmptySessionError(msg)
|
|
93
|
+
|
|
94
|
+
return _result_from(trajectory, rollout_jsonl)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def convert_loaded_codex_session(loaded: LoadedSession) -> ConversionResult:
|
|
98
|
+
"""Convert an already-read rollout into a validated ATIF trajectory.
|
|
99
|
+
|
|
100
|
+
The records are :func:`~atif_converter.infrastructure.harbor_adapter.read_session`'s;
|
|
101
|
+
nothing is read from disk here.
|
|
102
|
+
|
|
103
|
+
Raises
|
|
104
|
+
------
|
|
105
|
+
EmptySessionError: the converter found no convertible events.
|
|
106
|
+
ConversionError: any unexpected failure inside the converter.
|
|
107
|
+
"""
|
|
108
|
+
from atif_converter.infrastructure.codex_converter import convert_loaded_codex_rollout
|
|
109
|
+
|
|
110
|
+
rollout_jsonl = loaded.snapshot.session_jsonl
|
|
111
|
+
try:
|
|
112
|
+
trajectory = convert_loaded_codex_rollout(loaded)
|
|
113
|
+
except (
|
|
114
|
+
Exception
|
|
115
|
+
) as exc: # the converter is a port of untyped upstream code; classify at the seam
|
|
116
|
+
msg = f"Codex conversion failed for {rollout_jsonl}"
|
|
117
|
+
raise ConversionError(msg) from exc
|
|
118
|
+
return _result_from(trajectory, rollout_jsonl)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _result_from(trajectory: Trajectory | None, rollout_jsonl: Path) -> ConversionResult:
|
|
122
|
+
if trajectory is None:
|
|
123
|
+
msg = f"no convertible events in {rollout_jsonl}"
|
|
124
|
+
raise EmptySessionError(msg)
|
|
125
|
+
|
|
126
|
+
trajectory_dict: dict[str, Any] = trajectory.model_dump(mode="json", exclude_none=True)
|
|
127
|
+
|
|
128
|
+
errors = validate_trajectory(trajectory_dict)
|
|
129
|
+
if errors:
|
|
130
|
+
logger.warning("codex trajectory for {} failed validation: {}", rollout_jsonl, errors)
|
|
131
|
+
|
|
132
|
+
return ConversionResult(trajectory=trajectory_dict, validation_errors=errors)
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
|
|
3
|
+
"""Raw-side census of a Codex rollout: the input half of its LossReport.
|
|
4
|
+
|
|
5
|
+
The Codex counterpart of :mod:`atif_converter.infrastructure.census`, and it
|
|
6
|
+
counts the same way: over records already parsed out of a
|
|
7
|
+
:class:`~atif_converter.infrastructure.raw_records.SessionSnapshot`, with no
|
|
8
|
+
second parse, no disk access, no harbor, no env and no network.
|
|
9
|
+
|
|
10
|
+
Two things differ from the Claude Code census, both because a rollout is one
|
|
11
|
+
file rather than a main transcript plus a side-file tree:
|
|
12
|
+
|
|
13
|
+
* there are no subagent side-files to classify — a Codex sub-agent writes its
|
|
14
|
+
own rollout under its own session id, so it is a separate session, not a
|
|
15
|
+
side-file of this one;
|
|
16
|
+
* the convertible subset is finer than a record type. Only ``response_item``
|
|
17
|
+
records become steps, and only SOME payload types within them do
|
|
18
|
+
(:data:`~atif_converter.domain.codex_fidelity.CODEX_CONVERTIBLE_ITEM_TYPES`),
|
|
19
|
+
so the census counts the convertible items separately instead of declaring
|
|
20
|
+
every response item converted.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
from collections.abc import Iterable
|
|
26
|
+
from dataclasses import dataclass, field
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
from typing import Any
|
|
29
|
+
|
|
30
|
+
from atif_converter.domain.codex_fidelity import (
|
|
31
|
+
CODEX_CONVERTIBLE_ITEM_TYPES,
|
|
32
|
+
CodexRecordType,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
_KNOWN_TYPES: dict[str, CodexRecordType] = {
|
|
36
|
+
rt.value: rt for rt in CodexRecordType if rt is not CodexRecordType.OTHER
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(frozen=True, slots=True)
|
|
41
|
+
class CodexSessionCensus:
|
|
42
|
+
"""Raw record counts for one rollout, plus the convertible-item detail.
|
|
43
|
+
|
|
44
|
+
``record_counts`` covers EVERY record of the rollout, so it agrees
|
|
45
|
+
line-for-line with edges.jsonl. ``convertible_items`` is the subset of
|
|
46
|
+
``response_item`` records whose payload type reaches a step, and
|
|
47
|
+
``item_type_counts`` is the full response-item breakdown — which is what
|
|
48
|
+
makes a dropped ``tool_search_call`` visible as a number rather than as an
|
|
49
|
+
absence.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
session_jsonl: Path
|
|
53
|
+
record_counts: dict[CodexRecordType, int] = field(default_factory=dict)
|
|
54
|
+
item_type_counts: dict[str, int] = field(default_factory=dict)
|
|
55
|
+
convertible_items: int = 0
|
|
56
|
+
|
|
57
|
+
@property
|
|
58
|
+
def response_items(self) -> int:
|
|
59
|
+
"""Total ``response_item`` records, convertible or not."""
|
|
60
|
+
return self.record_counts.get(CodexRecordType.RESPONSE_ITEM, 0)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _record_type(record: dict[str, Any]) -> CodexRecordType:
|
|
64
|
+
raw_type = record.get("type")
|
|
65
|
+
if isinstance(raw_type, str):
|
|
66
|
+
return _KNOWN_TYPES.get(raw_type, CodexRecordType.OTHER)
|
|
67
|
+
return CodexRecordType.OTHER
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def codex_census_from_records(
|
|
71
|
+
session_jsonl: Path,
|
|
72
|
+
records: Iterable[tuple[dict[str, Any], str]],
|
|
73
|
+
) -> CodexSessionCensus:
|
|
74
|
+
"""Census one rollout's parsed records.
|
|
75
|
+
|
|
76
|
+
Only each record's ``type`` and, for a response item, its payload
|
|
77
|
+
``type`` are read, so a generator whose records are discarded as they are
|
|
78
|
+
consumed counts the same as a list.
|
|
79
|
+
"""
|
|
80
|
+
counts: dict[CodexRecordType, int] = {}
|
|
81
|
+
item_counts: dict[str, int] = {}
|
|
82
|
+
convertible = 0
|
|
83
|
+
for record, _source_file in records:
|
|
84
|
+
record_type = _record_type(record)
|
|
85
|
+
counts[record_type] = counts.get(record_type, 0) + 1
|
|
86
|
+
if record_type is not CodexRecordType.RESPONSE_ITEM:
|
|
87
|
+
continue
|
|
88
|
+
payload = record.get("payload")
|
|
89
|
+
item_type = payload.get("type") if isinstance(payload, dict) else None
|
|
90
|
+
key = item_type if isinstance(item_type, str) and item_type else "unknown"
|
|
91
|
+
item_counts[key] = item_counts.get(key, 0) + 1
|
|
92
|
+
if key in CODEX_CONVERTIBLE_ITEM_TYPES:
|
|
93
|
+
convertible += 1
|
|
94
|
+
|
|
95
|
+
return CodexSessionCensus(
|
|
96
|
+
session_jsonl=session_jsonl,
|
|
97
|
+
record_counts=counts,
|
|
98
|
+
item_type_counts=item_counts,
|
|
99
|
+
convertible_items=convertible,
|
|
100
|
+
)
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
# Ported from harbor 0.22.0, src/harbor/agents/installed/codex.py (Apache-2.0,
|
|
3
|
+
# Copyright the Harbor authors), so that atif-converter depends on harbor's
|
|
4
|
+
# PUBLIC trajectory models only.
|
|
5
|
+
|
|
6
|
+
"""Read one Codex rollout and convert it — the file half of the Codex port.
|
|
7
|
+
|
|
8
|
+
harbor's ``Codex._convert_events_to_trajectory`` takes a DIRECTORY: it globs
|
|
9
|
+
``*.jsonl`` inside it, converts ``max(...)`` of the matches (one file, chosen
|
|
10
|
+
by name, the rest dropped) and reads that file line by line. The adapter this
|
|
11
|
+
replaces satisfied that shape by symlinking one rollout alone into a scratch
|
|
12
|
+
directory. This module reads the rollout directly instead, and the two are
|
|
13
|
+
the same thing by construction: with exactly one file in the directory,
|
|
14
|
+
``max`` is that file, so "the directory's one file" and "this file" are one
|
|
15
|
+
read.
|
|
16
|
+
|
|
17
|
+
The line loop is harbor's, kept exact rather than replaced with
|
|
18
|
+
``read_text().splitlines()``: iterating the handle splits on newlines only,
|
|
19
|
+
where ``str.splitlines`` also splits on U+2028 / U+2029 / form feed, which are
|
|
20
|
+
legal unescaped inside a JSON string — a rollout carrying one would parse into
|
|
21
|
+
different records under the two loops.
|
|
22
|
+
|
|
23
|
+
What differs from harbor's read is the ``session_id`` fallback for a rollout
|
|
24
|
+
with no ``session_meta``: harbor uses the directory's name, which in its own
|
|
25
|
+
layout is the ``<DD>`` day directory; here that is the rollout's parent
|
|
26
|
+
directory name, the same value on a real Codex tree.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
from typing import Any
|
|
33
|
+
|
|
34
|
+
from atif_converter.domain.atif import Trajectory
|
|
35
|
+
from atif_converter.domain.codex_conversion import convert_codex_records
|
|
36
|
+
from atif_converter.infrastructure.raw_records import LoadedSession, parse_jsonl_records
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def read_codex_rollout(rollout_jsonl: Path) -> list[dict[str, Any]]:
|
|
40
|
+
"""The rollout's JSON records in file order, malformed lines skipped.
|
|
41
|
+
|
|
42
|
+
harbor's read loop: blank lines are ignored, a line that is not JSON is
|
|
43
|
+
logged at debug level and dropped, and everything that parses is kept
|
|
44
|
+
(harbor does not check that a parsed line is an object).
|
|
45
|
+
|
|
46
|
+
Raises
|
|
47
|
+
------
|
|
48
|
+
OSError: the file cannot be opened or read.
|
|
49
|
+
"""
|
|
50
|
+
with rollout_jsonl.open(encoding="utf-8") as handle:
|
|
51
|
+
return parse_jsonl_records(handle, rollout_jsonl)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def convert_codex_rollout(rollout_jsonl: Path) -> Trajectory | None:
|
|
55
|
+
"""Convert one Codex rollout JSONL into an ATIF trajectory.
|
|
56
|
+
|
|
57
|
+
``None`` exactly when harbor's converter returns ``None`` for the same
|
|
58
|
+
file staged alone: the rollout has no records, or none that becomes a
|
|
59
|
+
step. Validation and the domain error taxonomy are the application
|
|
60
|
+
layer's business, as they were for the adapter this replaces.
|
|
61
|
+
|
|
62
|
+
Raises
|
|
63
|
+
------
|
|
64
|
+
OSError: the file cannot be opened or read.
|
|
65
|
+
"""
|
|
66
|
+
raw_events = read_codex_rollout(rollout_jsonl)
|
|
67
|
+
return convert_codex_records(raw_events, fallback_session_id=rollout_jsonl.parent.name)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def convert_loaded_codex_rollout(loaded: LoadedSession) -> Trajectory | None:
|
|
71
|
+
"""Convert an already-read rollout: the records of :func:`~atif_converter.infrastructure.raw_records.load_session`.
|
|
72
|
+
|
|
73
|
+
Same result as :func:`convert_codex_rollout` on the same bytes, with no
|
|
74
|
+
second read. Only the main file's records are converted; a rollout has no
|
|
75
|
+
side-files, so anything else the loader discovered is the audit's
|
|
76
|
+
business, as it was before.
|
|
77
|
+
"""
|
|
78
|
+
rollout_jsonl = loaded.snapshot.session_jsonl
|
|
79
|
+
return convert_codex_records(loaded.main_records, fallback_session_id=rollout_jsonl.parent.name)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
__all__ = ["convert_codex_rollout", "convert_loaded_codex_rollout", "read_codex_rollout"]
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
|
|
3
|
+
"""The Claude Code conversion seam: our converter in, a validated ATIF dict out.
|
|
4
|
+
|
|
5
|
+
The ATIF validator and data classes are harbor's, vendored as
|
|
6
|
+
:mod:`atif_converter.domain.atif` (the module keeps its historical name; harbor
|
|
7
|
+
itself isn't a runtime dependency). The raw-JSONL -> ``Trajectory`` conversion
|
|
8
|
+
is ours, in :mod:`atif_converter.domain.claude_code_conversion` via
|
|
9
|
+
:mod:`atif_converter.infrastructure.claude_code_converter`, ported from harbor
|
|
10
|
+
and held to parity with harbor 0.24.0 by the oracle in this package's tests.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import TYPE_CHECKING, Any
|
|
18
|
+
|
|
19
|
+
from loguru import logger
|
|
20
|
+
|
|
21
|
+
if TYPE_CHECKING:
|
|
22
|
+
from atif_converter.domain.atif import Trajectory
|
|
23
|
+
|
|
24
|
+
from atif_converter.domain.blobs import Blob
|
|
25
|
+
from atif_converter.domain.errors import (
|
|
26
|
+
ConversionError,
|
|
27
|
+
EmptySessionError,
|
|
28
|
+
InvalidSessionInput,
|
|
29
|
+
)
|
|
30
|
+
from atif_converter.infrastructure.raw_records import LoadedSession, load_session
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass(frozen=True, slots=True)
|
|
34
|
+
class ConversionResult:
|
|
35
|
+
"""Output of one conversion: trajectory dict + validation verdict.
|
|
36
|
+
|
|
37
|
+
``convert_session`` returns the raw harbor output (``edges_lines`` and
|
|
38
|
+
``events_lines`` empty); ``convert_and_audit`` returns the ENRICHED
|
|
39
|
+
trajectory plus the ready-to-write edges.jsonl and session_events.jsonl
|
|
40
|
+
lines derived from the raw records, and the inline attachments it lifted
|
|
41
|
+
out of the trajectory (``blobs``, one per distinct content hash) for the
|
|
42
|
+
corpus to store.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
trajectory: dict[str, Any]
|
|
46
|
+
validation_errors: tuple[str, ...]
|
|
47
|
+
edges_lines: tuple[str, ...] = ()
|
|
48
|
+
#: session_events.jsonl lines (:mod:`atif_converter.domain.session_events`).
|
|
49
|
+
events_lines: tuple[str, ...] = ()
|
|
50
|
+
blobs: tuple[Blob, ...] = ()
|
|
51
|
+
|
|
52
|
+
@property
|
|
53
|
+
def is_valid(self) -> bool:
|
|
54
|
+
"""True when the ATIF ``TrajectoryValidator`` accepted the trajectory."""
|
|
55
|
+
return not self.validation_errors
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def require_transcript_file(path: Path) -> None:
|
|
59
|
+
"""Refuse anything that is not an existing ``.jsonl`` transcript.
|
|
60
|
+
|
|
61
|
+
Called at the TOP of each use case, before the fingerprint snapshot: the
|
|
62
|
+
snapshot stats every discovered file, so a path that does not exist reached
|
|
63
|
+
``OSError`` from inside the snapshot and surfaced as an uncaught
|
|
64
|
+
``FileNotFoundError`` — exit 1 from the CLI, indistinguishable from a crash
|
|
65
|
+
— instead of the terminal input verdict the caller can act on.
|
|
66
|
+
|
|
67
|
+
Raises
|
|
68
|
+
------
|
|
69
|
+
InvalidSessionInput: the path is absent, or not a ``.jsonl`` file.
|
|
70
|
+
"""
|
|
71
|
+
if path.suffix != ".jsonl" or not path.is_file():
|
|
72
|
+
msg = f"not a session JSONL file: {path}"
|
|
73
|
+
raise InvalidSessionInput(msg)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def validate_trajectory(trajectory: dict[str, Any]) -> tuple[str, ...]:
|
|
77
|
+
"""Run the ATIF ``TrajectoryValidator``; return its errors (empty = valid).
|
|
78
|
+
|
|
79
|
+
Used to re-validate after the enrichment pass mutates ``extra`` fields
|
|
80
|
+
(contract: TrajectoryValidator MUST pass post-enrichment).
|
|
81
|
+
"""
|
|
82
|
+
from atif_converter.domain.atif.trajectory_validator import (
|
|
83
|
+
TrajectoryValidator,
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
validator = TrajectoryValidator()
|
|
87
|
+
ok = validator.validate(trajectory)
|
|
88
|
+
return () if ok else tuple(str(e) for e in validator.errors)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def read_session(session_jsonl: Path) -> LoadedSession:
|
|
92
|
+
"""Read one session's files once, fingerprinted and parsed in the same pass.
|
|
93
|
+
|
|
94
|
+
The reader half of the seam: :func:`convert_loaded_session` converts what
|
|
95
|
+
this returns, and the use case audits the same records, so the files are
|
|
96
|
+
read once for everything. A transcript that is not UTF-8 is classified as
|
|
97
|
+
a ``ConversionError`` here, as it was when the converter did its own read,
|
|
98
|
+
so a materialize pass records one per-session failure and continues.
|
|
99
|
+
|
|
100
|
+
Raises
|
|
101
|
+
------
|
|
102
|
+
InvalidSessionInput: ``session_jsonl`` is not an existing ``.jsonl`` file.
|
|
103
|
+
ConversionError: a discovered file is not UTF-8.
|
|
104
|
+
"""
|
|
105
|
+
require_transcript_file(session_jsonl)
|
|
106
|
+
try:
|
|
107
|
+
return load_session(session_jsonl)
|
|
108
|
+
except UnicodeDecodeError as exc:
|
|
109
|
+
msg = f"conversion failed for {session_jsonl}"
|
|
110
|
+
raise ConversionError(msg) from exc
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _result_from(trajectory: Trajectory | None, session_jsonl: Path) -> ConversionResult:
|
|
114
|
+
"""Validate a converted trajectory into the seam's result, or raise on ``None``."""
|
|
115
|
+
if trajectory is None:
|
|
116
|
+
msg = f"no convertible events in {session_jsonl}"
|
|
117
|
+
raise EmptySessionError(msg)
|
|
118
|
+
|
|
119
|
+
trajectory_dict: dict[str, Any] = trajectory.model_dump(mode="json", exclude_none=True)
|
|
120
|
+
|
|
121
|
+
errors = validate_trajectory(trajectory_dict)
|
|
122
|
+
if errors:
|
|
123
|
+
logger.warning("trajectory for {} failed validation: {}", session_jsonl, errors)
|
|
124
|
+
|
|
125
|
+
return ConversionResult(trajectory=trajectory_dict, validation_errors=errors)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def convert_loaded_session(
|
|
129
|
+
loaded: LoadedSession,
|
|
130
|
+
*,
|
|
131
|
+
include_subagents: bool = True,
|
|
132
|
+
) -> ConversionResult:
|
|
133
|
+
"""Convert an already-read Claude Code session into a validated ATIF trajectory.
|
|
134
|
+
|
|
135
|
+
The records are :func:`read_session`'s; nothing is read from disk here.
|
|
136
|
+
|
|
137
|
+
Raises
|
|
138
|
+
------
|
|
139
|
+
EmptySessionError: the converter found no convertible events.
|
|
140
|
+
ConversionError: any unexpected failure inside the converter.
|
|
141
|
+
"""
|
|
142
|
+
from atif_converter.infrastructure.claude_code_converter import (
|
|
143
|
+
convert_loaded_claude_code_session,
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
session_jsonl = loaded.snapshot.session_jsonl
|
|
147
|
+
try:
|
|
148
|
+
trajectory = convert_loaded_claude_code_session(loaded, include_subagents=include_subagents)
|
|
149
|
+
except (
|
|
150
|
+
Exception
|
|
151
|
+
) as exc: # the converter is a port of untyped upstream code; classify at the seam
|
|
152
|
+
msg = f"conversion failed for {session_jsonl}"
|
|
153
|
+
raise ConversionError(msg) from exc
|
|
154
|
+
return _result_from(trajectory, session_jsonl)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def convert_session(
|
|
158
|
+
session_jsonl: Path,
|
|
159
|
+
*,
|
|
160
|
+
include_subagents: bool = True,
|
|
161
|
+
) -> ConversionResult:
|
|
162
|
+
"""Convert one Claude Code session JSONL into a validated ATIF trajectory.
|
|
163
|
+
|
|
164
|
+
Reads the session and its side files through our converter and validates
|
|
165
|
+
the result with the ATIF ``TrajectoryValidator``. The use case goes through
|
|
166
|
+
:func:`read_session` + :func:`convert_loaded_session` instead so the audit
|
|
167
|
+
shares the read; this is the plain path for a caller that wants the
|
|
168
|
+
trajectory alone.
|
|
169
|
+
|
|
170
|
+
Raises
|
|
171
|
+
------
|
|
172
|
+
InvalidSessionInput: ``session_jsonl`` is not an existing ``.jsonl`` file.
|
|
173
|
+
EmptySessionError: the converter found no convertible events.
|
|
174
|
+
ConversionError: any unexpected failure inside the converter.
|
|
175
|
+
"""
|
|
176
|
+
from atif_converter.infrastructure.claude_code_converter import convert_claude_code_session
|
|
177
|
+
|
|
178
|
+
require_transcript_file(session_jsonl)
|
|
179
|
+
|
|
180
|
+
try:
|
|
181
|
+
trajectory = convert_claude_code_session(session_jsonl, include_subagents=include_subagents)
|
|
182
|
+
except (
|
|
183
|
+
Exception
|
|
184
|
+
) as exc: # the converter is a port of untyped upstream code; classify at the seam
|
|
185
|
+
msg = f"conversion failed for {session_jsonl}"
|
|
186
|
+
raise ConversionError(msg) from exc
|
|
187
|
+
return _result_from(trajectory, session_jsonl)
|