agent-session-analytics 0.1.0__tar.gz → 0.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. {agent_session_analytics-0.1.0 → agent_session_analytics-0.2.1}/.gitignore +6 -0
  2. agent_session_analytics-0.2.1/CHANGELOG.md +73 -0
  3. {agent_session_analytics-0.1.0 → agent_session_analytics-0.2.1}/PKG-INFO +13 -5
  4. {agent_session_analytics-0.1.0 → agent_session_analytics-0.2.1}/README.md +12 -4
  5. {agent_session_analytics-0.1.0 → agent_session_analytics-0.2.1}/pyproject.toml +4 -3
  6. agent_session_analytics-0.2.1/src/agent_session_analytics/_adapters/__init__.py +44 -0
  7. agent_session_analytics-0.2.1/src/agent_session_analytics/_adapters/_jsonl.py +24 -0
  8. agent_session_analytics-0.2.1/src/agent_session_analytics/_adapters/claude.py +28 -0
  9. agent_session_analytics-0.2.1/src/agent_session_analytics/_adapters/codex.py +376 -0
  10. agent_session_analytics-0.2.1/src/agent_session_analytics/_adapters/copilot.py +9 -0
  11. agent_session_analytics-0.2.1/src/agent_session_analytics/_adapters/cursor.py +200 -0
  12. agent_session_analytics-0.2.1/src/agent_session_analytics/_adapters/pi_family.py +179 -0
  13. {agent_session_analytics-0.1.0 → agent_session_analytics-0.2.1}/src/agent_session_analytics/_db.py +110 -10
  14. agent_session_analytics-0.2.1/src/agent_session_analytics/_emitters.py +455 -0
  15. agent_session_analytics-0.2.1/src/agent_session_analytics/_ingest.py +208 -0
  16. {agent_session_analytics-0.1.0 → agent_session_analytics-0.2.1}/src/agent_session_analytics/_query.py +7 -1
  17. agent_session_analytics-0.2.1/src/agent_session_analytics/_tables.py +301 -0
  18. {agent_session_analytics-0.1.0 → agent_session_analytics-0.2.1}/src/agent_session_analytics/references/canonical-schema.md +38 -6
  19. {agent_session_analytics-0.1.0 → agent_session_analytics-0.2.1}/src/agent_session_analytics/references/harness-coverage.md +15 -3
  20. {agent_session_analytics-0.1.0 → agent_session_analytics-0.2.1}/src/agent_session_analytics/references/query-conventions.md +11 -0
  21. {agent_session_analytics-0.1.0 → agent_session_analytics-0.2.1}/tests/conftest.py +2 -2
  22. {agent_session_analytics-0.1.0 → agent_session_analytics-0.2.1}/tests/test_cli.py +139 -12
  23. agent_session_analytics-0.2.1/tests/test_codex_envelope.py +50 -0
  24. {agent_session_analytics-0.1.0 → agent_session_analytics-0.2.1}/tests/test_codex_ingest.py +6 -6
  25. agent_session_analytics-0.2.1/tests/test_duckdb_settings_edges.py +126 -0
  26. agent_session_analytics-0.2.1/tests/test_flattened_equivalence.py +286 -0
  27. agent_session_analytics-0.2.1/tests/test_flattened_tables.py +186 -0
  28. {agent_session_analytics-0.1.0 → agent_session_analytics-0.2.1}/tests/test_malformed_rows.py +8 -3
  29. {agent_session_analytics-0.1.0 → agent_session_analytics-0.2.1}/tests/test_paths.py +45 -6
  30. agent_session_analytics-0.2.1/tests/test_pi_family_tokens.py +46 -0
  31. agent_session_analytics-0.2.1/tests/test_user_turns.py +194 -0
  32. agent_session_analytics-0.2.1/tests/test_user_turns_edges.py +356 -0
  33. agent_session_analytics-0.1.0/CHANGELOG.md +0 -37
  34. agent_session_analytics-0.1.0/src/agent_session_analytics/_ingest.py +0 -1190
  35. {agent_session_analytics-0.1.0 → agent_session_analytics-0.2.1}/LICENSE +0 -0
  36. {agent_session_analytics-0.1.0 → agent_session_analytics-0.2.1}/src/agent_session_analytics/__init__.py +0 -0
  37. {agent_session_analytics-0.1.0 → agent_session_analytics-0.2.1}/src/agent_session_analytics/__main__.py +0 -0
  38. {agent_session_analytics-0.1.0 → agent_session_analytics-0.2.1}/src/agent_session_analytics/_cli.py +0 -0
  39. {agent_session_analytics-0.1.0 → agent_session_analytics-0.2.1}/src/agent_session_analytics/py.typed +0 -0
  40. {agent_session_analytics-0.1.0 → agent_session_analytics-0.2.1}/tests/__init__.py +0 -0
@@ -13,8 +13,14 @@ __pycache__/
13
13
  .cheese/
14
14
  .context/
15
15
 
16
+ # Skill archives: build output. `just build` writes them; CI publishes them on dist (ADR-012).
17
+ skills/*/scripts/*.pyz
18
+
16
19
  # Skill-creator iteration workspaces (eval runs, benchmarks, viewer HTML)
17
20
  skills/*-workspace/
18
21
 
22
+ # Live smoke run records (just live-smoke)
23
+ .live-smoke/
24
+
19
25
  # Serena MCP — machine-local; not tracked
20
26
  .serena/
@@ -0,0 +1,73 @@
1
+ # Changelog
2
+
3
+ This file records the notable changes to the `agent-session-analytics` distribution.
4
+ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
5
+ The project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
6
+ A `0.x` minor release can contain breaking changes.
7
+
8
+ The `publish agent-session-analytics` workflow reads the section for the tagged version.
9
+ It uses that section as the GitHub release notes.
10
+ To release, move the `Unreleased` entries into a new version section with the date.
11
+ Then set the same version in `packages/agent-session-analytics/pyproject.toml`.
12
+ Push a `agent-session-analytics-v<version>` tag after the release commit merges.
13
+
14
+ ## [Unreleased]
15
+
16
+ ## [0.2.1] - 2026-10-25
17
+
18
+ ### Changed
19
+
20
+ - The default database path is now `<cache>/agent-session-analytics/sessions.duckdb`, not `<cache>/dotfiles/session-analytics/sessions.duckdb`. If the new file does not exist and the legacy file exists, the package keeps using the legacy file. It never moves, copies, or deletes the legacy file. `SESSIONS_DB` still overrides both.
21
+
22
+ ## [0.2.0] - 2026-10-24
23
+
24
+ ### Changed
25
+
26
+ - Ingest builds `tool_uses`, `tool_results`, `stop_events`, and `model_turns` from narrow rows that the staging pass pre-flattens. DuckDB no longer runs `json_extract` over `raw_entries.message`. The `raw_entries` step runs on one thread. Peak memory of an ingest step drops from 11.1 GB to 2.6 GB on a 1 M-row log set. Tables, columns, and types stay the same.
27
+ - Every DuckDB run now sets `memory_limit` (default `4GB`, was `8GB`), `threads` (default `4`), and a 20GB spill cap. Ingest also sets `preserve_insertion_order=false`; read-only queries keep insertion order.
28
+ - `SESSIONS_DUCKDB_MEMORY_LIMIT` must be at least 16MiB and below 16EiB. Ingest checks the value before the stage step.
29
+ - An invalid `SESSIONS_DUCKDB_*` value makes `sql` exit 5 (a query error), not 4.
30
+ - `tool_uses.input` keeps an explicit JSON `null`, and `tool_results.content` prints floats the way DuckDB prints them.
31
+ - The schema version is now 3, so ingest rebuilds a version 1 or version 2 database.
32
+ - omp and pi `model_turns.cache_creation_tokens` now holds the cache-write count (`cacheWrite`). It was NULL.
33
+ - omp and pi `model_turns.context_tokens` now equals input + cache_read + cache_creation tokens, as it does for Claude.
34
+ - The Codex adapter keeps `item_completed` `UserMessage` prompts, `task_complete`, and `turn_aborted` events as `raw_entries` rows. Codex `sessions.entry_count` counts them.
35
+ - The omp and pi adapters keep the `attribution` of a user row in `raw_entries.message`. A row with `attribution` `agent` is not a real prompt.
36
+ - Codex `raw_entries.isSidechain` is now `true` for every row of a rollout whose `session_meta.source` has a `subagent` value (`thread_spawn`, `guardian`, or `review`) and `false` otherwise. It was NULL.
37
+
38
+ ### Added
39
+
40
+ - `SESSIONS_DUCKDB_THREADS` sets the DuckDB thread count.
41
+ - DuckDB spills to a private `<db>.spill-<pid>-<token>` directory, one per process. DuckDB creates it only when a run spills, and the process removes it at exit.
42
+ - A query-hygiene rule in `query-conventions.md`.
43
+ - The `user_turns` table: one row per real user prompt (claude, codex, omp, and pi) with `turn_index`, the full `text`, and `prev_assistant_text`. Blank prompts and agent-run Codex rollouts stay out.
44
+ - The `interrupt_events` table: main-session Claude `[Request interrupted` markers and Codex `turn_aborted` events with reason `interrupted` outside agent-run rollouts. The staging pass builds it. It can overlap `permission_denials`.
45
+
46
+ ### Fixed
47
+
48
+ - `ingest` `elapsed_seconds` now includes the stage step that discovers and normalizes the logs. It was the DuckDB load time only.
49
+
50
+ ## [0.1.0] - 2026-10-11
51
+
52
+ ### Added
53
+
54
+ - The `agent-session-analytics` console script with four JSON commands: `ingest`, `db-path`, `sql`, and `schema`.
55
+ - A schema version stamped in the database. `ingest` rebuilds a database whose version differs, even inside the one-hour freshness window.
56
+ - The engine moves out of the `skillz` skill and merges the dotfiles `session-analytics` engine at dotfiles commit `6ce36879`.
57
+ - Codex `token_usage` rows reach `model_turns`. They do not count in `sessions.entry_count`.
58
+ - The columns `model_turns.cache_creation_tokens`, `model_turns.context_tokens`, `sessions.version`, and `sessions.parent_session_id`.
59
+ - The references `canonical-schema.md`, `harness-coverage.md`, and `query-conventions.md` ship as package data.
60
+
61
+ ### Changed
62
+
63
+ - Codex `model_turns.tool_calls` is NULL, because Codex token rows carry no content.
64
+ - Cursor timestamps are UTC.
65
+
66
+ ### Removed
67
+
68
+ - The canned `query.sh` reports. Callers write their own SQL with `sql`.
69
+
70
+ [Unreleased]: https://github.com/paulnsorensen/skillz-that-grillz/compare/agent-session-analytics-v0.2.1...HEAD
71
+ [0.2.1]: https://github.com/paulnsorensen/skillz-that-grillz/compare/agent-session-analytics-v0.2.0...agent-session-analytics-v0.2.1
72
+ [0.2.0]: https://github.com/paulnsorensen/skillz-that-grillz/compare/agent-session-analytics-v0.1.0...agent-session-analytics-v0.2.0
73
+ [0.1.0]: https://github.com/paulnsorensen/skillz-that-grillz/releases/tag/agent-session-analytics-v0.1.0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: agent-session-analytics
3
- Version: 0.1.0
3
+ Version: 0.2.1
4
4
  Summary: Ingest coding-agent session logs into DuckDB and query them read-only.
5
5
  Project-URL: Homepage, https://github.com/paulnsorensen/skillz-that-grillz/tree/main/packages/agent-session-analytics
6
6
  Project-URL: Source, https://github.com/paulnsorensen/skillz-that-grillz/tree/main/packages/agent-session-analytics
@@ -96,14 +96,22 @@ A consumer that queries the database directly runs
96
96
  - `SESSIONS_DB` names the database. A relative value joins the current
97
97
  directory. The package does not expand `~` in a set value.
98
98
  - Otherwise an absolute `XDG_CACHE_HOME` selects
99
- `<cache>/dotfiles/session-analytics/sessions.duckdb`. A relative or unset
100
- value uses `~/.cache`.
101
- - `SESSIONS_DUCKDB_MEMORY_LIMIT` sets the DuckDB memory limit. The default is
102
- `8GB`.
99
+ `<cache>/agent-session-analytics/sessions.duckdb`. A relative or unset
100
+ value uses `~/.cache`. If that file does not exist and the legacy
101
+ `<cache>/dotfiles/session-analytics/sessions.duckdb` exists, the package uses
102
+ the legacy file. It never moves, copies, or deletes the legacy file.
103
+ - `SESSIONS_DUCKDB_MEMORY_LIMIT` sets the DuckDB memory limit, from 16MiB up to
104
+ but not including 16EiB. The default is `4GB`.
105
+ - `SESSIONS_DUCKDB_THREADS` sets the DuckDB thread count, from 1 to 256. The
106
+ default is `4`.
103
107
  - `CLAUDE_CONFIG_DIR`, `CODEX_HOME`, and `CURSOR_HOME` move the log roots.
104
108
 
105
109
  Ingest writes the transient `<db>.tmp` and `<db>.stage`, and a persistent
106
110
  `<db>.lock`. An `fcntl` lock serializes ingest runs. Ingest uses umask `077`.
111
+ DuckDB spills large operations to a per-process `<db>.spill-<pid>-<token>`
112
+ directory. DuckDB creates it only when a run spills, so a read-only query needs
113
+ no writable directory. Each process removes its own directory at exit; a killed
114
+ process can leave one behind, which is safe to delete. The spill cap is 20GB.
107
115
 
108
116
  ## Schema version
109
117
 
@@ -73,14 +73,22 @@ A consumer that queries the database directly runs
73
73
  - `SESSIONS_DB` names the database. A relative value joins the current
74
74
  directory. The package does not expand `~` in a set value.
75
75
  - Otherwise an absolute `XDG_CACHE_HOME` selects
76
- `<cache>/dotfiles/session-analytics/sessions.duckdb`. A relative or unset
77
- value uses `~/.cache`.
78
- - `SESSIONS_DUCKDB_MEMORY_LIMIT` sets the DuckDB memory limit. The default is
79
- `8GB`.
76
+ `<cache>/agent-session-analytics/sessions.duckdb`. A relative or unset
77
+ value uses `~/.cache`. If that file does not exist and the legacy
78
+ `<cache>/dotfiles/session-analytics/sessions.duckdb` exists, the package uses
79
+ the legacy file. It never moves, copies, or deletes the legacy file.
80
+ - `SESSIONS_DUCKDB_MEMORY_LIMIT` sets the DuckDB memory limit, from 16MiB up to
81
+ but not including 16EiB. The default is `4GB`.
82
+ - `SESSIONS_DUCKDB_THREADS` sets the DuckDB thread count, from 1 to 256. The
83
+ default is `4`.
80
84
  - `CLAUDE_CONFIG_DIR`, `CODEX_HOME`, and `CURSOR_HOME` move the log roots.
81
85
 
82
86
  Ingest writes the transient `<db>.tmp` and `<db>.stage`, and a persistent
83
87
  `<db>.lock`. An `fcntl` lock serializes ingest runs. Ingest uses umask `077`.
88
+ DuckDB spills large operations to a per-process `<db>.spill-<pid>-<token>`
89
+ directory. DuckDB creates it only when a run spills, so a read-only query needs
90
+ no writable directory. Each process removes its own directory at exit; a killed
91
+ process can leave one behind, which is safe to delete. The spill cap is 20GB.
84
92
 
85
93
  ## Schema version
86
94
 
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "agent-session-analytics"
3
- version = "0.1.0"
3
+ version = "0.2.1"
4
4
  description = "Ingest coding-agent session logs into DuckDB and query them read-only."
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.11"
@@ -29,10 +29,10 @@ Issues = "https://github.com/paulnsorensen/skillz-that-grillz/issues"
29
29
  Changelog = "https://github.com/paulnsorensen/skillz-that-grillz/blob/main/packages/agent-session-analytics/CHANGELOG.md"
30
30
 
31
31
  [dependency-groups]
32
- dev = ["pytest>=8", "pytest-xdist>=3", "basedpyright==1.40.1"]
32
+ dev = ["pytest>=8", "pytest-xdist>=3", "basedpyright==1.40.2"]
33
33
 
34
34
  [build-system]
35
- requires = ["hatchling>=1.27.0"]
35
+ requires = ["hatchling>=1.32.4"]
36
36
  build-backend = "hatchling.build"
37
37
 
38
38
  [tool.hatch.build.targets.wheel]
@@ -50,6 +50,7 @@ fromargs = { workspace = true }
50
50
 
51
51
  [tool.pytest.ini_options]
52
52
  testpaths = ["tests"]
53
+ markers = ["ac(id): ties a test to an acceptance-criterion ID"]
53
54
  addopts = "--strict-markers"
54
55
 
55
56
  [tool.basedpyright]
@@ -0,0 +1,44 @@
1
+ """Harness log adapters and their registry.
2
+
3
+ Each adapter is (name, discover, normalize):
4
+ discover() -> list of source paths, or [] when no logs are reachable.
5
+ normalize(path) -> yields canonical raw-entry dicts (already harness-tagged).
6
+ An adapter that finds nothing is non-fatal — the run logs it and continues.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from agent_session_analytics._adapters.claude import claude_discover, claude_normalize
12
+ from agent_session_analytics._adapters.codex import codex_discover, codex_normalize
13
+ from agent_session_analytics._adapters.copilot import copilot_discover
14
+ from agent_session_analytics._adapters.cursor import cursor_discover, cursor_normalize
15
+ from agent_session_analytics._adapters.pi_family import (
16
+ omp_discover,
17
+ omp_normalize,
18
+ pi_discover,
19
+ pi_normalize,
20
+ )
21
+
22
+ ADAPTERS = [
23
+ ("claude", claude_discover, claude_normalize),
24
+ ("codex", codex_discover, codex_normalize),
25
+ ("omp", omp_discover, omp_normalize),
26
+ ("pi", pi_discover, pi_normalize),
27
+ ("cursor", cursor_discover, cursor_normalize),
28
+ ("copilot", copilot_discover, None),
29
+ ]
30
+
31
+ __all__ = [
32
+ "ADAPTERS",
33
+ "claude_discover",
34
+ "claude_normalize",
35
+ "codex_discover",
36
+ "codex_normalize",
37
+ "copilot_discover",
38
+ "cursor_discover",
39
+ "cursor_normalize",
40
+ "omp_discover",
41
+ "omp_normalize",
42
+ "pi_discover",
43
+ "pi_normalize",
44
+ ]
@@ -0,0 +1,24 @@
1
+ """The JSON value type and the line reader that every harness adapter shares."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from collections.abc import Iterator
7
+ from typing import TYPE_CHECKING, cast
8
+
9
+ if TYPE_CHECKING:
10
+ from typing import TypeAlias
11
+
12
+ JsonValue: TypeAlias = "None | bool | int | float | str | list[JsonValue] | dict[str, JsonValue]"
13
+
14
+
15
+ def iter_jsonl(path: str) -> Iterator[JsonValue]:
16
+ with open(path, encoding="utf-8", errors="replace") as fh:
17
+ for line in fh:
18
+ line = line.strip()
19
+ if not line:
20
+ continue
21
+ try:
22
+ yield cast(JsonValue, json.loads(line))
23
+ except json.JSONDecodeError:
24
+ continue
@@ -0,0 +1,28 @@
1
+ """The Claude adapter: its logs already use the canonical envelope."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ from collections.abc import Iterator
7
+
8
+ from agent_session_analytics._adapters._jsonl import JsonValue, iter_jsonl
9
+ from agent_session_analytics._db import configured_path
10
+
11
+
12
+ def claude_discover() -> list[str]:
13
+ root = os.path.join(configured_path("CLAUDE_CONFIG_DIR", "~/.claude"), "projects")
14
+ if not os.path.isdir(root):
15
+ return []
16
+ out: list[str] = []
17
+ for dirpath, _dirs, files in os.walk(root):
18
+ out.extend(os.path.join(dirpath, f) for f in files if f.endswith(".jsonl"))
19
+ return out
20
+
21
+
22
+ def claude_normalize(path: str) -> Iterator[dict[str, JsonValue]]:
23
+ """Claude logs are already in the canonical envelope; just tag them."""
24
+ for entry in iter_jsonl(path):
25
+ if not isinstance(entry, dict):
26
+ continue
27
+ entry["harness"] = "claude"
28
+ yield entry
@@ -0,0 +1,376 @@
1
+ """The Codex adapter: rollout JSONL to the canonical envelope."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import os
7
+ import re
8
+ from collections.abc import Iterator
9
+ from typing import cast
10
+
11
+ from agent_session_analytics._adapters._jsonl import JsonValue, iter_jsonl
12
+ from agent_session_analytics._db import configured_path
13
+
14
+
15
+ def codex_discover() -> list[str]:
16
+ root = os.path.join(configured_path("CODEX_HOME", "~/.codex"), "sessions")
17
+ if not os.path.isdir(root):
18
+ return []
19
+ out: list[str] = []
20
+ for dirpath, _dirs, files in os.walk(root):
21
+ out.extend(os.path.join(dirpath, f) for f in files if f.endswith(".jsonl"))
22
+ return out
23
+
24
+
25
+ def _codex_payload_is_error(payload: JsonValue) -> bool:
26
+ if not isinstance(payload, dict):
27
+ return False
28
+ status = payload.get("status")
29
+ if isinstance(status, str) and status.casefold() in ("error", "failed", "failure"):
30
+ return True
31
+ for key in ("isError", "is_error"):
32
+ value = payload.get(key)
33
+ if value is True or (isinstance(value, str) and value.casefold() == "true"):
34
+ return True
35
+ code = payload.get("exit_code")
36
+ return isinstance(code, int) and not isinstance(code, bool) and code != 0
37
+
38
+
39
+ def _codex_content_block_payloads(blocks: list[JsonValue]) -> Iterator[dict[str, JsonValue]]:
40
+ if not blocks:
41
+ return
42
+ texts: list[str] = []
43
+ for block in blocks:
44
+ if not isinstance(block, dict) or block.get("type") != "input_text":
45
+ continue
46
+ block_text = block.get("text")
47
+ if not isinstance(block_text, str):
48
+ continue
49
+ texts.append(block_text)
50
+ if not texts:
51
+ return
52
+ header = texts[0]
53
+ if not re.fullmatch(
54
+ r"Script completed\nWall time(?:\s*:\s*|\s+)[^\n]*\nOutput:\n",
55
+ header,
56
+ ):
57
+ return
58
+ for block_text in texts[1:]:
59
+ try:
60
+ payload = cast(JsonValue, json.loads(block_text.strip()))
61
+ except ValueError:
62
+ continue
63
+ if isinstance(payload, dict):
64
+ yield payload
65
+
66
+
67
+ def _codex_structured_payloads(out: JsonValue) -> Iterator[dict[str, JsonValue]]:
68
+ if isinstance(out, dict):
69
+ yield out
70
+ return
71
+ if isinstance(out, list):
72
+ yield from _codex_content_block_payloads(out)
73
+ return
74
+ if not isinstance(out, str):
75
+ return
76
+ text = out.strip()
77
+ try:
78
+ parsed = cast(JsonValue, json.loads(text))
79
+ except ValueError:
80
+ parsed = None
81
+ if isinstance(parsed, dict):
82
+ yield parsed
83
+ return
84
+ if isinstance(parsed, list):
85
+ yield from _codex_content_block_payloads(parsed)
86
+ return
87
+ marker = re.search(
88
+ r"^Script completed\n(?:Wall time(?:\s*:\s*|\s+)[^\n]*\n)?Output:\s*(.+)\s*\Z",
89
+ out,
90
+ re.DOTALL,
91
+ )
92
+ if not marker:
93
+ return
94
+ decoder = json.JSONDecoder()
95
+ text = marker.group(1)
96
+ position = 0
97
+ while position < len(text):
98
+ while position < len(text) and text[position].isspace():
99
+ position += 1
100
+ if position == len(text):
101
+ return
102
+ try:
103
+ parsed, end = cast(tuple[JsonValue, int], decoder.raw_decode(text, position))
104
+ except ValueError:
105
+ return
106
+ if isinstance(parsed, dict):
107
+ yield parsed
108
+ elif isinstance(parsed, list):
109
+ yield from (item for item in parsed if isinstance(item, dict))
110
+ position = end
111
+
112
+
113
+ def _codex_output_is_error(out: JsonValue) -> str:
114
+ """Detect structured or legacy failure signals in a Codex tool output."""
115
+ if any(
116
+ _codex_payload_is_error(payload) for payload in _codex_structured_payloads(out)
117
+ ):
118
+ return "true"
119
+ if isinstance(out, str):
120
+ match = re.search(r"Process exited with code (-?\d+)\b", out)
121
+ if match and any(int(digit) != 0 for digit in match.group(1).lstrip("-")):
122
+ return "true"
123
+ return "false"
124
+
125
+
126
+ def _codex_command(name: JsonValue, parsed: JsonValue, raw_input: JsonValue) -> str | None:
127
+ """The executed command string for codex shell-ish tools, or None.
128
+
129
+ Mirrors claude's ``input.command`` so ``tool_uses.bash_cmd`` populates:
130
+ ``exec_command`` carries it in ``cmd``; legacy ``shell`` carries an argv
131
+ array (the ``-lc``/``-c`` payload is the real command); the ``exec``
132
+ custom tool's argument IS the raw code string.
133
+ """
134
+ if name == "exec_command" and isinstance(parsed, dict):
135
+ cmd = parsed.get("cmd")
136
+ return cmd if isinstance(cmd, str) else None
137
+ if name == "shell" and isinstance(parsed, dict):
138
+ cmd = parsed.get("command")
139
+ if isinstance(cmd, str):
140
+ return cmd
141
+ if isinstance(cmd, list) and cmd:
142
+ if len(cmd) == 3 and cmd[1] in ("-lc", "-c"):
143
+ return str(cmd[2])
144
+ return " ".join(str(c) for c in cmd)
145
+ return None
146
+ if name == "exec" and isinstance(raw_input, str):
147
+ return raw_input
148
+ return None
149
+
150
+
151
+ def _codex_token_usage(
152
+ payload: dict[str, JsonValue], last_total: JsonValue
153
+ ) -> tuple[dict[str, JsonValue] | None, JsonValue]:
154
+ """Map a token_count payload to (Claude-style usage, total_token_usage).
155
+
156
+ Returns usage None when last_token_usage is null or when total_token_usage
157
+ repeats the previous event (Codex re-emits it). The returned total is the
158
+ latest total seen on any token_count event, with or without usage.
159
+ """
160
+ info = payload.get("info")
161
+ if not isinstance(info, dict):
162
+ return None, last_total
163
+ total = info.get("total_token_usage")
164
+ new_total = last_total if total is None else total
165
+ last = info.get("last_token_usage")
166
+ if not isinstance(last, dict):
167
+ return None, new_total
168
+ if total is not None and total == last_total:
169
+ return None, new_total
170
+ usage: dict[str, JsonValue] = {
171
+ "input_tokens": last.get("input_tokens"),
172
+ "output_tokens": last.get("output_tokens"),
173
+ "cache_read_input_tokens": last.get("cached_input_tokens"),
174
+ }
175
+ return usage, new_total
176
+
177
+
178
+ def _codex_parent_session(payload: dict[str, JsonValue]) -> str | None:
179
+ """Parent thread id of a sub-agent rollout, or None.
180
+
181
+ A sub-agent session_meta carries ``source.subagent.thread_spawn``; other
182
+ sessions carry a plain string or no ``source``.
183
+ """
184
+ source = payload.get("source")
185
+ subagent = source.get("subagent") if isinstance(source, dict) else None
186
+ spawn = subagent.get("thread_spawn") if isinstance(subagent, dict) else None
187
+ parent = spawn.get("parent_thread_id") if isinstance(spawn, dict) else None
188
+ return parent if isinstance(parent, str) and parent else None
189
+
190
+
191
+ def _codex_is_subagent(payload: dict[str, JsonValue]) -> bool:
192
+ """Return True when session_meta marks an agent-run rollout.
193
+
194
+ Any ``source.subagent`` value counts: ``thread_spawn`` children, and
195
+ ``other`` (for example ``guardian``) or ``review`` runs that name no parent.
196
+ """
197
+ source = payload.get("source")
198
+ return isinstance(source, dict) and "subagent" in source
199
+
200
+
201
+ def _codex_event(payload: dict[str, JsonValue]) -> dict[str, JsonValue] | None:
202
+ """Map one non-token ``event_msg`` payload to canonical fields, or None to drop it.
203
+
204
+ An ``item_completed`` ``UserMessage`` item becomes a user entry with its
205
+ text blocks. ``task_complete`` becomes an assistant entry with subtype
206
+ ``task_complete`` and ``last_agent_message`` as its one text block (no block
207
+ when the turn ended without one). ``turn_aborted`` becomes a system entry
208
+ with subtype ``turn_aborted`` and the abort reason in ``stopReason``.
209
+ """
210
+ ptype = payload.get("type")
211
+ if ptype == "item_completed":
212
+ item = payload.get("item")
213
+ if not isinstance(item, dict) or item.get("type") != "UserMessage":
214
+ return None
215
+ content = item.get("content")
216
+ blocks: list[JsonValue] = [
217
+ {"type": "text", "text": block["text"]}
218
+ for block in (content if isinstance(content, list) else [])
219
+ if isinstance(block, dict) and block.get("type") == "text" and isinstance(block.get("text"), str)
220
+ ]
221
+ return {"type": "user", "message": {"content": blocks}} if blocks else None
222
+ if ptype == "task_complete":
223
+ last = payload.get("last_agent_message")
224
+ blocks = [{"type": "text", "text": last}] if isinstance(last, str) else []
225
+ return {"type": "assistant", "subtype": "task_complete", "message": {"content": blocks}}
226
+ if ptype == "turn_aborted":
227
+ reason = payload.get("reason")
228
+ return {
229
+ "type": "system",
230
+ "subtype": "turn_aborted",
231
+ "stopReason": reason if isinstance(reason, str) else None,
232
+ }
233
+ return None
234
+
235
+
236
+ def codex_normalize(path: str) -> Iterator[dict[str, object]]:
237
+ """Codex rollout JSONL -> canonical envelope.
238
+
239
+ session_meta carries id + cwd; a response_item/function_call becomes an
240
+ assistant tool_use block; a function_call_output OR custom_tool_call_output
241
+ becomes a user tool_result block (dropping the custom outputs was the ~50%
242
+ result-join gap — issue #704). Codex tool names (shell / apply_patch /
243
+ custom tools) are kept verbatim, and ``input.command`` is normalized to the
244
+ executed command string for shell-ish tools so ``bash_cmd`` extracts.
245
+ ``tool_search_output`` items have no matching call item and are dropped.
246
+ A forked rollout holds two ``session_meta`` records (own id first, parent
247
+ second); the first id, ``cwd``, and ``cli_version`` describe the file. A
248
+ token row before any id falls back to the file path, so ids stay unique.
249
+ ``session_meta.cli_version`` stamps ``version`` on every entry. A
250
+ sub-agent rollout names its parent in the first ``session_meta``; that id
251
+ stamps ``parentSessionId`` on every entry. Any ``source.subagent`` value
252
+ stamps ``isSidechain`` true on every entry, as Cursor does for sub-agent
253
+ files, so agent-run rollouts without a parent id stay identifiable. Each
254
+ non-duplicate
255
+ ``token_count`` event with a ``last_token_usage`` becomes one
256
+ ``token_usage`` entry (unique ``message.id``, empty content). Only
257
+ model_turns reads it, as one row; no other table sees it. User prompts,
258
+ ``task_complete``, and ``turn_aborted`` events map through _codex_event;
259
+ ``response_item`` user messages (preambles and injected context) stay out.
260
+ """
261
+ session_id = None
262
+ parent_session_id = None
263
+ is_subagent = False
264
+ cwd = None
265
+ cli_version = None
266
+ model = None
267
+ last_total = None
268
+ token_turns = 0
269
+
270
+ def envelope() -> dict[str, object]:
271
+ return {
272
+ "harness": "codex",
273
+ "timestamp": ts,
274
+ "sessionId": session_id,
275
+ "parentSessionId": parent_session_id,
276
+ "isSidechain": is_subagent,
277
+ "version": cli_version,
278
+ "cwd": cwd,
279
+ }
280
+
281
+ for entry in iter_jsonl(path):
282
+ if not isinstance(entry, dict):
283
+ continue
284
+ ts = entry.get("timestamp")
285
+ payload = entry.get("payload")
286
+ etype = entry.get("type")
287
+ if etype == "session_meta" and isinstance(payload, dict):
288
+ if session_id is None:
289
+ parent_session_id = _codex_parent_session(payload)
290
+ is_subagent = _codex_is_subagent(payload)
291
+ session_id = session_id or payload.get("id")
292
+ cwd = cwd or payload.get("cwd")
293
+ cli_version = cli_version or payload.get("cli_version") or None
294
+ continue
295
+ if etype == "turn_context" and isinstance(payload, dict):
296
+ cwd = payload.get("cwd") or cwd
297
+ model = payload.get("model") or model
298
+ continue
299
+ if (
300
+ etype == "event_msg"
301
+ and isinstance(payload, dict)
302
+ and payload.get("type") == "token_count"
303
+ ):
304
+ usage, last_total = _codex_token_usage(payload, last_total)
305
+ if usage is None:
306
+ continue
307
+ token_turns += 1
308
+ yield {
309
+ **envelope(),
310
+ "type": "token_usage",
311
+ "message": {
312
+ "id": f"codex-tokens:{session_id or path}:{token_turns}",
313
+ "model": model or "unknown",
314
+ "content": [],
315
+ "usage": usage,
316
+ },
317
+ }
318
+ continue
319
+ if etype == "event_msg" and isinstance(payload, dict):
320
+ event = _codex_event(payload)
321
+ if event is not None:
322
+ yield {
323
+ **envelope(),
324
+ **event,
325
+ }
326
+ continue
327
+ if etype != "response_item" or not isinstance(payload, dict):
328
+ continue
329
+ ptype = payload.get("type")
330
+ if ptype in ("function_call", "custom_tool_call"):
331
+ raw_input = payload.get("arguments") or payload.get("input")
332
+ try:
333
+ parsed: JsonValue = (
334
+ cast(JsonValue, json.loads(raw_input)) if isinstance(raw_input, str) else raw_input
335
+ )
336
+ except (json.JSONDecodeError, TypeError):
337
+ parsed = {"raw": raw_input}
338
+ if not isinstance(parsed, dict):
339
+ parsed = {"raw": parsed}
340
+ name = payload.get("name")
341
+ command = _codex_command(name, parsed, raw_input)
342
+ if command is not None:
343
+ parsed["command"] = command
344
+ yield {
345
+ **envelope(),
346
+ "type": "assistant",
347
+ "message": {
348
+ "content": [
349
+ {
350
+ "type": "tool_use",
351
+ "id": payload.get("call_id"),
352
+ "name": name,
353
+ "input": parsed,
354
+ }
355
+ ]
356
+ },
357
+ }
358
+ elif ptype in ("function_call_output", "custom_tool_call_output"):
359
+ out = payload.get("output")
360
+ is_error = _codex_output_is_error(out)
361
+ if isinstance(out, dict):
362
+ out = json.dumps(out)
363
+ yield {
364
+ **envelope(),
365
+ "type": "user",
366
+ "message": {
367
+ "content": [
368
+ {
369
+ "type": "tool_result",
370
+ "tool_use_id": payload.get("call_id"),
371
+ "content": out,
372
+ "is_error": is_error,
373
+ }
374
+ ]
375
+ },
376
+ }
@@ -0,0 +1,9 @@
1
+ """The Copilot adapter: discovery only, because the CLI keeps no local transcript."""
2
+
3
+ from __future__ import annotations
4
+
5
+
6
+ def copilot_discover() -> list[str]:
7
+ # GitHub Copilot CLI persists no local session transcript we can find
8
+ # (~/.copilot holds skills/ + mcp-config.json only). See harness-coverage.md.
9
+ return []