agent-session-analytics 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_session_analytics/__init__.py +5 -0
- agent_session_analytics/__main__.py +5 -0
- agent_session_analytics/_cli.py +57 -0
- agent_session_analytics/_db.py +158 -0
- agent_session_analytics/_ingest.py +1190 -0
- agent_session_analytics/_query.py +59 -0
- agent_session_analytics/py.typed +0 -0
- agent_session_analytics/references/canonical-schema.md +149 -0
- agent_session_analytics/references/harness-coverage.md +182 -0
- agent_session_analytics/references/query-conventions.md +91 -0
- agent_session_analytics-0.1.0.dist-info/METADATA +122 -0
- agent_session_analytics-0.1.0.dist-info/RECORD +15 -0
- agent_session_analytics-0.1.0.dist-info/WHEEL +4 -0
- agent_session_analytics-0.1.0.dist-info/entry_points.txt +2 -0
- agent_session_analytics-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Run one read-only query against the session database."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
|
|
7
|
+
from agent_session_analytics._db import (
|
|
8
|
+
DB_PATH,
|
|
9
|
+
SCHEMA_VERSION,
|
|
10
|
+
DatabaseError,
|
|
11
|
+
IngestError,
|
|
12
|
+
QueryError,
|
|
13
|
+
db_is_fresh,
|
|
14
|
+
db_is_stamped,
|
|
15
|
+
query_rows,
|
|
16
|
+
require_duckdb,
|
|
17
|
+
)
|
|
18
|
+
from agent_session_analytics._ingest import build_database
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _refresh_default_database() -> None:
|
|
22
|
+
"""Rebuild an old, absent, or wrong-version default database before the query."""
|
|
23
|
+
if db_is_fresh():
|
|
24
|
+
return
|
|
25
|
+
try:
|
|
26
|
+
_ = build_database()
|
|
27
|
+
except DatabaseError as error:
|
|
28
|
+
outcome = (
|
|
29
|
+
"refusing to query stale data"
|
|
30
|
+
if os.path.exists(DB_PATH)
|
|
31
|
+
else "no session database is available"
|
|
32
|
+
)
|
|
33
|
+
raise IngestError(
|
|
34
|
+
f"Automatic ingestion failed ({error}); {outcome}.", exit_code=error.exit_code
|
|
35
|
+
) from error
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def run_query(query: str) -> list[dict[str, object]]:
|
|
39
|
+
"""Return the rows of one read-only query.
|
|
40
|
+
|
|
41
|
+
The default database refreshes itself when it is old or has another schema
|
|
42
|
+
version. A ``SESSIONS_DB`` override never refreshes: a missing database or a
|
|
43
|
+
schema mismatch is an error, so the caller runs ``ingest`` on purpose.
|
|
44
|
+
"""
|
|
45
|
+
require_duckdb()
|
|
46
|
+
if os.environ.get("SESSIONS_DB"):
|
|
47
|
+
if not os.path.exists(DB_PATH):
|
|
48
|
+
raise IngestError(f"No session database at {DB_PATH}; run `agent-session-analytics ingest`.")
|
|
49
|
+
if not db_is_stamped():
|
|
50
|
+
raise IngestError(
|
|
51
|
+
f"The database at {DB_PATH} is not at schema version "
|
|
52
|
+
+ f"{SCHEMA_VERSION}; run `agent-session-analytics ingest --force`."
|
|
53
|
+
)
|
|
54
|
+
else:
|
|
55
|
+
_refresh_default_database()
|
|
56
|
+
try:
|
|
57
|
+
return query_rows(query)
|
|
58
|
+
except DatabaseError as error:
|
|
59
|
+
raise QueryError(str(error)) from error
|
|
File without changes
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
# Canonical Schema
|
|
2
|
+
|
|
3
|
+
Contents: [tool_uses](#tool_uses), [tool_results](#tool_results), [stop_events](#stop_events), [model_turns](#model_turns), [agent_spawns](#agent_spawns), [skill_invocations](#skill_invocations), [mcp_calls](#mcp_calls), [sessions](#sessions), [stop_hooks](#stop_hooks), [permission_denials](#permission_denials), [raw_entries](#raw_entries), [schema_info](#schema_info), [Type gotchas](#type-gotchas).
|
|
4
|
+
|
|
5
|
+
The shape every harness adapter normalizes into and every analytics pack queries
|
|
6
|
+
against. `agent-session-analytics ingest` loads one canonical row shape (the Claude envelope plus a
|
|
7
|
+
`harness` tag) into `${XDG_CACHE_HOME:-$HOME/.cache}/dotfiles/session-analytics/sessions.duckdb` by default, then flattens it into
|
|
8
|
+
the tables below. **Pack authors: write SQL against these tables; never reach
|
|
9
|
+
into a harness's native format.**
|
|
10
|
+
|
|
11
|
+
Every session-scoped table carries a `harness` column
|
|
12
|
+
(`claude`/`codex`/`omp`/`pi`/`cursor`/`copilot`). Filter or group by it to
|
|
13
|
+
compare sources; omit it to aggregate across all reachable harnesses.
|
|
14
|
+
|
|
15
|
+
## `tool_uses`
|
|
16
|
+
|
|
17
|
+
Flattened from assistant `message.content[]` blocks where `type='tool_use'`.
|
|
18
|
+
|
|
19
|
+
| Column | Type | Description |
|
|
20
|
+
|--------|------|-------------|
|
|
21
|
+
| harness | VARCHAR | Source harness |
|
|
22
|
+
| tool_name | VARCHAR | Tool invoked (Bash, Read, Edit, Agent, Skill, mcp__*, or a harness-native name like `shell`/`apply_patch`) |
|
|
23
|
+
| tool_use_id | VARCHAR | Unique ID for joining with `tool_results` |
|
|
24
|
+
| input | JSON | Full input object |
|
|
25
|
+
| bash_cmd | VARCHAR | Extracted command (Bash only) |
|
|
26
|
+
| skill_name | VARCHAR | Extracted skill (Skill only — claude) |
|
|
27
|
+
| skill_args | VARCHAR | Extracted args (Skill only — claude) |
|
|
28
|
+
| agent_type | VARCHAR | Extracted subagent_type (Agent only — claude) |
|
|
29
|
+
| agent_desc | VARCHAR | Extracted description (Agent only) |
|
|
30
|
+
| agent_mode | VARCHAR | Extracted mode (Agent only) |
|
|
31
|
+
| grep_pattern | VARCHAR | Extracted pattern (Grep only) |
|
|
32
|
+
| file_path | VARCHAR | Extracted file_path (Read/Edit/Write) |
|
|
33
|
+
| query | VARCHAR | Extracted query (ToolSearch) |
|
|
34
|
+
| timestamp | VARCHAR | ISO timestamp |
|
|
35
|
+
| sessionId | VARCHAR | Session identifier |
|
|
36
|
+
| cwd | VARCHAR | Working directory |
|
|
37
|
+
| gitBranch | VARCHAR | Git branch (claude only) |
|
|
38
|
+
|
|
39
|
+
## `tool_results`
|
|
40
|
+
|
|
41
|
+
Flattened from user `message.content[]` blocks where `type='tool_result'`.
|
|
42
|
+
|
|
43
|
+
| Column | Type | Description |
|
|
44
|
+
|--------|------|-------------|
|
|
45
|
+
| harness | VARCHAR | Source harness |
|
|
46
|
+
| tool_use_id | VARCHAR | Matches `tool_uses.tool_use_id` |
|
|
47
|
+
| content | VARCHAR | Result text (truncated to 500 chars) |
|
|
48
|
+
| is_error | VARCHAR | `'true'` if the call failed (string, not boolean). Never NULL — an absent source flag backfills to `'false'` |
|
|
49
|
+
| is_error_explicit | BOOLEAN | Whether the source block carried the flag. `false` means `is_error` was backfilled (claude omits the flag on most successes) |
|
|
50
|
+
| timestamp | VARCHAR | ISO timestamp |
|
|
51
|
+
| sessionId | VARCHAR | Session identifier |
|
|
52
|
+
|
|
53
|
+
## `stop_events`
|
|
54
|
+
|
|
55
|
+
Assistant messages where the model stopped generating (claude, cursor, omp, and pi;
|
|
56
|
+
omp and pi quota stalls show as `error`). Columns: `harness`,
|
|
57
|
+
`stop_reason`, `timestamp`, `sessionId`, `cwd`, `gitBranch`.
|
|
58
|
+
|
|
59
|
+
## `model_turns`
|
|
60
|
+
|
|
61
|
+
One row per assistant turn that names a model. Use it for round-trip latency,
|
|
62
|
+
batching, and quota analysis. Claude splits one message
|
|
63
|
+
into several raw entries; this table groups them by `message.id`.
|
|
64
|
+
|
|
65
|
+
| Column | Type | Description |
|
|
66
|
+
|--------|------|-------------|
|
|
67
|
+
| harness | VARCHAR | Source harness (claude, codex, omp, and pi; cursor names no model; codex rows use the latest `turn_context` model, or `unknown` before the first one) |
|
|
68
|
+
| model | VARCHAR | Model id; omp and pi use `<provider>/<model>`, or the bare model when the log names no provider |
|
|
69
|
+
| stop_reason | VARCHAR | Canonical stop reason; omp and pi `toolUse`/`stop` map to `tool_use`/`end_turn` |
|
|
70
|
+
| error_message | VARCHAR | Provider error text on an `error` stop (omp and pi) |
|
|
71
|
+
| input_tokens / output_tokens / cache_read_tokens / cache_creation_tokens | BIGINT | Token usage for the turn. Codex rows come from `token_count` events: one row per event with a non-null `last_token_usage`, skipping an event whose `total_token_usage` repeats the previous one. Codex `input_tokens` includes cached tokens. Claude `input_tokens` excludes cache reads and cache creation. Only Claude reports `cache_creation_tokens` |
|
|
72
|
+
| context_tokens | BIGINT | One context-size measure for both harnesses. Claude: `input_tokens + cache_read_tokens + cache_creation_tokens`. Other harnesses, including Codex: `input_tokens` |
|
|
73
|
+
| prompt_tokens | BIGINT | Full context size sent for the turn (omp and pi) |
|
|
74
|
+
| duration_ms / ttft_ms | DOUBLE | Model round-trip time and time to first token (omp and pi) |
|
|
75
|
+
| tool_calls | BIGINT | Tool calls issued in the turn; 1 means the turn did not batch. NULL for Codex: its token rows carry no content, so Codex tool calls and `stop_reason` are not measured |
|
|
76
|
+
| timestamp, sessionId, cwd | VARCHAR | Join keys |
|
|
77
|
+
|
|
78
|
+
## `agent_spawns`
|
|
79
|
+
|
|
80
|
+
Subset of `tool_uses` for `Agent` (claude) and `Task` (cursor) calls. Columns: `harness`,
|
|
81
|
+
`agent_type` (defaults to `general-purpose`), `description`, `mode`, `anchor_id`,
|
|
82
|
+
`timestamp`, `sessionId`, `cwd`. `anchor_id` is the spawn's `tool_use_id`.
|
|
83
|
+
|
|
84
|
+
## `skill_invocations`
|
|
85
|
+
|
|
86
|
+
Subset of `tool_uses` for `Skill` calls (claude). Columns: `harness`,
|
|
87
|
+
`skill_name`, `args`, `anchor_id`, `timestamp`, `sessionId`, `cwd`.
|
|
88
|
+
`anchor_id` is the invocation's `tool_use_id`. Window queries use it to exclude the anchor event.
|
|
89
|
+
`anchor_id` is NULL when the log names no tool call id.
|
|
90
|
+
|
|
91
|
+
## `mcp_calls`
|
|
92
|
+
|
|
93
|
+
Subset of `tool_uses` where `tool_name LIKE 'mcp__%'`. Same columns as
|
|
94
|
+
`tool_uses`. Names are harness-specific: Claude and Cursor use
|
|
95
|
+
`mcp__<server>__<method>`, while Pi and OMP preserve
|
|
96
|
+
`mcp__<server>_<method>`. Consumers must branch on `harness`; for Pi-family
|
|
97
|
+
rows, split the suffix once at the first underscore.
|
|
98
|
+
|
|
99
|
+
## `sessions`
|
|
100
|
+
|
|
101
|
+
One row per `(harness, sessionId, cwd, branch)`. Columns: `harness`,
|
|
102
|
+
`sessionId`, `first_seen`, `last_seen`, `project` (cwd), `branch`, `version`,
|
|
103
|
+
`parent_session_id`, `entry_count`. `version` is the harness CLI version (Claude `version`, Codex
|
|
104
|
+
`session_meta.cli_version`) of the latest entry by timestamp. It is NULL when
|
|
105
|
+
the log names none.
|
|
106
|
+
`entry_count` counts the rows of the session in `raw_entries` except Codex `token_usage` rows. Those rows are synthetic and feed only `model_turns`.
|
|
107
|
+
`parent_session_id` is the Codex `source.subagent.thread_spawn.parent_thread_id` of a sub-agent rollout. It is NULL for any other session.
|
|
108
|
+
A forked Codex sub-agent rollout counts as a standalone session with its own `sessionId`. A Claude sidechain keeps the parent `sessionId`.
|
|
109
|
+
|
|
110
|
+
## `stop_hooks`
|
|
111
|
+
|
|
112
|
+
System entries with subtype `stop_hook_summary` (claude only — codex/omp emit
|
|
113
|
+
none). Columns: `harness`, `timestamp`, `sessionId`, `hookCount`,
|
|
114
|
+
`hookInfos` (JSON), `hookErrors` (JSON), `preventedContinuation`, `stopReason`,
|
|
115
|
+
`hasOutput`, `level`.
|
|
116
|
+
|
|
117
|
+
## `permission_denials`
|
|
118
|
+
|
|
119
|
+
Pre-filtered `tool_results` for permission/hook denials (claude-dominant).
|
|
120
|
+
Columns: `harness`, `content`, `sessionId`, `timestamp`.
|
|
121
|
+
|
|
122
|
+
## `raw_entries`
|
|
123
|
+
|
|
124
|
+
The full unflattened canonical rows (post-adapter). Carries `harness` plus every
|
|
125
|
+
column in the ingest engine's `RAW_COLUMNS`. Use only when the materialized tables
|
|
126
|
+
lack a field you need.
|
|
127
|
+
|
|
128
|
+
## `schema_info`
|
|
129
|
+
|
|
130
|
+
One row with one integer column, `version`. It holds the schema version of the
|
|
131
|
+
database. `agent-session-analytics ingest` rebuilds a database whose version differs
|
|
132
|
+
from the package version, even inside the one-hour freshness window.
|
|
133
|
+
|
|
134
|
+
## Type gotchas
|
|
135
|
+
|
|
136
|
+
- `is_error` is VARCHAR `'true'`/`'false'` — compare as strings. It is never
|
|
137
|
+
NULL: an absent source flag is backfilled to `'false'` (`is_error_explicit`
|
|
138
|
+
records which rows were backfilled). Measured on claude logs: ~99.5% of
|
|
139
|
+
absent-flag results carry non-error content, so the backfill is sound;
|
|
140
|
+
claude error rates remain floors, not exact estimates.
|
|
141
|
+
- `is_error_explicit` is a real BOOLEAN — `-json` prints it bare
|
|
142
|
+
(`true`/`false`), unlike the VARCHAR `is_error`.
|
|
143
|
+
- Timestamps are VARCHAR ISO strings — cast for date math (`timestamp::TIMESTAMP`,
|
|
144
|
+
`timestamp::DATE`).
|
|
145
|
+
- DuckDB CLI `-json` preserves column types: BOOLEAN/BIGINT print bare, the
|
|
146
|
+
schema's many VARCHAR columns print as strings.
|
|
147
|
+
- Empty result sets print `[]`.
|
|
148
|
+
- `input` is JSON — use `json_extract_string()` for fields not already
|
|
149
|
+
materialized as columns.
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
# Harness Coverage
|
|
2
|
+
|
|
3
|
+
Contents: [Coverage status](#coverage-status), [Per-harness format notes](#per-harness-format-notes), [Signal-quality caveats](#signal-quality-caveats).
|
|
4
|
+
|
|
5
|
+
Which coding-agent harnesses the analytics layer ingests, where their session
|
|
6
|
+
logs live, and how each is parsed. `agent-session-analytics ingest` runs one **normalizing adapter**
|
|
7
|
+
per harness; every adapter is discovery-gated and best-effort. A harness with no
|
|
8
|
+
accessible logs is recorded here and skipped non-fatally — full coverage of what
|
|
9
|
+
is reachable, not parsing the unparseable.
|
|
10
|
+
|
|
11
|
+
Every canonical table carries a `harness` column (`claude` / `codex` / `omp` /
|
|
12
|
+
`pi` / `cursor` / `copilot`) so one query can compare sources. See
|
|
13
|
+
the schema reference for the table shapes.
|
|
14
|
+
|
|
15
|
+
## Coverage status
|
|
16
|
+
|
|
17
|
+
| Harness | Log location | Format | Adapter | Status |
|
|
18
|
+
|---------|-------------|--------|---------|--------|
|
|
19
|
+
| claude | `~/.claude/projects/**/*.jsonl` | JSONL, one turn per line; assistant/user `message.content[]` blocks | `claude_normalize` (pass-through, already canonical) | parsed |
|
|
20
|
+
| codex | `~/.codex/sessions/**/*.jsonl` | JSONL rollout; `session_meta` + `response_item`/`event_msg` payloads | `codex_normalize` | parsed |
|
|
21
|
+
| omp | `~/.omp/agent/sessions/<flattened-project-dir>/*.jsonl` | JSONL; `session` header + `message` entries with `toolCall` / `toolResult` | `omp_normalize` | parsed |
|
|
22
|
+
| pi | `~/.pi/agent/sessions/<flattened-project-dir>/*.jsonl` | Pi-family JSONL | `pi_normalize` | parsed |
|
|
23
|
+
| cursor | `~/.cursor/projects/<project-slug>/agent-transcripts/<uuid>/<uuid>.jsonl` (+ `subagents/*.jsonl`) | JSONL, one message per line; `role`/`message.content[]` blocks, plus `turn_ended` status lines | `cursor_normalize` | parsed |
|
|
24
|
+
| copilot | `~/.copilot/` | holds `skills/` + `mcp-config.json` only; no local transcript found | none | **no accessible logs** |
|
|
25
|
+
|
|
26
|
+
## Per-harness format notes
|
|
27
|
+
|
|
28
|
+
### claude
|
|
29
|
+
|
|
30
|
+
Native format is already the canonical envelope (`type`, `timestamp`,
|
|
31
|
+
`sessionId`, `cwd`, `gitBranch`, `message.content[]`). The adapter only tags
|
|
32
|
+
`harness='claude'`. Subagent JSONL lives in `subagents/` subdirectories — picked
|
|
33
|
+
up by the recursive walk, but those turns have no direct user interaction (no
|
|
34
|
+
stop events, no denials).
|
|
35
|
+
|
|
36
|
+
Claude omits `is_error` on most successful tool results. The flattener
|
|
37
|
+
backfills those to `'false'` and marks them `is_error_explicit = false`.
|
|
38
|
+
Measured: ~99.5% of absent-flag results carry non-error content.
|
|
39
|
+
Claude error rates are therefore floors, because a handful of harness-side truncation notices lack the flag.
|
|
40
|
+
|
|
41
|
+
**`bash_cmd` is the model-typed command, pre-hook.** A PreToolUse
|
|
42
|
+
`updatedInput` rewrite can execute a different command while the transcript
|
|
43
|
+
records the original. Hook rewrite coverage is therefore not measurable from
|
|
44
|
+
Claude transcripts.
|
|
45
|
+
|
|
46
|
+
### codex
|
|
47
|
+
|
|
48
|
+
A forked Codex sub-agent writes its own rollout, so it counts as a standalone session. A Claude sidechain keeps the parent `sessionId`.
|
|
49
|
+
|
|
50
|
+
Rollout JSONL. Each line is `{timestamp, type, payload}`:
|
|
51
|
+
|
|
52
|
+
- `session_meta` — `payload.id` (session id) + `payload.cwd`. Threaded onto every
|
|
53
|
+
following row in the file.
|
|
54
|
+
- `turn_context` — refreshes `payload.cwd`.
|
|
55
|
+
- `response_item / function_call` and `custom_tool_call` → an assistant
|
|
56
|
+
`tool_use` block. The tool name (`shell`, `exec_command`, `exec`,
|
|
57
|
+
`apply_patch`, custom tools) is kept verbatim; `arguments` is JSON-parsed
|
|
58
|
+
into `input`. For shell-ish tools, `input.command` is **normalized to the
|
|
59
|
+
executed command string**, so `bash_cmd` populates.
|
|
60
|
+
`exec_command` copies `cmd`. Legacy `shell` argv arrays collapse to the
|
|
61
|
+
`-lc`/`-c` payload (or a space-join).
|
|
62
|
+
The `exec` custom tool's raw code-string argument becomes the command.
|
|
63
|
+
- `response_item / function_call_output` **and `custom_tool_call_output`** → a
|
|
64
|
+
user `tool_result` block. (Dropping the custom outputs was the ~50%
|
|
65
|
+
result-join gap — issue #704.) `tool_search_output` items have no matching
|
|
66
|
+
call item and are dropped.
|
|
67
|
+
|
|
68
|
+
Codex has no `Skill` / `Agent` tool primitives, so `skill_invocations` and
|
|
69
|
+
`agent_spawns` stay claude-centric. `reasoning` items (encrypted) are dropped.
|
|
70
|
+
|
|
71
|
+
- Structured `exit_code` (including negative values), failed status, `isError`/`is_error` true, and legacy `Process exited with code N` markers set `is_error`. Pending and successful session outputs remain non-errors. Wrapper parsing trusts complete JSON or an `Output:` JSON envelope, not arbitrary prose. Content-block arrays require an execution header in the first `input_text` block. The parser processes later JSON blocks independently.
|
|
72
|
+
- Wrapper-level errors are lower bounds. Nested function or MCP calls can produce different counts. The adapter omits native Codex timing and user records. It reads two other records. `session_meta.cli_version` becomes `raw_entries.version` on every entry of the session. The first `session_meta` of a sub-agent rollout names its parent in `source.subagent.thread_spawn.parent_thread_id`; that id becomes `raw_entries.parentSessionId` and `sessions.parent_session_id`. Each `event_msg` `token_count` event with a non-null `last_token_usage` becomes one `raw_entries` row of `type` `token_usage`. Only `model_turns` reads that row; `tool_uses`, `stop_events`, and `sessions.entry_count` do not. The row takes its model from the latest `turn_context`, or `unknown` before the first one, and its tokens from `last_token_usage`. The adapter skips an event whose `total_token_usage` repeats the previous event. Other `event_msg` records stay omitted.
|
|
73
|
+
- Codex `is_error` values are adapter-derived. `explicit_error_flag_pct` does not establish source evidence for inferred wrapper, status, or exit-code signals.
|
|
74
|
+
|
|
75
|
+
### omp and pi
|
|
76
|
+
|
|
77
|
+
Both harnesses use Pi-family session JSONL, one file per session under a flattened-path project dir
|
|
78
|
+
(e.g. `-Dev-dotfiles`). The `session` header entry (`{type:'session', id, cwd,
|
|
79
|
+
timestamp, title}`) supplies sessionId + cwd for every row. `message` entries:
|
|
80
|
+
|
|
81
|
+
- assistant `toolCall` content blocks → `tool_use` (`arguments` becomes `input`,
|
|
82
|
+
so `bash_cmd` extracts from `input.command`); `text`/`thinking` blocks pass
|
|
83
|
+
through;
|
|
84
|
+
- role `toolResult` → a user `tool_result` block, joined on `toolCallId`;
|
|
85
|
+
- role `user` passes through;
|
|
86
|
+
- assistant `model`, `usage`, `duration`, `ttft`, `contextSnapshot.promptTokens`,
|
|
87
|
+
`stopReason`, and `errorMessage` map to Claude key names and feed
|
|
88
|
+
`model_turns`. `stopReason` `error` and `aborted` also land in `stop_events`.
|
|
89
|
+
|
|
90
|
+
Error flag: every `toolResult` message carries a **msg-level `isError` boolean**.
|
|
91
|
+
It is one convention for builtin and MCP tools.
|
|
92
|
+
For MCP tools a duplicate flag lives
|
|
93
|
+
at `details.xdev.inner.isError`. It matches the msg-level flag across all sessions, so the adapter reads only the msg-level one.
|
|
94
|
+
|
|
95
|
+
OMP-specific caveats:
|
|
96
|
+
|
|
97
|
+
- **MCP naming** is a third scheme: `mcp__tilth_search` = `mcp__` + server +
|
|
98
|
+
*single* underscore + tool.
|
|
99
|
+
These rows land in `mcp_calls`, because the `mcp__%` prefix filter matches.
|
|
100
|
+
A query that splits server and method on a double-underscore separator misparses omp names.
|
|
101
|
+
Split on the prefix + first `_` instead when filtering `harness='omp'`.
|
|
102
|
+
- **Shaken content**: context-compacted tool results are stored as a stub like
|
|
103
|
+
`[shaken ~275 tokens — recover: artifact://46 (region 2)]`. Kept verbatim —
|
|
104
|
+
content-based metrics (result length, error-text matching) undercount for
|
|
105
|
+
shaken rows.
|
|
106
|
+
- **Tool-call id reuse**: `toolCall` ids (`write_0|fc_...`) are not globally
|
|
107
|
+
unique. A small fraction (~0.3%) repeat across sessions, so session-agnostic
|
|
108
|
+
`tool_use_id` joins can slightly overcount.
|
|
109
|
+
Join on `sessionId` too when exactness matters.
|
|
110
|
+
|
|
111
|
+
### cursor
|
|
112
|
+
|
|
113
|
+
Each transcript file (top-level or `subagents/<uuid>.jsonl`) is one session,
|
|
114
|
+
session id = the uuid filename. Lines are `{"role": "user"|"assistant",
|
|
115
|
+
"message": {"content": [...]}}` or `{"type": "turn_ended", "status": ...}`;
|
|
116
|
+
content blocks are only `text` and `tool_use` (`{"type":"tool_use","name":...,
|
|
117
|
+
"input":{...}}`).
|
|
118
|
+
|
|
119
|
+
`cwd` is decoded from the project-slug directory name (dashes stand in for
|
|
120
|
+
slashes).
|
|
121
|
+
A directory whose own name contains a dash is ambiguous if every dash
|
|
122
|
+
is a separator.
|
|
123
|
+
So the adapter walks left-to-right. At each step it takes the longest prefix of remaining segments that exists on disk
|
|
124
|
+
(`Users-paul-Dev-easy-cheese` → `/Users/paul/Dev/easy-cheese` when that path is
|
|
125
|
+
real). Unresolved tails fall back to a naive split. `gitBranch` is always null
|
|
126
|
+
(not recorded).
|
|
127
|
+
|
|
128
|
+
Cursor tool_use blocks carry no id. There are no `tool_result` blocks at
|
|
129
|
+
all: no result content, no `is_error`, no per-call timestamps.
|
|
130
|
+
The adapter synthesizes a deterministic `tool_use_id` (`session:line:block_idx`), since
|
|
131
|
+
nothing needs to join against it.
|
|
132
|
+
It stamps every row with the most recent
|
|
133
|
+
`<timestamp>Weekday, Mon D, YYYY, H:MM AM (UTC±N)</timestamp>` tag seen in a
|
|
134
|
+
prior user turn (embedded alongside `<user_query>`).
|
|
135
|
+
The adapter converts the tag to UTC and carries it forward, so cursor timestamps are turn-granularity, not per-call.
|
|
136
|
+
Because of this, cursor has 0% `results_joined_pct` by construction (not a
|
|
137
|
+
measurement gap) and is excluded from any error-rate query.
|
|
138
|
+
|
|
139
|
+
`CallMcpTool` calls that carry both `server` and `toolName`
|
|
140
|
+
(`{"name":"CallMcpTool","input":{"server":..., "toolName":...}}`) are remapped
|
|
141
|
+
to `mcp__<server>__<toolName>` to match the `mcp__%` filter and the
|
|
142
|
+
double-underscore split convention used by claude/codex. Incomplete wrappers
|
|
143
|
+
(missing either key — observed on a subset of live transcripts) keep the native
|
|
144
|
+
`CallMcpTool` name rather than collapsing to `mcp__None__None`. The remapped
|
|
145
|
+
row's `input` stays the wrapper object (`server` / `toolName` / `arguments`);
|
|
146
|
+
packs that read MCP args from `input.query` / `input.cwd` will see those fields
|
|
147
|
+
under `input.arguments` for cursor.
|
|
148
|
+
|
|
149
|
+
Cursor dispatches sub-agents with `Task` (not Claude's `Agent`). `agent_spawns`
|
|
150
|
+
includes both names so cursor Task rows land there; `subagent_type` still fills
|
|
151
|
+
`agent_type`. Each `subagents/<uuid>.jsonl` file is its own `sessionId`, but the
|
|
152
|
+
adapter sets `isSidechain=true` and `parentUuid` to the parent transcript uuid
|
|
153
|
+
so reconstruction can join the same way Claude packs use `isSidechain`.
|
|
154
|
+
Discovery only walks `**/agent-transcripts/**/*.jsonl` — other JSONL under
|
|
155
|
+
`~/.cursor/projects` is ignored.
|
|
156
|
+
|
|
157
|
+
`turn_ended` lines (`{"status":"success"|"error"}`) become a stop_events row
|
|
158
|
+
with `stop_reason` set to the status string. User-abort errors
|
|
159
|
+
(`"error": "User aborted request"`) map to `aborted` instead of sharing
|
|
160
|
+
`error` with model failures. The flattening SQL's stop_reason filter lists
|
|
161
|
+
claude's `end_turn`/`stop_sequence`/`max_tokens` and cursor's
|
|
162
|
+
`success`/`error`/`aborted`.
|
|
163
|
+
|
|
164
|
+
### copilot — no accessible logs
|
|
165
|
+
|
|
166
|
+
The GitHub Copilot CLI keeps no local session transcript we can locate
|
|
167
|
+
(`~/.copilot` holds only `skills/` and `mcp-config.json`). Discover returns `[]`.
|
|
168
|
+
Re-evaluate if a transcript store appears.
|
|
169
|
+
|
|
170
|
+
## Signal-quality caveats
|
|
171
|
+
|
|
172
|
+
Some metrics are only reliable on harnesses that record the underlying field —
|
|
173
|
+
e.g. token/cost data is absent from most logs (`token-economics` degrades to
|
|
174
|
+
"insufficient signal"), and Codex/OMP/Pi lack Claude's hook + permission-denial
|
|
175
|
+
entries, so `stop_hooks` / `permission_denials` are effectively claude-only.
|
|
176
|
+
Packs must degrade gracefully rather than fabricate.
|
|
177
|
+
|
|
178
|
+
`agent-session-analytics ingest` returns a per-harness **coverage** list after every rebuild. It has two fields.
|
|
179
|
+
`results_joined_pct` counts tool calls with a joined result. A low value means per-tool error rates for that harness are floors, not estimates.
|
|
180
|
+
`explicit_error_flag_pct` counts results whose error flag came from the source rather than the `'false'` backfill.
|
|
181
|
+
Read it before quoting cross-harness error-rate comparisons.
|
|
182
|
+
An empty `model_turns` result or zero errors is not evidence of speed or no failures. Unsupported native events and missing result joins can produce both outcomes. Inspect native Codex JSONL read-only before making either claim.
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# Query & Pack-Authoring Conventions
|
|
2
|
+
|
|
3
|
+
Conventions every analytics **pack** relies on. A pack is a skill-owned file at
|
|
4
|
+
`skills/<skill>/references/<domain>.md` describing one analytics domain. One
|
|
5
|
+
fresh read-only pack context runs exactly one pack, reading the *queries* from
|
|
6
|
+
the pack and the *schema* from the schema reference of this package.
|
|
7
|
+
|
|
8
|
+
## Pack file shape
|
|
9
|
+
|
|
10
|
+
```
|
|
11
|
+
# Pack: <domain>
|
|
12
|
+
# target_param: {TARGET} — the placeholder queries substitute
|
|
13
|
+
# target_kind: skill or agent, from the dispatch prompt
|
|
14
|
+
# start table: {START_TABLE} and {NAME_COLUMN}, set from target_kind (see Substitution)
|
|
15
|
+
# harness: <how the pack uses the harness filter>
|
|
16
|
+
# queries: ordered list of {name, sql} — sql references the canonical schema
|
|
17
|
+
# output_format: a markdown template the digest fills
|
|
18
|
+
|
|
19
|
+
## 1. <query name>
|
|
20
|
+
```sql
|
|
21
|
+
SELECT ... FROM tool_uses WHERE ... ;
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
...
|
|
25
|
+
|
|
26
|
+
## Output Format
|
|
27
|
+
|
|
28
|
+
```
|
|
29
|
+
<markdown template — what the ~2 KB digest looks like>
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Keep a pack to ~4-6 queries. The digest the agent returns must fit ~2 KB.
|
|
35
|
+
|
|
36
|
+
## Harness filtering
|
|
37
|
+
|
|
38
|
+
Every spawn carries a `harness=<all|claude|codex|omp|pi|cursor|copilot>`
|
|
39
|
+
parameter. In pack SQL:
|
|
40
|
+
|
|
41
|
+
- `harness='all'` → omit the harness predicate (aggregate every reachable source).
|
|
42
|
+
- a specific harness → add `WHERE harness = '<name>'` (or `AND harness = ...`).
|
|
43
|
+
|
|
44
|
+
State in the pack header which mode it expects. Domains that depend on
|
|
45
|
+
claude-only fields (`stop_hooks`, `permission_denials`, `skill_invocations`)
|
|
46
|
+
should say so and degrade to "insufficient signal" on other harnesses rather
|
|
47
|
+
than report zero as if it were meaningful. `agent_spawns` includes Claude
|
|
48
|
+
`Agent` and Cursor `Task`.
|
|
49
|
+
|
|
50
|
+
## Substitution
|
|
51
|
+
|
|
52
|
+
Queries use the placeholders that `target_param` and the pack header name
|
|
53
|
+
(e.g. `{TARGET}`, `{START_TABLE}`, `{NAME_COLUMN}`). The caller substitutes the
|
|
54
|
+
literal values before running. Quote the target as a string literal in SQL
|
|
55
|
+
(`WHERE {NAME_COLUMN} = '{TARGET}'`).
|
|
56
|
+
|
|
57
|
+
A pack that serves both skills and agents takes `target_kind` from the dispatch
|
|
58
|
+
prompt. It sets `{START_TABLE}` and `{NAME_COLUMN}` from this mapping:
|
|
59
|
+
|
|
60
|
+
- `skill`: `skill_invocations`, `skill_name`
|
|
61
|
+
- `agent`: `agent_spawns`, `agent_type`
|
|
62
|
+
|
|
63
|
+
## Empty results
|
|
64
|
+
|
|
65
|
+
If a query returns `[]`, note "no data" for that section and continue — never
|
|
66
|
+
block on one empty result. A pack that returns all-empty should say
|
|
67
|
+
"insufficient signal", not invent findings.
|
|
68
|
+
|
|
69
|
+
## Running queries
|
|
70
|
+
|
|
71
|
+
A pack context queries the supplied `database` input. It never runs `ingest`
|
|
72
|
+
or `db-path`. Run each query with the DuckDB CLI:
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
duckdb -init /dev/null -readonly <database> -json -c "SQL"
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
The `-init /dev/null` flag skips `~/.duckdbrc`. The `-readonly` flag blocks
|
|
79
|
+
writes. The `-json` flag gives machine-readable output.
|
|
80
|
+
|
|
81
|
+
## Signal-quality honesty
|
|
82
|
+
|
|
83
|
+
Three domains are known low/medium-signal and must degrade gracefully:
|
|
84
|
+
|
|
85
|
+
- `token-economics` — token/cost fields are usually absent from logs.
|
|
86
|
+
- `routing-accuracy` — no intent ground-truth exists; correlational at best.
|
|
87
|
+
- `knowledge-gaps` — medium-signal inference.
|
|
88
|
+
|
|
89
|
+
Record the caveat in the pack and emit "insufficient signal" rather than
|
|
90
|
+
fabricate a confident finding. This pairs with the confidence axis of the calling
|
|
91
|
+
skill's calibration reference (`<don't know>` is never surfaced).
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: agent-session-analytics
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Ingest coding-agent session logs into DuckDB and query them read-only.
|
|
5
|
+
Project-URL: Homepage, https://github.com/paulnsorensen/skillz-that-grillz/tree/main/packages/agent-session-analytics
|
|
6
|
+
Project-URL: Source, https://github.com/paulnsorensen/skillz-that-grillz/tree/main/packages/agent-session-analytics
|
|
7
|
+
Project-URL: Issues, https://github.com/paulnsorensen/skillz-that-grillz/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/paulnsorensen/skillz-that-grillz/blob/main/packages/agent-session-analytics/CHANGELOG.md
|
|
9
|
+
Author-email: Paul Sorensen <paulnsorensen@gmail.com>
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: agent,analytics,duckdb,session-logs
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Environment :: Console
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Typing :: Typed
|
|
20
|
+
Requires-Python: >=3.11
|
|
21
|
+
Requires-Dist: fromargs<0.3,>=0.2.0
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# agent-session-analytics
|
|
25
|
+
|
|
26
|
+
`agent-session-analytics` ingests the session logs of coding agents into one DuckDB
|
|
27
|
+
database. It then answers SQL queries that cannot write or run commands. It
|
|
28
|
+
supports the Claude Code, Codex, Cursor, OMP, and pi harnesses.
|
|
29
|
+
|
|
30
|
+
The package ships engine primitives only. It has no canned reports. A caller
|
|
31
|
+
writes its own SQL against the schema.
|
|
32
|
+
|
|
33
|
+
## Install
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
uv tool install agent-session-analytics
|
|
37
|
+
# or
|
|
38
|
+
pip install agent-session-analytics
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
The only runtime dependency is `fromargs`. The package calls the `duckdb`
|
|
42
|
+
command-line tool and does not need the Python `duckdb` module. Install DuckDB
|
|
43
|
+
with one of these commands:
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
uv tool install duckdb-cli
|
|
47
|
+
brew install duckdb
|
|
48
|
+
mise use -g duckdb
|
|
49
|
+
curl https://install.duckdb.org | sh
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
The package never downloads DuckDB. When `duckdb` is not on `PATH`, `ingest`
|
|
53
|
+
and `sql` exit 3 and print these options.
|
|
54
|
+
|
|
55
|
+
## Commands
|
|
56
|
+
|
|
57
|
+
Every command prints one JSON document on stdout. Every error is one JSON line
|
|
58
|
+
on stderr: `{"error": <message>, "exit_code": <n>}`.
|
|
59
|
+
|
|
60
|
+
| Command | Result |
|
|
61
|
+
|---|---|
|
|
62
|
+
| `ingest [--force]` | Build or refresh the database. A database inside the one-hour freshness window with the current schema version keeps its data. |
|
|
63
|
+
| `db-path` | The resolved database path, as `{"path": "<abs path>"}`. |
|
|
64
|
+
| `sql <query>` | The rows of one query in DuckDB safe mode. |
|
|
65
|
+
| `schema` | The canonical schema reference. |
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
agent-session-analytics ingest
|
|
69
|
+
agent-session-analytics db-path | jq -r .path
|
|
70
|
+
agent-session-analytics sql "SELECT harness, count(*) AS sessions FROM sessions GROUP BY harness"
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
`sql` opens the database read-only and runs in DuckDB safe mode. Safe mode
|
|
74
|
+
rejects shell commands, file writes, and reads of files outside the database.
|
|
75
|
+
A rejected query exits 5.
|
|
76
|
+
|
|
77
|
+
`sql` ingests an old or absent default database first. It does not ingest a
|
|
78
|
+
`SESSIONS_DB` override. It never queries stale data after a failed ingest.
|
|
79
|
+
|
|
80
|
+
A consumer that queries the database directly runs
|
|
81
|
+
`duckdb -init /dev/null -readonly <database> -json -c "<sql>"`, where
|
|
82
|
+
`<database>` is the `path` field of `db-path`.
|
|
83
|
+
|
|
84
|
+
## Exit codes
|
|
85
|
+
|
|
86
|
+
| Code | Meaning |
|
|
87
|
+
|---|---|
|
|
88
|
+
| 0 | Success |
|
|
89
|
+
| 2 | Bad input |
|
|
90
|
+
| 3 | The `duckdb` command is missing |
|
|
91
|
+
| 4 | Ingest failed or timed out, or the database is missing, unreadable, or has another schema version |
|
|
92
|
+
| 5 | DuckDB rejects the query, or the query times out |
|
|
93
|
+
|
|
94
|
+
## Database path
|
|
95
|
+
|
|
96
|
+
- `SESSIONS_DB` names the database. A relative value joins the current
|
|
97
|
+
directory. The package does not expand `~` in a set value.
|
|
98
|
+
- Otherwise an absolute `XDG_CACHE_HOME` selects
|
|
99
|
+
`<cache>/dotfiles/session-analytics/sessions.duckdb`. A relative or unset
|
|
100
|
+
value uses `~/.cache`.
|
|
101
|
+
- `SESSIONS_DUCKDB_MEMORY_LIMIT` sets the DuckDB memory limit. The default is
|
|
102
|
+
`8GB`.
|
|
103
|
+
- `CLAUDE_CONFIG_DIR`, `CODEX_HOME`, and `CURSOR_HOME` move the log roots.
|
|
104
|
+
|
|
105
|
+
Ingest writes the transient `<db>.tmp` and `<db>.stage`, and a persistent
|
|
106
|
+
`<db>.lock`. An `fcntl` lock serializes ingest runs. Ingest uses umask `077`.
|
|
107
|
+
|
|
108
|
+
## Schema version
|
|
109
|
+
|
|
110
|
+
The database holds a one-row `schema_info` table with an integer `version`.
|
|
111
|
+
The package version of that integer is `SCHEMA_VERSION`. `ingest` rebuilds a
|
|
112
|
+
database whose stored version differs, even inside the freshness window. `sql`
|
|
113
|
+
treats a mismatch like a stale database. Only this repository changes the
|
|
114
|
+
schema.
|
|
115
|
+
|
|
116
|
+
## References
|
|
117
|
+
|
|
118
|
+
These files ship as package data in `agent_session_analytics/references/`:
|
|
119
|
+
|
|
120
|
+
- `canonical-schema.md` — tables and columns. `agent-session-analytics schema` prints it.
|
|
121
|
+
- `harness-coverage.md` — what each harness log provides.
|
|
122
|
+
- `query-conventions.md` — rules for writing queries.
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
agent_session_analytics/__init__.py,sha256=JYunnFVOxSPkQ-qpK6yaT26Ldo8u-xy25j4UKtSxtDg,144
|
|
2
|
+
agent_session_analytics/__main__.py,sha256=WcZXdTlpnj_PI7T13h3ExSWAiuWY_8Bfx2ZpzRJux7A,71
|
|
3
|
+
agent_session_analytics/_cli.py,sha256=HOZZfYkRW7FVTle33Sve9aJJ_lFJvHBOHBuOrDRkQfQ,1773
|
|
4
|
+
agent_session_analytics/_db.py,sha256=4Kycvtb1hXVO4Jmr2e2SkHi37nq8lhr-rv1PlRKz7Es,5469
|
|
5
|
+
agent_session_analytics/_ingest.py,sha256=AsHkZEuLj37hXDOTYZdRKpLWlY3X-MA5FDE1pSciDAs,45360
|
|
6
|
+
agent_session_analytics/_query.py,sha256=mkLo5gA_lC43YoThz2a64mc8PpZPvyYMYVbqqE-dteg,1866
|
|
7
|
+
agent_session_analytics/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
agent_session_analytics/references/canonical-schema.md,sha256=g_ZO4j00V_VHm4d6cc4RHd0R2gLcZ7HwUJx-b-yN0nw,8397
|
|
9
|
+
agent_session_analytics/references/harness-coverage.md,sha256=zYlORs58ReZMVn2ru5R4TTwnaOR54e7GHw1hsSGKQ0I,11909
|
|
10
|
+
agent_session_analytics/references/query-conventions.md,sha256=LefSUBa0yIeAKKLKvreo5OTiLzzSIVeYHbXlYf_AMbk,3241
|
|
11
|
+
agent_session_analytics-0.1.0.dist-info/METADATA,sha256=8J4NED3p4t67qkeRC3pDXWksY3-jidh-ElxhbWzgjhI,4770
|
|
12
|
+
agent_session_analytics-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
13
|
+
agent_session_analytics-0.1.0.dist-info/entry_points.txt,sha256=lYN2Drp1wruwNv9njqFhRIvUr6rbCei9HfMjlLkmxYw,73
|
|
14
|
+
agent_session_analytics-0.1.0.dist-info/licenses/LICENSE,sha256=lT2oNYDRyYDND9T_fYwnp7OrBEzjeEM5-1Nn9u6FL-g,1070
|
|
15
|
+
agent_session_analytics-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Paul Sorensen
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|