atif-sql 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- atif_analytics/__init__.py +9 -0
- atif_analytics/application/__init__.py +3 -0
- atif_analytics/application/analyze.py +158 -0
- atif_analytics/application/prompts.py +750 -0
- atif_analytics/application/use_cases/__init__.py +3 -0
- atif_analytics/application/use_cases/_shared.py +264 -0
- atif_analytics/application/use_cases/classify.py +430 -0
- atif_analytics/application/use_cases/conflicts.py +448 -0
- atif_analytics/application/use_cases/friction.py +634 -0
- atif_analytics/application/use_cases/perceived.py +451 -0
- atif_analytics/domain/__init__.py +3 -0
- atif_analytics/domain/authorship.py +166 -0
- atif_analytics/domain/config.py +30 -0
- atif_analytics/domain/costs.py +50 -0
- atif_analytics/domain/friction.py +117 -0
- atif_analytics/domain/layout.py +94 -0
- atif_analytics/domain/models.py +385 -0
- atif_analytics/domain/ports.py +100 -0
- atif_analytics/domain/transcript.py +359 -0
- atif_analytics/infrastructure/__init__.py +3 -0
- atif_analytics/infrastructure/corpus_reader.py +546 -0
- atif_analytics/infrastructure/parquet_cache.py +186 -0
- atif_analytics/infrastructure/settings.py +114 -0
- atif_analytics/infrastructure/sqlite_state/__init__.py +3 -0
- atif_analytics/infrastructure/sqlite_state/checkpointer.py +326 -0
- atif_analytics/infrastructure/sqlite_state/retry_queue.py +296 -0
- atif_analytics/py.typed +0 -0
- atif_cli/__init__.py +3 -0
- atif_cli/__main__.py +8 -0
- atif_cli/app.py +2459 -0
- atif_cli/converter_adapter.py +151 -0
- atif_cli/corpus.py +409 -0
- atif_cli/cron.py +311 -0
- atif_cli/duck_errors.py +97 -0
- atif_cli/embed_lake.py +101 -0
- atif_cli/errors.py +89 -0
- atif_cli/lake.py +373 -0
- atif_cli/lake_sessions.py +270 -0
- atif_cli/output.py +297 -0
- atif_cli/py.typed +0 -0
- atif_converter/__init__.py +3 -0
- atif_converter/application/__init__.py +3 -0
- atif_converter/application/convert_and_audit.py +294 -0
- atif_converter/application/convert_codex.py +210 -0
- atif_converter/domain/__init__.py +9 -0
- atif_converter/domain/agents.py +40 -0
- atif_converter/domain/atif/__init__.py +55 -0
- atif_converter/domain/atif/agent.py +37 -0
- atif_converter/domain/atif/content.py +161 -0
- atif_converter/domain/atif/final_metrics.py +45 -0
- atif_converter/domain/atif/metrics.py +49 -0
- atif_converter/domain/atif/observation.py +21 -0
- atif_converter/domain/atif/observation_result.py +47 -0
- atif_converter/domain/atif/step.py +144 -0
- atif_converter/domain/atif/subagent_trajectory_ref.py +90 -0
- atif_converter/domain/atif/tool_call.py +36 -0
- atif_converter/domain/atif/trajectory.py +211 -0
- atif_converter/domain/atif/trajectory_validator.py +299 -0
- atif_converter/domain/blobs.py +510 -0
- atif_converter/domain/claude_code_conversion.py +1077 -0
- atif_converter/domain/codex_conversion.py +1036 -0
- atif_converter/domain/codex_edges.py +192 -0
- atif_converter/domain/codex_enrichment.py +419 -0
- atif_converter/domain/codex_fidelity.py +124 -0
- atif_converter/domain/codex_nested_calls.py +165 -0
- atif_converter/domain/edges.py +113 -0
- atif_converter/domain/enrichment.py +412 -0
- atif_converter/domain/errors.py +60 -0
- atif_converter/domain/fidelity.py +171 -0
- atif_converter/domain/model_prices.json +8538 -0
- atif_converter/domain/pricing.py +790 -0
- atif_converter/domain/result_signals.py +774 -0
- atif_converter/domain/schema_version.py +90 -0
- atif_converter/domain/session_events.py +408 -0
- atif_converter/infrastructure/__init__.py +3 -0
- atif_converter/infrastructure/census.py +89 -0
- atif_converter/infrastructure/claude_code_converter.py +127 -0
- atif_converter/infrastructure/codex_adapter.py +132 -0
- atif_converter/infrastructure/codex_census.py +100 -0
- atif_converter/infrastructure/codex_converter.py +82 -0
- atif_converter/infrastructure/harbor_adapter.py +187 -0
- atif_converter/infrastructure/raw_records.py +426 -0
- atif_converter/infrastructure/source_archive.py +154 -0
- atif_converter/py.typed +0 -0
- atif_corpus/__init__.py +13 -0
- atif_corpus/application/__init__.py +7 -0
- atif_corpus/application/materialize.py +1826 -0
- atif_corpus/domain/__init__.py +9 -0
- atif_corpus/domain/agents.py +33 -0
- atif_corpus/domain/generation.py +34 -0
- atif_corpus/domain/layout.py +168 -0
- atif_corpus/domain/ports.py +175 -0
- atif_corpus/domain/session_id.py +68 -0
- atif_corpus/domain/sessions.py +228 -0
- atif_corpus/domain/slug.py +72 -0
- atif_corpus/domain/source_layout.py +143 -0
- atif_corpus/domain/watermark.py +76 -0
- atif_corpus/infrastructure/__init__.py +7 -0
- atif_corpus/infrastructure/atomic.py +266 -0
- atif_corpus/infrastructure/compress_artifacts.py +161 -0
- atif_corpus/infrastructure/fake_converter.py +174 -0
- atif_corpus/infrastructure/scanner.py +316 -0
- atif_corpus/infrastructure/settings.py +130 -0
- atif_corpus/infrastructure/source_archive.py +202 -0
- atif_corpus/py.typed +0 -0
- atif_duck/__init__.py +43 -0
- atif_duck/domain/__init__.py +32 -0
- atif_duck/domain/artifacts.py +66 -0
- atif_duck/domain/authorship.py +165 -0
- atif_duck/domain/catalog.py +626 -0
- atif_duck/domain/columnar.py +149 -0
- atif_duck/domain/embedding_guard.py +88 -0
- atif_duck/domain/examples.py +265 -0
- atif_duck/domain/lake.py +427 -0
- atif_duck/domain/lake_sessions.py +97 -0
- atif_duck/domain/raw_readers.py +140 -0
- atif_duck/domain/session_id.py +69 -0
- atif_duck/domain/sql_literal.py +50 -0
- atif_duck/infrastructure/__init__.py +34 -0
- atif_duck/infrastructure/analytics.py +469 -0
- atif_duck/infrastructure/authorship.py +139 -0
- atif_duck/infrastructure/columnar.py +641 -0
- atif_duck/infrastructure/lake.py +1383 -0
- atif_duck/infrastructure/lake_sessions.py +194 -0
- atif_duck/infrastructure/lake_settings.py +63 -0
- atif_duck/infrastructure/lake_steps.py +225 -0
- atif_duck/infrastructure/projections.py +221 -0
- atif_duck/infrastructure/registry.py +2194 -0
- atif_duck/infrastructure/stored_artifacts.py +115 -0
- atif_duck/py.typed +0 -0
- atif_embed/__init__.py +12 -0
- atif_embed/application/__init__.py +3 -0
- atif_embed/application/embed.py +293 -0
- atif_embed/application/prune.py +80 -0
- atif_embed/domain/__init__.py +3 -0
- atif_embed/domain/discovery.py +70 -0
- atif_embed/domain/embedding_guard.py +83 -0
- atif_embed/domain/errors.py +94 -0
- atif_embed/domain/ports.py +223 -0
- atif_embed/domain/sql_literal.py +49 -0
- atif_embed/domain/text_stamp.py +71 -0
- atif_embed/infrastructure/__init__.py +3 -0
- atif_embed/infrastructure/cohere_bedrock.py +433 -0
- atif_embed/infrastructure/corpus_text_rows.py +273 -0
- atif_embed/infrastructure/gemma_local.py +355 -0
- atif_embed/infrastructure/lake_text_rows.py +203 -0
- atif_embed/infrastructure/lance_store.py +440 -0
- atif_embed/infrastructure/providers.py +30 -0
- atif_embed/infrastructure/settings.py +154 -0
- atif_embed/py.typed +0 -0
- atif_models/__init__.py +14 -0
- atif_models/domain/__init__.py +9 -0
- atif_models/domain/ports.py +143 -0
- atif_models/domain/registry.py +151 -0
- atif_models/domain/schema.py +109 -0
- atif_models/infrastructure/__init__.py +8 -0
- atif_models/infrastructure/openai_bedrock.py +314 -0
- atif_models/infrastructure/settings.py +85 -0
- atif_models/py.typed +0 -0
- atif_sql-0.2.0.dist-info/METADATA +272 -0
- atif_sql-0.2.0.dist-info/RECORD +164 -0
- atif_sql-0.2.0.dist-info/WHEEL +4 -0
- atif_sql-0.2.0.dist-info/entry_points.txt +2 -0
- atif_sql-0.2.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
|
|
3
|
+
"""The session id boundary: which directory names may become a session.
|
|
4
|
+
|
|
5
|
+
A session id is the one piece of text from OUTSIDE this workspace that ends up
|
|
6
|
+
inside a corpus path: it is derived from a transcript filename under the
|
|
7
|
+
source root, becomes the ``sessions/<session_id>/`` directory name, and from
|
|
8
|
+
there names the files atif-duck opens. Everything else in a statement is a
|
|
9
|
+
constant. So the id is checked once, at the boundary, and a name that fails
|
|
10
|
+
the check is skipped and reported rather than carried any further.
|
|
11
|
+
|
|
12
|
+
The rule is deliberately narrow: an ASCII letter or digit first, then letters,
|
|
13
|
+
digits, ``.``, ``_`` and ``-``, at most 255 characters (the longest name most
|
|
14
|
+
filesystems accept). Every id both supported agents produce passes: Claude
|
|
15
|
+
Code names a session with a version 4 UUID, Codex with a version 7 UUID
|
|
16
|
+
(``019e9e38-39a6-7170-86fb-46e6f1cb1931``). Quotes, whitespace, path
|
|
17
|
+
separators, ``;``, ``$``, ``?`` and anything outside ASCII fail.
|
|
18
|
+
|
|
19
|
+
DELIBERATE TWIN of :mod:`atif_corpus.domain.session_id`, the same way
|
|
20
|
+
`sql_literal` and `embedding_guard` twin their atif-embed copies: the independence
|
|
21
|
+
contract forbids the two packages from importing each other, and both sides
|
|
22
|
+
must apply the SAME rule: the corpus writer before it writes, this registry
|
|
23
|
+
before it reads a corpus an older version wrote and before any session name
|
|
24
|
+
reaches a path. Each package's tests read the other's copy as source text and
|
|
25
|
+
fail if the pattern or the limit drifts.
|
|
26
|
+
|
|
27
|
+
Pure module: stdlib only.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
import re
|
|
33
|
+
|
|
34
|
+
#: The accepted shape, as a pattern string so the twin pin can compare it.
|
|
35
|
+
SESSION_ID_PATTERN: str = r"^[A-Za-z0-9][A-Za-z0-9._-]*$"
|
|
36
|
+
|
|
37
|
+
#: Longest accepted id, in characters.
|
|
38
|
+
SESSION_ID_MAX_CHARS: int = 255
|
|
39
|
+
|
|
40
|
+
_SESSION_ID_RE = re.compile(SESSION_ID_PATTERN)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def session_id_rejection(candidate: str) -> str | None:
|
|
44
|
+
"""Why ``candidate`` cannot be a session id, or ``None`` when it can.
|
|
45
|
+
|
|
46
|
+
The reason is one short phrase meant for a log line; a caller that only
|
|
47
|
+
needs the verdict uses :func:`is_valid_session_id`.
|
|
48
|
+
"""
|
|
49
|
+
if len(candidate) > SESSION_ID_MAX_CHARS:
|
|
50
|
+
return f"longer than {SESSION_ID_MAX_CHARS} characters"
|
|
51
|
+
# fullmatch, not match: with ``$`` alone a trailing newline would pass.
|
|
52
|
+
if _SESSION_ID_RE.fullmatch(candidate) is None:
|
|
53
|
+
return (
|
|
54
|
+
"contains a character outside [A-Za-z0-9._-] or does not start with a letter or digit"
|
|
55
|
+
)
|
|
56
|
+
return None
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def is_valid_session_id(candidate: str) -> bool:
|
|
60
|
+
"""Whether ``candidate`` passes the boundary."""
|
|
61
|
+
return session_id_rejection(candidate) is None
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
__all__ = [
|
|
65
|
+
"SESSION_ID_MAX_CHARS",
|
|
66
|
+
"SESSION_ID_PATTERN",
|
|
67
|
+
"is_valid_session_id",
|
|
68
|
+
"session_id_rejection",
|
|
69
|
+
]
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
|
|
3
|
+
"""Single-quoted SQL literal escaping, and the type that marks text as SQL.
|
|
4
|
+
|
|
5
|
+
Two things live here because they are the two ends of the same boundary.
|
|
6
|
+
|
|
7
|
+
:data:`SqlFragment` is a ``NewType`` over ``str``: a value typed as one is
|
|
8
|
+
text that may be spliced into a statement. Only three kinds of code produce it:
|
|
9
|
+
:func:`sql_literal` here, the catalog constants, and the projection module's
|
|
10
|
+
column expressions. A path, a session id, or anything else that arrived from
|
|
11
|
+
outside is a plain ``str`` and reaches DuckDB as a bound parameter instead. The
|
|
12
|
+
type is advisory at runtime (a NewType is its underlying ``str``); the AST test
|
|
13
|
+
over the registry modules is what enforces the rule.
|
|
14
|
+
|
|
15
|
+
:func:`sql_literal` is for the few statement kinds DuckDB refuses to prepare:
|
|
16
|
+
``ATTACH``, and the producer's one-row session projection. It doubles every
|
|
17
|
+
embedded single quote, which is the SQL standard's own escape. It lives in
|
|
18
|
+
``domain`` so every module that still needs it reaches the SAME implementation.
|
|
19
|
+
|
|
20
|
+
Pure module: stdlib only, no duckdb.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
from typing import NewType
|
|
26
|
+
|
|
27
|
+
#: Text that may be spliced into a SQL statement. See the module docstring.
|
|
28
|
+
SqlFragment = NewType("SqlFragment", str)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def sql_literal(value: str) -> SqlFragment:
|
|
32
|
+
"""Escape ``value`` as a single-quoted SQL string literal.
|
|
33
|
+
|
|
34
|
+
Doubles every embedded single quote, per the SQL standard's own escape.
|
|
35
|
+
|
|
36
|
+
Parameters
|
|
37
|
+
----------
|
|
38
|
+
value
|
|
39
|
+
Value to embed in a statement.
|
|
40
|
+
|
|
41
|
+
Returns
|
|
42
|
+
-------
|
|
43
|
+
SqlFragment
|
|
44
|
+
``value`` wrapped in single quotes, embedded quotes doubled.
|
|
45
|
+
"""
|
|
46
|
+
escaped = value.replace("'", "''")
|
|
47
|
+
return SqlFragment(f"'{escaped}'")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
__all__ = ["SqlFragment", "sql_literal"]
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
|
|
3
|
+
"""Infrastructure layer: DuckDB registry over the materialized ATIF corpus.
|
|
4
|
+
|
|
5
|
+
Also the columnar producer (:class:`ColumnarArtifactProducer`) that writes
|
|
6
|
+
the typed per-session parquet artifacts the registry prefers to read, and
|
|
7
|
+
:func:`columnar_coverage`, which reports how many sessions carry them.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from atif_duck.infrastructure.columnar import (
|
|
11
|
+
ColumnarArtifactProducer,
|
|
12
|
+
ColumnarCoverage,
|
|
13
|
+
columnar_coverage,
|
|
14
|
+
session_has_columnar,
|
|
15
|
+
)
|
|
16
|
+
from atif_duck.infrastructure.registry import (
|
|
17
|
+
RawSources,
|
|
18
|
+
register,
|
|
19
|
+
register_macros,
|
|
20
|
+
register_raw,
|
|
21
|
+
register_views,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
__all__ = [
|
|
25
|
+
"ColumnarArtifactProducer",
|
|
26
|
+
"ColumnarCoverage",
|
|
27
|
+
"RawSources",
|
|
28
|
+
"columnar_coverage",
|
|
29
|
+
"register",
|
|
30
|
+
"register_macros",
|
|
31
|
+
"register_raw",
|
|
32
|
+
"register_views",
|
|
33
|
+
"session_has_columnar",
|
|
34
|
+
]
|
|
@@ -0,0 +1,469 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
|
|
3
|
+
"""v2 analytics views + macros over the atif-analytics parquet outputs.
|
|
4
|
+
|
|
5
|
+
atif-analytics writes its artifacts under ``<corpus_root>/analytics/`` (one
|
|
6
|
+
sharded dir per LLM pipeline); this module binds them as the analytics
|
|
7
|
+
views and macros the catalog declares.
|
|
8
|
+
|
|
9
|
+
The trajectory pipeline and the structural ones (cluster, terms,
|
|
10
|
+
community) were removed on 2026-09-27, and with them ``message_trajectory``,
|
|
11
|
+
``message_clusters``, ``cluster_terms``, ``session_communities``,
|
|
12
|
+
``community_profile`` and the ``autonomy_trend``, ``success_rate_by_work``,
|
|
13
|
+
``sentiment_arc``, ``cluster_top_terms`` and ``community_top_topics`` macros.
|
|
14
|
+
Their parquets may still sit on disk in an older corpus; nothing binds them.
|
|
15
|
+
|
|
16
|
+
atif-duck may not import atif-analytics (independence contract), so the
|
|
17
|
+
artifact names are pinned here against the documented layout — the shape is
|
|
18
|
+
contract, not code sharing.
|
|
19
|
+
|
|
20
|
+
Gating: each view is created only when its backing parquet is populated on
|
|
21
|
+
disk; each macro is registered only when EVERY view it binds against was
|
|
22
|
+
registered. Missing analytics parquets are the default state until the
|
|
23
|
+
pipelines run, so skips log at DEBUG.
|
|
24
|
+
|
|
25
|
+
Column semantics worth stating once (repeated per macro below):
|
|
26
|
+
|
|
27
|
+
* ``conflicts_over_time`` / ``perceived_*`` join ``messages`` (the
|
|
28
|
+
edges-derived uuid-keyed view) to recover conversation time.
|
|
29
|
+
* ``conflicts_over_time``'s ``root_session_id`` equals ``session_id``: harbor
|
|
30
|
+
INLINES subagent transcripts (fidelity gap 4), so one conversation is one
|
|
31
|
+
session and there is no parent session to collapse onto.
|
|
32
|
+
* ``friction_rate`` / ``perceived_rate`` count their user-message
|
|
33
|
+
denominator from ``human_turns``: main-chain user steps a HUMAN wrote, so
|
|
34
|
+
Stop hook feedback, task notifications and harness text are not in it.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
from __future__ import annotations
|
|
38
|
+
|
|
39
|
+
from typing import TYPE_CHECKING
|
|
40
|
+
|
|
41
|
+
from loguru import logger
|
|
42
|
+
|
|
43
|
+
if TYPE_CHECKING:
|
|
44
|
+
from pathlib import Path
|
|
45
|
+
|
|
46
|
+
import duckdb
|
|
47
|
+
|
|
48
|
+
# ---------------------------------------------------------------------------
|
|
49
|
+
# Artifact layout (pinned against atif-analytics' AnalyticsLayout — contract)
|
|
50
|
+
# ---------------------------------------------------------------------------
|
|
51
|
+
|
|
52
|
+
#: Prefix of the per-view parquet reader each analytics view selects from.
|
|
53
|
+
#: ``CREATE VIEW`` cannot take a bound parameter, so the parquet paths go
|
|
54
|
+
#: through the connection's ``read_parquet`` relation API (a Python value, not
|
|
55
|
+
#: statement text), the relation is registered under ``<prefix><view_name>``,
|
|
56
|
+
#: and the view itself interpolates constants only.
|
|
57
|
+
_ANALYTICS_READER_PREFIX: str = "v_raw_analytics_"
|
|
58
|
+
|
|
59
|
+
#: view name -> path relative to <corpus_root>/analytics/. Directory entries
|
|
60
|
+
#: are sharded caches (part-*.parquet); file entries are single parquets.
|
|
61
|
+
_ANALYTICS_SOURCES: dict[str, str] = {
|
|
62
|
+
"session_classifications": "session_classifications",
|
|
63
|
+
"session_conflicts": "session_conflicts",
|
|
64
|
+
"user_friction": "user_friction",
|
|
65
|
+
"perceived_errors": "perceived_errors",
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
#: Per-view projections; a view absent here selects ``*``.
|
|
69
|
+
#:
|
|
70
|
+
#: ``session_classifications`` names its columns because its shards span two
|
|
71
|
+
#: schemas. Rows written before 2026-09-27 also carry the LLM ``autonomy_tier``
|
|
72
|
+
#: and ``success`` labels, which were dropped as unreliable (the same outcome
|
|
73
|
+
#: drew both labels hundreds of times each on one corpus). The two names
|
|
74
|
+
#: survive as always-NULL compatibility columns so a downstream reader that
|
|
75
|
+
#: selects them still binds; ``session_outcomes`` carries the deterministic
|
|
76
|
+
#: replacement. ``category`` is an alias of ``work_category``.
|
|
77
|
+
_VIEW_PROJECTIONS: dict[str, str] = {
|
|
78
|
+
"session_classifications": (
|
|
79
|
+
"session_id, work_category, goal, confidence, classified_at, "
|
|
80
|
+
"CAST(NULL AS VARCHAR) AS autonomy_tier, CAST(NULL AS VARCHAR) AS success, "
|
|
81
|
+
"work_category AS category"
|
|
82
|
+
),
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
#: Per-macro view dependencies, keyed on the views this module registers. A
|
|
86
|
+
#: macro whose entry is unsatisfied is skipped: DuckDB binds a macro body at
|
|
87
|
+
#: CREATE time, so registering it against an absent view fails outright.
|
|
88
|
+
_ANALYTICS_MACRO_REQUIREMENTS: dict[str, tuple[str, ...]] = {
|
|
89
|
+
"work_mix": ("session_classifications",),
|
|
90
|
+
"friction_counts": ("user_friction",),
|
|
91
|
+
"friction_rate": ("user_friction",),
|
|
92
|
+
"friction_examples": ("user_friction",),
|
|
93
|
+
"conflicts_over_time": ("session_conflicts",),
|
|
94
|
+
"perceived_counts": ("perceived_errors",),
|
|
95
|
+
"perceived_rate": ("perceived_errors",),
|
|
96
|
+
"perceived_examples": ("perceived_errors",),
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
#: Smallest byte count a readable parquet file can have: the format frames every
|
|
101
|
+
#: file as ``PAR1`` + footer + 4-byte footer length + ``PAR1``. A shorter file is
|
|
102
|
+
#: what a crashed writer leaves behind, and DuckDB errors on it rather than
|
|
103
|
+
#: reading zero rows, so a view built over one fails to register. Declared here
|
|
104
|
+
#: rather than imported because atif-duck may never import atif-analytics.
|
|
105
|
+
_MIN_PARQUET_BYTES = 16
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _backing_files(path: Path) -> list[Path]:
|
|
109
|
+
"""The parquet files backing one artifact (sharded dir or single file)."""
|
|
110
|
+
if not path.exists():
|
|
111
|
+
return []
|
|
112
|
+
if path.is_dir():
|
|
113
|
+
return sorted(
|
|
114
|
+
p for p in path.glob("part-*.parquet") if p.stat().st_size > _MIN_PARQUET_BYTES
|
|
115
|
+
)
|
|
116
|
+
return [path] if path.stat().st_size > _MIN_PARQUET_BYTES else []
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def register_analytics(
|
|
120
|
+
con: duckdb.DuckDBPyConnection, corpus_root: Path, *more_roots: Path
|
|
121
|
+
) -> set[str]:
|
|
122
|
+
"""Bind the analytics parquets as views; return the set registered.
|
|
123
|
+
|
|
124
|
+
Each view is created only when its backing parquet is populated.
|
|
125
|
+
Derived views (``session_goals``, ``conflicts_summary``) follow their
|
|
126
|
+
upstream. Idempotent against a partially-populated corpus.
|
|
127
|
+
|
|
128
|
+
With ``more_roots`` (a query over every corpus of the lake) each view
|
|
129
|
+
reads the parquets under every root's ``analytics/``. The rows carry no
|
|
130
|
+
corpus column (the catalog's shapes do not change with the scope); join
|
|
131
|
+
``sessions`` on ``session_id`` for it. That join is exact while each
|
|
132
|
+
session id lives in one corpus, which holds for corpora converted from
|
|
133
|
+
different transcripts. A copied corpus repeats its ids, and there each
|
|
134
|
+
analytics row matches the session in both corpora.
|
|
135
|
+
"""
|
|
136
|
+
roots = (corpus_root, *more_roots)
|
|
137
|
+
registered: set[str] = set()
|
|
138
|
+
|
|
139
|
+
for view_name, rel in _ANALYTICS_SOURCES.items():
|
|
140
|
+
parts = [part for root in roots for part in _backing_files(root / "analytics" / rel)]
|
|
141
|
+
if not parts:
|
|
142
|
+
logger.debug(
|
|
143
|
+
"register_analytics: skipping {} (no parquet under {} root(s))",
|
|
144
|
+
view_name,
|
|
145
|
+
len(roots),
|
|
146
|
+
)
|
|
147
|
+
continue
|
|
148
|
+
projection = _VIEW_PROJECTIONS.get(view_name, "*")
|
|
149
|
+
reader = f"{_ANALYTICS_READER_PREFIX}{view_name}"
|
|
150
|
+
try:
|
|
151
|
+
# The parquet paths never enter SQL text: the relation API takes
|
|
152
|
+
# them as a Python list, and the relation is registered as the
|
|
153
|
+
# view's reader. It stays a lazy read_parquet scan (the optimizer
|
|
154
|
+
# inlines the reader into the view), so the file is still opened
|
|
155
|
+
# per query. Not ``con.sql("... read_parquet(?)", params=...)``:
|
|
156
|
+
# that shape measured about 3x the memory of a plain view.
|
|
157
|
+
# ``union_by_name``: shards written before and after a schema
|
|
158
|
+
# change (session_classifications, 2026-09-27) differ in columns,
|
|
159
|
+
# and a positional union would misread one as the other.
|
|
160
|
+
con.read_parquet([str(p) for p in parts], union_by_name=True).create_view(
|
|
161
|
+
reader, replace=True
|
|
162
|
+
)
|
|
163
|
+
# View name and projection are module constants (_ANALYTICS_SOURCES /
|
|
164
|
+
# _VIEW_PROJECTIONS); the reader name is a constant prefix plus that
|
|
165
|
+
# view name.
|
|
166
|
+
con.execute(
|
|
167
|
+
f"CREATE OR REPLACE VIEW {view_name} AS " # noqa: S608 # nosec B608 - constants only; paths are bound into the reader relation
|
|
168
|
+
f"SELECT {projection} FROM {reader};"
|
|
169
|
+
)
|
|
170
|
+
logger.debug("Registered analytics view: {}", view_name)
|
|
171
|
+
registered.add(view_name)
|
|
172
|
+
except Exception:
|
|
173
|
+
# register-or-fail-loud for real DDL errors — a populated parquet
|
|
174
|
+
# that cannot bind is corruption, not a fresh-install state.
|
|
175
|
+
logger.exception(
|
|
176
|
+
"Failed to register analytics view {} from {} file(s)", view_name, len(parts)
|
|
177
|
+
)
|
|
178
|
+
raise
|
|
179
|
+
|
|
180
|
+
if "session_classifications" in registered:
|
|
181
|
+
con.execute(
|
|
182
|
+
"""
|
|
183
|
+
CREATE OR REPLACE VIEW session_goals AS
|
|
184
|
+
SELECT session_id, goal, confidence, classified_at
|
|
185
|
+
FROM session_classifications;
|
|
186
|
+
"""
|
|
187
|
+
)
|
|
188
|
+
registered.add("session_goals")
|
|
189
|
+
logger.debug("Registered analytics view: session_goals")
|
|
190
|
+
|
|
191
|
+
if "perceived_errors" in registered:
|
|
192
|
+
# Sessions with zero perceived-error rows simply do not appear here
|
|
193
|
+
# (clean sessions write nothing — LangSmith parity); callers wanting
|
|
194
|
+
# every session LEFT JOIN onto ``sessions`` and coalesce. max_severity
|
|
195
|
+
# orders minor < moderate < major; signals is the DISTINCT ordered
|
|
196
|
+
# list of evidence categories seen in the session.
|
|
197
|
+
con.execute(
|
|
198
|
+
"""
|
|
199
|
+
CREATE OR REPLACE VIEW perceived_summary AS
|
|
200
|
+
SELECT session_id,
|
|
201
|
+
count(*) AS n_errors,
|
|
202
|
+
CASE max(CASE severity
|
|
203
|
+
WHEN 'major' THEN 3
|
|
204
|
+
WHEN 'moderate' THEN 2
|
|
205
|
+
ELSE 1 END)
|
|
206
|
+
WHEN 3 THEN 'major'
|
|
207
|
+
WHEN 2 THEN 'moderate'
|
|
208
|
+
ELSE 'minor' END AS max_severity,
|
|
209
|
+
list_sort(list(DISTINCT signal)) AS signals
|
|
210
|
+
FROM perceived_errors
|
|
211
|
+
GROUP BY session_id;
|
|
212
|
+
"""
|
|
213
|
+
)
|
|
214
|
+
registered.add("perceived_summary")
|
|
215
|
+
logger.debug("Registered analytics view: perceived_summary")
|
|
216
|
+
|
|
217
|
+
if "session_conflicts" in registered:
|
|
218
|
+
# A session with zero conflict rows is ABSENT here, not present with
|
|
219
|
+
# conflict_count = 0. A caller that wants every session must LEFT JOIN
|
|
220
|
+
# onto ``sessions`` and coalesce, or its denominator silently drops the
|
|
221
|
+
# conflict-free sessions — which are the majority.
|
|
222
|
+
con.execute(
|
|
223
|
+
"""
|
|
224
|
+
CREATE OR REPLACE VIEW conflicts_summary AS
|
|
225
|
+
SELECT session_id, count(*) AS conflict_count
|
|
226
|
+
FROM session_conflicts
|
|
227
|
+
GROUP BY session_id;
|
|
228
|
+
"""
|
|
229
|
+
)
|
|
230
|
+
registered.add("conflicts_summary")
|
|
231
|
+
logger.debug("Registered analytics view: conflicts_summary")
|
|
232
|
+
|
|
233
|
+
return registered
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def register_analytics_macros(
|
|
237
|
+
con: duckdb.DuckDBPyConnection,
|
|
238
|
+
registered_views: set[str],
|
|
239
|
+
) -> None:
|
|
240
|
+
"""Create the analytics macros whose backing views are registered.
|
|
241
|
+
|
|
242
|
+
Every signature must match its ``ANALYTICS_MACRO_SIGNATURES`` entry in
|
|
243
|
+
:mod:`atif_duck.domain.catalog`; the drift test parses this function's
|
|
244
|
+
DDL text and fails on any divergence.
|
|
245
|
+
|
|
246
|
+
Must run after :func:`register_analytics`, the base ``register_views``
|
|
247
|
+
and ``register_authorship`` — several macros join the transcript-derived
|
|
248
|
+
views (``messages``, ``human_turns``).
|
|
249
|
+
"""
|
|
250
|
+
analytics_macros: list[tuple[str, str]] = [
|
|
251
|
+
# Work-category mix in the last N days.
|
|
252
|
+
(
|
|
253
|
+
"work_mix",
|
|
254
|
+
"""
|
|
255
|
+
CREATE OR REPLACE MACRO work_mix(since_days) AS TABLE (
|
|
256
|
+
SELECT work_category, count(*) AS n
|
|
257
|
+
FROM session_classifications
|
|
258
|
+
WHERE classified_at >= current_timestamp - (since_days * INTERVAL 1 DAY)
|
|
259
|
+
GROUP BY 1
|
|
260
|
+
ORDER BY n DESC
|
|
261
|
+
);
|
|
262
|
+
""",
|
|
263
|
+
),
|
|
264
|
+
# Counts per friction label over the last N days by message ``ts``
|
|
265
|
+
# (the user's utterance time). NULL = full corpus. Excludes
|
|
266
|
+
# label='none' (the majority sentinel would swamp the output).
|
|
267
|
+
(
|
|
268
|
+
"friction_counts",
|
|
269
|
+
"""
|
|
270
|
+
CREATE OR REPLACE MACRO friction_counts(since_days) AS TABLE (
|
|
271
|
+
SELECT label,
|
|
272
|
+
count(*) AS n,
|
|
273
|
+
count(DISTINCT session_id) AS sessions,
|
|
274
|
+
avg(confidence) AS avg_confidence,
|
|
275
|
+
sum(CASE WHEN source='regex' THEN 1 ELSE 0 END) AS n_regex,
|
|
276
|
+
sum(CASE WHEN source='llm' THEN 1 ELSE 0 END) AS n_llm
|
|
277
|
+
FROM user_friction
|
|
278
|
+
WHERE label != 'none'
|
|
279
|
+
AND (since_days IS NULL
|
|
280
|
+
OR ts >= current_timestamp - (since_days * INTERVAL 1 DAY))
|
|
281
|
+
GROUP BY label
|
|
282
|
+
ORDER BY n DESC
|
|
283
|
+
);
|
|
284
|
+
""",
|
|
285
|
+
),
|
|
286
|
+
# Per-session friction pressure vs the human turn count. The
|
|
287
|
+
# denominator counts ``human_turns`` (main-chain user steps a human
|
|
288
|
+
# wrote), so hook feedback and harness text never dilute the rate.
|
|
289
|
+
(
|
|
290
|
+
"friction_rate",
|
|
291
|
+
"""
|
|
292
|
+
CREATE OR REPLACE MACRO friction_rate(since_days) AS TABLE (
|
|
293
|
+
WITH hits AS (
|
|
294
|
+
SELECT session_id,
|
|
295
|
+
count(*) FILTER (WHERE label != 'none') AS n_friction,
|
|
296
|
+
count(*) FILTER (WHERE label = 'status_ping') AS n_status,
|
|
297
|
+
count(*) FILTER (WHERE label = 'unmet_expectation') AS n_unmet,
|
|
298
|
+
count(*) FILTER (WHERE label = 'confusion') AS n_confusion,
|
|
299
|
+
count(*) FILTER (WHERE label = 'interruption') AS n_interruption,
|
|
300
|
+
count(*) FILTER (WHERE label = 'correction') AS n_correction,
|
|
301
|
+
count(*) FILTER (WHERE label = 'frustration') AS n_frustration
|
|
302
|
+
FROM user_friction
|
|
303
|
+
WHERE since_days IS NULL
|
|
304
|
+
OR ts >= current_timestamp - (since_days * INTERVAL 1 DAY)
|
|
305
|
+
GROUP BY session_id
|
|
306
|
+
),
|
|
307
|
+
user_msgs AS (
|
|
308
|
+
SELECT ht.session_id,
|
|
309
|
+
count(*) AS n_user_msgs
|
|
310
|
+
FROM human_turns ht
|
|
311
|
+
WHERE since_days IS NULL
|
|
312
|
+
OR ht.ts >= current_timestamp - (since_days * INTERVAL 1 DAY)
|
|
313
|
+
GROUP BY 1
|
|
314
|
+
)
|
|
315
|
+
SELECT h.session_id,
|
|
316
|
+
h.n_friction,
|
|
317
|
+
h.n_status, h.n_unmet, h.n_confusion,
|
|
318
|
+
h.n_interruption, h.n_correction, h.n_frustration,
|
|
319
|
+
COALESCE(um.n_user_msgs, 0) AS n_user_msgs,
|
|
320
|
+
h.n_friction::DOUBLE / NULLIF(um.n_user_msgs, 0) AS rate
|
|
321
|
+
FROM hits h
|
|
322
|
+
LEFT JOIN user_msgs um USING (session_id)
|
|
323
|
+
WHERE h.n_friction > 0
|
|
324
|
+
ORDER BY h.n_friction DESC
|
|
325
|
+
);
|
|
326
|
+
""",
|
|
327
|
+
),
|
|
328
|
+
# Top-N example user messages for one friction label.
|
|
329
|
+
(
|
|
330
|
+
"friction_examples",
|
|
331
|
+
"""
|
|
332
|
+
CREATE OR REPLACE MACRO friction_examples(label_name, n) AS TABLE (
|
|
333
|
+
SELECT session_id, ts, text_snippet, rationale, source, confidence
|
|
334
|
+
FROM user_friction
|
|
335
|
+
WHERE label = label_name
|
|
336
|
+
ORDER BY confidence DESC, ts DESC
|
|
337
|
+
LIMIT n
|
|
338
|
+
);
|
|
339
|
+
""",
|
|
340
|
+
),
|
|
341
|
+
# Conflicts on a conversation-time axis, NOT a detection-time one:
|
|
342
|
+
# ``detected_at`` is the worker run-time clock, so the macro
|
|
343
|
+
# recovers ``m.ts`` by joining the later turn of the pair
|
|
344
|
+
# (``turn_b_uuid``) back to ``messages.uuid``. INNER JOIN on
|
|
345
|
+
# purpose — the recoverable subset is the trustworthy subset.
|
|
346
|
+
# root_session_id = session_id here: harbor inlines subagent
|
|
347
|
+
# transcripts, so one conversation is one session.
|
|
348
|
+
(
|
|
349
|
+
"conflicts_over_time",
|
|
350
|
+
"""
|
|
351
|
+
CREATE OR REPLACE MACRO conflicts_over_time(since_days) AS TABLE (
|
|
352
|
+
SELECT m.ts AS conversation_ts,
|
|
353
|
+
sc.session_id,
|
|
354
|
+
sc.session_id AS root_session_id,
|
|
355
|
+
sc.turn_a_uuid,
|
|
356
|
+
sc.turn_b_uuid,
|
|
357
|
+
sc.conflict_kind,
|
|
358
|
+
sc.severity,
|
|
359
|
+
sc.agent_position,
|
|
360
|
+
sc.user_position,
|
|
361
|
+
sc.confidence,
|
|
362
|
+
sc.detected_at
|
|
363
|
+
FROM session_conflicts sc
|
|
364
|
+
JOIN messages m
|
|
365
|
+
ON m.uuid = sc.turn_b_uuid
|
|
366
|
+
WHERE since_days IS NULL
|
|
367
|
+
OR m.ts >= current_timestamp - (since_days * INTERVAL 1 DAY)
|
|
368
|
+
ORDER BY m.ts DESC
|
|
369
|
+
);
|
|
370
|
+
""",
|
|
371
|
+
),
|
|
372
|
+
# Counts per perceived-error signal over the last N days on a REAL
|
|
373
|
+
# conversation-time axis (conflicts_over_time precedent):
|
|
374
|
+
# ``detected_at`` is the worker run-time clock, so the macro
|
|
375
|
+
# recovers ``m.ts`` by joining the anchor USER turn (turn_uuid)
|
|
376
|
+
# back to ``messages.uuid``. INNER JOIN on purpose — the
|
|
377
|
+
# recoverable subset is the trustworthy subset. NULL = full corpus.
|
|
378
|
+
(
|
|
379
|
+
"perceived_counts",
|
|
380
|
+
"""
|
|
381
|
+
CREATE OR REPLACE MACRO perceived_counts(since_days) AS TABLE (
|
|
382
|
+
SELECT pe.signal,
|
|
383
|
+
count(*) AS n,
|
|
384
|
+
count(DISTINCT pe.session_id) AS sessions,
|
|
385
|
+
count(*) FILTER (WHERE pe.severity = 'major') AS n_major,
|
|
386
|
+
count(*) FILTER (WHERE pe.severity = 'moderate') AS n_moderate,
|
|
387
|
+
count(*) FILTER (WHERE pe.severity = 'minor') AS n_minor,
|
|
388
|
+
avg(pe.confidence) AS avg_confidence
|
|
389
|
+
FROM perceived_errors pe
|
|
390
|
+
JOIN messages m
|
|
391
|
+
ON m.uuid = pe.turn_uuid
|
|
392
|
+
WHERE since_days IS NULL
|
|
393
|
+
OR m.ts >= current_timestamp - (since_days * INTERVAL 1 DAY)
|
|
394
|
+
GROUP BY pe.signal
|
|
395
|
+
ORDER BY n DESC
|
|
396
|
+
);
|
|
397
|
+
""",
|
|
398
|
+
),
|
|
399
|
+
# Per-session perceived-error pressure vs the human turn count
|
|
400
|
+
# (friction_rate precedent: the denominator counts ``human_turns``). Sessions with zero perceived errors do
|
|
401
|
+
# not appear — LEFT JOIN onto sessions and coalesce for the
|
|
402
|
+
# full-population rate.
|
|
403
|
+
(
|
|
404
|
+
"perceived_rate",
|
|
405
|
+
"""
|
|
406
|
+
CREATE OR REPLACE MACRO perceived_rate(since_days) AS TABLE (
|
|
407
|
+
WITH hits AS (
|
|
408
|
+
SELECT pe.session_id,
|
|
409
|
+
count(*) AS n_errors,
|
|
410
|
+
count(*) FILTER (WHERE pe.severity = 'major') AS n_major,
|
|
411
|
+
count(*) FILTER (WHERE pe.severity = 'moderate') AS n_moderate,
|
|
412
|
+
count(*) FILTER (WHERE pe.severity = 'minor') AS n_minor
|
|
413
|
+
FROM perceived_errors pe
|
|
414
|
+
JOIN messages m
|
|
415
|
+
ON m.uuid = pe.turn_uuid
|
|
416
|
+
WHERE since_days IS NULL
|
|
417
|
+
OR m.ts >= current_timestamp - (since_days * INTERVAL 1 DAY)
|
|
418
|
+
GROUP BY pe.session_id
|
|
419
|
+
),
|
|
420
|
+
user_msgs AS (
|
|
421
|
+
SELECT ht.session_id,
|
|
422
|
+
count(*) AS n_user_msgs
|
|
423
|
+
FROM human_turns ht
|
|
424
|
+
WHERE since_days IS NULL
|
|
425
|
+
OR ht.ts >= current_timestamp - (since_days * INTERVAL 1 DAY)
|
|
426
|
+
GROUP BY 1
|
|
427
|
+
)
|
|
428
|
+
SELECT h.session_id,
|
|
429
|
+
h.n_errors,
|
|
430
|
+
h.n_major, h.n_moderate, h.n_minor,
|
|
431
|
+
COALESCE(um.n_user_msgs, 0) AS n_user_msgs,
|
|
432
|
+
h.n_errors::DOUBLE / NULLIF(um.n_user_msgs, 0) AS rate
|
|
433
|
+
FROM hits h
|
|
434
|
+
LEFT JOIN user_msgs um USING (session_id)
|
|
435
|
+
ORDER BY h.n_errors DESC
|
|
436
|
+
);
|
|
437
|
+
""",
|
|
438
|
+
),
|
|
439
|
+
# Top-N examples for one perceived-error signal, highest-confidence
|
|
440
|
+
# first (friction_examples precedent).
|
|
441
|
+
(
|
|
442
|
+
"perceived_examples",
|
|
443
|
+
"""
|
|
444
|
+
CREATE OR REPLACE MACRO perceived_examples(signal_name, n) AS TABLE (
|
|
445
|
+
SELECT session_id, turn_uuid, severity, evidence,
|
|
446
|
+
agent_error_summary, confidence, detected_at
|
|
447
|
+
FROM perceived_errors
|
|
448
|
+
WHERE signal = signal_name
|
|
449
|
+
ORDER BY confidence DESC, detected_at DESC
|
|
450
|
+
LIMIT n
|
|
451
|
+
);
|
|
452
|
+
""",
|
|
453
|
+
),
|
|
454
|
+
]
|
|
455
|
+
|
|
456
|
+
for macro_name, ddl in analytics_macros:
|
|
457
|
+
required = _ANALYTICS_MACRO_REQUIREMENTS.get(macro_name, ())
|
|
458
|
+
if any(view not in registered_views for view in required):
|
|
459
|
+
logger.debug("Skipped analytics macro {} (view missing among {})", macro_name, required)
|
|
460
|
+
continue
|
|
461
|
+
try:
|
|
462
|
+
con.execute(ddl)
|
|
463
|
+
logger.debug("Registered analytics macro: {}", macro_name)
|
|
464
|
+
except Exception:
|
|
465
|
+
logger.exception("Failed to register analytics macro {}", macro_name)
|
|
466
|
+
raise
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
__all__ = ["register_analytics", "register_analytics_macros"]
|