atif-sql 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. atif_analytics/__init__.py +9 -0
  2. atif_analytics/application/__init__.py +3 -0
  3. atif_analytics/application/analyze.py +158 -0
  4. atif_analytics/application/prompts.py +750 -0
  5. atif_analytics/application/use_cases/__init__.py +3 -0
  6. atif_analytics/application/use_cases/_shared.py +264 -0
  7. atif_analytics/application/use_cases/classify.py +430 -0
  8. atif_analytics/application/use_cases/conflicts.py +448 -0
  9. atif_analytics/application/use_cases/friction.py +634 -0
  10. atif_analytics/application/use_cases/perceived.py +451 -0
  11. atif_analytics/domain/__init__.py +3 -0
  12. atif_analytics/domain/authorship.py +166 -0
  13. atif_analytics/domain/config.py +30 -0
  14. atif_analytics/domain/costs.py +50 -0
  15. atif_analytics/domain/friction.py +117 -0
  16. atif_analytics/domain/layout.py +94 -0
  17. atif_analytics/domain/models.py +385 -0
  18. atif_analytics/domain/ports.py +100 -0
  19. atif_analytics/domain/transcript.py +359 -0
  20. atif_analytics/infrastructure/__init__.py +3 -0
  21. atif_analytics/infrastructure/corpus_reader.py +546 -0
  22. atif_analytics/infrastructure/parquet_cache.py +186 -0
  23. atif_analytics/infrastructure/settings.py +114 -0
  24. atif_analytics/infrastructure/sqlite_state/__init__.py +3 -0
  25. atif_analytics/infrastructure/sqlite_state/checkpointer.py +326 -0
  26. atif_analytics/infrastructure/sqlite_state/retry_queue.py +296 -0
  27. atif_analytics/py.typed +0 -0
  28. atif_cli/__init__.py +3 -0
  29. atif_cli/__main__.py +8 -0
  30. atif_cli/app.py +2459 -0
  31. atif_cli/converter_adapter.py +151 -0
  32. atif_cli/corpus.py +409 -0
  33. atif_cli/cron.py +311 -0
  34. atif_cli/duck_errors.py +97 -0
  35. atif_cli/embed_lake.py +101 -0
  36. atif_cli/errors.py +89 -0
  37. atif_cli/lake.py +373 -0
  38. atif_cli/lake_sessions.py +270 -0
  39. atif_cli/output.py +297 -0
  40. atif_cli/py.typed +0 -0
  41. atif_converter/__init__.py +3 -0
  42. atif_converter/application/__init__.py +3 -0
  43. atif_converter/application/convert_and_audit.py +294 -0
  44. atif_converter/application/convert_codex.py +210 -0
  45. atif_converter/domain/__init__.py +9 -0
  46. atif_converter/domain/agents.py +40 -0
  47. atif_converter/domain/atif/__init__.py +55 -0
  48. atif_converter/domain/atif/agent.py +37 -0
  49. atif_converter/domain/atif/content.py +161 -0
  50. atif_converter/domain/atif/final_metrics.py +45 -0
  51. atif_converter/domain/atif/metrics.py +49 -0
  52. atif_converter/domain/atif/observation.py +21 -0
  53. atif_converter/domain/atif/observation_result.py +47 -0
  54. atif_converter/domain/atif/step.py +144 -0
  55. atif_converter/domain/atif/subagent_trajectory_ref.py +90 -0
  56. atif_converter/domain/atif/tool_call.py +36 -0
  57. atif_converter/domain/atif/trajectory.py +211 -0
  58. atif_converter/domain/atif/trajectory_validator.py +299 -0
  59. atif_converter/domain/blobs.py +510 -0
  60. atif_converter/domain/claude_code_conversion.py +1077 -0
  61. atif_converter/domain/codex_conversion.py +1036 -0
  62. atif_converter/domain/codex_edges.py +192 -0
  63. atif_converter/domain/codex_enrichment.py +419 -0
  64. atif_converter/domain/codex_fidelity.py +124 -0
  65. atif_converter/domain/codex_nested_calls.py +165 -0
  66. atif_converter/domain/edges.py +113 -0
  67. atif_converter/domain/enrichment.py +412 -0
  68. atif_converter/domain/errors.py +60 -0
  69. atif_converter/domain/fidelity.py +171 -0
  70. atif_converter/domain/model_prices.json +8538 -0
  71. atif_converter/domain/pricing.py +790 -0
  72. atif_converter/domain/result_signals.py +774 -0
  73. atif_converter/domain/schema_version.py +90 -0
  74. atif_converter/domain/session_events.py +408 -0
  75. atif_converter/infrastructure/__init__.py +3 -0
  76. atif_converter/infrastructure/census.py +89 -0
  77. atif_converter/infrastructure/claude_code_converter.py +127 -0
  78. atif_converter/infrastructure/codex_adapter.py +132 -0
  79. atif_converter/infrastructure/codex_census.py +100 -0
  80. atif_converter/infrastructure/codex_converter.py +82 -0
  81. atif_converter/infrastructure/harbor_adapter.py +187 -0
  82. atif_converter/infrastructure/raw_records.py +426 -0
  83. atif_converter/infrastructure/source_archive.py +154 -0
  84. atif_converter/py.typed +0 -0
  85. atif_corpus/__init__.py +13 -0
  86. atif_corpus/application/__init__.py +7 -0
  87. atif_corpus/application/materialize.py +1826 -0
  88. atif_corpus/domain/__init__.py +9 -0
  89. atif_corpus/domain/agents.py +33 -0
  90. atif_corpus/domain/generation.py +34 -0
  91. atif_corpus/domain/layout.py +168 -0
  92. atif_corpus/domain/ports.py +175 -0
  93. atif_corpus/domain/session_id.py +68 -0
  94. atif_corpus/domain/sessions.py +228 -0
  95. atif_corpus/domain/slug.py +72 -0
  96. atif_corpus/domain/source_layout.py +143 -0
  97. atif_corpus/domain/watermark.py +76 -0
  98. atif_corpus/infrastructure/__init__.py +7 -0
  99. atif_corpus/infrastructure/atomic.py +266 -0
  100. atif_corpus/infrastructure/compress_artifacts.py +161 -0
  101. atif_corpus/infrastructure/fake_converter.py +174 -0
  102. atif_corpus/infrastructure/scanner.py +316 -0
  103. atif_corpus/infrastructure/settings.py +130 -0
  104. atif_corpus/infrastructure/source_archive.py +202 -0
  105. atif_corpus/py.typed +0 -0
  106. atif_duck/__init__.py +43 -0
  107. atif_duck/domain/__init__.py +32 -0
  108. atif_duck/domain/artifacts.py +66 -0
  109. atif_duck/domain/authorship.py +165 -0
  110. atif_duck/domain/catalog.py +626 -0
  111. atif_duck/domain/columnar.py +149 -0
  112. atif_duck/domain/embedding_guard.py +88 -0
  113. atif_duck/domain/examples.py +265 -0
  114. atif_duck/domain/lake.py +427 -0
  115. atif_duck/domain/lake_sessions.py +97 -0
  116. atif_duck/domain/raw_readers.py +140 -0
  117. atif_duck/domain/session_id.py +69 -0
  118. atif_duck/domain/sql_literal.py +50 -0
  119. atif_duck/infrastructure/__init__.py +34 -0
  120. atif_duck/infrastructure/analytics.py +469 -0
  121. atif_duck/infrastructure/authorship.py +139 -0
  122. atif_duck/infrastructure/columnar.py +641 -0
  123. atif_duck/infrastructure/lake.py +1383 -0
  124. atif_duck/infrastructure/lake_sessions.py +194 -0
  125. atif_duck/infrastructure/lake_settings.py +63 -0
  126. atif_duck/infrastructure/lake_steps.py +225 -0
  127. atif_duck/infrastructure/projections.py +221 -0
  128. atif_duck/infrastructure/registry.py +2194 -0
  129. atif_duck/infrastructure/stored_artifacts.py +115 -0
  130. atif_duck/py.typed +0 -0
  131. atif_embed/__init__.py +12 -0
  132. atif_embed/application/__init__.py +3 -0
  133. atif_embed/application/embed.py +293 -0
  134. atif_embed/application/prune.py +80 -0
  135. atif_embed/domain/__init__.py +3 -0
  136. atif_embed/domain/discovery.py +70 -0
  137. atif_embed/domain/embedding_guard.py +83 -0
  138. atif_embed/domain/errors.py +94 -0
  139. atif_embed/domain/ports.py +223 -0
  140. atif_embed/domain/sql_literal.py +49 -0
  141. atif_embed/domain/text_stamp.py +71 -0
  142. atif_embed/infrastructure/__init__.py +3 -0
  143. atif_embed/infrastructure/cohere_bedrock.py +433 -0
  144. atif_embed/infrastructure/corpus_text_rows.py +273 -0
  145. atif_embed/infrastructure/gemma_local.py +355 -0
  146. atif_embed/infrastructure/lake_text_rows.py +203 -0
  147. atif_embed/infrastructure/lance_store.py +440 -0
  148. atif_embed/infrastructure/providers.py +30 -0
  149. atif_embed/infrastructure/settings.py +154 -0
  150. atif_embed/py.typed +0 -0
  151. atif_models/__init__.py +14 -0
  152. atif_models/domain/__init__.py +9 -0
  153. atif_models/domain/ports.py +143 -0
  154. atif_models/domain/registry.py +151 -0
  155. atif_models/domain/schema.py +109 -0
  156. atif_models/infrastructure/__init__.py +8 -0
  157. atif_models/infrastructure/openai_bedrock.py +314 -0
  158. atif_models/infrastructure/settings.py +85 -0
  159. atif_models/py.typed +0 -0
  160. atif_sql-0.2.0.dist-info/METADATA +272 -0
  161. atif_sql-0.2.0.dist-info/RECORD +164 -0
  162. atif_sql-0.2.0.dist-info/WHEEL +4 -0
  163. atif_sql-0.2.0.dist-info/entry_points.txt +2 -0
  164. atif_sql-0.2.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,69 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+
3
+ """The session id boundary: which directory names may become a session.
4
+
5
+ A session id is the one piece of text from OUTSIDE this workspace that ends up
6
+ inside a corpus path: it is derived from a transcript filename under the
7
+ source root, becomes the ``sessions/<session_id>/`` directory name, and from
8
+ there names the files atif-duck opens. Everything else in a statement is a
9
+ constant. So the id is checked once, at the boundary, and a name that fails
10
+ the check is skipped and reported rather than carried any further.
11
+
12
+ The rule is deliberately narrow: an ASCII letter or digit first, then letters,
13
+ digits, ``.``, ``_`` and ``-``, at most 255 characters (the longest name most
14
+ filesystems accept). Every id both supported agents produce passes: Claude
15
+ Code names a session with a version 4 UUID, Codex with a version 7 UUID
16
+ (``019e9e38-39a6-7170-86fb-46e6f1cb1931``). Quotes, whitespace, path
17
+ separators, ``;``, ``$``, ``?`` and anything outside ASCII fail.
18
+
19
+ DELIBERATE TWIN of :mod:`atif_corpus.domain.session_id`, the same way
20
+ `sql_literal` and `embedding_guard` twin their atif-embed copies: the independence
21
+ contract forbids the two packages from importing each other, and both sides
22
+ must apply the SAME rule: the corpus writer before it writes, this registry
23
+ before it reads a corpus an older version wrote and before any session name
24
+ reaches a path. Each package's tests read the other's copy as source text and
25
+ fail if the pattern or the limit drifts.
26
+
27
+ Pure module: stdlib only.
28
+ """
29
+
30
+ from __future__ import annotations
31
+
32
+ import re
33
+
34
+ #: The accepted shape, as a pattern string so the twin pin can compare it.
35
+ SESSION_ID_PATTERN: str = r"^[A-Za-z0-9][A-Za-z0-9._-]*$"
36
+
37
+ #: Longest accepted id, in characters.
38
+ SESSION_ID_MAX_CHARS: int = 255
39
+
40
+ _SESSION_ID_RE = re.compile(SESSION_ID_PATTERN)
41
+
42
+
43
+ def session_id_rejection(candidate: str) -> str | None:
44
+ """Why ``candidate`` cannot be a session id, or ``None`` when it can.
45
+
46
+ The reason is one short phrase meant for a log line; a caller that only
47
+ needs the verdict uses :func:`is_valid_session_id`.
48
+ """
49
+ if len(candidate) > SESSION_ID_MAX_CHARS:
50
+ return f"longer than {SESSION_ID_MAX_CHARS} characters"
51
+ # fullmatch, not match: with ``$`` alone a trailing newline would pass.
52
+ if _SESSION_ID_RE.fullmatch(candidate) is None:
53
+ return (
54
+ "contains a character outside [A-Za-z0-9._-] or does not start with a letter or digit"
55
+ )
56
+ return None
57
+
58
+
59
+ def is_valid_session_id(candidate: str) -> bool:
60
+ """Whether ``candidate`` passes the boundary."""
61
+ return session_id_rejection(candidate) is None
62
+
63
+
64
+ __all__ = [
65
+ "SESSION_ID_MAX_CHARS",
66
+ "SESSION_ID_PATTERN",
67
+ "is_valid_session_id",
68
+ "session_id_rejection",
69
+ ]
@@ -0,0 +1,50 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+
3
+ """Single-quoted SQL literal escaping, and the type that marks text as SQL.
4
+
5
+ Two things live here because they are the two ends of the same boundary.
6
+
7
+ :data:`SqlFragment` is a ``NewType`` over ``str``: a value typed as one is
8
+ text that may be spliced into a statement. Only three kinds of code produce it:
9
+ :func:`sql_literal` here, the catalog constants, and the projection module's
10
+ column expressions. A path, a session id, or anything else that arrived from
11
+ outside is a plain ``str`` and reaches DuckDB as a bound parameter instead. The
12
+ type is advisory at runtime (a NewType is its underlying ``str``); the AST test
13
+ over the registry modules is what enforces the rule.
14
+
15
+ :func:`sql_literal` is for the few statement kinds DuckDB refuses to prepare:
16
+ ``ATTACH``, and the producer's one-row session projection. It doubles every
17
+ embedded single quote, which is the SQL standard's own escape. It lives in
18
+ ``domain`` so every module that still needs it reaches the SAME implementation.
19
+
20
+ Pure module: stdlib only, no duckdb.
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ from typing import NewType
26
+
27
+ #: Text that may be spliced into a SQL statement. See the module docstring.
28
+ SqlFragment = NewType("SqlFragment", str)
29
+
30
+
31
+ def sql_literal(value: str) -> SqlFragment:
32
+ """Escape ``value`` as a single-quoted SQL string literal.
33
+
34
+ Doubles every embedded single quote, per the SQL standard's own escape.
35
+
36
+ Parameters
37
+ ----------
38
+ value
39
+ Value to embed in a statement.
40
+
41
+ Returns
42
+ -------
43
+ SqlFragment
44
+ ``value`` wrapped in single quotes, embedded quotes doubled.
45
+ """
46
+ escaped = value.replace("'", "''")
47
+ return SqlFragment(f"'{escaped}'")
48
+
49
+
50
+ __all__ = ["SqlFragment", "sql_literal"]
@@ -0,0 +1,34 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+
3
+ """Infrastructure layer: DuckDB registry over the materialized ATIF corpus.
4
+
5
+ Also the columnar producer (:class:`ColumnarArtifactProducer`) that writes
6
+ the typed per-session parquet artifacts the registry prefers to read, and
7
+ :func:`columnar_coverage`, which reports how many sessions carry them.
8
+ """
9
+
10
+ from atif_duck.infrastructure.columnar import (
11
+ ColumnarArtifactProducer,
12
+ ColumnarCoverage,
13
+ columnar_coverage,
14
+ session_has_columnar,
15
+ )
16
+ from atif_duck.infrastructure.registry import (
17
+ RawSources,
18
+ register,
19
+ register_macros,
20
+ register_raw,
21
+ register_views,
22
+ )
23
+
24
+ __all__ = [
25
+ "ColumnarArtifactProducer",
26
+ "ColumnarCoverage",
27
+ "RawSources",
28
+ "columnar_coverage",
29
+ "register",
30
+ "register_macros",
31
+ "register_raw",
32
+ "register_views",
33
+ "session_has_columnar",
34
+ ]
@@ -0,0 +1,469 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+
3
+ """v2 analytics views + macros over the atif-analytics parquet outputs.
4
+
5
+ atif-analytics writes its artifacts under ``<corpus_root>/analytics/`` (one
6
+ sharded dir per LLM pipeline); this module binds them as the analytics
7
+ views and macros the catalog declares.
8
+
9
+ The trajectory pipeline and the structural ones (cluster, terms,
10
+ community) were removed on 2026-09-27, and with them ``message_trajectory``,
11
+ ``message_clusters``, ``cluster_terms``, ``session_communities``,
12
+ ``community_profile`` and the ``autonomy_trend``, ``success_rate_by_work``,
13
+ ``sentiment_arc``, ``cluster_top_terms`` and ``community_top_topics`` macros.
14
+ Their parquets may still sit on disk in an older corpus; nothing binds them.
15
+
16
+ atif-duck may not import atif-analytics (independence contract), so the
17
+ artifact names are pinned here against the documented layout — the shape is
18
+ contract, not code sharing.
19
+
20
+ Gating: each view is created only when its backing parquet is populated on
21
+ disk; each macro is registered only when EVERY view it binds against was
22
+ registered. Missing analytics parquets are the default state until the
23
+ pipelines run, so skips log at DEBUG.
24
+
25
+ Column semantics worth stating once (repeated per macro below):
26
+
27
+ * ``conflicts_over_time`` / ``perceived_*`` join ``messages`` (the
28
+ edges-derived uuid-keyed view) to recover conversation time.
29
+ * ``conflicts_over_time``'s ``root_session_id`` equals ``session_id``: harbor
30
+ INLINES subagent transcripts (fidelity gap 4), so one conversation is one
31
+ session and there is no parent session to collapse onto.
32
+ * ``friction_rate`` / ``perceived_rate`` count their user-message
33
+ denominator from ``human_turns``: main-chain user steps a HUMAN wrote, so
34
+ Stop hook feedback, task notifications and harness text are not in it.
35
+ """
36
+
37
+ from __future__ import annotations
38
+
39
+ from typing import TYPE_CHECKING
40
+
41
+ from loguru import logger
42
+
43
+ if TYPE_CHECKING:
44
+ from pathlib import Path
45
+
46
+ import duckdb
47
+
48
+ # ---------------------------------------------------------------------------
49
+ # Artifact layout (pinned against atif-analytics' AnalyticsLayout — contract)
50
+ # ---------------------------------------------------------------------------
51
+
52
+ #: Prefix of the per-view parquet reader each analytics view selects from.
53
+ #: ``CREATE VIEW`` cannot take a bound parameter, so the parquet paths go
54
+ #: through the connection's ``read_parquet`` relation API (a Python value, not
55
+ #: statement text), the relation is registered under ``<prefix><view_name>``,
56
+ #: and the view itself interpolates constants only.
57
+ _ANALYTICS_READER_PREFIX: str = "v_raw_analytics_"
58
+
59
+ #: view name -> path relative to <corpus_root>/analytics/. Directory entries
60
+ #: are sharded caches (part-*.parquet); file entries are single parquets.
61
+ _ANALYTICS_SOURCES: dict[str, str] = {
62
+ "session_classifications": "session_classifications",
63
+ "session_conflicts": "session_conflicts",
64
+ "user_friction": "user_friction",
65
+ "perceived_errors": "perceived_errors",
66
+ }
67
+
68
+ #: Per-view projections; a view absent here selects ``*``.
69
+ #:
70
+ #: ``session_classifications`` names its columns because its shards span two
71
+ #: schemas. Rows written before 2026-09-27 also carry the LLM ``autonomy_tier``
72
+ #: and ``success`` labels, which were dropped as unreliable (the same outcome
73
+ #: drew both labels hundreds of times each on one corpus). The two names
74
+ #: survive as always-NULL compatibility columns so a downstream reader that
75
+ #: selects them still binds; ``session_outcomes`` carries the deterministic
76
+ #: replacement. ``category`` is an alias of ``work_category``.
77
+ _VIEW_PROJECTIONS: dict[str, str] = {
78
+ "session_classifications": (
79
+ "session_id, work_category, goal, confidence, classified_at, "
80
+ "CAST(NULL AS VARCHAR) AS autonomy_tier, CAST(NULL AS VARCHAR) AS success, "
81
+ "work_category AS category"
82
+ ),
83
+ }
84
+
85
+ #: Per-macro view dependencies, keyed on the views this module registers. A
86
+ #: macro whose entry is unsatisfied is skipped: DuckDB binds a macro body at
87
+ #: CREATE time, so registering it against an absent view fails outright.
88
+ _ANALYTICS_MACRO_REQUIREMENTS: dict[str, tuple[str, ...]] = {
89
+ "work_mix": ("session_classifications",),
90
+ "friction_counts": ("user_friction",),
91
+ "friction_rate": ("user_friction",),
92
+ "friction_examples": ("user_friction",),
93
+ "conflicts_over_time": ("session_conflicts",),
94
+ "perceived_counts": ("perceived_errors",),
95
+ "perceived_rate": ("perceived_errors",),
96
+ "perceived_examples": ("perceived_errors",),
97
+ }
98
+
99
+
100
+ #: Smallest byte count a readable parquet file can have: the format frames every
101
+ #: file as ``PAR1`` + footer + 4-byte footer length + ``PAR1``. A shorter file is
102
+ #: what a crashed writer leaves behind, and DuckDB errors on it rather than
103
+ #: reading zero rows, so a view built over one fails to register. Declared here
104
+ #: rather than imported because atif-duck may never import atif-analytics.
105
+ _MIN_PARQUET_BYTES = 16
106
+
107
+
108
+ def _backing_files(path: Path) -> list[Path]:
109
+ """The parquet files backing one artifact (sharded dir or single file)."""
110
+ if not path.exists():
111
+ return []
112
+ if path.is_dir():
113
+ return sorted(
114
+ p for p in path.glob("part-*.parquet") if p.stat().st_size > _MIN_PARQUET_BYTES
115
+ )
116
+ return [path] if path.stat().st_size > _MIN_PARQUET_BYTES else []
117
+
118
+
119
+ def register_analytics(
120
+ con: duckdb.DuckDBPyConnection, corpus_root: Path, *more_roots: Path
121
+ ) -> set[str]:
122
+ """Bind the analytics parquets as views; return the set registered.
123
+
124
+ Each view is created only when its backing parquet is populated.
125
+ Derived views (``session_goals``, ``conflicts_summary``) follow their
126
+ upstream. Idempotent against a partially-populated corpus.
127
+
128
+ With ``more_roots`` (a query over every corpus of the lake) each view
129
+ reads the parquets under every root's ``analytics/``. The rows carry no
130
+ corpus column (the catalog's shapes do not change with the scope); join
131
+ ``sessions`` on ``session_id`` for it. That join is exact while each
132
+ session id lives in one corpus, which holds for corpora converted from
133
+ different transcripts. A copied corpus repeats its ids, and there each
134
+ analytics row matches the session in both corpora.
135
+ """
136
+ roots = (corpus_root, *more_roots)
137
+ registered: set[str] = set()
138
+
139
+ for view_name, rel in _ANALYTICS_SOURCES.items():
140
+ parts = [part for root in roots for part in _backing_files(root / "analytics" / rel)]
141
+ if not parts:
142
+ logger.debug(
143
+ "register_analytics: skipping {} (no parquet under {} root(s))",
144
+ view_name,
145
+ len(roots),
146
+ )
147
+ continue
148
+ projection = _VIEW_PROJECTIONS.get(view_name, "*")
149
+ reader = f"{_ANALYTICS_READER_PREFIX}{view_name}"
150
+ try:
151
+ # The parquet paths never enter SQL text: the relation API takes
152
+ # them as a Python list, and the relation is registered as the
153
+ # view's reader. It stays a lazy read_parquet scan (the optimizer
154
+ # inlines the reader into the view), so the file is still opened
155
+ # per query. Not ``con.sql("... read_parquet(?)", params=...)``:
156
+ # that shape measured about 3x the memory of a plain view.
157
+ # ``union_by_name``: shards written before and after a schema
158
+ # change (session_classifications, 2026-09-27) differ in columns,
159
+ # and a positional union would misread one as the other.
160
+ con.read_parquet([str(p) for p in parts], union_by_name=True).create_view(
161
+ reader, replace=True
162
+ )
163
+ # View name and projection are module constants (_ANALYTICS_SOURCES /
164
+ # _VIEW_PROJECTIONS); the reader name is a constant prefix plus that
165
+ # view name.
166
+ con.execute(
167
+ f"CREATE OR REPLACE VIEW {view_name} AS " # noqa: S608 # nosec B608 - constants only; paths are bound into the reader relation
168
+ f"SELECT {projection} FROM {reader};"
169
+ )
170
+ logger.debug("Registered analytics view: {}", view_name)
171
+ registered.add(view_name)
172
+ except Exception:
173
+ # register-or-fail-loud for real DDL errors — a populated parquet
174
+ # that cannot bind is corruption, not a fresh-install state.
175
+ logger.exception(
176
+ "Failed to register analytics view {} from {} file(s)", view_name, len(parts)
177
+ )
178
+ raise
179
+
180
+ if "session_classifications" in registered:
181
+ con.execute(
182
+ """
183
+ CREATE OR REPLACE VIEW session_goals AS
184
+ SELECT session_id, goal, confidence, classified_at
185
+ FROM session_classifications;
186
+ """
187
+ )
188
+ registered.add("session_goals")
189
+ logger.debug("Registered analytics view: session_goals")
190
+
191
+ if "perceived_errors" in registered:
192
+ # Sessions with zero perceived-error rows simply do not appear here
193
+ # (clean sessions write nothing — LangSmith parity); callers wanting
194
+ # every session LEFT JOIN onto ``sessions`` and coalesce. max_severity
195
+ # orders minor < moderate < major; signals is the DISTINCT ordered
196
+ # list of evidence categories seen in the session.
197
+ con.execute(
198
+ """
199
+ CREATE OR REPLACE VIEW perceived_summary AS
200
+ SELECT session_id,
201
+ count(*) AS n_errors,
202
+ CASE max(CASE severity
203
+ WHEN 'major' THEN 3
204
+ WHEN 'moderate' THEN 2
205
+ ELSE 1 END)
206
+ WHEN 3 THEN 'major'
207
+ WHEN 2 THEN 'moderate'
208
+ ELSE 'minor' END AS max_severity,
209
+ list_sort(list(DISTINCT signal)) AS signals
210
+ FROM perceived_errors
211
+ GROUP BY session_id;
212
+ """
213
+ )
214
+ registered.add("perceived_summary")
215
+ logger.debug("Registered analytics view: perceived_summary")
216
+
217
+ if "session_conflicts" in registered:
218
+ # A session with zero conflict rows is ABSENT here, not present with
219
+ # conflict_count = 0. A caller that wants every session must LEFT JOIN
220
+ # onto ``sessions`` and coalesce, or its denominator silently drops the
221
+ # conflict-free sessions — which are the majority.
222
+ con.execute(
223
+ """
224
+ CREATE OR REPLACE VIEW conflicts_summary AS
225
+ SELECT session_id, count(*) AS conflict_count
226
+ FROM session_conflicts
227
+ GROUP BY session_id;
228
+ """
229
+ )
230
+ registered.add("conflicts_summary")
231
+ logger.debug("Registered analytics view: conflicts_summary")
232
+
233
+ return registered
234
+
235
+
236
+ def register_analytics_macros(
237
+ con: duckdb.DuckDBPyConnection,
238
+ registered_views: set[str],
239
+ ) -> None:
240
+ """Create the analytics macros whose backing views are registered.
241
+
242
+ Every signature must match its ``ANALYTICS_MACRO_SIGNATURES`` entry in
243
+ :mod:`atif_duck.domain.catalog`; the drift test parses this function's
244
+ DDL text and fails on any divergence.
245
+
246
+ Must run after :func:`register_analytics`, the base ``register_views``
247
+ and ``register_authorship`` — several macros join the transcript-derived
248
+ views (``messages``, ``human_turns``).
249
+ """
250
+ analytics_macros: list[tuple[str, str]] = [
251
+ # Work-category mix in the last N days.
252
+ (
253
+ "work_mix",
254
+ """
255
+ CREATE OR REPLACE MACRO work_mix(since_days) AS TABLE (
256
+ SELECT work_category, count(*) AS n
257
+ FROM session_classifications
258
+ WHERE classified_at >= current_timestamp - (since_days * INTERVAL 1 DAY)
259
+ GROUP BY 1
260
+ ORDER BY n DESC
261
+ );
262
+ """,
263
+ ),
264
+ # Counts per friction label over the last N days by message ``ts``
265
+ # (the user's utterance time). NULL = full corpus. Excludes
266
+ # label='none' (the majority sentinel would swamp the output).
267
+ (
268
+ "friction_counts",
269
+ """
270
+ CREATE OR REPLACE MACRO friction_counts(since_days) AS TABLE (
271
+ SELECT label,
272
+ count(*) AS n,
273
+ count(DISTINCT session_id) AS sessions,
274
+ avg(confidence) AS avg_confidence,
275
+ sum(CASE WHEN source='regex' THEN 1 ELSE 0 END) AS n_regex,
276
+ sum(CASE WHEN source='llm' THEN 1 ELSE 0 END) AS n_llm
277
+ FROM user_friction
278
+ WHERE label != 'none'
279
+ AND (since_days IS NULL
280
+ OR ts >= current_timestamp - (since_days * INTERVAL 1 DAY))
281
+ GROUP BY label
282
+ ORDER BY n DESC
283
+ );
284
+ """,
285
+ ),
286
+ # Per-session friction pressure vs the human turn count. The
287
+ # denominator counts ``human_turns`` (main-chain user steps a human
288
+ # wrote), so hook feedback and harness text never dilute the rate.
289
+ (
290
+ "friction_rate",
291
+ """
292
+ CREATE OR REPLACE MACRO friction_rate(since_days) AS TABLE (
293
+ WITH hits AS (
294
+ SELECT session_id,
295
+ count(*) FILTER (WHERE label != 'none') AS n_friction,
296
+ count(*) FILTER (WHERE label = 'status_ping') AS n_status,
297
+ count(*) FILTER (WHERE label = 'unmet_expectation') AS n_unmet,
298
+ count(*) FILTER (WHERE label = 'confusion') AS n_confusion,
299
+ count(*) FILTER (WHERE label = 'interruption') AS n_interruption,
300
+ count(*) FILTER (WHERE label = 'correction') AS n_correction,
301
+ count(*) FILTER (WHERE label = 'frustration') AS n_frustration
302
+ FROM user_friction
303
+ WHERE since_days IS NULL
304
+ OR ts >= current_timestamp - (since_days * INTERVAL 1 DAY)
305
+ GROUP BY session_id
306
+ ),
307
+ user_msgs AS (
308
+ SELECT ht.session_id,
309
+ count(*) AS n_user_msgs
310
+ FROM human_turns ht
311
+ WHERE since_days IS NULL
312
+ OR ht.ts >= current_timestamp - (since_days * INTERVAL 1 DAY)
313
+ GROUP BY 1
314
+ )
315
+ SELECT h.session_id,
316
+ h.n_friction,
317
+ h.n_status, h.n_unmet, h.n_confusion,
318
+ h.n_interruption, h.n_correction, h.n_frustration,
319
+ COALESCE(um.n_user_msgs, 0) AS n_user_msgs,
320
+ h.n_friction::DOUBLE / NULLIF(um.n_user_msgs, 0) AS rate
321
+ FROM hits h
322
+ LEFT JOIN user_msgs um USING (session_id)
323
+ WHERE h.n_friction > 0
324
+ ORDER BY h.n_friction DESC
325
+ );
326
+ """,
327
+ ),
328
+ # Top-N example user messages for one friction label.
329
+ (
330
+ "friction_examples",
331
+ """
332
+ CREATE OR REPLACE MACRO friction_examples(label_name, n) AS TABLE (
333
+ SELECT session_id, ts, text_snippet, rationale, source, confidence
334
+ FROM user_friction
335
+ WHERE label = label_name
336
+ ORDER BY confidence DESC, ts DESC
337
+ LIMIT n
338
+ );
339
+ """,
340
+ ),
341
+ # Conflicts on a conversation-time axis, NOT a detection-time one:
342
+ # ``detected_at`` is the worker run-time clock, so the macro
343
+ # recovers ``m.ts`` by joining the later turn of the pair
344
+ # (``turn_b_uuid``) back to ``messages.uuid``. INNER JOIN on
345
+ # purpose — the recoverable subset is the trustworthy subset.
346
+ # root_session_id = session_id here: harbor inlines subagent
347
+ # transcripts, so one conversation is one session.
348
+ (
349
+ "conflicts_over_time",
350
+ """
351
+ CREATE OR REPLACE MACRO conflicts_over_time(since_days) AS TABLE (
352
+ SELECT m.ts AS conversation_ts,
353
+ sc.session_id,
354
+ sc.session_id AS root_session_id,
355
+ sc.turn_a_uuid,
356
+ sc.turn_b_uuid,
357
+ sc.conflict_kind,
358
+ sc.severity,
359
+ sc.agent_position,
360
+ sc.user_position,
361
+ sc.confidence,
362
+ sc.detected_at
363
+ FROM session_conflicts sc
364
+ JOIN messages m
365
+ ON m.uuid = sc.turn_b_uuid
366
+ WHERE since_days IS NULL
367
+ OR m.ts >= current_timestamp - (since_days * INTERVAL 1 DAY)
368
+ ORDER BY m.ts DESC
369
+ );
370
+ """,
371
+ ),
372
+ # Counts per perceived-error signal over the last N days on a REAL
373
+ # conversation-time axis (conflicts_over_time precedent):
374
+ # ``detected_at`` is the worker run-time clock, so the macro
375
+ # recovers ``m.ts`` by joining the anchor USER turn (turn_uuid)
376
+ # back to ``messages.uuid``. INNER JOIN on purpose — the
377
+ # recoverable subset is the trustworthy subset. NULL = full corpus.
378
+ (
379
+ "perceived_counts",
380
+ """
381
+ CREATE OR REPLACE MACRO perceived_counts(since_days) AS TABLE (
382
+ SELECT pe.signal,
383
+ count(*) AS n,
384
+ count(DISTINCT pe.session_id) AS sessions,
385
+ count(*) FILTER (WHERE pe.severity = 'major') AS n_major,
386
+ count(*) FILTER (WHERE pe.severity = 'moderate') AS n_moderate,
387
+ count(*) FILTER (WHERE pe.severity = 'minor') AS n_minor,
388
+ avg(pe.confidence) AS avg_confidence
389
+ FROM perceived_errors pe
390
+ JOIN messages m
391
+ ON m.uuid = pe.turn_uuid
392
+ WHERE since_days IS NULL
393
+ OR m.ts >= current_timestamp - (since_days * INTERVAL 1 DAY)
394
+ GROUP BY pe.signal
395
+ ORDER BY n DESC
396
+ );
397
+ """,
398
+ ),
399
+ # Per-session perceived-error pressure vs the human turn count
400
+ # (friction_rate precedent: the denominator counts ``human_turns``). Sessions with zero perceived errors do
401
+ # not appear — LEFT JOIN onto sessions and coalesce for the
402
+ # full-population rate.
403
+ (
404
+ "perceived_rate",
405
+ """
406
+ CREATE OR REPLACE MACRO perceived_rate(since_days) AS TABLE (
407
+ WITH hits AS (
408
+ SELECT pe.session_id,
409
+ count(*) AS n_errors,
410
+ count(*) FILTER (WHERE pe.severity = 'major') AS n_major,
411
+ count(*) FILTER (WHERE pe.severity = 'moderate') AS n_moderate,
412
+ count(*) FILTER (WHERE pe.severity = 'minor') AS n_minor
413
+ FROM perceived_errors pe
414
+ JOIN messages m
415
+ ON m.uuid = pe.turn_uuid
416
+ WHERE since_days IS NULL
417
+ OR m.ts >= current_timestamp - (since_days * INTERVAL 1 DAY)
418
+ GROUP BY pe.session_id
419
+ ),
420
+ user_msgs AS (
421
+ SELECT ht.session_id,
422
+ count(*) AS n_user_msgs
423
+ FROM human_turns ht
424
+ WHERE since_days IS NULL
425
+ OR ht.ts >= current_timestamp - (since_days * INTERVAL 1 DAY)
426
+ GROUP BY 1
427
+ )
428
+ SELECT h.session_id,
429
+ h.n_errors,
430
+ h.n_major, h.n_moderate, h.n_minor,
431
+ COALESCE(um.n_user_msgs, 0) AS n_user_msgs,
432
+ h.n_errors::DOUBLE / NULLIF(um.n_user_msgs, 0) AS rate
433
+ FROM hits h
434
+ LEFT JOIN user_msgs um USING (session_id)
435
+ ORDER BY h.n_errors DESC
436
+ );
437
+ """,
438
+ ),
439
+ # Top-N examples for one perceived-error signal, highest-confidence
440
+ # first (friction_examples precedent).
441
+ (
442
+ "perceived_examples",
443
+ """
444
+ CREATE OR REPLACE MACRO perceived_examples(signal_name, n) AS TABLE (
445
+ SELECT session_id, turn_uuid, severity, evidence,
446
+ agent_error_summary, confidence, detected_at
447
+ FROM perceived_errors
448
+ WHERE signal = signal_name
449
+ ORDER BY confidence DESC, detected_at DESC
450
+ LIMIT n
451
+ );
452
+ """,
453
+ ),
454
+ ]
455
+
456
+ for macro_name, ddl in analytics_macros:
457
+ required = _ANALYTICS_MACRO_REQUIREMENTS.get(macro_name, ())
458
+ if any(view not in registered_views for view in required):
459
+ logger.debug("Skipped analytics macro {} (view missing among {})", macro_name, required)
460
+ continue
461
+ try:
462
+ con.execute(ddl)
463
+ logger.debug("Registered analytics macro: {}", macro_name)
464
+ except Exception:
465
+ logger.exception("Failed to register analytics macro {}", macro_name)
466
+ raise
467
+
468
+
469
+ __all__ = ["register_analytics", "register_analytics_macros"]