atif-sql 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. atif_analytics/__init__.py +9 -0
  2. atif_analytics/application/__init__.py +3 -0
  3. atif_analytics/application/analyze.py +158 -0
  4. atif_analytics/application/prompts.py +750 -0
  5. atif_analytics/application/use_cases/__init__.py +3 -0
  6. atif_analytics/application/use_cases/_shared.py +264 -0
  7. atif_analytics/application/use_cases/classify.py +430 -0
  8. atif_analytics/application/use_cases/conflicts.py +448 -0
  9. atif_analytics/application/use_cases/friction.py +634 -0
  10. atif_analytics/application/use_cases/perceived.py +451 -0
  11. atif_analytics/domain/__init__.py +3 -0
  12. atif_analytics/domain/authorship.py +166 -0
  13. atif_analytics/domain/config.py +30 -0
  14. atif_analytics/domain/costs.py +50 -0
  15. atif_analytics/domain/friction.py +117 -0
  16. atif_analytics/domain/layout.py +94 -0
  17. atif_analytics/domain/models.py +385 -0
  18. atif_analytics/domain/ports.py +100 -0
  19. atif_analytics/domain/transcript.py +359 -0
  20. atif_analytics/infrastructure/__init__.py +3 -0
  21. atif_analytics/infrastructure/corpus_reader.py +546 -0
  22. atif_analytics/infrastructure/parquet_cache.py +186 -0
  23. atif_analytics/infrastructure/settings.py +114 -0
  24. atif_analytics/infrastructure/sqlite_state/__init__.py +3 -0
  25. atif_analytics/infrastructure/sqlite_state/checkpointer.py +326 -0
  26. atif_analytics/infrastructure/sqlite_state/retry_queue.py +296 -0
  27. atif_analytics/py.typed +0 -0
  28. atif_cli/__init__.py +3 -0
  29. atif_cli/__main__.py +8 -0
  30. atif_cli/app.py +2459 -0
  31. atif_cli/converter_adapter.py +151 -0
  32. atif_cli/corpus.py +409 -0
  33. atif_cli/cron.py +311 -0
  34. atif_cli/duck_errors.py +97 -0
  35. atif_cli/embed_lake.py +101 -0
  36. atif_cli/errors.py +89 -0
  37. atif_cli/lake.py +373 -0
  38. atif_cli/lake_sessions.py +270 -0
  39. atif_cli/output.py +297 -0
  40. atif_cli/py.typed +0 -0
  41. atif_converter/__init__.py +3 -0
  42. atif_converter/application/__init__.py +3 -0
  43. atif_converter/application/convert_and_audit.py +294 -0
  44. atif_converter/application/convert_codex.py +210 -0
  45. atif_converter/domain/__init__.py +9 -0
  46. atif_converter/domain/agents.py +40 -0
  47. atif_converter/domain/atif/__init__.py +55 -0
  48. atif_converter/domain/atif/agent.py +37 -0
  49. atif_converter/domain/atif/content.py +161 -0
  50. atif_converter/domain/atif/final_metrics.py +45 -0
  51. atif_converter/domain/atif/metrics.py +49 -0
  52. atif_converter/domain/atif/observation.py +21 -0
  53. atif_converter/domain/atif/observation_result.py +47 -0
  54. atif_converter/domain/atif/step.py +144 -0
  55. atif_converter/domain/atif/subagent_trajectory_ref.py +90 -0
  56. atif_converter/domain/atif/tool_call.py +36 -0
  57. atif_converter/domain/atif/trajectory.py +211 -0
  58. atif_converter/domain/atif/trajectory_validator.py +299 -0
  59. atif_converter/domain/blobs.py +510 -0
  60. atif_converter/domain/claude_code_conversion.py +1077 -0
  61. atif_converter/domain/codex_conversion.py +1036 -0
  62. atif_converter/domain/codex_edges.py +192 -0
  63. atif_converter/domain/codex_enrichment.py +419 -0
  64. atif_converter/domain/codex_fidelity.py +124 -0
  65. atif_converter/domain/codex_nested_calls.py +165 -0
  66. atif_converter/domain/edges.py +113 -0
  67. atif_converter/domain/enrichment.py +412 -0
  68. atif_converter/domain/errors.py +60 -0
  69. atif_converter/domain/fidelity.py +171 -0
  70. atif_converter/domain/model_prices.json +8538 -0
  71. atif_converter/domain/pricing.py +790 -0
  72. atif_converter/domain/result_signals.py +774 -0
  73. atif_converter/domain/schema_version.py +90 -0
  74. atif_converter/domain/session_events.py +408 -0
  75. atif_converter/infrastructure/__init__.py +3 -0
  76. atif_converter/infrastructure/census.py +89 -0
  77. atif_converter/infrastructure/claude_code_converter.py +127 -0
  78. atif_converter/infrastructure/codex_adapter.py +132 -0
  79. atif_converter/infrastructure/codex_census.py +100 -0
  80. atif_converter/infrastructure/codex_converter.py +82 -0
  81. atif_converter/infrastructure/harbor_adapter.py +187 -0
  82. atif_converter/infrastructure/raw_records.py +426 -0
  83. atif_converter/infrastructure/source_archive.py +154 -0
  84. atif_converter/py.typed +0 -0
  85. atif_corpus/__init__.py +13 -0
  86. atif_corpus/application/__init__.py +7 -0
  87. atif_corpus/application/materialize.py +1826 -0
  88. atif_corpus/domain/__init__.py +9 -0
  89. atif_corpus/domain/agents.py +33 -0
  90. atif_corpus/domain/generation.py +34 -0
  91. atif_corpus/domain/layout.py +168 -0
  92. atif_corpus/domain/ports.py +175 -0
  93. atif_corpus/domain/session_id.py +68 -0
  94. atif_corpus/domain/sessions.py +228 -0
  95. atif_corpus/domain/slug.py +72 -0
  96. atif_corpus/domain/source_layout.py +143 -0
  97. atif_corpus/domain/watermark.py +76 -0
  98. atif_corpus/infrastructure/__init__.py +7 -0
  99. atif_corpus/infrastructure/atomic.py +266 -0
  100. atif_corpus/infrastructure/compress_artifacts.py +161 -0
  101. atif_corpus/infrastructure/fake_converter.py +174 -0
  102. atif_corpus/infrastructure/scanner.py +316 -0
  103. atif_corpus/infrastructure/settings.py +130 -0
  104. atif_corpus/infrastructure/source_archive.py +202 -0
  105. atif_corpus/py.typed +0 -0
  106. atif_duck/__init__.py +43 -0
  107. atif_duck/domain/__init__.py +32 -0
  108. atif_duck/domain/artifacts.py +66 -0
  109. atif_duck/domain/authorship.py +165 -0
  110. atif_duck/domain/catalog.py +626 -0
  111. atif_duck/domain/columnar.py +149 -0
  112. atif_duck/domain/embedding_guard.py +88 -0
  113. atif_duck/domain/examples.py +265 -0
  114. atif_duck/domain/lake.py +427 -0
  115. atif_duck/domain/lake_sessions.py +97 -0
  116. atif_duck/domain/raw_readers.py +140 -0
  117. atif_duck/domain/session_id.py +69 -0
  118. atif_duck/domain/sql_literal.py +50 -0
  119. atif_duck/infrastructure/__init__.py +34 -0
  120. atif_duck/infrastructure/analytics.py +469 -0
  121. atif_duck/infrastructure/authorship.py +139 -0
  122. atif_duck/infrastructure/columnar.py +641 -0
  123. atif_duck/infrastructure/lake.py +1383 -0
  124. atif_duck/infrastructure/lake_sessions.py +194 -0
  125. atif_duck/infrastructure/lake_settings.py +63 -0
  126. atif_duck/infrastructure/lake_steps.py +225 -0
  127. atif_duck/infrastructure/projections.py +221 -0
  128. atif_duck/infrastructure/registry.py +2194 -0
  129. atif_duck/infrastructure/stored_artifacts.py +115 -0
  130. atif_duck/py.typed +0 -0
  131. atif_embed/__init__.py +12 -0
  132. atif_embed/application/__init__.py +3 -0
  133. atif_embed/application/embed.py +293 -0
  134. atif_embed/application/prune.py +80 -0
  135. atif_embed/domain/__init__.py +3 -0
  136. atif_embed/domain/discovery.py +70 -0
  137. atif_embed/domain/embedding_guard.py +83 -0
  138. atif_embed/domain/errors.py +94 -0
  139. atif_embed/domain/ports.py +223 -0
  140. atif_embed/domain/sql_literal.py +49 -0
  141. atif_embed/domain/text_stamp.py +71 -0
  142. atif_embed/infrastructure/__init__.py +3 -0
  143. atif_embed/infrastructure/cohere_bedrock.py +433 -0
  144. atif_embed/infrastructure/corpus_text_rows.py +273 -0
  145. atif_embed/infrastructure/gemma_local.py +355 -0
  146. atif_embed/infrastructure/lake_text_rows.py +203 -0
  147. atif_embed/infrastructure/lance_store.py +440 -0
  148. atif_embed/infrastructure/providers.py +30 -0
  149. atif_embed/infrastructure/settings.py +154 -0
  150. atif_embed/py.typed +0 -0
  151. atif_models/__init__.py +14 -0
  152. atif_models/domain/__init__.py +9 -0
  153. atif_models/domain/ports.py +143 -0
  154. atif_models/domain/registry.py +151 -0
  155. atif_models/domain/schema.py +109 -0
  156. atif_models/infrastructure/__init__.py +8 -0
  157. atif_models/infrastructure/openai_bedrock.py +314 -0
  158. atif_models/infrastructure/settings.py +85 -0
  159. atif_models/py.typed +0 -0
  160. atif_sql-0.2.0.dist-info/METADATA +272 -0
  161. atif_sql-0.2.0.dist-info/RECORD +164 -0
  162. atif_sql-0.2.0.dist-info/WHEEL +4 -0
  163. atif_sql-0.2.0.dist-info/entry_points.txt +2 -0
  164. atif_sql-0.2.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,3 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+
3
+ """The four LLM analytics pipelines: classify, conflicts, friction, perceived."""
@@ -0,0 +1,264 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+
3
+ """Cross-use-case plumbing shared by the four LLM pipelines.
4
+
5
+ The four are ``PIPELINE_NAMES`` in
6
+ :mod:`atif_analytics.infrastructure.sqlite_state.checkpointer`: classify,
7
+ conflicts, user_friction, perceived.
8
+
9
+ The provider factory + per-pipeline usage/cost logging (one provider
10
+ instance per pipeline run, usage accumulated in the provider's
11
+ :class:`~atif_models.domain.ports.UsageAccumulator` and logged once at the
12
+ end), and the storage-port default builders.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import asyncio
18
+ from contextlib import contextmanager
19
+ from typing import TYPE_CHECKING, Any
20
+
21
+ from loguru import logger
22
+
23
+ from atif_analytics.infrastructure.parquet_cache import ParquetCache
24
+ from atif_analytics.infrastructure.sqlite_state.checkpointer import SqliteCheckpoint
25
+ from atif_analytics.infrastructure.sqlite_state.retry_queue import SqliteRetryQueue
26
+
27
+ if TYPE_CHECKING:
28
+ from collections.abc import Callable, Coroutine, Generator, Sequence
29
+ from pathlib import Path
30
+
31
+ from atif_analytics.infrastructure.settings import AnalyticsSettings
32
+ from atif_models.domain.ports import LlmStructuredProvider
33
+ from atif_models.domain.registry import ModelSpec
34
+
35
+
36
+ def build_provider(
37
+ settings: AnalyticsSettings, pipeline: str
38
+ ) -> tuple[LlmStructuredProvider, ModelSpec]:
39
+ """One provider per pipeline run, sized per atif-models settings.
40
+
41
+ Size assignments (atif-models ``LlmSettings``): classify=small,
42
+ conflicts=large, friction=small, perceived=medium — all resolved through
43
+ the atif-models registry (no model id is written down here).
44
+ """
45
+ from atif_models.infrastructure.openai_bedrock import OpenAiBedrockProvider
46
+
47
+ llm = settings.llm()
48
+ spec = llm.spec_for(pipeline)
49
+ provider = OpenAiBedrockProvider(
50
+ spec,
51
+ region=llm.llm_region,
52
+ concurrency=llm.llm_concurrency,
53
+ )
54
+ logger.info(
55
+ "{}: provider={} model={} concurrency={}",
56
+ pipeline,
57
+ provider.provider,
58
+ spec.model_id,
59
+ llm.llm_concurrency,
60
+ )
61
+ return provider, spec
62
+
63
+
64
+ @contextmanager
65
+ def pipeline_usage(
66
+ provider: LlmStructuredProvider, spec: ModelSpec, pipeline: str
67
+ ) -> Generator[None]:
68
+ """Log the accumulated usage + estimated cost once per pipeline run."""
69
+ from atif_models.domain.registry import estimate_cost as spec_cost
70
+
71
+ try:
72
+ yield
73
+ finally:
74
+ usage = getattr(provider, "usage", None)
75
+ if usage is not None:
76
+ summary = usage.summary()
77
+ cost = spec_cost(
78
+ spec,
79
+ input_tokens=summary["input_tokens"],
80
+ output_tokens=summary["output_tokens"],
81
+ )
82
+ logger.info(
83
+ "{}: usage calls={} in={} out={} reasoning={} cached={} est_cost={}",
84
+ pipeline,
85
+ summary["calls"],
86
+ summary["input_tokens"],
87
+ summary["output_tokens"],
88
+ summary["reasoning_tokens"],
89
+ summary["cached_tokens"],
90
+ f"${cost:.4f}" if cost is not None else "n/a",
91
+ )
92
+
93
+
94
+ class RunBudget:
95
+ """Cross-pipeline dollar ceiling over the providers' UsageAccumulators.
96
+
97
+ One instance per ``run_analyze`` LLM lane. Each pipeline registers its
98
+ ``(provider, spec)`` via :meth:`watch` before dispatching; :meth:`spent_usd`
99
+ prices every watched accumulator's RUNNING ACTUALS (not estimates), so
100
+ :meth:`exhausted` is checked cheaply between dispatch batches. When it
101
+ trips, pipelines stop dispatching — nothing is stamped (no checkpoint, no
102
+ cache row, no retry entry) for units that never started, so the next run
103
+ picks them up cleanly.
104
+
105
+ OVERSHOOT BOUND. Actual spend is only observable after a call returns, so
106
+ the ceiling is a stop-dispatch trigger, not a hard cap. Worst case the run
107
+ ends at ``max_cost_usd`` plus the cost of the units already in flight when
108
+ the crossing became visible — at most :data:`BUDGET_CHECK_BATCH` units on
109
+ the most expensive watched model: :func:`gather_under_budget` sends units
110
+ in batches of that size and checks the budget at each batch boundary, so
111
+ the overshoot does NOT scale with the session ceiling or the write chunk
112
+ size. Pinned by ``test_overshoot_never_exceeds_the_documented_budget_batch``.
113
+ """
114
+
115
+ def __init__(self, max_cost_usd: float) -> None:
116
+ self.max_cost_usd = max_cost_usd
117
+ self._watched: list[tuple[LlmStructuredProvider, ModelSpec]] = []
118
+
119
+ def watch(self, provider: LlmStructuredProvider, spec: ModelSpec) -> None:
120
+ """Track one pipeline's provider accumulator for the rest of the run."""
121
+ self._watched.append((provider, spec))
122
+
123
+ def spent_usd(self) -> float:
124
+ """Total actual spend so far across every watched accumulator."""
125
+ from atif_models.domain.registry import estimate_cost as spec_cost
126
+
127
+ total = 0.0
128
+ for provider, spec in self._watched:
129
+ usage = getattr(provider, "usage", None)
130
+ if usage is None:
131
+ continue
132
+ summary = usage.summary()
133
+ cost = spec_cost(
134
+ spec,
135
+ input_tokens=summary["input_tokens"],
136
+ output_tokens=summary["output_tokens"],
137
+ )
138
+ total += cost or 0.0
139
+ return total
140
+
141
+ def exhausted(self) -> bool:
142
+ """True once actual spend has crossed the ceiling."""
143
+ return self.spent_usd() >= self.max_cost_usd
144
+
145
+
146
+ #: Units dispatched between budget re-checks. This is the overshoot unit of
147
+ #: the cost ceiling: the ceiling can only be observed after a call returns,
148
+ #: so whatever is in flight at the crossing is already bought. Small enough
149
+ #: that a stage's overshoot is a handful of calls rather than its whole
150
+ #: remaining batch; large enough to keep the provider's concurrency limiter
151
+ #: (default 16) fed.
152
+ BUDGET_CHECK_BATCH: int = 8
153
+
154
+
155
+ async def gather_under_budget[T, R](
156
+ units: Sequence[T],
157
+ *,
158
+ make_coro: Callable[[T], Coroutine[Any, Any, R]],
159
+ budget: RunBudget | None,
160
+ batch_size: int = BUDGET_CHECK_BATCH,
161
+ ) -> tuple[list[T], list[R | BaseException]]:
162
+ """Dispatch ``units`` in budget-checked batches; return what actually ran.
163
+
164
+ Returns ``(dispatched_units, results)`` — parallel lists, results in
165
+ ``asyncio.gather(return_exceptions=True)`` form. Dispatch stops at the
166
+ first batch boundary where the budget is exhausted, so the caller's
167
+ zip over the pair never stamps a unit that was never sent.
168
+
169
+ ``make_coro`` is called lazily per batch: a coroutine built for a unit
170
+ that never gets dispatched would be an un-awaited coroutine warning.
171
+ """
172
+ dispatched: list[T] = []
173
+ results: list[R | BaseException] = []
174
+ for start in range(0, len(units), max(1, batch_size)):
175
+ if budget is not None and budget.exhausted():
176
+ break
177
+ batch = list(units[start : start + max(1, batch_size)])
178
+ batch_results = await asyncio.gather(
179
+ *(make_coro(unit) for unit in batch), return_exceptions=True
180
+ )
181
+ dispatched.extend(batch)
182
+ results.extend(batch_results)
183
+ return dispatched, results
184
+
185
+
186
+ def build_cache(target: Path) -> ParquetCache:
187
+ """Default cache port over one sharded parquet dir."""
188
+ return ParquetCache(target)
189
+
190
+
191
+ def write_refusal_audit(
192
+ refusals_dir: Path,
193
+ *,
194
+ pipeline: str,
195
+ refusals: list[tuple[str, str]],
196
+ ) -> None:
197
+ """Append ``(unit_id, reason)`` refusal audit rows to ``analytics/refusals``.
198
+
199
+ Refusal-skips must be queryable, not just log lines. Used by pipelines
200
+ whose OUTPUT schema cannot hold an in-band sentinel row without bending
201
+ view semantics — conflicts rows are uuid-keyed PAIRS, so a sentinel pair
202
+ would poison the conversation-time joins. (classify writes an in-band
203
+ sentinel instead: its row is session-keyed, so a documented
204
+ ``goal='[refused]'`` row keeps ``session_classifications`` complete
205
+ without touching any view.) Empty ``refusals`` is a no-op.
206
+ """
207
+ if not refusals:
208
+ return
209
+ from datetime import UTC, datetime
210
+
211
+ import polars as pl
212
+
213
+ now = datetime.now(UTC)
214
+ ParquetCache(refusals_dir).write_part(
215
+ pl.DataFrame(
216
+ {
217
+ "pipeline": [pipeline] * len(refusals),
218
+ "unit_id": [uid for uid, _ in refusals],
219
+ "reason": [reason[:500] for _, reason in refusals],
220
+ "refused_at": [now] * len(refusals),
221
+ },
222
+ schema={
223
+ "pipeline": pl.Utf8,
224
+ "unit_id": pl.Utf8,
225
+ "reason": pl.Utf8,
226
+ "refused_at": pl.Datetime("us", "UTC"),
227
+ },
228
+ )
229
+ )
230
+
231
+
232
+ def build_checkpoint(state_db: Path) -> SqliteCheckpoint:
233
+ """Default checkpoint port over the corpus ``state.db``."""
234
+ return SqliteCheckpoint(state_db)
235
+
236
+
237
+ def build_retry_queue(state_db: Path) -> SqliteRetryQueue:
238
+ """Default retry-queue port over the corpus ``state.db``."""
239
+ return SqliteRetryQueue(state_db)
240
+
241
+
242
+ def write_chunk_size(batch_size: int) -> int:
243
+ """Crash-resilience write chunking: ``max(batch_size*4, 256)`` rows per part.
244
+
245
+ This bounds how much completed work a crash can lose, NOT how much the
246
+ run can spend — the budget ceiling is enforced per
247
+ :data:`BUDGET_CHECK_BATCH` units inside each chunk, because the default
248
+ chunk (384) is wider than the whole session ceiling (50).
249
+ """
250
+ return max(batch_size * 4, 256)
251
+
252
+
253
+ __all__ = [
254
+ "BUDGET_CHECK_BATCH",
255
+ "RunBudget",
256
+ "build_cache",
257
+ "build_checkpoint",
258
+ "build_provider",
259
+ "build_retry_queue",
260
+ "gather_under_budget",
261
+ "pipeline_usage",
262
+ "write_chunk_size",
263
+ "write_refusal_audit",
264
+ ]
@@ -0,0 +1,430 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+
3
+ """Session classification pipeline over the materialized corpus.
4
+
5
+ Reads complete session transcripts from the MATERIALIZED corpus and emits one
6
+ row per session into the ``session_classifications`` sharded cache with
7
+ work_category, goal, and confidence fields. It runs on the SMALL model tier.
8
+
9
+ Only ``interactive`` sessions are classified (``session_kind`` in
10
+ :mod:`atif_analytics.domain.transcript`, the rule atif-duck's
11
+ ``session_outcomes.kind`` applies). A turn audit is a machine prompt and a
12
+ verdict, and a one-shot job's goal is its one prompt; both are checkpointed
13
+ as skipped without a call, and a session that later grows past one human
14
+ turn is re-admitted by the checkpoint. What a session achieved is
15
+ ``session_outcomes.outcome``, computed from the steps, not asked of a model:
16
+ the LLM ``success`` and ``autonomy_tier`` labels were dropped on 2026-09-27
17
+ because they were not reproducible (802 success / 586 failure on the same
18
+ turn-audit outcome).
19
+ Pull-once / write-many shape: anti-join against the parquet, dispatch
20
+ parallel structured-output calls under the provider's concurrency limiter,
21
+ write results in chunks of ``max(batch_size * 4, 256)`` for crash-resilience.
22
+
23
+ Transcript text is sent to a third-party model provider on this path: a
24
+ session's full rendered body, tool arguments and tool results included,
25
+ leaves this machine.
26
+
27
+ Error routing (CONTRACT-V2 taxonomy): :class:`RefusalError` is terminal —
28
+ the session is checkpointed with a DOCUMENTED sentinel audit row
29
+ (``work_category='unknown'``, ``goal='[refused]'``, ``confidence=0.0``) so the refusal is queryable and never re-billed;
30
+ :class:`ProviderUnavailable` (and any other exception) goes to the retry
31
+ queue for a later run.
32
+ """
33
+
34
+ from __future__ import annotations
35
+
36
+ import asyncio
37
+ import time
38
+ from datetime import UTC, datetime
39
+ from typing import TYPE_CHECKING, Any
40
+
41
+ import polars as pl
42
+ from loguru import logger
43
+
44
+ from atif_analytics.application.prompts import CLASSIFY_SYSTEM_PROMPT
45
+ from atif_analytics.application.use_cases._shared import (
46
+ RunBudget,
47
+ build_cache,
48
+ build_checkpoint,
49
+ build_provider,
50
+ build_retry_queue,
51
+ gather_under_budget,
52
+ pipeline_usage,
53
+ write_chunk_size,
54
+ )
55
+ from atif_analytics.domain.costs import estimate_cost_tokens, tokens_for_chars
56
+ from atif_analytics.domain.models import SessionClassification
57
+ from atif_analytics.infrastructure.corpus_reader import CorpusReader
58
+ from atif_analytics.infrastructure.sqlite_state import checkpointer
59
+
60
+ if TYPE_CHECKING:
61
+ from atif_analytics.infrastructure.parquet_cache import ParquetCache
62
+ from atif_analytics.infrastructure.settings import AnalyticsSettings
63
+ from atif_analytics.infrastructure.sqlite_state.checkpointer import SqliteCheckpoint
64
+ from atif_analytics.infrastructure.sqlite_state.retry_queue import SqliteRetryQueue
65
+ from atif_models.domain.ports import LlmStructuredProvider
66
+ from atif_models.domain.registry import ModelSpec
67
+
68
+ _PARQUET_SCHEMA: dict[str, Any] = {
69
+ "session_id": pl.Utf8,
70
+ "work_category": pl.Utf8,
71
+ "goal": pl.Utf8,
72
+ "confidence": pl.Float32,
73
+ "classified_at": pl.Datetime("us", "UTC"),
74
+ }
75
+
76
+ #: Dry-run OUTPUT budget per session (~150 structured-output tokens: two
77
+ #: labels and a one-sentence goal). Input
78
+ #: is MEASURED from the rendered transcript (chars/4), never a static prior:
79
+ #: a real transcript runs an order of magnitude past any fixed guess.
80
+ _AVG_OUT_TOKENS = 150
81
+
82
+ #: The one session kind classify spends on.
83
+ _CLASSIFIED_KIND = "interactive"
84
+
85
+
86
+ def _already_done(cache: ParquetCache) -> set[str]:
87
+ """Session ids already present in the cache (anti-join key)."""
88
+ done_df = cache.read_all(columns=["session_id"])
89
+ if done_df is not None and done_df.height > 0:
90
+ return set(done_df["session_id"].to_list())
91
+ return set()
92
+
93
+
94
+ async def _classify_async(
95
+ settings: AnalyticsSettings,
96
+ *,
97
+ reader: CorpusReader,
98
+ provider: LlmStructuredProvider,
99
+ since_days: int | None,
100
+ limit: int | None,
101
+ cache: ParquetCache,
102
+ checkpoint: SqliteCheckpoint,
103
+ retry: SqliteRetryQueue,
104
+ budget: RunBudget | None,
105
+ ) -> int:
106
+ """Async implementation behind :func:`classify_sessions`."""
107
+ already = _already_done(cache)
108
+
109
+ # Checkpoint skip: compare current (last_ts, mtime) against the last run.
110
+ bounds = reader.session_bounds(since_days=since_days, limit=limit)
111
+ unchanged_pending, skipped = checkpointer.filter_unchanged(
112
+ ((sid, lt, mt) for sid, (lt, mt) in bounds.items()),
113
+ pipeline="classify",
114
+ checkpoint_db_path=checkpoint.db_path,
115
+ )
116
+ keep = set(unchanged_pending)
117
+
118
+ # Retry queue: pull pending retries first so they're re-admitted even
119
+ # when the checkpoint would otherwise skip them.
120
+ retry_ids = set(retry.drain(pipeline="classify"))
121
+ if retry_ids:
122
+ logger.info("classify: draining {} retry-queue entries", len(retry_ids))
123
+ keep |= retry_ids
124
+
125
+ # Retry queue is the SINGLE re-admission gate for failed units: a session
126
+ # with a live entry (attempts exhausted OR backoff not yet elapsed) is
127
+ # never checkpointed, so the checkpoint path would re-admit — and re-bill —
128
+ # it fresh on every run. Skip it here; drain re-admits when due.
129
+ blocked = retry.blocked_units(pipeline="classify")
130
+ if blocked & keep:
131
+ logger.info(
132
+ "classify: skipping {} sessions with live retry entries (backoff/attempts cap)",
133
+ len(blocked & keep),
134
+ )
135
+ keep -= blocked
136
+
137
+ # Session ceiling applied DURING the walk, not after: bounds is
138
+ # newest-first and the cap keeps the freshest sessions, so admitting
139
+ # past it only to slice back would render transcripts (up to 800K chars
140
+ # each) for sessions that can never run this tick. On a backlogged
141
+ # corpus that is the difference between 50 rendered strings and
142
+ # thousands held at once.
143
+ max_sessions = settings.llm_max_sessions_per_run
144
+ pending: list[tuple[str, str]] = []
145
+ empty_sids: list[str] = []
146
+ not_interactive: list[str] = []
147
+ deferred = 0
148
+ for sid in bounds:
149
+ if sid in already and sid not in retry_ids:
150
+ continue
151
+ if sid not in keep:
152
+ continue
153
+ if len(pending) >= max_sessions:
154
+ deferred += 1
155
+ continue
156
+ # Kind gate BEFORE rendering: a turn audit or a one-shot job is
157
+ # checkpointed at current bounds and never billed; growth re-admits.
158
+ if reader.session_kind(sid) != _CLASSIFIED_KIND:
159
+ not_interactive.append(sid)
160
+ continue
161
+ text = reader.session_text(sid)
162
+ if not text:
163
+ # A session that renders to nothing yields no row and no LLM
164
+ # call, so without a checkpoint it re-renders every tick
165
+ # forever. Stamp it at current bounds; growth re-admits it.
166
+ empty_sids.append(sid)
167
+ continue
168
+ pending.append((sid, text))
169
+
170
+ if empty_sids or not_interactive:
171
+ checkpoint.mark_completed(
172
+ pipeline="classify",
173
+ rows=[(sid, *bounds.get(sid, (None, None))) for sid in (*empty_sids, *not_interactive)],
174
+ )
175
+ logger.info(
176
+ "classify: checkpointed {} empty-transcript and {} non-interactive "
177
+ "(turn audit / one-shot job) sessions as skipped",
178
+ len(empty_sids),
179
+ len(not_interactive),
180
+ )
181
+
182
+ if deferred:
183
+ logger.warning(
184
+ "classify: capped at {} sessions (llm_max_sessions_per_run); {} deferred",
185
+ max_sessions,
186
+ deferred,
187
+ )
188
+
189
+ if not pending:
190
+ logger.info("classify: no pending sessions (skipped={} via checkpoint)", skipped)
191
+ return 0
192
+ if skipped:
193
+ logger.info("classify: skipped {} sessions via checkpoint", skipped)
194
+
195
+ chunk_size = write_chunk_size(settings.batch_size)
196
+ logger.info("classify: {} pending, chunks of {}", len(pending), chunk_size)
197
+
198
+ written = 0
199
+ i = 0
200
+ chunk_index = 0
201
+ total_chunks = (len(pending) + chunk_size - 1) // chunk_size
202
+ while i < len(pending):
203
+ chunk_index += 1
204
+ t0 = time.monotonic()
205
+ # Dispatch in budget-checked sub-batches. One gather over the whole
206
+ # write chunk would buy every remaining call before the ceiling was
207
+ # observable again: chunk_size is 384 by default against a session
208
+ # ceiling of 50, so the chunk loop alone checks exactly once, before
209
+ # any call. ``i`` advances by what was actually SENT, so a budget
210
+ # stop mid-chunk leaves the rest of the chunk unstamped rather than
211
+ # silently skipped.
212
+ chunk, results = await gather_under_budget(
213
+ pending[i : i + chunk_size],
214
+ make_coro=lambda unit: provider.classify_structured(
215
+ system=CLASSIFY_SYSTEM_PROMPT,
216
+ prompt=unit[1],
217
+ schema=SessionClassification,
218
+ ),
219
+ budget=budget,
220
+ )
221
+ elapsed = time.monotonic() - t0
222
+ if not chunk:
223
+ # Abort remaining work cleanly: unstarted sessions get no
224
+ # checkpoint/cache/retry stamp, so the next run picks them up.
225
+ logger.warning(
226
+ "classify: cost ceiling hit (${:.2f} >= ${:.2f}) — "
227
+ "aborting with {} sessions unstarted (nothing stamped)",
228
+ budget.spent_usd() if budget is not None else 0.0,
229
+ budget.max_cost_usd if budget is not None else 0.0,
230
+ len(pending) - i,
231
+ )
232
+ break
233
+ i += len(chunk)
234
+
235
+ now = datetime.now(UTC)
236
+ ok_rows: list[dict[str, Any]] = []
237
+ refused_sids: list[str] = []
238
+ errors = 0
239
+ for (sid, _), res in zip(chunk, results, strict=True):
240
+ if isinstance(res, BaseException):
241
+ from atif_models.domain.ports import RefusalError
242
+
243
+ if isinstance(res, RefusalError):
244
+ # Terminal: write the DOCUMENTED refusal sentinel row
245
+ # (work_category='unknown', goal='[refused]', confidence
246
+ # 0.0) so refusal-skips are queryable, then checkpoint so
247
+ # we never re-bill. The fake category sits at confidence
248
+ # 0.0 and goal='[refused]' is the audit marker.
249
+ # (conflicts, whose rows are uuid-keyed pairs, uses the
250
+ # refusals sidecar instead.)
251
+ logger.info("classify: {} refused (terminal) — sentinel row", sid)
252
+ refused_sids.append(sid)
253
+ ok_rows.append(
254
+ {
255
+ "session_id": sid,
256
+ "work_category": "unknown",
257
+ "goal": "[refused]",
258
+ "confidence": 0.0,
259
+ "classified_at": now,
260
+ }
261
+ )
262
+ continue
263
+ errors += 1
264
+ logger.warning("classify: {} failed (queued for retry): {}", sid, res)
265
+ retry.enqueue(pipeline="classify", unit_id=sid, error=str(res))
266
+ continue
267
+ ok_rows.append(
268
+ {
269
+ "session_id": sid,
270
+ "work_category": res.work_category,
271
+ "goal": res.goal,
272
+ "confidence": float(res.confidence),
273
+ "classified_at": now,
274
+ }
275
+ )
276
+
277
+ if ok_rows:
278
+ cache.write_part(pl.DataFrame(ok_rows, schema=_PARQUET_SCHEMA))
279
+
280
+ # Checkpoint at CURRENT bounds so a later rerun with no new steps is
281
+ # a no-op; refused sessions checkpoint too (terminal, via their
282
+ # sentinel rows already in ok_rows). Clear the completed sessions
283
+ # from the retry queue.
284
+ done_sids = [row["session_id"] for row in ok_rows]
285
+ if done_sids:
286
+ checkpoint.mark_completed(
287
+ pipeline="classify",
288
+ rows=[(sid, *bounds.get(sid, (None, None))) for sid in done_sids],
289
+ )
290
+ retry.mark_done(pipeline="classify", unit_ids=done_sids)
291
+
292
+ written += len(ok_rows)
293
+ logger.info(
294
+ "classify chunk {}/{}: {} ok, {} refused, {} errors, {:.1f}s",
295
+ chunk_index,
296
+ total_chunks,
297
+ len(ok_rows),
298
+ len(refused_sids),
299
+ errors,
300
+ elapsed,
301
+ )
302
+
303
+ logger.info("classify: wrote {} total rows", written)
304
+ return written
305
+
306
+
307
+ def _dry_run_plan(
308
+ settings: AnalyticsSettings,
309
+ *,
310
+ reader: CorpusReader,
311
+ cache: ParquetCache,
312
+ spec: ModelSpec,
313
+ since_days: int | None,
314
+ limit: int | None,
315
+ ) -> dict[str, Any]:
316
+ """The classify ``--dry-run`` plan dict, with measured token counts.
317
+
318
+ Input tokens are MEASURED per pending session from the actual rendered
319
+ transcript (chars/4) instead of a static average, and the plan surfaces
320
+ the budget ceilings plus the capped session count — what WOULD run.
321
+ The kind gate applies here as in the real run: only interactive
322
+ sessions count toward the cap and the estimate.
323
+ """
324
+ already = _already_done(cache)
325
+ bounds = reader.session_bounds(since_days=since_days, limit=limit)
326
+ pending = [sid for sid in bounds if sid not in already]
327
+ max_sessions = settings.llm_max_sessions_per_run
328
+ capped: list[str] = [] # newest-first, same cap as the real run
329
+ skipped_kind = 0
330
+ for sid in pending:
331
+ if len(capped) >= max_sessions:
332
+ break
333
+ if reader.session_kind(sid) != _CLASSIFIED_KIND:
334
+ skipped_kind += 1
335
+ continue
336
+ capped.append(sid)
337
+ in_tokens = sum(tokens_for_chars(len(text)) for text in reader.session_texts(capped))
338
+ out_tokens = len(capped) * _AVG_OUT_TOKENS
339
+ pricing = (spec.pricing_in or 0.0, spec.pricing_out or 0.0)
340
+ cost = estimate_cost_tokens(in_tokens, out_tokens, pricing)
341
+ logger.info(
342
+ "classify --dry-run: {} sessions pending ({} would run under the cap, {} "
343
+ "skipped as turn audits or one-shot jobs). Estimated cost ~${:.2f} (model={})",
344
+ len(pending),
345
+ len(capped),
346
+ skipped_kind,
347
+ cost,
348
+ spec.model_id,
349
+ )
350
+ return {
351
+ "pipeline": "classify",
352
+ "candidates": len(pending),
353
+ "skipped_not_interactive": skipped_kind,
354
+ "capped_candidates": len(capped),
355
+ "llm_calls": len(capped),
356
+ "estimated_input_tokens": in_tokens,
357
+ "estimated_output_tokens": out_tokens,
358
+ "estimated_cost_usd": round(cost, 4),
359
+ "max_sessions_per_run": max_sessions,
360
+ "max_cost_usd_per_run": settings.llm_max_cost_usd_per_run,
361
+ "model": spec.model_id,
362
+ "since_days": since_days,
363
+ "limit": limit,
364
+ "dry_run": True,
365
+ }
366
+
367
+
368
+ def classify_sessions(
369
+ settings: AnalyticsSettings,
370
+ *,
371
+ since_days: int | None = None,
372
+ limit: int | None = None,
373
+ dry_run: bool = True,
374
+ reader: CorpusReader | None = None,
375
+ provider: LlmStructuredProvider | None = None,
376
+ spec: ModelSpec | None = None,
377
+ cache: ParquetCache | None = None,
378
+ checkpoint: SqliteCheckpoint | None = None,
379
+ retry: SqliteRetryQueue | None = None,
380
+ budget: RunBudget | None = None,
381
+ ) -> int | dict[str, Any]:
382
+ """Classify pending sessions; return the row count (or a dry-run plan dict).
383
+
384
+ ``dry_run`` defaults to True (CONTRACT-V2 cost guard). The storage
385
+ ports and the reader default to the settings-derived adapters;
386
+ ``provider`` defaults to the registry-resolved OpenAI Bedrock adapter
387
+ (one instance per run, usage logged at the end). ``budget`` is the
388
+ cross-pipeline cost ceiling from ``run_analyze``; it is re-checked every
389
+ :data:`~atif_analytics.application.use_cases._shared.BUDGET_CHECK_BATCH`
390
+ sessions, and when it trips the remaining sessions are left unstamped.
391
+ Overshoot is bounded by the batch in flight at the crossing, not by the
392
+ session ceiling.
393
+ """
394
+ layout = settings.layout()
395
+ reader = (
396
+ reader
397
+ if reader is not None
398
+ else CorpusReader(settings.corpus_root, caps=settings.transcript_caps())
399
+ )
400
+ cache = cache if cache is not None else build_cache(layout.classifications_dir)
401
+
402
+ if provider is None or spec is None:
403
+ provider, spec = build_provider(settings, "classify")
404
+
405
+ if dry_run:
406
+ return _dry_run_plan(
407
+ settings, reader=reader, cache=cache, spec=spec, since_days=since_days, limit=limit
408
+ )
409
+
410
+ checkpoint = checkpoint if checkpoint is not None else build_checkpoint(layout.state_db_path)
411
+ retry = retry if retry is not None else build_retry_queue(layout.state_db_path)
412
+ if budget is not None:
413
+ budget.watch(provider, spec)
414
+ with pipeline_usage(provider, spec, "classify"):
415
+ return asyncio.run(
416
+ _classify_async(
417
+ settings,
418
+ reader=reader,
419
+ provider=provider,
420
+ since_days=since_days,
421
+ limit=limit,
422
+ cache=cache,
423
+ checkpoint=checkpoint,
424
+ retry=retry,
425
+ budget=budget,
426
+ )
427
+ )
428
+
429
+
430
+ __all__ = ["classify_sessions"]