claude-sql 0.7.0__tar.gz → 1.0.0__tar.gz

{claude_sql-0.7.0 → claude_sql-1.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: claude-sql
-Version: 0.7.0
+Version: 1.0.0
 Summary: Zero-copy SQL + semantic search + LLM analytics over ~/.claude/ transcripts.
 Keywords: claude,claude-code,anthropic,duckdb,sql,semantic-search,embeddings,bedrock,transcripts,analytics,observability
 Author: Laith Al-Saadoon
@@ -8,7 +8,7 @@ Author-email: Laith Al-Saadoon <lalsaado@amazon.com>
 License: Apache-2.0
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.13
-Classifier: Development Status :: 4 - Beta
+Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Operating System :: MacOS
@@ -37,6 +37,7 @@ Requires-Dist: pyyaml>=6.0.3
 Requires-Dist: scikit-learn>=1.5
 Requires-Dist: scipy>=1.13
 Requires-Dist: tenacity>=9.1.4
+Requires-Dist: tiktoken>=0.12.0
 Requires-Dist: umap-learn>=0.5.12
 Requires-Python: >=3.13
 Project-URL: Homepage, https://github.com/theagenticguy/claude-sql

{claude_sql-0.7.0 → claude_sql-1.0.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "claude-sql"
-version = "0.7.0"
+version = "1.0.0"
 description = "Zero-copy SQL + semantic search + LLM analytics over ~/.claude/ transcripts."
 readme = "README.md"
 license = { text = "Apache-2.0" }
@@ -16,7 +16,7 @@ keywords = [
 classifiers = [
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3.13",
-    "Development Status :: 4 - Beta",
+    "Development Status :: 5 - Production/Stable",
     "Intended Audience :: Developers",
     "Operating System :: POSIX :: Linux",
     "Operating System :: MacOS",
@@ -48,6 +48,7 @@ dependencies = [
     "scikit-learn>=1.5",
     "scipy>=1.13",
     "tenacity>=9.1.4",
+    "tiktoken>=0.12.0",
     "umap-learn>=0.5.12",
 ]
 
@@ -304,7 +305,7 @@ version_provider = "uv"
 version_scheme = "pep440"
 tag_format = "v$version"
 update_changelog_on_bump = true
-major_version_zero = true
+major_version_zero = false
 annotated_tag = true
 changelog_incremental = true
 changelog_merge_prerelease = true

claude_sql-1.0.0/src/claude_sql/classify_worker.py

@@ -0,0 +1,254 @@
+"""Session classification pipeline.
+
+Reads complete Claude Code session transcripts and emits one row per session
+into ``settings.classifications_parquet_path`` with autonomy_tier,
+work_category, success, goal, and confidence fields. Pull-once / write-many
+shape: anti-join against the parquet, dispatch parallel Bedrock calls under
+``settings.llm_concurrency``, write results in chunks of
+``max(batch_size * 4, 256)`` for crash-resilience.
+
+All Bedrock plumbing — client construction, retry, structured-output
+parsing, the per-pipeline cache-stat accumulator — lives in
+:mod:`claude_sql.llm_shared`.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING, Any
+
+import anyio
+import polars as pl
+from loguru import logger
+
+from claude_sql import checkpointer, retry_queue
+from claude_sql.llm_shared import (
+    CLASSIFY_SYSTEM_PROMPT,
+    _build_bedrock_client,
+    _count_pending_sessions,
+    _estimate_cost,
+    classify_one,
+    pipeline_cache_stats,
+)
+from claude_sql.parquet_shards import read_all, write_part
+from claude_sql.schemas import SESSION_CLASSIFICATION_SCHEMA
+from claude_sql.session_text import iter_session_texts, session_bounds
+
+if TYPE_CHECKING:
+    import duckdb
+
+    from claude_sql.config import Settings
+
+
+async def _classify_sessions_async(
+    con: duckdb.DuckDBPyConnection,
+    settings: Settings,
+    *,
+    since_days: int | None,
+    limit: int | None,
+    thinking_mode: str,
+) -> int:
+    """Async implementation behind :func:`classify_sessions`."""
+    already: set[str] = set()
+    done_df = read_all(settings.classifications_parquet_path)
+    if done_df is not None and done_df.height > 0:
+        already = set(done_df["session_id"].to_list())
+
+    # Checkpoint skip: compare current (last_ts, mtime) against the last run.
+    bounds = session_bounds(con, since_days=since_days, limit=limit)
+    unchanged_pending, skipped = checkpointer.filter_unchanged(
+        ((sid, lt, mt) for sid, (lt, mt) in bounds.items()),
+        pipeline="classify",
+        checkpoint_db_path=settings.checkpoint_db_path,
+    )
+    keep = set(unchanged_pending)
+
+    # Retry queue: pull pending retries first so they're re-enqueued into
+    # `keep` even when the checkpoint would otherwise skip them.
+    retry_ids = set(retry_queue.drain(settings.checkpoint_db_path, pipeline="classify"))
+    if retry_ids:
+        logger.info("classify: draining {} retry-queue entries", len(retry_ids))
+        keep |= retry_ids
+
+    pending: list[tuple[str, str]] = []
+    for sid, text in iter_session_texts(con, settings=settings, since_days=since_days, limit=limit):
+        if sid in already and sid not in retry_ids:
+            continue
+        if sid not in keep:
+            continue
+        pending.append((sid, text))
+
+    if not pending:
+        logger.info("classify: no pending sessions (skipped={} via checkpoint)", skipped)
+        return 0
+    if skipped:
+        logger.info("classify: skipped {} sessions via checkpoint", skipped)
+
+    client = _build_bedrock_client(settings)
+    sem = anyio.CapacityLimiter(settings.llm_concurrency)
+    chunk_size = max(settings.batch_size * 4, 256)
+    logger.info(
+        "classify: {} pending, model={}, thinking={}, concurrency={}, chunks of {}",
+        len(pending),
+        settings.sonnet_model_id,
+        thinking_mode,
+        settings.llm_concurrency,
+        chunk_size,
+    )
+
+    written = 0
+    for i in range(0, len(pending), chunk_size):
+        chunk = pending[i : i + chunk_size]
+        t0 = time.monotonic()
+        coros = [
+            classify_one(
+                client,
+                settings.sonnet_model_id,
+                SESSION_CLASSIFICATION_SCHEMA,
+                text,
+                max_tokens=settings.classify_max_tokens,
+                thinking_mode=thinking_mode,
+                sem=sem,
+                system=CLASSIFY_SYSTEM_PROMPT,
+                pipeline="classify",
+            )
+            for _, text in chunk
+        ]
+        results = await asyncio.gather(*coros, return_exceptions=True)
+        elapsed = time.monotonic() - t0
+
+        now = datetime.now(UTC)
+        ok_rows: list[dict[str, Any]] = []
+        errors = 0
+        for (sid, _), res in zip(chunk, results, strict=True):
+            if isinstance(res, BaseException):
+                errors += 1
+                logger.warning("classify: {} failed (queued for retry): {}", sid, res)
+                retry_queue.enqueue(
+                    settings.checkpoint_db_path,
+                    pipeline="classify",
+                    unit_id=sid,
+                    error=str(res),
+                )
+                continue
+            res_dict: dict[str, Any] = res
+            ok_rows.append(
+                {
+                    "session_id": sid,
+                    "autonomy_tier": res_dict.get("autonomy_tier"),
+                    "work_category": res_dict.get("work_category"),
+                    "success": res_dict.get("success"),
+                    "goal": res_dict.get("goal"),
+                    "confidence": float(res_dict.get("confidence", 0.0)),
+                    "classified_at": now,
+                }
+            )
+
+        if ok_rows:
+            df = pl.DataFrame(
+                ok_rows,
+                schema={
+                    "session_id": pl.Utf8,
+                    "autonomy_tier": pl.Utf8,
+                    "work_category": pl.Utf8,
+                    "success": pl.Utf8,
+                    "goal": pl.Utf8,
+                    "confidence": pl.Float32,
+                    "classified_at": pl.Datetime("us", "UTC"),
+                },
+            )
+            write_part(settings.classifications_parquet_path, df)
+
+        # Checkpoint the sessions we just classified — at their CURRENT bounds,
+        # so a later re-run with no new messages is a no-op. Also clear those
+        # sessions from the retry queue.
+        if ok_rows:
+            ok_sids = [row["session_id"] for row in ok_rows]
+            checkpointer.mark_completed(
+                settings.checkpoint_db_path,
+                pipeline="classify",
+                rows=[(sid, *bounds.get(sid, (None, None))) for sid in ok_sids],
+            )
+            retry_queue.mark_done(
+                settings.checkpoint_db_path,
+                pipeline="classify",
+                unit_ids=ok_sids,
+            )
+
+        written += len(ok_rows)
+        logger.info(
+            "classify chunk {}/{}: {} ok, {} errors, {:.1f}s ({:.1f} sess/s)",
+            i // chunk_size + 1,
+            (len(pending) + chunk_size - 1) // chunk_size,
+            len(ok_rows),
+            errors,
+            elapsed,
+            len(ok_rows) / elapsed if elapsed > 0 else 0,
+        )
+
+    logger.info("classify: wrote {} total rows", written)
+    return written
+
+
+def classify_sessions(
+    con: duckdb.DuckDBPyConnection,
+    settings: Settings,
+    *,
+    since_days: int | None = None,
+    limit: int | None = None,
+    dry_run: bool = False,
+    no_thinking: bool = False,
+) -> int | dict[str, Any]:
+    """Classify pending sessions and return count of successful classifications.
+
+    In ``--dry-run`` mode, returns a plan dict with keys ``{pipeline,
+    candidates, llm_calls, avg_input_tokens, avg_output_tokens,
+    estimated_cost_usd, model, thinking, since_days, limit}`` instead of the
+    row count, so the CLI can emit it as structured JSON.
+    """
+    thinking_mode = "disabled" if no_thinking else settings.classify_thinking
+
+    if dry_run:
+        already: set[str] = set()
+        done_df = read_all(settings.classifications_parquet_path)
+        if done_df is not None and done_df.height > 0:
+            already = set(done_df["session_id"].to_list())
+        pending_count = _count_pending_sessions(
+            con, already=already, since_days=since_days, limit=limit
+        )
+        # Back-of-envelope: avg 8K input tokens, 300 output per session.
+        cost = _estimate_cost(pending_count, 8000, 300, settings.sonnet_pricing)
+        logger.info(
+            "classify --dry-run: {} sessions pending.  Estimated cost ~${:.2f} "
+            "(thinking={}, model={})",
+            pending_count,
+            cost,
+            thinking_mode,
+            settings.sonnet_model_id,
+        )
+        return {
+            "pipeline": "classify",
+            "candidates": pending_count,
+            "llm_calls": pending_count,
+            "avg_input_tokens": 8000,
+            "avg_output_tokens": 300,
+            "estimated_cost_usd": round(cost, 4),
+            "model": settings.sonnet_model_id,
+            "thinking": thinking_mode,
+            "since_days": since_days,
+            "limit": limit,
+            "dry_run": True,
+        }
+
+    with pipeline_cache_stats("classify"):
+        return asyncio.run(
+            _classify_sessions_async(
+                con,
+                settings,
+                since_days=since_days,
+                limit=limit,
+                thinking_mode=thinking_mode,
+            )
+        )