atif-sql 0.2.0__py3-none-any.whl

atif_cli/cron.py

@@ -0,0 +1,311 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""The ``atif-sql cron`` subcommand group: install (print-only) + status.
+
+Companion surface for ``scripts/atif-sql-refresh.sh``. Two commands:
+
+* ``cron install`` prints the crontab block for the lanes and NEVER
+  writes the crontab (CONTRACT-V2 §Cron: "no silent crontab writes") — the
+  human pastes it after checking ``crontab -l``.
+* ``cron status`` reports, per lane: whether the flock is currently held
+  (and by which pid), the last completed run parsed from the refresh log,
+  and the last skip. Plus the log tail.
+
+Lean by construction: stdlib + cyclopts only, no atif_* imports beyond
+:mod:`atif_cli.output` — this module loads eagerly with ``atif_cli.app``
+(sub-apps must exist at registration time) and therefore sits on the fast
+path pinned by the lean-import test.
+
+The log/lock parsing is pure (:func:`parse_last_runs`, :func:`lock_state`
+takes injected probe results) so the unit tests never need a live cron.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Annotated
+
+import cyclopts
+
+from atif_cli.errors import EXIT_CODES
+from atif_cli.output import OutputFormat, emit_json, resolve_format
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+cron_app = cyclopts.App(
+    name="cron",
+    help="Inspect and (manually) install the atif-sql refresh cron lanes.",
+)
+
+#: The lanes and their crontab schedules — the single source the
+#: ``install`` block is rendered from. Cadence rationale lives in the
+#: refresh script's header (materialize is the cheap incremental lane; llm
+#: is the one that spends; compact is the nightly lake maintenance).
+LANES: tuple[tuple[str, str], ...] = (
+    ("materialize", "*/10 * * * *"),
+    ("llm", "20 10 * * *"),
+    ("compact", "40 3 * * *"),
+)
+
+#: Lanes cut from the refresh script, with the date. The script still accepts
+#: each name and exits 0 with a one-line "lane removed" log, so a crontab
+#: line nobody deleted yet stays quiet; ``cron install`` names them so the
+#: human deletes the line.
+REMOVED_LANES: tuple[tuple[str, str], ...] = (("structural", "2026-09-27"),)
+
+#: ``date -Is`` prefix on every refresh-log line (fixed-offset ISO-8601).
+_TS = r"(?P<ts>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[+-]\d{2}:\d{2})"
+_COMPLETE_RE = re.compile(_TS + r" refresh complete \(mode=(?P<mode>[a-z]+), exit=(?P<exit>\d+)\)")
+_SKIP_RE = re.compile(_TS + r" skip\[(?P<mode>[a-z]+)\]:")
+
+
+@dataclass(frozen=True, slots=True)
+class LaneRun:
+    """One parsed lane event: the run's ISO timestamp plus its exit code."""
+
+    timestamp: str
+    exit_code: int
+
+
+@dataclass(frozen=True, slots=True)
+class LaneStatus:
+    """Everything ``cron status`` reports for one lane."""
+
+    lane: str
+    schedule: str
+    lock_held: bool
+    holder_pid: int | None
+    last_complete: LaneRun | None
+    last_skip: str | None
+
+
+def crontab_block(script: Path, log: Path) -> str:
+    """Render the crontab block for ``script`` logging to ``log``.
+
+    Pure. One line per lane, schedules from :data:`LANES`, stdout+stderr
+    appended to the cron-side log (the script keeps its own structured log
+    under ``scripts/.run/`` regardless).
+    """
+    width = max(len(schedule) for _, schedule in LANES)
+    lines = [f"{schedule:<{width}} {script} {lane} >> {log} 2>&1" for lane, schedule in LANES]
+    return "\n".join(lines)
+
+
+def parse_last_runs(lines: Iterable[str]) -> dict[str, dict[str, LaneRun | str]]:
+    """Extract per-lane last-completion and last-skip events from log lines.
+
+    Pure. Later lines win (the log is append-only, so iteration order IS
+    chronological order). Returns ``{lane: {"complete": LaneRun, "skip": ts}}``
+    with keys present only when the event was seen.
+    """
+    out: dict[str, dict[str, LaneRun | str]] = {}
+    for line in lines:
+        if match := _COMPLETE_RE.match(line):
+            out.setdefault(match["mode"], {})["complete"] = LaneRun(
+                timestamp=match["ts"], exit_code=int(match["exit"])
+            )
+        elif match := _SKIP_RE.match(line):
+            out.setdefault(match["mode"], {})["skip"] = match["ts"]
+    return out
+
+
+def lock_state(*, probe_acquired: bool, pidfile_text: str | None) -> tuple[bool, int | None]:
+    """Classify one lane's lock: ``(held, holder_pid)``.
+
+    Pure — the caller injects the nonblocking-flock probe result and the
+    pidfile's text. ``probe_acquired=True`` means WE got the lock, so no run
+    holds it. The pid is reported only for a held lock; a stale pidfile next
+    to a free lock is normal (the last run's pid, not a holder).
+    """
+    if probe_acquired:
+        return (False, None)
+    pid: int | None = None
+    if pidfile_text is not None:
+        stripped = pidfile_text.strip()
+        if stripped.isdigit():
+            pid = int(stripped)
+    return (True, pid)
+
+
+def _probe_lock(lock_path: Path) -> bool:
+    """Try to acquire ``lock_path`` nonblocking; True = acquired (lane idle).
+
+    The kernel drops the flock when the fd closes, so a successful probe
+    perturbs nothing. An absent lock file means no run ever started — also
+    "acquired" for classification purposes.
+    """
+    import fcntl
+
+    if not lock_path.exists():
+        return True
+    with lock_path.open("w") as handle:
+        try:
+            fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except OSError:
+            return False
+        fcntl.flock(handle, fcntl.LOCK_UN)
+        return True
+
+
+def _scripts_dir() -> Path | None:
+    """Locate ``scripts/atif-sql-refresh.sh`` by walking up from this file.
+
+    Works for the editable install this workspace runs (site-packages paths
+    resolve back into ``packages/atif-cli/src``). Returns ``None`` when the
+    tree isn't reachable — e.g. a future ``uv tool install`` — in which case
+    the commands require the explicit ``--script`` flag.
+    """
+    for parent in Path(__file__).resolve().parents:
+        candidate = parent / "scripts" / "atif-sql-refresh.sh"
+        if candidate.is_file():
+            return candidate.parent
+    return None
+
+
+def _resolve_script(script: Path | None) -> Path:
+    """Resolve the refresh-script path from the flag or by discovery."""
+    if script is not None:
+        return script
+    scripts_dir = _scripts_dir()
+    if scripts_dir is None:
+        import sys
+
+        print(
+            "error: cannot locate scripts/atif-sql-refresh.sh from this install; "
+            "pass --script explicitly",
+            file=sys.stderr,
+        )
+        raise SystemExit(EXIT_CODES["invalid_input"])
+    return scripts_dir / "atif-sql-refresh.sh"
+
+
+@cron_app.command
+def install(*, script: Path | None = None) -> None:
+    """Print the crontab block for the refresh lanes — never write it.
+
+    CONTRACT-V2 §Cron: no silent crontab writes. Check ``crontab -l`` for an
+    existing block, then paste this one via ``crontab -e``.
+
+    Parameters
+    ----------
+    script
+        Path to ``atif-sql-refresh.sh``; default: discovered from the repo
+        this CLI is (editably) installed from.
+    """
+    resolved = _resolve_script(script)
+    log = resolved.parent / ".run" / "atif-sql-refresh.cron.log"
+    print("# atif-sql refresh lanes — paste into `crontab -e` (check `crontab -l` first)")
+    for lane, removed_on in REMOVED_LANES:
+        print(f"# the {lane} lane was removed {removed_on}: delete any `{lane}` line")
+    print(crontab_block(resolved, log))
+
+
+@cron_app.command
+def status(
+    *,
+    script: Path | None = None,
+    tail: int = 10,
+    fmt: Annotated[OutputFormat, cyclopts.Parameter(name="--format")] = OutputFormat.AUTO,
+) -> None:
+    """Report each lane's lock holder + last run/skip parsed from the log.
+
+    Read-only: the lock probe acquires-and-releases nonblocking, perturbing
+    no running lane.
+
+    Parameters
+    ----------
+    script
+        Path to ``atif-sql-refresh.sh`` (its ``.run/`` sibling holds the
+        locks and log); default: discovered from the repo.
+    tail
+        How many trailing log lines to include (0 disables).
+    fmt
+        ``auto`` = human lines on a TTY, JSON on a pipe.
+    """
+    resolved = _resolve_script(script)
+    run_dir = resolved.parent / ".run"
+    log_path = run_dir / "atif-sql-refresh.log"
+    log_lines = log_path.read_text().splitlines() if log_path.is_file() else []
+    runs = parse_last_runs(log_lines)
+
+    lanes: list[LaneStatus] = []
+    for lane, schedule in LANES:
+        pidfile = run_dir / f"atif-sql-refresh-{lane}.pid"
+        held, pid = lock_state(
+            probe_acquired=_probe_lock(run_dir / f"atif-sql-refresh-{lane}.lock"),
+            pidfile_text=pidfile.read_text() if pidfile.is_file() else None,
+        )
+        lane_runs = runs.get(lane, {})
+        complete = lane_runs.get("complete")
+        skip = lane_runs.get("skip")
+        lanes.append(
+            LaneStatus(
+                lane=lane,
+                schedule=schedule,
+                lock_held=held,
+                holder_pid=pid,
+                last_complete=complete if isinstance(complete, LaneRun) else None,
+                last_skip=skip if isinstance(skip, str) else None,
+            )
+        )
+
+    tail_lines = log_lines[-tail:] if tail > 0 else []
+    if resolve_format(fmt) is OutputFormat.TABLE:
+        for lane_status in lanes:
+            lock = (
+                f"RUNNING (pid {lane_status.holder_pid or '?'})"
+                if lane_status.lock_held
+                else "idle"
+            )
+            last = (
+                f"{lane_status.last_complete.timestamp} exit={lane_status.last_complete.exit_code}"
+                if lane_status.last_complete
+                else "never"
+            )
+            print(f"{lane_status.lane:<12} [{lane_status.schedule:<12}] {lock:<20} last: {last}")
+            if lane_status.last_skip:
+                print(f"{'':<12} last skip: {lane_status.last_skip}")
+        if tail_lines:
+            print(f"\n--- last {len(tail_lines)} log lines ({log_path}) ---")
+            for line in tail_lines:
+                print(line)
+        return
+    emit_json(
+        {
+            "log": str(log_path),
+            "lanes": [
+                {
+                    "lane": s.lane,
+                    "schedule": s.schedule,
+                    "lock_held": s.lock_held,
+                    "holder_pid": s.holder_pid,
+                    "last_complete": (
+                        {"timestamp": s.last_complete.timestamp, "exit": s.last_complete.exit_code}
+                        if s.last_complete
+                        else None
+                    ),
+                    "last_skip": s.last_skip,
+                }
+                for s in lanes
+            ],
+            "tail": tail_lines,
+        },
+        fmt,
+    )
+
+
+__all__ = [
+    "LANES",
+    "REMOVED_LANES",
+    "LaneRun",
+    "LaneStatus",
+    "cron_app",
+    "crontab_block",
+    "install",
+    "lock_state",
+    "parse_last_runs",
+    "status",
+]

atif_cli/duck_errors.py

@@ -0,0 +1,97 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""DuckDB error classification for the ``query`` and ``search`` commands.
+
+The one piece of the exit-code taxonomy that needs ``import duckdb`` — kept
+out of :mod:`atif_cli.errors` (pure) and :mod:`atif_cli.output` (rendering)
+so the lean import path never touches the driver. Import this module ONLY
+inside command bodies that already hold a DuckDB connection.
+
+The mapping: ``ParserException`` -> parse_error (64), ``CatalogException`` ->
+catalog_error (65), everything else under ``duckdb.Error`` -> runtime_error
+(70). The distinction agents act on is 64 vs 65 — a 64 means rewrite the SQL,
+a 65 means the object does not exist, so run ``atif-sql schema`` first.
+
+:data:`REGISTRATION_ERRORS` widens the caught set on the registration path.
+``EmbeddingProviderMismatch`` derives from ``Exception``, not
+``duckdb.Error``, so a bare ``except duckdb.Error`` lets it escape as an
+unhandled traceback with exit 1 — a code outside :data:`EXIT_CODES`
+entirely.
+"""
+
+from __future__ import annotations
+
+import duckdb
+
+from atif_cli.errors import EXIT_CODES, ClassifiedError
+from atif_duck.domain.embedding_guard import EmbeddingProviderMismatch
+
+#: What ``register`` can raise: DuckDB DDL failures plus the embedding-store
+#: identity guard. Catch this tuple, then hand the exception to
+#: :func:`classify_registration_error`.
+REGISTRATION_ERRORS: tuple[type[Exception], ...] = (duckdb.Error, EmbeddingProviderMismatch)
+
+
+def classify_duckdb_error(exc: duckdb.Error) -> ClassifiedError:
+    """Classify a ``duckdb.Error`` into a stable kind + exit code."""
+    message = str(exc)
+    if isinstance(exc, duckdb.ParserException):
+        return ClassifiedError(
+            kind="parse_error",
+            exit_code=EXIT_CODES["parse_error"],
+            message=message,
+            hint="check SQL syntax; try `atif-sql schema --format json` for view/macro names",
+        )
+    if isinstance(exc, duckdb.CatalogException):
+        return ClassifiedError(
+            kind="catalog_error",
+            exit_code=EXIT_CODES["catalog_error"],
+            message=message,
+            hint="unknown view or column; run `atif-sql schema --format json` for the catalog",
+        )
+    if isinstance(exc, duckdb.IOException):
+        return ClassifiedError(
+            kind="runtime_error",
+            exit_code=EXIT_CODES["runtime_error"],
+            message=message,
+            hint="is the corpus materialized? run `atif-sql materialize` then `atif-sql status`",
+        )
+    return ClassifiedError(
+        kind="runtime_error",
+        exit_code=EXIT_CODES["runtime_error"],
+        message=message,
+        hint=None,
+    )
+
+
+def classify_registration_error(exc: Exception) -> ClassifiedError:
+    """Classify anything :data:`REGISTRATION_ERRORS` catches.
+
+    ``EmbeddingProviderMismatch`` maps to ``embedding_mismatch`` at exit 65
+    (the catalog-class code: the store's stamped identity does not match the
+    active embedder, so the requested object cannot be bound). Everything
+    else falls through to :func:`classify_duckdb_error`.
+    """
+    if isinstance(exc, EmbeddingProviderMismatch):
+        return ClassifiedError(
+            kind="embedding_mismatch",
+            exit_code=EXIT_CODES["embedding_mismatch"],
+            message=str(exc),
+            hint="delete the Lance store and re-run `atif-sql embed --all --no-dry-run`, "
+            "or point ATIF_SQL_EMBED_MODEL_ID at the model that wrote it",
+        )
+    if isinstance(exc, duckdb.Error):
+        return classify_duckdb_error(exc)
+    return ClassifiedError(
+        kind="runtime_error",
+        exit_code=EXIT_CODES["runtime_error"],
+        message=str(exc),
+        hint=None,
+    )
+
+
+__all__ = [
+    "REGISTRATION_ERRORS",
+    "classify_duckdb_error",
+    "classify_registration_error",
+]

atif_cli/embed_lake.py

@@ -0,0 +1,101 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""atif-embed's ``LakeStepsPort``, implemented over atif-duck's lake.
+
+atif-embed and atif-duck may not import each other, so the composition root
+joins them: :class:`DuckLakeSteps` opens the corpus's steps through
+:func:`atif_duck.infrastructure.lake_steps.open_lake_steps` and hands them to
+atif-embed in the port's own types. Imported only inside command bodies, so
+the fast paths never load DuckDB.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Self
+
+from loguru import logger
+
+from atif_embed.domain.ports import LakePosition
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+    from pathlib import Path
+    from types import TracebackType
+
+    from atif_duck.infrastructure.lake import LakeLayout
+    from atif_duck.infrastructure.lake_steps import LakeSteps
+
+
+@dataclass(slots=True)
+class _Snapshot:
+    """One open :class:`~atif_duck.infrastructure.lake_steps.LakeSteps`, in the port's types."""
+
+    steps: LakeSteps
+
+    @property
+    def position(self) -> LakePosition:
+        return LakePosition(lineage=self.steps.lineage, snapshot_id=self.steps.snapshot_id)
+
+    def can_read_changes_since(self, since: LakePosition) -> bool:
+        return since.lineage == self.steps.lineage and self.steps.can_read_changes_since(
+            since.snapshot_id
+        )
+
+    def step_texts(
+        self, *, min_chars: int, changed_since: LakePosition | None
+    ) -> Iterator[tuple[str, str]]:
+        since = None if changed_since is None else changed_since.snapshot_id
+        return self.steps.step_texts(min_chars=min_chars, since_snapshot=since)
+
+    def step_keys(self) -> frozenset[str]:
+        return self.steps.step_keys()
+
+    def close(self) -> None:
+        self.steps.close()
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        self.close()
+
+
+@dataclass(frozen=True, slots=True)
+class DuckLakeSteps:
+    """``LakeStepsPort`` over the lake at ``layout``."""
+
+    layout: LakeLayout
+    memory_limit_bytes: int | None = None
+
+    def open_steps(self, corpus_root: Path) -> _Snapshot | None:
+        """The corpus's steps at the published snapshot, or ``None`` (reason logged) to fall back."""
+        from atif_duck.infrastructure.lake_steps import LakeSteps, open_lake_steps
+
+        opened = open_lake_steps(
+            self.layout, corpus_root, memory_limit_bytes=self.memory_limit_bytes
+        )
+        if not isinstance(opened, LakeSteps):
+            logger.info("embed: {}; reading the per-session artifacts instead", opened.reason)
+            return None
+        return _Snapshot(opened)
+
+
+def lake_steps_port() -> DuckLakeSteps:
+    """The port over the lake ``ATIF_SQL_LAKE_ROOT`` names, capped like ``query``."""
+    from atif_cli.app import query_memory_limit_bytes
+    from atif_duck.infrastructure.lake import LakeLayout
+    from atif_duck.infrastructure.lake_settings import LakeSettings
+
+    return DuckLakeSteps(
+        layout=LakeLayout(LakeSettings().lake_root),
+        memory_limit_bytes=query_memory_limit_bytes(),
+    )
+
+
+__all__ = ["DuckLakeSteps", "lake_steps_port"]

atif_cli/errors.py

@@ -0,0 +1,89 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""CLI error taxonomy: stable exit codes + the classified-error shape.
+
+:data:`EXIT_CODES` is the workspace's ONE exit-code table. The numbers follow
+the BSD ``sysexits.h`` range so a shell reading them gets the conventional
+meaning: 64 for malformed input or SQL, 65 for a catalog miss (unknown
+view/macro/column), 70 for a runtime error. No other package owns a mapping —
+atif-converter raises typed :class:`~atif_converter.domain.errors.DomainError`
+subclasses and deliberately carries no codes, so ``convert`` translates them
+here and every command exits from the same dict. A usage error cyclopts
+refuses exits ``invalid_input``: ``atif_cli.app.main`` maps cyclopts 5's own
+exit 2, which this table gives to ``empty_session``.
+
+A number in this dict is a WIRE CONTRACT. An agent branches on it, so
+reassigning an existing key to a different number is a breaking change even
+though nothing in this repo fails.
+
+Pure module: no duckdb, no atif_* imports — the concrete ``duckdb.Error``
+classifier lives in :mod:`atif_cli.duck_errors` so this module stays on the
+lean import path (pinned by the fresh-interpreter lean-import test).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+#: Exit codes agents can rely on. Add keys freely; never renumber one.
+EXIT_CODES: dict[str, int] = {
+    "ok": 0,
+    "empty_session": 2,  # convert: session parsed but produced no trajectory
+    "no_embeddings": 2,  # search: store empty — run `atif-sql embed --all --no-dry-run`
+    "invalid_input": 64,  # malformed user-supplied flags / paths
+    "parse_error": 64,  # malformed SQL
+    "catalog_error": 65,  # unknown view/macro/column
+    "validation_error": 65,  # convert: trajectory failed TrajectoryValidator
+    "embedding_mismatch": 65,  # query/search: Lance store written by another provider
+    "runtime_error": 70,  # everything else duckdb (or the adapter) raises
+    "sandbox_refused": 70,  # query: a statement kind the sandbox never runs (COPY, EXPORT,
+    # ATTACH, INSTALL, LOAD, PREPARE, ...). Same number as runtime_error, which is what
+    # every other sandbox refusal exits with; the distinct kind names the reason.
+    "root_refused": 77,  # query/search/analyze: running as uid 0 without ATIF_SQL_ALLOW_ROOT=1.
+    # EX_NOPERM by convention: the process has too MUCH permission for the file modes
+    # that protect the corpus to mean anything.
+    "extension_missing": 78,  # search: the Lance store exists but the lance DuckDB extension
+    # is not installed; `atif-sql embed --install-extension` is the operator action, so the
+    # same EX_CONFIG contract as terminal_state.
+    "terminal_state": 78,  # embed: store/config state requires OPERATOR action (EX_CONFIG);
+    # retrying without intervention cannot succeed — unattended lanes suppress
+    # retries on this code instead of burning identical ticks
+    "suspicious_scan": 78,  # materialize: the source scan found 0 sessions over a
+    # non-empty corpus, so marking them source-removed was refused. Same EX_CONFIG contract as
+    # terminal_state — a wrong source_root needs an operator, and a retry over the
+    # same wrong root cannot succeed — but a distinct `kind` string, because the
+    # remedy is a path, not a store. A separate code matters most for what it is
+    # NOT: exit 1 here is an uncaught traceback, indistinguishable from a crash.
+    "lake_mismatch": 65,  # lake verify: a session's lake rows differ from its artifacts
+    # (EX_DATAERR). `atif-sql lake rebuild` reloads every corpus from its artifacts.
+    "lake_unavailable": 78,  # lake verify/compact: no lake, a stale one, or no ducklake
+    # extension; an operator runs `atif-sql lake rebuild` (EX_CONFIG, like terminal_state).
+    "harbor_missing": 127,  # RETIRED, kept so this table never renumbers. It named
+    # the loss of the private harbor method the converter used to be built on; the
+    # conversion is ours now (atif_converter.domain.*_conversion), so no code path
+    # raises it. EX_NOTFOUND by convention: a surface, not a datum.
+}
+
+
+@dataclass(frozen=True, slots=True)
+class ClassifiedError:
+    """The structured shape of a CLI error after classification."""
+
+    kind: str  # "invalid_input" | "parse_error" | "catalog_error" | "runtime_error"
+    exit_code: int
+    message: str
+    hint: str | None = None
+
+    def to_payload(self) -> dict[str, Any]:
+        """The JSON error envelope emitted on non-TTY stderr."""
+        return {
+            "error": {
+                "kind": self.kind,
+                "message": self.message,
+                "hint": self.hint,
+            }
+        }
+
+
+__all__ = ["EXIT_CODES", "ClassifiedError"]