grask 0.1.0rc1__py3-none-any.whl

grask/SKILL.md

@@ -0,0 +1,62 @@
+---
+name: grask
+description: Serve the next pending grask probe — a multiple-choice question about code the developer recently shipped — using the native question UI. Use when the user types /grask.
+---
+
+# Serving a grask probe
+
+Grask quizzes the developer on mechanisms they shipped without fully
+understanding. You are the delivery surface only: you serve the question and
+relay the answer. You never grade, never guess, and never see the answer key.
+
+## Hard rules
+
+- Never open or query grask's database directly. The two subcommands below are
+  the entire interface.
+- Never speculate about which option is correct — not in text, not in labels,
+  not in previews, not before or after the pick. Grading happens in
+  `grask record`.
+- One native question, one round. No confidence round, no follow-ups.
+
+## Flow
+
+1. Run:
+
+   ```
+   grask serve --json
+   ```
+
+   If the output is `{"pending": null}`, say there is nothing pending and stop.
+   If the command is not found, grask is not installed on this PATH — say so and
+   stop rather than hunting for a checkout to `cd` into.
+
+2. Ask ONE native question, preview-style (like plan-mode option picks), built
+   entirely from the served JSON:
+
+   - `question`: a context line from `topic` and `created_at`, then the full
+     question text — e.g.
+     `from 2026-07-21 · idempotency of the retry path — What would happen if …?`
+     The question must be readable inside the picker itself; do not rely on
+     markdown printed before it.
+   - One option per stored option, in stored order:
+     - `label`: the letter plus the first few distinguishing words of the
+       option (e.g. `a) dedup to a no-op`). Keep labels short; they are not the
+       full text.
+     - `preview`: the full stored option text, verbatim and unabridged. The
+       side-by-side preview pane is where the developer reads the option.
+   - In exactly one option's preview (or the question text), append a footer
+     note: "Other" accepts `skip`, or `wrong: <what's off>` if the question
+     misreads what happened.
+
+3. Record the result:
+   - Picked letter L: `grask record <probe_id> --pick L`
+   - Skipped: `grask record <probe_id> --skip`
+   - Premise rejected: `grask record <probe_id> --wrong --objection
+     "<their words>"` (omit `--objection` if they gave no reason).
+
+   Show the result: ✓ or ✗ from `outcome`, then the `explanation` verbatim. If
+   the command prints `{"error": ...}`, show the error and stop — do not retry
+   with different flags.
+
+4. Run `serve` again. If another probe is pending, say so and offer to
+   continue — do not auto-serve it.

grask/__init__.py

@@ -0,0 +1,3 @@
+"""grask — one question about your own code, at the end of a session."""
+
+__version__ = "0.1.0"

grask/ask.py

@@ -0,0 +1,224 @@
+"""One probe, asked and graded mechanically.
+
+Pure logic. The console is injected, exactly as `capture_session` injects its
+stages, so the whole flow is drivable by a scripted console. No argparse, no
+terminal control codes, no TTY — those belong to `cli.py`, and keeping them out
+is what leaves the delivery question open. `ask` does not care whether a human
+typed `grask` or a hook called it.
+
+There is no model call anywhere in this path. Picking an option IS the answer:
+the verdict is `pick == correct_idx`, the explanation was written at generation
+time, and the only `error` outcome left is a stored row too malformed to serve.
+That is a property of the design, not of careful coding — a judge cannot be
+slow, expensive, or cowardly if there is no judge.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Protocol
+
+from grask.probe import Rubric
+
+# Written to a TEXT column and grouped on by hand later, so the strings are the
+# schema. `premise_rejected` is its own outcome rather than a flavour of skip:
+# it is the zealot rate, and a rate you cannot query is a rate nobody checks.
+PASSED = "passed"
+FAILED = "failed"
+SKIPPED = "skipped"
+PREMISE_REJECTED = "premise_rejected"
+ERROR = "error"
+
+WRONG = "/wrong"
+
+# Option letters, and therefore the hard cap on how many options a stored row
+# may carry and still be served.
+LETTERS = "abcde"
+
+
+class Console(Protocol):
+    """Everything the interrogation needs from the outside world."""
+
+    def show(self, text: str) -> None: ...
+
+    def prompt(self, text: str) -> str: ...
+
+
+@dataclass(frozen=True)
+class PendingProbe:
+    """A stored probe, ready to be asked.
+
+    Lives here rather than in `storage.py` so that the import runs
+    storage -> ask, matching how storage already imports `Probe` and `Seed` from
+    the modules that produce them, and avoiding the cycle the other direction
+    would create.
+
+    `correct_idx` is `int | None` because the row is trusted nowhere: a mangled
+    row surfaces as `None`, an empty tuple, or an out-of-range index, and `ask`
+    records `error` instead of guessing.
+    """
+
+    probe_id: int
+    question: str
+    options: tuple[str, ...]
+    correct_idx: int | None
+    explanation: str
+    rubric: Rubric
+    created_at: str
+
+
+@dataclass(frozen=True)
+class AnswerTurn:
+    """One developer turn: the question, and the option text they chose."""
+
+    turn: int
+    question: str
+    answer: str
+
+
+@dataclass(frozen=True)
+class Interrogation:
+    """Everything that happened, ready to be stored. No database in sight."""
+
+    probe_id: int
+    outcome: str
+    # Always None since the confidence round was cut; the column stays nullable
+    # so historical rows keep their numbers.
+    confidence: int | None
+    objection: str | None
+    turns: tuple[AnswerTurn, ...]
+    cost_usd: float | None
+
+
+OBJECTION_PROMPT = "what's wrong with it? (enter to skip)"
+MALFORMED = "this probe's stored options are unusable; recording an error."
+
+
+def context_line(pending: PendingProbe) -> str:
+    """One line of orientation above the question.
+
+    Without it the developer reads a question about work they cannot place, which
+    is the version of this tool that feels like a quiz. `created_at` is the
+    probe's, which is the session's within a few seconds of it.
+    """
+    when = pending.created_at[:10]
+    return f"from {when} · {pending.rubric.topic}"
+
+
+def render_options(options: tuple[str, ...]) -> str:
+    return "\n".join(f"  {LETTERS[i]}) {option}" for i, option in enumerate(options))
+
+
+def pick_prompt(count: int) -> str:
+    return f"pick   [a-{LETTERS[count - 1]}]   ·   enter = skip   ·   /wrong"
+
+
+def pick_hint(count: int) -> str:
+    return f"type a single letter, a-{LETTERS[count - 1]}."
+
+
+def _unservable(pending: PendingProbe) -> bool:
+    """A row the ask cannot honestly grade.
+
+    Storage already filters `options IS NULL`, so what arrives here failed to
+    parse or carries an index that names no option. Grading either would invent
+    a verdict, which is worse than admitting the row is broken.
+    """
+    return (
+        len(pending.options) < 2
+        or len(pending.options) > len(LETTERS)
+        or pending.correct_idx is None
+        or not 0 <= pending.correct_idx < len(pending.options)
+        or not pending.explanation.strip()
+    )
+
+
+def resolution(
+    pending: PendingProbe,
+    outcome: str,
+    *,
+    objection: str | None = None,
+    turns: tuple[AnswerTurn, ...] = (),
+) -> Interrogation:
+    """An Interrogation for `pending`, however it ended.
+
+    Shared by the interactive loop and the non-interactive record path so both
+    write identically shaped rows. `cost_usd` is always 0.0: there is no model
+    call anywhere in an ask, whichever surface drove it.
+    """
+    return Interrogation(
+        probe_id=pending.probe_id,
+        outcome=outcome,
+        confidence=None,
+        objection=objection,
+        turns=turns,
+        cost_usd=0.0,
+    )
+
+
+def grade(pending: PendingProbe, pick: str) -> Interrogation:
+    """(pending, pick letter) -> a mechanically graded Interrogation.
+
+    The one place a pick becomes a verdict; `ask` and `grask record` both land
+    here. Raises ValueError rather than guessing on bad input — the caller
+    decides how to surface it. Never call this on a row `_unservable` flags.
+    """
+    valid = LETTERS[: len(pending.options)]
+    letter = pick.strip().lower()
+    if len(letter) != 1 or letter not in valid:
+        raise ValueError(f"pick must be a single letter, a-{valid[-1]}")
+    picked = valid.index(letter)
+    return resolution(
+        pending,
+        PASSED if picked == pending.correct_idx else FAILED,
+        turns=(AnswerTurn(turn=0, question=pending.question, answer=pending.options[picked]),),
+    )
+
+
+def ask(pending: PendingProbe, console: Console) -> Interrogation:
+    """Run one probe to a verdict: pick, mechanical grade."""
+
+    def done(
+        outcome: str,
+        *,
+        objection: str | None = None,
+        turns: tuple[AnswerTurn, ...] = (),
+    ) -> Interrogation:
+        return resolution(
+            pending,
+            outcome,
+            objection=objection,
+            turns=turns,
+        )
+
+    def ask_objection() -> str | None:
+        """Shared `/wrong` handling: prompt once for an optional reason.
+
+        Optional because requiring an argument to escape is how you get an escape
+        hatch nobody uses. The outcome is the signal; the text is a bonus.
+        """
+        typed = console.prompt(OBJECTION_PROMPT).strip()
+        return typed or None
+
+    if _unservable(pending):
+        console.show(MALFORMED)
+        return done(ERROR)
+
+    console.show(context_line(pending))
+    console.show(pending.question)
+    console.show(render_options(pending.options))
+
+    valid = LETTERS[: len(pending.options)]
+    while True:
+        typed = console.prompt(pick_prompt(len(pending.options))).strip().lower()
+        if not typed:
+            return done(SKIPPED)
+        if typed == WRONG:
+            return done(PREMISE_REJECTED, objection=ask_objection())
+        if len(typed) == 1 and typed in valid:
+            break
+        console.show(pick_hint(len(pending.options)))
+
+    graded = grade(pending, typed)
+    console.show(f"{'✓' if graded.outcome == PASSED else '✗'} {pending.explanation}")
+    return graded

grask/capture.py

@@ -0,0 +1,158 @@
+"""The whole pipeline, end to end, for one session.
+
+Runs detached from a SessionEnd hook with nothing watching its exit code. That
+single fact decides the error handling: an exception here reaches no one, so
+every failure has to become a row and a log line instead. `capture_session` does
+not raise. If it ever does, a session ends and grask silently forgets it.
+
+Order is extract → triage → seed → probe, cheapest first. Stage 0 is free and
+filters sessions with no human in them; triage is one call and filters the
+majority; only what survives both pays for stages 2 and 3.
+"""
+
+from __future__ import annotations
+
+import sys
+import traceback
+from datetime import UTC, datetime
+from pathlib import Path
+
+from grask.dialogue import extract_dialogue as _extract_dialogue
+from grask.probe import probe as _probe
+from grask.seed import seed as _seed
+from grask.select import select
+from grask.storage import Store, grask_home
+from grask.transcript import extract
+from grask.triage import triage as _triage
+
+
+def log(message: str) -> None:
+    """Append to the capture log. Never raises — this is the failure path itself."""
+    try:
+        path = grask_home() / "grask.log"
+        path.parent.mkdir(parents=True, exist_ok=True)
+        stamp = datetime.now(UTC).isoformat()
+        with path.open("a", encoding="utf-8") as handle:
+            handle.write(f"{stamp} {message}\n")
+    except Exception:
+        pass
+
+
+def capture_session(
+    transcript_path: Path,
+    store: Store,
+    *,
+    triage=_triage,
+    seed=_seed,
+    probe=_probe,
+    extract_dialogue=_extract_dialogue,
+) -> None:
+    """Triage one ended session and persist whatever it earned.
+
+    The stages are injectable so the control flow can be tested without spending
+    money on a model. Defaults are the real thing.
+    """
+    session_id = Path(transcript_path).stem
+    try:
+        if store.has_session(session_id):
+            return
+
+        session = extract(Path(transcript_path))
+
+        if not session.turns:
+            # Stage 0's floor. No human said anything, so there is nothing to ask
+            # about and no reason to pay triage to confirm it.
+            store.record_session(
+                session_id=session.session_id,
+                transcript_path=str(transcript_path),
+                cwd=session.cwd,
+                git_branch=session.git_branch,
+                verdict="silent",
+            )
+            return
+
+        verdict = triage(session)
+
+        # triage() never raises; it reports failure by returning silent with
+        # `.error` set. Recording that as silence would make a broken model call
+        # look like a boring session, and the failure-rate number would be a lie.
+        if verdict.error:
+            log(f"{session_id} triage error: {verdict.error}")
+            store.record_session(
+                session_id=session.session_id,
+                transcript_path=str(transcript_path),
+                cwd=session.cwd,
+                git_branch=session.git_branch,
+                verdict="error",
+                cost_usd=verdict.cost_usd,
+                duration_ms=verdict.duration_ms,
+            )
+            return
+
+        if not verdict.kept:
+            store.record_session(
+                session_id=session.session_id,
+                transcript_path=str(transcript_path),
+                cwd=session.cwd,
+                git_branch=session.git_branch,
+                verdict="silent",
+                cost_usd=verdict.cost_usd,
+                duration_ms=verdict.duration_ms,
+            )
+            return
+
+        # The verdict carries the selected moment's fields but not the Moment
+        # itself, and stage 2 needs the object. `select` is pure, so re-running it
+        # over the same moments returns the same one triage chose.
+        moment = select(verdict.moments)
+        if moment is None:
+            raise RuntimeError("verdict is 'ask' but no moment survives selection")
+
+        dialogue = extract_dialogue(Path(transcript_path))
+        the_seed = seed(dialogue, moment)
+        the_probe = probe(the_seed, dialogue)
+
+        store.record_session(
+            session_id=session.session_id,
+            transcript_path=str(transcript_path),
+            cwd=session.cwd,
+            git_branch=session.git_branch,
+            verdict="ask",
+            signal=verdict.signal,
+            topic=verdict.topic,
+            cost_usd=verdict.cost_usd,
+            duration_ms=verdict.duration_ms,
+        )
+        seed_id = store.add_seed(the_seed)
+        store.add_probe(seed_id, the_probe)
+
+    except Exception:
+        log(f"{session_id} capture failed:\n{traceback.format_exc()}")
+        try:
+            store.record_session(
+                session_id=session_id,
+                transcript_path=str(transcript_path),
+                cwd=None,
+                git_branch=None,
+                verdict="error",
+            )
+        except Exception:
+            log(f"{session_id} could not even record the error:\n{traceback.format_exc()}")
+
+
+def main(argv: list[str] | None = None) -> int:
+    """The detached worker: `python -m grask.capture <transcript_path>`."""
+    args = sys.argv[1:] if argv is None else argv
+    if not args:
+        log("worker started with no transcript path")
+        return 0
+    try:
+        with Store() as store:
+            capture_session(Path(args[0]), store)
+    except Exception:
+        log(f"worker failed before capture:\n{traceback.format_exc()}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

grask/capture_run.py

@@ -0,0 +1,236 @@
+"""Run the capture pipeline over transcripts the hook never saw.
+
+The SessionEnd hook only ever captures sessions that end from now on, and the
+sessions that end from now on are overwhelmingly grask's own. Waiting for the
+queue to fill measures grask on grask. This walks the corpus already on disk
+instead, skips grask's own project by default, and pays for a bounded number of
+real sessions from other work.
+
+Costs money, so it does not spend by default: without `--go` it prints the plan
+and an estimate and stops.
+
+Usage:
+    uv run python -m grask.capture_run [--limit N] [--go]
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Callable, Sequence
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from grask.capture import capture_session
+from grask.storage import Store
+from grask.transcript import extract, find_transcripts
+
+DEFAULT_WORKERS = 6
+DEFAULT_LIMIT = 40
+
+# Any transcript whose project directory contains one of these is not corpus.
+# "grask" keeps the tool from being evaluated on its own construction; the
+# scratchpad dirs are agent side-sessions with no developer in them at all.
+DEFAULT_EXCLUDE = ("grask", "scratchpad")
+
+# Measured across this runner's own batches, which is the only population that
+# predicts this runner: 60 sessions selected, 25 kept, $16.36 spent.
+#
+# Deliberately one all-in number per session rather than a triage/seed/probe
+# decomposition. The decomposition looks more principled and fits worse: the two
+# observed batches ($1.62/10 and $14.74/50) cannot be reconciled by any single
+# pair of per-stage costs, because session length varies more than stage price
+# does. A model that cannot fit the two points it was built from should not be
+# dressed up as three constants.
+#
+# The first version of this was fitted to n=14 from a different population (the
+# hook's grask-only sessions) and under-quoted a 50-session run by 64%. Erring
+# low is the harmful direction — it under-quotes spend the developer then
+# authorises — so treat the band, not the point, as the estimate.
+COST_PER_SESSION = 0.27
+COST_BAND = 0.4  # observed batch error against the point estimate, both ways
+KEPT_RATE = 0.42  # 25 of 60; drives the probe-count estimate, not the cost one
+
+
+@dataclass
+class Plan:
+    """What a run would touch, and what it declined to."""
+
+    selected: list[Path] = field(default_factory=list)
+    excluded: int = 0
+    already_captured: int = 0
+    no_human_turns: int = 0
+    over_limit: int = 0
+
+    def estimate_usd(self) -> float:
+        return len(self.selected) * COST_PER_SESSION
+
+    def estimate_range_usd(self) -> tuple[float, float]:
+        """The number to quote. A point estimate here has already misled once."""
+        point = self.estimate_usd()
+        return point * (1 - COST_BAND), point * (1 + COST_BAND)
+
+    def estimate_probes(self) -> int:
+        return round(len(self.selected) * KEPT_RATE)
+
+
+def plan(
+    paths: Sequence[Path],
+    *,
+    exclude: Sequence[str],
+    is_captured: Callable[[str], bool],
+    has_human_turns: Callable[[Path], bool],
+    limit: int | None,
+) -> Plan:
+    """Decide which transcripts to spend on. Pure — every side effect is injected.
+
+    The three filters are ordered by what they cost to evaluate: the project name
+    is free, the captured check is one indexed read, and the human-turn check
+    parses the whole file. The limit applies last, so `--limit 40` means forty
+    sessions that will actually reach triage rather than forty rows of scratchpad.
+    """
+    result = Plan()
+    for path in paths:
+        project = path.parent.name
+        if any(marker in project for marker in exclude):
+            result.excluded += 1
+            continue
+        if is_captured(path.stem):
+            result.already_captured += 1
+            continue
+        if not has_human_turns(path):
+            result.no_human_turns += 1
+            continue
+        if limit is not None and len(result.selected) >= limit:
+            result.over_limit += 1
+            continue
+        result.selected.append(path)
+    return result
+
+
+def has_human_turns(path: Path) -> bool:
+    """Stage 0's filter, run early so empty sessions never eat the limit."""
+    try:
+        return bool(extract(path).turns)
+    except OSError:
+        return False
+
+
+def capture_one(path: Path) -> None:
+    """One session, one connection.
+
+    sqlite3 connections are not shareable across threads, so each task opens its
+    own. Writes are brief next to the model calls they follow, and the driver's
+    default busy timeout absorbs the overlap.
+    """
+    with Store() as store:
+        capture_session(path, store)
+
+
+def outcomes(store: Store, session_ids: Sequence[str]) -> str:
+    """Read back what the run actually produced.
+
+    `capture_session` reports nothing — it is built to run detached, where the
+    only honest channel is a row. So the report is a query, not a return value.
+    """
+    if not session_ids:
+        return "Nothing ran."
+    marks = ",".join("?" * len(session_ids))
+    verdicts = store.conn.execute(
+        f"SELECT verdict, COUNT(*) n, SUM(COALESCE(cost_usd, 0)) c "
+        f"FROM sessions WHERE session_id IN ({marks}) GROUP BY verdict",
+        tuple(session_ids),
+    ).fetchall()
+    probes = store.conn.execute(
+        f"SELECT COUNT(*) n, SUM(COALESCE(p.cost_usd, 0) + COALESCE(s.cost_usd, 0)) c "
+        f"FROM probes p JOIN seeds s ON s.id = p.seed_id "
+        f"WHERE s.session_id IN ({marks})",
+        tuple(session_ids),
+    ).fetchone()
+
+    spent = sum(row["c"] or 0.0 for row in verdicts) + (probes["c"] or 0.0)
+    lines = [
+        "=" * 72,
+        f"CAPTURED {len(session_ids)} sessions",
+        "=" * 72,
+    ]
+    for row in sorted(verdicts, key=lambda r: -r["n"]):
+        lines.append(f"  {row['verdict']:<8} {row['n']:>4d}   ${row['c'] or 0.0:.2f}")
+    lines += [
+        "",
+        f"  probes minted  {probes['n']:>4d}",
+        f"  spent          ${spent:.2f}",
+    ]
+    if not probes["n"]:
+        lines.append("")
+        lines.append("  No probes. Either triage kept nothing or every kept session errored —")
+        lines.append("  check grask.log before spending on a second batch.")
+    return "\n".join(lines)
+
+
+def describe(result: Plan, *, limit: int | None) -> str:
+    low, high = result.estimate_range_usd()
+    lines = [
+        "=" * 72,
+        f"BACKFILL PLAN: {len(result.selected)} sessions to capture",
+        "=" * 72,
+        f"  skipped, excluded project     {result.excluded:>5d}",
+        f"  skipped, already captured     {result.already_captured:>5d}",
+        f"  skipped, no human turns       {result.no_human_turns:>5d}",
+        f"  skipped, over the limit       {result.over_limit:>5d}" if limit is not None else "",
+        "",
+        f"  estimated cost  ${low:.2f} - ${high:.2f}"
+        f"   (~${COST_PER_SESSION:.2f}/session, +/-{COST_BAND:.0%})",
+        f"  expected probes {result.estimate_probes():>5d}   (at {KEPT_RATE:.0%} kept)",
+        "",
+        "-" * 72,
+    ]
+    for path in result.selected:
+        lines.append(f"  {path.stem[:8]}  {path.parent.name}")
+    return "\n".join(line for line in lines if line != "")
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--limit", type=int, default=DEFAULT_LIMIT)
+    parser.add_argument("--workers", type=int, default=DEFAULT_WORKERS)
+    parser.add_argument("--root", type=Path, default=None, help="transcript root")
+    parser.add_argument(
+        "--exclude",
+        action="append",
+        default=None,
+        help=f"project-name substring to skip; repeatable (default: {DEFAULT_EXCLUDE})",
+    )
+    parser.add_argument("--go", action="store_true", help="actually spend; otherwise dry run")
+    args = parser.parse_args(argv)
+
+    exclude = tuple(args.exclude) if args.exclude else DEFAULT_EXCLUDE
+    with Store() as store:
+        result = plan(
+            find_transcripts(args.root),
+            exclude=exclude,
+            is_captured=store.has_session,
+            has_human_turns=has_human_turns,
+            limit=args.limit,
+        )
+
+    print(describe(result, limit=args.limit))
+    if not result.selected:
+        return 0
+    if not args.go:
+        print("\nDry run. Re-run with --go to spend.")
+        return 0
+
+    print(f"\ncapturing on {args.workers} workers...", file=sys.stderr)
+    with ThreadPoolExecutor(max_workers=args.workers) as pool:
+        list(pool.map(capture_one, result.selected))
+
+    with Store() as store:
+        print()
+        print(outcomes(store, [p.stem for p in result.selected]))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())