chipgraph 0.1.0__py3-none-any.whl

chipgraph/packs/assist/ask/answer.py

@@ -0,0 +1,266 @@
+"""The API-runtime answerer behind `chipgraph ask`: retrieve, ask a model, check, retry.
+
+`answer_question` runs the same contract as the Claude Code path (`/chipgraph:ask`): the
+model sees only the sources `ask_context` returned, must reply with an `AskAnswer` JSON
+object, and the reply goes through `ask_check`. A rejected reply is sent back once with the
+reasons; an answer that still fails is never shown: the result is "I don't know" with the
+reasons. With no source at all the model is not called.
+
+The provider comes from the profile's `models.providers` (`make_provider`), wrapped in a
+`GuardedProvider` (the `nda` rule, retry, accounting); the model id is `models.tiers.small`.
+With no provider configured, `run_ask` returns the retrieved sources only (`no_provider`).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from chipgraph.adapters.llm import AnthropicProvider, FakeProvider, GuardedProvider, LlmError
+from chipgraph.app.context import AppContext
+from chipgraph.app.errors import AppError
+from chipgraph.core.config.models import Profile
+from chipgraph.core.contracts.types import DataLabel
+from chipgraph.core.plugin_api.protocols import LlmProvider
+from chipgraph.core.plugin_api.types import LlmMessage, LlmRequest
+from chipgraph.packs.assist.ask._project import AskProject
+from chipgraph.packs.assist.ask.check import check_answer
+from chipgraph.packs.assist.ask.contract import (
+    UNKNOWN_ANSWER,
+    AskAnswer,
+    AskCheck,
+    AskContext,
+)
+from chipgraph.packs.assist.ask.retrieve import DEFAULT_LIMIT, retrieve
+
+MAX_ATTEMPTS = 2
+"""The first reply, plus one retry with the checker's reasons."""
+
+MAX_TOKENS = 1024
+
+AskStatus = Literal["answered", "unknown", "rejected", "no_provider"]
+
+SYSTEM_PROMPT = """\
+You answer one question about a chip design project, using ONLY the sources you are given
+(entities of its Design Model and lines of its documents). Rules:
+
+- Every fact in your answer must come from a source. Cite the sources you used, each by
+  its exact `citation` string ('model:<key>' or 'path:line'), or by its `defined_at`.
+- Never use outside knowledge and never guess a number, a name or a behaviour.
+- If the sources do not answer the question, set "unknown" to true, say briefly what is
+  missing in "answer", and cite nothing.
+
+Reply with exactly one JSON object and nothing else:
+{"answer": "<short answer>", "citations": ["<citation>", ...], "unknown": false}
+"""
+
+
+class AskResult(BaseModel):
+    """What `chipgraph ask` prints: the checked answer (or none), and the sources."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    schema_version: int = 1
+    status: AskStatus = Field(
+        description=(
+            "'answered' or 'unknown' (a verified answer), 'rejected' (no reply passed the "
+            "check: the answer is 'I don't know'), 'no_provider' (sources only)."
+        )
+    )
+    question: str
+    answer: AskAnswer | None = Field(default=None, description="The verified answer.")
+    check: AskCheck | None = Field(default=None, description="The last check that ran.")
+    attempts: int = Field(default=0, description="Model calls made.")
+    context: AskContext = Field(description="The retrieved sources.")
+    provider: str | None = Field(default=None, description="The provider used, if any.")
+    model: str | None = Field(default=None, description="The model id used, if any.")
+
+
+def make_provider(profile: Profile, name: str | None = None) -> LlmProvider | None:
+    """The `LlmProvider` configured in `models.providers`, or None when there is none.
+
+    `name` picks one provider; it is required when several are configured. `anthropic`
+    is the Anthropic API, `fake` the scripted test provider, and any other name an
+    Anthropic-compatible endpoint (`base_url` in its config).
+    """
+    providers = profile.models.providers
+    if name is None:
+        if not providers:
+            return None
+        if len(providers) > 1:
+            raise AppError(
+                f"several providers in models.providers ({', '.join(sorted(providers))}); "
+                "pick one with --provider"
+            )
+        [name] = providers
+    elif name not in providers:
+        known = ", ".join(sorted(providers)) or "none"
+        raise AppError(f"no provider {name!r} in models.providers (configured: {known})")
+    if name == "fake":
+        return FakeProvider(name="fake")
+    try:
+        return AnthropicProvider.from_models(profile.models, name)
+    except LlmError as exc:
+        raise AppError(str(exc)) from exc
+
+
+def run_ask(
+    ctx: AppContext,
+    question: str,
+    *,
+    limit: int = DEFAULT_LIMIT,
+    provider_name: str | None = None,
+) -> AskResult:
+    """`chipgraph ask`: answer with the profile's provider, or return the sources only."""
+    profile = ctx.require_profile().profile
+    project = AskProject.load(ctx)
+    provider = make_provider(profile, provider_name)
+    if provider is None:
+        return AskResult(
+            status="no_provider", question=question, context=retrieve(project, question, limit)
+        )
+    model = profile.models.tiers.get("small")
+    if not model:
+        raise AppError(
+            f"provider {provider.name!r} is configured but models.tiers.small is not: set the "
+            "model id /ask should use"
+        )
+    guarded = GuardedProvider(provider, data=profile.data)
+    return asyncio.run(answer_question(project, question, guarded, model=model, limit=limit))
+
+
+async def answer_question(
+    project: AskProject,
+    question: str,
+    provider: LlmProvider,
+    *,
+    model: str,
+    limit: int = DEFAULT_LIMIT,
+) -> AskResult:
+    """Retrieve, ask `provider`, check the reply, retry once with the reasons."""
+    context = retrieve(project, question, limit)
+
+    def result(
+        status: AskStatus, answer: AskAnswer, check: AskCheck | None, attempts: int
+    ) -> AskResult:
+        return AskResult(
+            status=status,
+            question=question,
+            answer=answer,
+            check=check,
+            attempts=attempts,
+            context=context,
+            provider=provider.name,
+            model=model,
+        )
+
+    if context.no_sources:
+        return result("unknown", AskAnswer(answer=UNKNOWN_ANSWER, unknown=True), None, 0)
+
+    messages = [
+        LlmMessage(role="system", content=SYSTEM_PROMPT),
+        LlmMessage(role="user", content=_user_prompt(context)),
+    ]
+    labels = _labels(project, context)
+    check: AskCheck | None = None
+    for attempt in range(1, MAX_ATTEMPTS + 1):
+        request = LlmRequest(
+            model=model, messages=tuple(messages), max_tokens=MAX_TOKENS, labels=labels
+        )
+        try:
+            response = await provider.complete(request)
+        except LlmError as exc:
+            raise AppError(f"/ask: the model call failed: {exc}") from exc
+        parsed, parse_error = parse_answer(response.text)
+        if parsed is None:
+            check = AskCheck(ok=False, reasons=(parse_error or "not an answer object",))
+        else:
+            check = check_answer(project, parsed)
+        if check.ok and check.answer is not None:
+            status: AskStatus = "unknown" if check.answer.unknown else "answered"
+            return result(status, check.answer, check, attempt)
+        messages.append(LlmMessage(role="assistant", content=response.text))
+        messages.append(LlmMessage(role="user", content=_retry_prompt(check)))
+
+    fallback = AskAnswer(
+        answer="I don't know: no answer could be verified against this project's sources.",
+        unknown=True,
+    )
+    return result("rejected", fallback, check, MAX_ATTEMPTS)
+
+
+def parse_answer(text: str) -> tuple[AskAnswer | None, str | None]:
+    """An `AskAnswer` from a model reply (a JSON object, maybe fenced), or the error."""
+    raw = text.strip()
+    if raw.startswith("```"):
+        raw = raw.split("\n", 1)[1] if "\n" in raw else ""
+        raw = raw.rsplit("```", 1)[0]
+    start, end = raw.find("{"), raw.rfind("}")
+    if start < 0 or end < start:
+        return None, "the reply is not a JSON object"
+    try:
+        data = json.loads(raw[start : end + 1])
+    except json.JSONDecodeError as exc:
+        return None, f"the reply is not valid JSON: {exc.msg}"
+    if not isinstance(data, dict):
+        return None, "the reply is not a JSON object"
+    answer = data.get("answer", "")
+    citations = data.get("citations", [])
+    unknown = data.get("unknown", False)
+    if not isinstance(answer, str):
+        return None, "'answer' must be a string"
+    if not isinstance(citations, list) or not all(isinstance(c, str) for c in citations):
+        return None, "'citations' must be a list of strings"
+    if not isinstance(unknown, bool):
+        return None, "'unknown' must be true or false"
+    return AskAnswer(answer=answer, citations=tuple(citations), unknown=unknown), None
+
+
+def _user_prompt(context: AskContext) -> str:
+    sources = [s.model_dump(mode="json", exclude_defaults=True) for s in context.sources]
+    return (
+        f"Question: {context.question}\n\n"
+        f"Sources ({len(sources)}), best first:\n"
+        f"{json.dumps(sources, indent=1)}\n\n"
+        f"{context.note}"
+    )
+
+
+def _retry_prompt(check: AskCheck) -> str:
+    reasons = "\n".join(f"- {r}" for r in check.reasons)
+    return (
+        "Your answer was rejected by the citation check:\n"
+        f"{reasons}\n"
+        "Reply again with one JSON object. Cite only `citation` or `defined_at` strings from "
+        "the sources above; if they do not answer the question, set unknown to true."
+    )
+
+
+def _labels(project: AskProject, context: AskContext) -> tuple[DataLabel, ...]:
+    """The data labels of every file behind the sources (never 'nda': those are excluded)."""
+    files: set[str] = set()
+    for source in context.sources:
+        for citation in (source.citation, source.defined_at):
+            if citation and not citation.startswith("model:"):
+                files.add(citation.rsplit(":", 1)[0])
+        if source.kind == "model":
+            entity = project.model.get(source.citation.removeprefix("model:"))
+            if entity is not None and entity.source.file:
+                files.add(entity.source.file)
+    labels = {project.labels.label_for(f) for f in files} or {project.labels.default}
+    return tuple(sorted(labels))
+
+
+__all__ = [
+    "MAX_ATTEMPTS",
+    "SYSTEM_PROMPT",
+    "AskResult",
+    "AskStatus",
+    "answer_question",
+    "make_provider",
+    "parse_answer",
+    "run_ask",
+]

chipgraph/packs/assist/ask/check.py

@@ -0,0 +1,182 @@
+"""`ask_check`: verify an `/ask` answer's citations before anyone sees the answer.
+
+Rules (DESIGN 4.8: no evidence, no claim):
+
+- A citation is either `path:line` / `path:start-end` (the path must be an indexed
+  document and every line must exist; a range spans at most `MAX_RANGE` lines) or
+  `model:<key>` (the key must exist in the Design Model). A bare model key is accepted and
+  written back as `model:<key>`. Nothing from an `nda` file is citable.
+- Every citation must be valid: one invented citation rejects the answer.
+- An answer with `unknown: false` needs at least one valid citation and some text.
+- An answer with `unknown: true` needs no citation (and any it gives must be valid).
+
+The result is the verified answer (citations in canonical form, duplicates dropped) or the
+reasons it was rejected, with a verdict per citation so the answerer can fix it.
+"""
+
+from __future__ import annotations
+
+import difflib
+import re
+
+from chipgraph.app.context import AppContext
+from chipgraph.packs.assist.ask._project import AskProject
+from chipgraph.packs.assist.ask.contract import (
+    UNKNOWN_ANSWER,
+    AskAnswer,
+    AskCheck,
+    CitationCheck,
+)
+
+MAX_RANGE = 30
+"""A `path:start-end` citation may span at most this many lines."""
+
+MAX_CITATIONS = 20
+"""An answer may carry at most this many citations."""
+
+_FILE_LINE = re.compile(r"^(?P<path>.+?):(?P<start>\d+)(?:-(?P<end>\d+))?$")
+_MAX_TEXT = 400
+
+
+def ask_check(ctx: AppContext, answer: AskAnswer) -> AskCheck:
+    """Verify `answer` against the project of `ctx` (see the module docstring)."""
+    return check_answer(AskProject.load(ctx), answer)
+
+
+def check_answer(project: AskProject, answer: AskAnswer) -> AskCheck:
+    """`ask_check` on an already loaded `AskProject`."""
+    reasons: list[str] = []
+    checks = [check_citation(project, c) for c in answer.citations[:MAX_CITATIONS]]
+    if len(answer.citations) > MAX_CITATIONS:
+        reasons.append(
+            f"too many citations ({len(answer.citations)}); cite at most {MAX_CITATIONS}"
+        )
+    for check in checks:
+        if not check.valid:
+            reasons.append(f"invalid citation {check.citation!r}: {check.reason}")
+    valid = [c for c in checks if c.valid]
+    text = answer.answer.strip()
+    if not answer.unknown:
+        if not text:
+            reasons.append("the answer text is empty")
+        if not valid:
+            reasons.append(
+                "an answer needs at least one valid citation ('path:line' or 'model:<key>' "
+                "from ask_context); if the sources do not answer the question, set unknown: true"
+            )
+    if reasons:
+        return AskCheck(ok=False, citations=tuple(checks), reasons=tuple(reasons))
+
+    canonical: list[str] = []
+    for check in valid:
+        if check.normalized is not None and check.normalized not in canonical:
+            canonical.append(check.normalized)
+    verified = AskAnswer(
+        answer=text or UNKNOWN_ANSWER,
+        citations=tuple(canonical),
+        unknown=answer.unknown,
+    )
+    return AskCheck(ok=True, answer=verified, citations=tuple(checks))
+
+
+def check_citation(project: AskProject, citation: str) -> CitationCheck:
+    """The verdict on one citation string."""
+    raw = citation
+    text = citation.strip()
+    if not text:
+        return CitationCheck(citation=raw, valid=False, reason="empty citation")
+    if text.startswith("model:"):
+        return _check_key(project, raw, text.removeprefix("model:").strip())
+    match = _FILE_LINE.match(text)
+    if match is not None:
+        path = _normalise_path(match.group("path"))
+        if project.docs.is_indexed(path) or project.model.get(text) is None:
+            return _check_lines(project, raw, path, match.group("start"), match.group("end"))
+    if project.model.get(text) is not None:
+        return _check_key(project, raw, text)
+    return CitationCheck(
+        citation=raw,
+        valid=False,
+        reason=(
+            "not a 'path:line' or 'model:<key>' citation"
+            + _did_you_mean(text, list(project.model.entities))
+        ),
+    )
+
+
+def _check_key(project: AskProject, raw: str, key: str) -> CitationCheck:
+    entity = project.model.get(key)
+    if entity is None:
+        return CitationCheck(
+            citation=raw,
+            valid=False,
+            kind="model",
+            reason=f"no model key {key!r}" + _did_you_mean(key, list(project.model.entities)),
+        )
+    if not project.visible(entity):
+        return CitationCheck(
+            citation=raw,
+            valid=False,
+            kind="model",
+            reason=f"{key!r} comes from an 'nda' file, which /ask may not cite",
+        )
+    return CitationCheck(
+        citation=raw,
+        valid=True,
+        kind="model",
+        normalized=f"model:{key}",
+        text=_clip(project.summary(entity)),
+    )
+
+
+def _check_lines(
+    project: AskProject, raw: str, path: str, start_text: str, end_text: str | None
+) -> CitationCheck:
+    counts = project.docs.line_counts()
+    if path not in counts:
+        label = project.labels.label_for(path)
+        why = (
+            f"{path} is labelled 'nda' and is never indexed for /ask"
+            if label == "nda"
+            else f"{path} is not an indexed document" + _did_you_mean(path, list(counts))
+        )
+        return CitationCheck(citation=raw, valid=False, kind="document", reason=why)
+    start = int(start_text)
+    end = int(end_text) if end_text is not None else start
+    total = counts[path]
+    if start < 1 or end < start:
+        reason = f"bad line range {start}-{end}"
+    elif end > total:
+        reason = f"{path} has {total} lines; line {end} does not exist"
+    elif end - start + 1 > MAX_RANGE:
+        reason = f"a range may span at most {MAX_RANGE} lines; cite the exact lines"
+    else:
+        rows = project.docs.lines(path, start, end)
+        normalized = f"{path}:{start}" if start == end else f"{path}:{start}-{end}"
+        return CitationCheck(
+            citation=raw,
+            valid=True,
+            kind="document",
+            normalized=normalized,
+            text=_clip("\n".join(t for _, t in rows)),
+        )
+    return CitationCheck(citation=raw, valid=False, kind="document", reason=reason)
+
+
+def _normalise_path(path: str) -> str:
+    text = path.strip().replace("\\", "/")
+    while text.startswith("./"):
+        text = text[2:]
+    return text
+
+
+def _did_you_mean(name: str, choices: list[str]) -> str:
+    matches = difflib.get_close_matches(name, choices, n=3, cutoff=0.6)
+    return f"; did you mean: {', '.join(matches)}" if matches else ""
+
+
+def _clip(text: str) -> str:
+    return text if len(text) <= _MAX_TEXT else text[: _MAX_TEXT - 3] + "..."
+
+
+__all__ = ["MAX_CITATIONS", "MAX_RANGE", "ask_check", "check_answer", "check_citation"]

chipgraph/packs/assist/ask/contract.py

@@ -0,0 +1,115 @@
+"""The `/ask` data contracts: retrieved sources, the answer, and the citation check.
+
+An answer is `{"answer": "...", "citations": ["file:line" | "model:<key>", ...],
+"unknown": bool}` (`AskAnswer`). `ask_check` verifies it and returns an `AskCheck`: the
+verified answer, or the reasons it was rejected. Every model is JSON-dumpable, frozen, and
+carries a `schema_version`.
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+UNKNOWN_ANSWER = "I don't know: no source in this project's Design Model or documents answers it."
+"""The answer text used when there is no source, or no answer could be verified."""
+
+
+class DocLine(BaseModel):
+    """One line of an indexed document."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    line: int = Field(ge=1, description="1-based line number.")
+    text: str = Field(description="The line's text.")
+
+
+class AskSource(BaseModel):
+    """One source `ask_context` returned: a model entity or a document line."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    citation: str = Field(description="What to cite: 'model:<key>' or 'path:line'.")
+    kind: Literal["model", "document"] = Field(description="A model entity or a document line.")
+    text: str = Field(description="The entity's facts, or the document line.")
+    origin: Literal["lookup", "search"] = Field(
+        description="'lookup': a typed model query on a name in the question; 'search': FTS5."
+    )
+    defined_at: str | None = Field(
+        default=None,
+        description="For a model entity: the indexed 'path:line' it was read from, also citable.",
+    )
+    context: tuple[DocLine, ...] = Field(
+        default=(), description="For a document line: the lines around it."
+    )
+
+
+class AskContext(BaseModel):
+    """The result of `ask_context`: the sources an answer may use, and only those."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    schema_version: int = 1
+    question: str = Field(description="The question, as asked.")
+    query: str | None = Field(description="The FTS5 query used, or None when no term was left.")
+    sources: tuple[AskSource, ...] = Field(default=(), description="Best first.")
+    no_sources: bool = Field(description="True when nothing matched: the answer is 'unknown'.")
+    note: str = Field(description="How to use the sources.")
+
+
+class AskAnswer(BaseModel):
+    """An answer to one `/ask` question: text, citations, and whether it is 'unknown'."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    schema_version: int = 1
+    answer: str = Field(description="The answer text.")
+    citations: tuple[str, ...] = Field(
+        default=(), description="'path:line' (or 'path:start-end') or 'model:<key>'."
+    )
+    unknown: bool = Field(
+        default=False, description="True when the sources do not answer the question."
+    )
+
+
+class CitationCheck(BaseModel):
+    """The verdict on one citation."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    citation: str = Field(description="The citation as given.")
+    valid: bool = Field(description="True when it names an indexed line or a model key.")
+    kind: Literal["model", "document"] | None = Field(
+        default=None, description="What it names, when that could be told."
+    )
+    normalized: str | None = Field(
+        default=None, description="The canonical form: 'path:line[-end]' or 'model:<key>'."
+    )
+    text: str | None = Field(default=None, description="What it points at, when valid.")
+    reason: str | None = Field(default=None, description="Why it is invalid.")
+
+
+class AskCheck(BaseModel):
+    """The result of `ask_check`: the verified answer (when `ok`) or the reasons."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    schema_version: int = 1
+    ok: bool = Field(description="True when the answer is accepted.")
+    answer: AskAnswer | None = Field(
+        default=None, description="The verified answer (canonical citations), when ok."
+    )
+    citations: tuple[CitationCheck, ...] = Field(default=(), description="One per citation.")
+    reasons: tuple[str, ...] = Field(default=(), description="Why it was rejected.")
+
+
+__all__ = [
+    "UNKNOWN_ANSWER",
+    "AskAnswer",
+    "AskCheck",
+    "AskContext",
+    "AskSource",
+    "CitationCheck",
+    "DocLine",
+]