chipgraph 0.1.0__py3-none-any.whl

chipgraph/packs/assist/ask/documents.py

@@ -0,0 +1,157 @@
+"""Which project files `/ask` may search, and indexing them into the model store.
+
+`ModelStore.write()` rebuilds the database from scratch, which drops every document added
+with `ModelStore.add_document`. So `chipgraph ingest` calls `index_documents` right after
+it writes the model: every ingest re-indexes the same, deterministic document set.
+
+The document set is:
+
+- every input file ingest read (the chip-level spec, each block's spec, each filelist and
+  the RTL sources it lists), and
+- the project's own Markdown docs: `README.md`, `AGENTS.md`, `doc/**/*.md`, `docs/**/*.md`,
+
+minus files labelled `nda` (every `/ask` model may be a cloud model, so an `nda` file is
+never indexed; ingest reports each one it skipped), vendored paths (a `vendor`,
+`third_party`, ... path segment), paths outside the project, and binary or very large files.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from pathlib import Path, PurePosixPath
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from chipgraph.core.model.store import ModelStore
+from chipgraph.core.state.artifacts import LabelRules
+
+MARKDOWN_DOCS: tuple[str, ...] = ("README.md", "AGENTS.md", "doc/**/*.md", "docs/**/*.md")
+"""Globs (relative to the project root) of the Markdown docs indexed besides ingest inputs."""
+
+VENDOR_SEGMENTS = frozenset({"vendor", "third_party", "thirdparty", "external", "extern"})
+"""A path with one of these segments is vendored code: never indexed."""
+
+MAX_DOCUMENT_BYTES = 2 * 1024 * 1024
+"""Files larger than this (generated netlists, dumps) are not indexed."""
+
+SkipReason = Literal["nda", "vendor", "outside_root", "missing", "binary", "too_large"]
+
+
+class SkippedDocument(BaseModel):
+    """A candidate document that was not indexed, and why."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    path: str = Field(description="Repo-relative POSIX path (or the path as given).")
+    reason: SkipReason = Field(description="Why it was not indexed.")
+
+
+class DocumentIndexReport(BaseModel):
+    """What `index_documents` indexed for `/ask`, and what it skipped."""
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    schema_version: int = 1
+    indexed: tuple[str, ...] = Field(default=(), description="Indexed paths, sorted.")
+    lines: int = Field(default=0, description="Total number of indexed lines.")
+    skipped: tuple[SkippedDocument, ...] = Field(
+        default=(), description="Candidates not indexed, sorted by path."
+    )
+
+    @property
+    def nda_skipped(self) -> tuple[str, ...]:
+        """The paths skipped because they are labelled `nda`."""
+        return tuple(s.path for s in self.skipped if s.reason == "nda")
+
+
+def document_candidates(root: Path, input_paths: Iterable[str]) -> list[str]:
+    """Every candidate document path: ingest inputs plus the Markdown docs, sorted, unique."""
+    candidates = {_normalise(p) for p in input_paths}
+    for pattern in MARKDOWN_DOCS:
+        for path in root.glob(pattern):
+            if path.is_file():
+                candidates.add(_relative(path, root))
+    return sorted(candidates)
+
+
+def index_documents(
+    root: Path,
+    store: ModelStore,
+    input_paths: Iterable[str],
+    labels: LabelRules,
+) -> DocumentIndexReport:
+    """Index the `/ask` document set of the project at `root` into `store`.
+
+    Call it after `store.write()`, which drops earlier documents. Deterministic: the same
+    files give the same rows, in the same order.
+    """
+    indexed: list[str] = []
+    skipped: list[SkippedDocument] = []
+    lines = 0
+    for rel in document_candidates(root, input_paths):
+        reason = _skip_reason(rel, root, labels)
+        if reason is not None:
+            skipped.append(SkippedDocument(path=rel, reason=reason))
+            continue
+        data = (root / rel).read_bytes()
+        if len(data) > MAX_DOCUMENT_BYTES:
+            skipped.append(SkippedDocument(path=rel, reason="too_large"))
+            continue
+        if b"\x00" in data:
+            skipped.append(SkippedDocument(path=rel, reason="binary"))
+            continue
+        text = data.decode("utf-8", errors="replace")
+        store.add_document(rel, text)
+        indexed.append(rel)
+        lines += len(text.splitlines())
+    return DocumentIndexReport(indexed=tuple(indexed), lines=lines, skipped=tuple(skipped))
+
+
+def is_vendor_path(rel: str) -> bool:
+    """True when `rel` has a vendored-code path segment."""
+    return any(part.lower() in VENDOR_SEGMENTS for part in PurePosixPath(rel).parts)
+
+
+def _skip_reason(rel: str, root: Path, labels: LabelRules) -> SkipReason | None:
+    posix = PurePosixPath(rel)
+    if posix.is_absolute() or ".." in posix.parts:
+        return "outside_root"
+    if labels.label_for(rel) == "nda":
+        return "nda"
+    if is_vendor_path(rel):
+        return "vendor"
+    path = root / rel
+    if not path.is_file():
+        return "missing"
+    try:
+        path.resolve().relative_to(root.resolve())
+    except ValueError:
+        return "outside_root"  # a symlink out of the project
+    return None
+
+
+def _normalise(rel: str) -> str:
+    text = rel.replace("\\", "/")
+    while text.startswith("./"):
+        text = text[2:]
+    return text
+
+
+def _relative(path: Path, root: Path) -> str:
+    try:
+        return path.resolve().relative_to(root.resolve()).as_posix()
+    except ValueError:
+        return path.as_posix()
+
+
+__all__ = [
+    "MARKDOWN_DOCS",
+    "MAX_DOCUMENT_BYTES",
+    "VENDOR_SEGMENTS",
+    "DocumentIndexReport",
+    "SkippedDocument",
+    "document_candidates",
+    "index_documents",
+    "is_vendor_path",
+]

chipgraph/packs/assist/ask/fts.py

@@ -0,0 +1,80 @@
+"""A safe FTS5 query from a free-text question (FTS5 syntax never reaches SQLite raw).
+
+The question is tokenised into words (Unicode letters, digits, `_`, with inner `-` or
+`.`, so `REQ-TIM-004`, `CTRL.EN` and `pin_out` stay one term). English stop words and
+one-character words are dropped, a plural gets its singular as an extra term, every term
+is quoted as an FTS5 string, and the terms are ORed. A quoted string is never an FTS5
+operator (`NEAR`, `AND`, `*`, `^`, `col:`), and the tokeniser splits it into a phrase
+exactly as it split the indexed text. A question with no usable term gives `None`: no
+search at all.
+"""
+
+from __future__ import annotations
+
+import re
+
+MAX_TERMS = 32
+"""At most this many terms go into one query (the first ones in the question)."""
+
+_TERM = re.compile(r"\w+(?:[-.]\w+)*")
+
+_STOP_WORDS_TEXT = """
+    a about above after again all also am an and any are as at be been before being below
+    between both but by can could did do does doing done down during each either else
+    for from further get gets give given had has have having he her here hers how however
+    i if in into is it its itself just let me more most much must my no nor not now of
+    off on once only or other our out over own per please same she should show so some
+    such tell than that the their them then there these they this those through to too
+    under until up upon us very via was we were what whats when where whether which while
+    who whom whose why will with within without would yes you your
+"""
+STOP_WORDS = frozenset(_STOP_WORDS_TEXT.split())
+"""English function words and question words: never a search term on their own.
+
+`out`, `in` and `no` are stop words too: as ports (`pin_out`) they are kept inside the
+longer term, and a typed lookup still finds an entity named exactly that.
+"""
+
+
+def question_terms(question: str) -> list[str]:
+    """The search terms of `question`, lowercased, in order, without duplicates."""
+    terms: list[str] = []
+    seen: set[str] = set()
+    for match in _TERM.finditer(question):
+        word = match.group(0).strip("-.").lower()
+        if len(word) < 2 or word in STOP_WORDS:
+            continue
+        for term in (word, _singular(word)):
+            if term and term not in seen and term not in STOP_WORDS:
+                seen.add(term)
+                terms.append(term)
+    return terms[:MAX_TERMS]
+
+
+def fts_query(question: str) -> str | None:
+    """A safe FTS5 MATCH expression for `question`: quoted terms ORed, or `None`."""
+    terms = question_terms(question)
+    if not terms:
+        return None
+    return " OR ".join(_quote(term) for term in terms)
+
+
+def _quote(term: str) -> str:
+    # Terms only hold word characters, `-` and `.`; doubling `"` keeps this safe anyway.
+    return '"' + term.replace('"', '""') + '"'
+
+
+def _singular(word: str) -> str | None:
+    """A naive singular for an English plural (`registers` -> `register`), else None."""
+    if not word.isalpha() or len(word) < 5:
+        return None
+    if word.endswith("ies"):
+        return word[:-3] + "y"
+    if word.endswith("sses") or word.endswith("xes"):
+        return word[:-2]
+    if word.endswith("s") and not word.endswith(("ss", "us", "is")):
+        return word[:-1]
+    return None
+
+
+__all__ = ["MAX_TERMS", "STOP_WORDS", "fts_query", "question_terms"]

chipgraph/packs/assist/ask/retrieve.py

@@ -0,0 +1,331 @@
+"""`ask_context`: the sources an `/ask` answer may use (DESIGN 4.5: typed queries, no grep).
+
+Two kinds of retrieval, merged:
+
+1. **Typed lookups** (`ModelQuery`): a word of the question that names something the model
+   knows (a block, module, register, `REG.FIELD`, port, requirement id, clock, reset, ...)
+   gives that entity, with its facts and relations; a kind word (`register`, `interrupt`,
+   `open items`, `memory map`, ...) lists the entities of that kind in the blocks the
+   question is about. A port named `clk` exists in every module, so an entity in a block
+   the question names ranks above one in another block.
+2. **Full-text search** (FTS5 over entities and documents) with a safe query built from
+   the question (`fts.fts_query`). A document hit cites `path:line` and carries the lines
+   around it.
+
+Entities and documents from `nda` files are never returned. An empty result is a valid
+answer: `no_sources` is then true and the answer must be "I don't know".
+"""
+
+from __future__ import annotations
+
+import re
+import sqlite3
+
+from chipgraph.app.context import AppContext
+from chipgraph.core.model.entities import EntityBase
+from chipgraph.packs.assist.ask._project import AskProject
+from chipgraph.packs.assist.ask.contract import AskContext, AskSource, DocLine
+from chipgraph.packs.assist.ask.fts import STOP_WORDS, fts_query
+
+DEFAULT_LIMIT = 12
+MAX_LIMIT = 50
+CONTEXT_LINES = 2
+"""Lines shown before and after a document hit."""
+
+_WORD = re.compile(r"\w+(?:[-.]\w+)*")
+
+_KIND_PRIORITY: dict[str, int] = {
+    "requirement": 10,
+    "register": 9,
+    "field": 8,
+    "interrupt": 8,
+    "memory_region": 7,
+    "block": 6,
+    "module": 6,
+    "port": 5,
+    "clock": 5,
+    "reset": 5,
+    "parameter": 4,
+    "interface": 4,
+    "open_item": 4,
+    "project": 3,
+}
+
+_KIND_WORDS: dict[str, str] = {
+    "register": "register",
+    "registers": "register",
+    "field": "field",
+    "fields": "field",
+    "bitfield": "field",
+    "bitfields": "field",
+    "requirement": "requirement",
+    "requirements": "requirement",
+    "interrupt": "interrupt",
+    "interrupts": "interrupt",
+    "port": "port",
+    "ports": "port",
+    "signal": "port",
+    "signals": "port",
+    "module": "module",
+    "modules": "module",
+    "parameter": "parameter",
+    "parameters": "parameter",
+    "clock": "clock",
+    "clocks": "clock",
+    "reset": "reset",
+    "resets": "reset",
+    "block": "block",
+    "blocks": "block",
+    "bus": "interface",
+    "buses": "interface",
+    "interface": "interface",
+    "interfaces": "interface",
+    "address": "memory_region",
+    "addresses": "memory_region",
+}
+_PHRASE_KINDS: tuple[tuple[str, str], ...] = (
+    ("open item", "open_item"),
+    ("open question", "open_item"),
+    ("memory map", "memory_region"),
+    ("address map", "memory_region"),
+)
+
+_GENERIC_WORDS = frozenset(
+    {
+        "access", "address", "base", "block", "clock", "config", "control", "count", "data",
+        "enable", "field", "interrupt", "line", "mode", "module", "offset", "port", "register",
+        "reset", "size", "status", "type", "value", "width",
+    }
+)  # fmt: skip
+"""Words that are also common register or field names: they match an entity only when
+written exactly as its name."""
+
+_DIRECT_BONUS = 10
+_KIND_WORD_BONUS = 6
+_CONTEXT_BONUS = 5
+_OFF_CONTEXT_PENALTY = 6
+_MAX_PER_KIND_LISTING = 8
+
+
+def ask_context(ctx: AppContext, question: str, limit: int = DEFAULT_LIMIT) -> AskContext:
+    """The sources for `question` in the project of `ctx`, best first, at most `limit`."""
+    project = AskProject.load(ctx)
+    return retrieve(project, question, limit)
+
+
+def retrieve(project: AskProject, question: str, limit: int = DEFAULT_LIMIT) -> AskContext:
+    """`ask_context` on an already loaded `AskProject`."""
+    limit = max(1, min(limit, MAX_LIMIT))
+    typed = _typed_sources(project, question, limit)
+    query = fts_query(question)
+    searched = _search_sources(project, query, limit) if query is not None else []
+
+    sources: list[AskSource] = []
+    seen: set[str] = set()
+
+    def take(items: list[AskSource], cap: int) -> None:
+        for source in items:
+            if len(sources) >= cap:
+                return
+            if source.citation not in seen:
+                seen.add(source.citation)
+                sources.append(source)
+
+    take(typed, max(1, (limit + 1) // 2))
+    take(searched, limit)
+    take(typed, limit)
+
+    no_sources = not sources
+    note = (
+        "No source matches the question: answer that you do not know (unknown: true)."
+        if no_sources
+        else "Answer only from these sources. Cite every claim with a source's `citation` "
+        "(or its `defined_at`) exactly as given. If they do not answer the question, set "
+        "unknown: true and say what is missing. Do not use outside knowledge."
+    )
+    return AskContext(
+        question=question,
+        query=query,
+        sources=tuple(sources),
+        no_sources=no_sources,
+        note=note,
+    )
+
+
+# --- typed lookups ----------------------------------------------------------------------
+
+
+def _typed_sources(project: AskProject, question: str, limit: int) -> list[AskSource]:
+    """Typed lookups on the names and kind words of the question, best first.
+
+    Score = kind priority, + `_DIRECT_BONUS` for an entity the question names, +
+    `_KIND_WORD_BONUS` when its kind is named too (or it is listed for a kind word in the
+    blocks the question is about), +/- the block context. An entity that only shares its
+    name with a named block (the `timer` memory region when the question says "timer")
+    is not "named" unless its kind is.
+    """
+    words = [m.group(0).strip("-.") for m in _WORD.finditer(question)]
+    words = [w for w in words if len(w) >= 2]
+    kinds = _kinds_named(words)
+    names: dict[str, list[EntityBase]] = {}
+    for entity in project.model.entities.values():
+        if entity.kind in _KIND_PRIORITY and project.visible(entity):
+            names.setdefault(entity.name.lower(), []).append(entity)
+
+    direct: dict[str, EntityBase] = {}
+    for word in words:
+        lowered = word.lower()
+        if lowered in STOP_WORDS:
+            continue
+        for entity in names.get(lowered, ()):
+            # A generic word ("reset value", "status") names an entity only when it is
+            # written exactly as that entity's name (`RESET`, `STATUS`).
+            if lowered in _GENERIC_WORDS and word != entity.name:
+                continue
+            direct.setdefault(entity.key, entity)
+        if "." in word:
+            for entity in _dotted_fields(project, word):
+                direct.setdefault(entity.key, entity)
+    block_names = {e.name.lower() for e in direct.values() if e.kind == "block"}
+    context_blocks, context_modules = _context(project, list(direct.values()))
+
+    # A kind word lists that kind in the blocks the question is about; with no such
+    # block, only when the whole project has a few of them (its one clock, its resets).
+    listed: dict[str, EntityBase] = {}
+    for kind in kinds:
+        members = [
+            e
+            for e in sorted(project.model.by_kind(kind), key=lambda e: e.key)
+            if project.visible(e) and (not context_blocks or project.blocks_of(e) & context_blocks)
+        ]
+        if context_blocks or len(members) <= _MAX_PER_KIND_LISTING:
+            for entity in members[:_MAX_PER_KIND_LISTING]:
+                listed.setdefault(entity.key, entity)
+
+    scored: list[tuple[int, str, EntityBase]] = []
+    for key, entity in {**listed, **direct}.items():
+        kind_named = entity.kind in kinds
+        named = key in direct and (
+            entity.kind == "block" or entity.name.lower() not in block_names or kind_named
+        )
+        score = _KIND_PRIORITY.get(entity.kind, 1)
+        if named:
+            score += _DIRECT_BONUS
+        if kind_named and (named or context_blocks):
+            score += _KIND_WORD_BONUS
+        score += _context_adjustment(project, entity, context_blocks, context_modules)
+        scored.append((score, key, entity))
+    scored.sort(key=lambda item: (-item[0], item[1]))
+    return [_model_source(project, entity, "lookup") for _, _, entity in scored[:limit]]
+
+
+def _dotted_fields(project: AskProject, word: str) -> list[EntityBase]:
+    """`CTRL.EN` (register.field) or `timer.CTRL` (block.register) as model entities."""
+    parts = word.lower().split(".")
+    found = []
+    for entity in project.model.entities.values():
+        if entity.kind not in ("field", "register") or not project.visible(entity):
+            continue
+        key_parts = entity.key.partition(":")[2].lower().split(".")
+        if key_parts[-len(parts) :] == parts:
+            found.append(entity)
+    return found
+
+
+def _context(project: AskProject, direct: list[EntityBase]) -> tuple[set[str], set[str]]:
+    """The blocks and modules the question is about.
+
+    The blocks and modules it names; if it names none, the blocks of the registers,
+    fields, requirements and memory regions it names.
+    """
+    modules = {e.key for e in direct if e.kind == "module"}
+    blocks: set[str] = set()
+    for entity in direct:
+        if entity.kind in ("block", "module"):
+            blocks |= project.blocks_of(entity)
+    if not blocks:
+        for entity in direct:
+            if entity.kind in ("register", "field", "requirement", "memory_region"):
+                blocks |= project.blocks_of(entity)
+    return blocks, modules
+
+
+def _context_adjustment(
+    project: AskProject,
+    entity: EntityBase,
+    context_blocks: set[str],
+    context_modules: set[str],
+) -> int:
+    if entity.kind in ("block", "project"):
+        return 0
+    module = project.module_of(entity)
+    blocks = project.blocks_of(entity)
+    if context_modules and module is not None:
+        in_context = module in context_modules
+    elif context_blocks and blocks:
+        in_context = bool(blocks & context_blocks)
+    else:
+        return 0
+    return _CONTEXT_BONUS if in_context else -_OFF_CONTEXT_PENALTY
+
+
+def _kinds_named(words: list[str]) -> list[str]:
+    """The entity kinds the question names ("registers", "open items", "memory map")."""
+    lowered = [w.lower() for w in words]
+    kinds: list[str] = []
+    for index, word in enumerate(lowered):
+        following = lowered[index + 1] if index + 1 < len(lowered) else ""
+        pair = f"{word} {following}"
+        named = next((k for p, k in _PHRASE_KINDS if pair.startswith(p)), None)
+        if named is None:
+            if word == "reset" and following.startswith("value"):
+                continue  # "reset value" asks about a register, not about a reset
+            named = _KIND_WORDS.get(word)
+        if named is not None and named not in kinds:
+            kinds.append(named)
+    return kinds
+
+
+def _model_source(project: AskProject, entity: EntityBase, origin: str) -> AskSource:
+    return AskSource(
+        citation=f"model:{entity.key}",
+        kind="model",
+        text=project.summary(entity),
+        origin=origin,  # type: ignore[arg-type]
+        defined_at=project.defined_at(entity),
+    )
+
+
+# --- full-text search -------------------------------------------------------------------
+
+
+def _search_sources(project: AskProject, query: str, limit: int) -> list[AskSource]:
+    try:
+        hits = project.store.search(query, limit=limit * 2)
+    except sqlite3.Error:
+        return []  # a query the FTS5 parser still refuses: no search result, never an error
+    sources: list[AskSource] = []
+    for hit in hits:
+        if hit.key is not None:
+            entity = project.get(hit.key)
+            if entity is not None:
+                sources.append(_model_source(project, entity, "search"))
+        elif hit.file is not None and hit.line is not None:
+            if project.labels.label_for(hit.file) == "nda":
+                continue
+            sources.append(_document_source(project, hit.file, hit.line, hit.text))
+    return sources
+
+
+def _document_source(project: AskProject, path: str, line: int, text: str) -> AskSource:
+    around = project.docs.lines(path, max(1, line - CONTEXT_LINES), line + CONTEXT_LINES)
+    return AskSource(
+        citation=f"{path}:{line}",
+        kind="document",
+        text=text,
+        origin="search",
+        context=tuple(DocLine(line=n, text=t) for n, t in around if n != line),
+    )
+
+
+__all__ = ["CONTEXT_LINES", "DEFAULT_LIMIT", "MAX_LIMIT", "ask_context", "retrieve"]