gapit 0.2.2__py3-none-any.whl

gapit/db_build_ops.py

@@ -0,0 +1,210 @@
+"""The custom-database build use-case, shared by the CLI and MCP.
+
+Turns a user-supplied FASTA into a fully built gapit-native database via the
+existing pipeline (records.jsonl -> sequences + BLAST index + manifest,
+written last). This module is orchestration plus a metadata merge only — the
+building blocks live in records/dbbuild/fasta/dbcodec/db (SPEC.md §11 covers
+the build pipeline itself). The typer command (:mod:`gapit.cmd_db_build`)
+and the MCP tool (:mod:`gapit.mcp_tools`) are thin callers.
+
+Header kind is detected PER RECORD (mixed files allowed): a ``gapit|``
+prefix decodes through the strict tagged codec, anything else through the
+abricate ``~~~`` rules (a plain id carries no ``~~~`` and decodes to itself).
+``db`` is always the TARGET name and ``source_id`` keeps the original id
+token; record order is input order. Sequences are stored verbatim — no
+provider-style normalization, this is the user's curated truth.
+"""
+
+import csv
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel
+
+from gapit import config
+from gapit.db import mol_type
+from gapit.dbbuild import build_database
+from gapit.dbcodec import decode_seqid
+from gapit.errors import DatabaseError, InputError, UsageError
+from gapit.fasta import FastaRecord, iter_fasta
+from gapit.records import Record, write_records
+
+Dbtype = Literal["nucl", "prot"]
+
+
+class BuildReceipt(BaseModel, frozen=True):
+    """One-line JSON success receipt for `db build` (mirrors db fetch)."""
+
+    db: str
+    records: int
+    dbtype: Dbtype
+    destination: str
+
+
+@dataclass(frozen=True, slots=True)
+class Metadata:
+    """Parsed --tsv: which merge columns the header carries, and the row
+    values per gene (accession, ';'-joined function; '' for absent columns)."""
+
+    columns: frozenset[str]
+    rows: dict[str, tuple[str, str]]
+
+
+def _split_function(joined: str) -> tuple[str, ...]:
+    """';'-joined classes -> tuple, empty pieces dropped."""
+    return tuple(part for part in joined.split(";") if part)
+
+
+def _to_record(fasta: FastaRecord, name: str, default_product: str) -> Record:
+    """One FASTA record -> one Record.
+
+    decode_seqid routes by header kind; the product prefers the FASTA
+    description, then --description, then the gene (the provider-side
+    ``description or gene`` pattern). Malformed ``gapit|`` headers raise
+    their native HEADER_MALFORMED DatabaseError.
+    """
+    header = decode_seqid(fasta.id, default_db=name)
+    return Record(
+        db=name,
+        gene=header.gene,
+        sequence=fasta.sequence,
+        accession=header.accession,
+        function=_split_function(header.function),
+        product=fasta.description or default_product or header.gene,
+        source_id=fasta.id,
+    )
+
+
+def _read_metadata(path: Path, warn: Callable[[str], None]) -> Metadata:
+    """Parse the metadata TSV: header row mandatory and must carry ``gene``
+    (InputError METADATA_MALFORMED otherwise); ``accession``/``function``
+    are optional per-file, extra columns are ignored. Duplicate genes keep
+    the FIRST row (warn); short rows read their missing cells as empty.
+    """
+    if not path.is_file():
+        raise InputError(
+            f"metadata file not found or unreadable: {path}",
+            code="INPUT_NOT_FOUND",
+            context={"file": str(path)},
+        )
+    with path.open(encoding="utf-8", newline="") as handle:
+        rows = csv.reader(handle, delimiter="\t")
+        header = next((row for row in rows if row), None)
+        if header is None or "gene" not in header:
+            raise InputError(
+                f"metadata TSV needs a header row with a 'gene' column: {path}",
+                code="METADATA_MALFORMED",
+                context={"file": str(path), "needed": "gene"},
+            )
+        columns = {column: index for index, column in enumerate(header)}
+
+        def cell(padded: list[str], column: str) -> str:
+            return padded[columns[column]]
+
+        metadata: dict[str, tuple[str, str]] = {}
+        for row in rows:
+            if not any(value.strip() for value in row):
+                continue
+            padded = row + [""] * (len(header) - len(row))
+            gene = cell(padded, "gene")
+            if not gene:
+                continue
+            if gene in metadata:
+                warn(f"duplicate gene {gene!r} in metadata TSV: keeping the first row")
+                continue
+            metadata[gene] = (
+                cell(padded, "accession") if "accession" in columns else "",
+                cell(padded, "function") if "function" in columns else "",
+            )
+    return Metadata(columns=frozenset(columns), rows=metadata)
+
+
+def _merge(records: list[Record], metadata: Metadata, warn: Callable[[str], None]) -> list[Record]:
+    """Overwrite accession/function on records whose gene has a metadata row
+    (only the columns the TSV actually carries); genes only present in the
+    TSV are warned about and skipped."""
+    merged: list[Record] = []
+    for record in records:
+        row = metadata.rows.get(record.gene)
+        if row is None:
+            merged.append(record)
+            continue
+        update: dict[str, object] = {}
+        if "accession" in metadata.columns:
+            update["accession"] = row[0]
+        if "function" in metadata.columns:
+            update["function"] = _split_function(row[1])
+        merged.append(record.model_copy(update=update))
+    fasta_genes = {record.gene for record in records}
+    for gene in sorted(set(metadata.rows) - fasta_genes):
+        warn(f"gene {gene!r} in metadata TSV not found in FASTA: skipped")
+    return merged
+
+
+def _resolve_dbtype(flag: Dbtype | None, records: list[Record]) -> Dbtype:
+    """Explicit --dbtype wins; else the abricate mol_type heuristic over the
+    concatenated input sequences (db.mol_type, SPEC.md §2)."""
+    if flag is not None:
+        return flag
+    return mol_type("".join(record.sequence for record in records))
+
+
+def perform_build(
+    name: str,
+    fasta: Path,
+    tsv: Path | None,
+    dbtype: Dbtype | None,
+    description: str,
+    datadir: Path | None,
+    force: bool,
+    *,
+    warn: Callable[[str], None],
+    quiet: bool = True,
+) -> BuildReceipt:
+    """Run the custom-build pipeline and return the receipt — the shared CLI
+    + MCP path. Warnings go to the caller-supplied ``warn`` (CLI: stderr;
+    MCP: dropped — stderr is reserved for the protocol)."""
+
+    # Security/frozen rule: `Path(datadir) / name` REPLACES the base when name
+    # is absolute (and `..` escapes it); plain names only, all else allowed.
+    if not name or name in (".", "..") or "/" in name or "\\" in name:
+        raise UsageError(
+            f"database name must be a plain name without path separators: {name!r}",
+            code="USAGE_ERROR",
+            context={"name": name},
+        )
+    if not fasta.is_file():
+        raise InputError(
+            f"FASTA file not found or unreadable: {fasta}",
+            code="INPUT_NOT_FOUND",
+            context={"file": str(fasta)},
+        )
+    db_dir = config.ensure_datadir(datadir) / name
+    if (db_dir / "gapit-manifest.json").is_file() and not force:
+        raise DatabaseError(
+            f"won't overwrite existing database {name} (use --force)",
+            code="DB_ALREADY_EXISTS",
+            context={"db": name},
+        )
+    records = [_to_record(fasta_record, name, description) for fasta_record in iter_fasta(fasta)]
+    if tsv is not None:
+        records = _merge(records, _read_metadata(tsv, warn), warn)
+    db_dir.mkdir(parents=True, exist_ok=True)
+    write_records(records, db_dir / "records.jsonl")
+    manifest = build_database(
+        db_dir,
+        name=name,
+        dbtype=_resolve_dbtype(dbtype, records),
+        source_urls=("local",),
+        fetched_at=datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        quiet=quiet,
+    )
+    return BuildReceipt(
+        db=name,
+        records=manifest.n_records,
+        dbtype=manifest.dbtype,
+        destination=str(db_dir),
+    )

gapit/db_ops.py

@@ -0,0 +1,128 @@
+"""Database provider use-cases shared by the CLI and MCP: fetch and list.
+
+``perform_fetch`` installs provider database(s) under the datadir (bundled
+snapshot first, ``from_source`` forces upstream); the ``db_list_*``
+callables build the gapit.dblist/1 provider listing. The typer commands
+(:mod:`gapit.cmd_db`) and the MCP tools (:mod:`gapit.mcp_tools`) are thin
+callers — this module owns the behavior and imports no CLI plumbing.
+"""
+
+from collections.abc import Iterator
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from gapit import config
+from gapit.errors import UsageError
+from gapit.providers import REGISTRY
+from gapit.providers.common import Dbtype, fetch_provider
+from gapit.records import read_manifest
+
+# Bare `gapit db fetch` installs these, in order — the two providers whose
+# snapshots ship inside the wheel (Wave G: zero-network bootstrap).
+DEFAULT_DBS = ("card", "vfdb")
+
+
+class ProviderReceipt(BaseModel, frozen=True):
+    """One-line JSON success receipt for `db fetch` (no biological data)."""
+
+    db: str
+    records: int
+    dbtype: Dbtype
+    destination: str
+
+
+class DbListEntry(BaseModel, frozen=True):
+    """One provider row in the gapit.dblist/1 listing document."""
+
+    name: str
+    description: str
+    dbtype: str
+    installed: bool
+    records: int | None = None
+
+
+class DbListDocument(BaseModel, frozen=True):
+    """gapit.dblist/1 — `gapit db list --json` output.
+
+    A CLI listing, deliberately local to this module: NOT registered in
+    `gapit schema` (the public schema surface stays unchanged this wave).
+    """
+
+    model_config = ConfigDict(populate_by_name=True)
+
+    schema_name: Literal["gapit.dblist/1"] = Field(default="gapit.dblist/1", alias="schema")
+    providers: tuple[DbListEntry, ...]
+
+
+def perform_fetch(
+    name: str | None,
+    datadir: Path | None,
+    *,
+    force: bool = False,
+    from_source: bool = False,
+    quiet: bool = True,
+    debug: bool = False,
+) -> Iterator[ProviderReceipt]:
+    """Fetch provider database(s) into <datadir>/NAME — the shared CLI + MCP
+    path. NAME None installs every database in DEFAULT_DBS order; each
+    receipt yields as its install completes (streaming, like the CLI's
+    per-db stdout lines).
+    """
+
+    def fetch_one(provider_name: str) -> ProviderReceipt:
+        provider = REGISTRY.get(provider_name)
+        if provider is None:
+            raise UsageError(
+                f"unknown provider: {provider_name} (available: {', '.join(sorted(REGISTRY))})",
+                code="USAGE_ERROR",
+                context={"provider": provider_name},
+            )
+        db_dir = config.ensure_datadir(datadir) / provider_name
+        manifest = fetch_provider(
+            provider,
+            db_dir,
+            fetched_at=datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
+            force=force,
+            quiet=quiet,
+            from_source=from_source,
+            debug=debug,
+        )
+        return ProviderReceipt(
+            db=provider_name,
+            records=manifest.n_records,
+            dbtype=manifest.dbtype,
+            destination=str(db_dir),
+        )
+
+    for provider_name in (name,) if name is not None else DEFAULT_DBS:
+        yield fetch_one(provider_name)
+
+
+def db_list_entries(root: Path) -> list[DbListEntry]:
+    """Provider rows for the gapit.dblist/1 listing — shared CLI + MCP path."""
+    entries: list[DbListEntry] = []
+    for provider_name in sorted(REGISTRY):
+        provider = REGISTRY[provider_name]
+        manifest_path = root / provider_name / "gapit-manifest.json"
+        installed = manifest_path.is_file()
+        entries.append(
+            DbListEntry(
+                name=provider_name,
+                description=provider.description,
+                dbtype=provider.dbtype,
+                installed=installed,
+                records=read_manifest(manifest_path).n_records if installed else None,
+            )
+        )
+    return entries
+
+
+def db_list_json(root: Path) -> str:
+    """The gapit.dblist/1 document JSON — shared CLI + MCP path."""
+    entries = db_list_entries(root)
+    return DbListDocument(providers=tuple(entries)).model_dump_json(
+        indent=2, by_alias=True, exclude_none=True
+    )

gapit/db_query_ops.py

@@ -0,0 +1,252 @@
+"""Read-only queries over installed databases, shared by the CLI and MCP.
+
+- ``perform_search``: case-insensitive lookup across the ``records.jsonl``
+  truth stores of every installed database (SPEC.md §11).
+- ``perform_outdated``: installed-database staleness report against a
+  threshold (``days``) and the bundled snapshot dates. Staleness is a
+  REPORT, never an error state.
+
+Both are read-only: no builds, no network, no datadir writes. The typer
+commands (:mod:`gapit.cmd_db_search`, :mod:`gapit.cmd_db_outdated`) and the
+MCP tools (:mod:`gapit.mcp_tools`) are thin callers.
+"""
+
+import enum
+from collections.abc import Callable, Iterable
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from gapit import config
+from gapit.errors import DatabaseError, InputError, UsageError
+from gapit.proctools import note
+from gapit.providers import REGISTRY
+from gapit.providers.common import bundled_snapshot_manifest
+from gapit.records import Record, installed_db_dirs, read_manifest, read_records
+
+DEFAULT_LIMIT = 100
+DEFAULT_STALE_DAYS = 90
+
+
+class SearchField(enum.Enum):
+    """Record fields `db search` can match against."""
+
+    gene = "gene"
+    accession = "accession"
+    function = "function"
+    product = "product"
+    any = "any"
+
+
+class SearchEntry(BaseModel, frozen=True):
+    """One search hit: the JSONL line (and the TSV row's source), a CLI
+    listing shape — not a registered schema."""
+
+    db: str
+    gene: str
+    accession: str
+    function: tuple[str, ...]
+    product: str
+    length: int
+
+
+def _candidates(record: Record, field: SearchField) -> tuple[str, ...]:
+    """The string values of the chosen field (function contributes one per class)."""
+    match field:
+        case SearchField.gene:
+            return (record.gene,)
+        case SearchField.accession:
+            return (record.accession,)
+        case SearchField.function:
+            return record.function
+        case SearchField.product:
+            return (record.product,)
+        case SearchField.any:
+            return (record.gene, record.accession, *record.function, record.product)
+
+
+def _matches(record: Record, needle: str, field: SearchField, exact: bool) -> bool:
+    """Case-insensitive substring by default; --exact is full-field equality."""
+    values = _candidates(record, field)
+    if exact:
+        return any(value.lower() == needle for value in values)
+    return any(needle in value.lower() for value in values)
+
+
+def search_tsv_row(record: Record) -> str:
+    """The hit row: DB\tGENE\tACCESSION\tFUNCTION\tPRODUCT\tLENGTH."""
+    return (
+        f"{record.db}\t{record.gene}\t{record.accession}\t"
+        f"{';'.join(record.function)}\t{record.product}\t{len(record.sequence)}"
+    )
+
+
+def search_json_line(record: Record) -> str:
+    """The hit as one JSONL line (same fields as the TSV row, snake_case)."""
+    return SearchEntry(
+        db=record.db,
+        gene=record.gene,
+        accession=record.accession,
+        function=record.function,
+        product=record.product,
+        length=len(record.sequence),
+    ).model_dump_json()
+
+
+def perform_search(
+    term: str,
+    datadir: Path | None,
+    *,
+    db: str | None = None,
+    field: SearchField = SearchField.any,
+    exact: bool = False,
+    limit: int = DEFAULT_LIMIT,
+    render: Callable[[Record], str] = search_tsv_row,
+    quiet: bool = True,
+) -> tuple[list[str], int]:
+    """Scan the installed records.jsonl stores — the shared CLI + MCP path.
+
+    Returns ``(hit lines up to limit, total matching count)``; the caller
+    owns the truncation note (a stderr diagnostic) and the printing.
+    """
+    root = config.resolve_datadir(datadir)
+    db_dirs = installed_db_dirs(root)
+    if db is not None and db not in {db_dir.name for db_dir in db_dirs}:
+        installed = " ".join(sorted(db_dir.name for db_dir in db_dirs))
+        raise UsageError(
+            f"unknown database: {db} (installed: {installed or 'none'})",
+            code="USAGE_ERROR",
+            context={"db": db, "installed": installed},
+        )
+    needle = term.lower()
+    hits: list[str] = []
+    total = 0
+    for db_dir in db_dirs:
+        if db is not None and db_dir.name != db:
+            continue
+        records_path = db_dir / "records.jsonl"
+        if not records_path.is_file():
+            if db is not None:
+                raise DatabaseError(
+                    f"database {db_dir.name} is incomplete: records.jsonl missing",
+                    code="DB_INCOMPLETE",
+                    context={"db": db_dir.name},
+                )
+            note(quiet, f"skipping {db_dir.name}: no records.jsonl (incomplete database)")
+            continue
+        for record in read_records(records_path):
+            if _matches(record, needle, field, exact):
+                total += 1
+                if limit == 0 or len(hits) < limit:
+                    hits.append(render(record))
+    return hits, total
+
+
+class DbOutdatedEntry(BaseModel, frozen=True):
+    """One installed-database row in the gapit.dboutdated/1 report."""
+
+    db: str
+    fetched_at: str
+    age_days: float
+    status: Literal["ok", "stale", "snapshot-update", "stale+snapshot-update"]
+    upstream_version: str
+
+
+class DbOutdatedDocument(BaseModel, frozen=True):
+    """gapit.dboutdated/1 — `gapit db outdated --json` output.
+
+    A CLI listing, deliberately local to this module: NOT registered in
+    `gapit schema` (mirrors the gapit.dblist/1 precedent).
+    """
+
+    model_config = ConfigDict(populate_by_name=True)
+
+    schema_name: Literal["gapit.dboutdated/1"] = Field(default="gapit.dboutdated/1", alias="schema")
+    databases: tuple[DbOutdatedEntry, ...]
+
+
+def _utc_timestamp(value: str, where: str) -> datetime:
+    """Parse a manifest fetched_at into aware UTC; anything else is a
+    malformed manifest (trusted metadata: writers emit ISO-8601 with Z)."""
+    try:
+        parsed = datetime.fromisoformat(value)
+    except ValueError as exc:
+        raise InputError(
+            f"malformed manifest {where}: invalid fetched_at: {value}",
+            code="MANIFEST_MALFORMED",
+            context={"file": where},
+        ) from exc
+    if parsed.tzinfo is None:
+        raise InputError(
+            f"malformed manifest {where}: fetched_at has no UTC offset: {value}",
+            code="MANIFEST_MALFORMED",
+            context={"file": where},
+        )
+    return parsed.astimezone(UTC)
+
+
+def _snapshot_newer(name: str, fetched: datetime) -> bool:
+    """True when the provider's bundled snapshot manifest is newer than the
+    installed ``fetched`` (unknown provider / no bundle -> False)."""
+    provider = REGISTRY.get(name)
+    if provider is None:
+        return False
+    archived = bundled_snapshot_manifest(provider)
+    return archived is not None and fetched < _utc_timestamp(
+        archived.fetched_at, f"bundled snapshot of {name}"
+    )
+
+
+def perform_outdated(
+    datadir: Path | None, *, days: int = DEFAULT_STALE_DAYS
+) -> list[DbOutdatedEntry]:
+    """Compute the staleness report entries — the shared CLI + MCP path.
+
+    A database is `stale` past ``days`` and `snapshot-update` when its
+    provider's bundled snapshot is newer than the installed copy.
+    """
+    root = config.resolve_datadir(datadir)
+    db_dirs = installed_db_dirs(root)
+    if not db_dirs:
+        raise DatabaseError(
+            f"no installed databases in datadir: {root}",
+            code="DATADIR_EMPTY",
+            context={"datadir": str(root)},
+        )
+    now = datetime.now(UTC)
+    entries: list[DbOutdatedEntry] = []
+    for db_dir in db_dirs:
+        manifest_path = db_dir / "gapit-manifest.json"
+        manifest = read_manifest(manifest_path)
+        fetched = _utc_timestamp(manifest.fetched_at, str(manifest_path))
+        age = (now - fetched).total_seconds() / 86400
+        match (age > days, _snapshot_newer(db_dir.name, fetched)):
+            case (True, True):
+                status = "stale+snapshot-update"
+            case (True, False):
+                status = "stale"
+            case (False, True):
+                status = "snapshot-update"
+            case (False, False):
+                status = "ok"
+        entries.append(
+            DbOutdatedEntry(
+                db=db_dir.name,
+                fetched_at=manifest.fetched_at,
+                age_days=round(age, 2),
+                status=status,
+                upstream_version=manifest.upstream_version,
+            )
+        )
+    return entries
+
+
+def outdated_tsv_lines(entries: Iterable[DbOutdatedEntry]) -> list[str]:
+    """The NAME/FETCHED_AT/AGE_DAYS/STATUS table lines (CLI stdout = MCP text)."""
+    lines = ["NAME\tFETCHED_AT\tAGE_DAYS\tSTATUS"]
+    lines.extend(
+        f"{entry.db}\t{entry.fetched_at}\t{entry.age_days:.2f}\t{entry.status}" for entry in entries
+    )
+    return lines