gapit 0.2.2__py3-none-any.whl

gapit/dbbuild.py

@@ -0,0 +1,207 @@
+"""Wave A3 build pipeline: records.jsonl -> a fully built gapit-native database.
+
+``build_database`` turns a directory's truth source (``records.jsonl``) into
+the three artifacts consumers rely on — the ``sequences`` FASTA projection,
+its BLAST index, and the ``gapit-manifest.json`` provenance sidecar written
+LAST (it certifies the artifacts). Every step is deterministic, atomic where
+it matters, and self-verifying: the generated headers must decode back
+through :mod:`gapit.dbcodec` before anything is indexed. No minimap2 index
+is persisted: reads mode indexes the ``sequences`` FASTA in memory only
+(SPEC.md §10).
+"""
+
+import hashlib
+import os
+from collections.abc import Sequence
+from pathlib import Path
+from tempfile import NamedTemporaryFile
+from typing import Literal
+
+from gapit.db import DbHeader, make_blast_db
+from gapit.dbcodec import decode_seqid, encode_seqid
+from gapit.errors import DatabaseError
+from gapit.fasta import iter_fasta
+from gapit.proctools import note, run_tool
+from gapit.records import Manifest, Record, count_records, read_records, write_manifest
+
+_WRAP_COLUMNS = 60
+_CHUNK_BYTES = 1 << 20
+
+
+def generate_sequences(records_path: Path, sequences_path: Path) -> None:
+    """Stream ``records.jsonl`` into the ``sequences`` FASTA projection.
+
+    One header line per record (``encode_seqid(...)`` + space + product),
+    sequence wrapped at 60 columns, LF endings, UTF-8. Bytes land in a temp
+    file in the target directory and are ``os.replace``d only after the whole
+    file was written, so a failure never leaves a partial ``sequences``.
+
+    Raises DatabaseError ``BUILD_INVALID`` (exit 4) for a record whose product
+    contains a line break or whose sequence is empty, and for a records file
+    with zero records (makeblastdb cannot index nothing); the per-record
+    errors carry the record's gene in context. Duplicate ``(db, gene)`` pairs
+    are allowed and preserved — upstream databases contain them and
+    records.jsonl order is the truth.
+    """
+    temp_path: Path | None = None
+    n_written = 0
+    try:
+        with NamedTemporaryFile(
+            mode="w",
+            encoding="utf-8",
+            newline="\n",
+            dir=sequences_path.parent,
+            prefix=f".{sequences_path.name}.",
+            suffix=".tmp",
+            delete=False,
+        ) as temp:
+            temp_path = Path(temp.name)
+            for record in read_records(records_path):
+                _check_record(record)
+                seqid = encode_seqid(record.db, record.gene, record.accession, record.function)
+                temp.write(f">{seqid} {record.product}\n")
+                for offset in range(0, len(record.sequence), _WRAP_COLUMNS):
+                    temp.write(f"{record.sequence[offset : offset + _WRAP_COLUMNS]}\n")
+                n_written += 1
+        if n_written == 0:
+            raise DatabaseError(
+                f"cannot build {sequences_path}: {records_path} contains no records",
+                code="BUILD_INVALID",
+                context={"file": str(records_path)},
+            )
+        assert temp_path is not None
+        os.replace(temp_path, sequences_path)
+    finally:
+        if temp_path is not None:
+            temp_path.unlink(missing_ok=True)
+
+
+def _check_record(record: Record) -> None:
+    """Generation-time validation: the two record shapes that would produce a
+    broken FASTA (headers must stay one line; empty sequences cannot be
+    indexed)."""
+    if "\n" in record.product or "\r" in record.product:
+        raise DatabaseError(
+            f"record {record.gene!r}: product contains a line break",
+            code="BUILD_INVALID",
+            context={"gene": record.gene},
+        )
+    if not record.sequence:
+        raise DatabaseError(
+            f"record {record.gene!r}: empty sequence",
+            code="BUILD_INVALID",
+            context={"gene": record.gene},
+        )
+
+
+def _self_check_failed(gene: str, reason: str) -> DatabaseError:
+    """The uniform self-check failure: gene located, machine-stable reason."""
+    return DatabaseError(
+        f"self-check failed at gene {gene!r} ({reason})",
+        code="BUILD_SELF_CHECK_FAILED",
+        context={"gene": gene, "reason": reason},
+    )
+
+
+def verify_sequences(sequences_path: Path, records_path: Path, name: str) -> None:
+    """Self-check: every generated FASTA record decodes back to its record.
+
+    Streams ``sequences`` and ``records.jsonl`` in parallel; the header (via
+    :func:`gapit.dbcodec.decode_seqid` with ``default_db=name``) and the
+    sequence must match record-for-record, in order. The FASTA description
+    (product) is deliberately not compared: the reader strips leading and
+    trailing whitespace by design, so it is a display field, not a truth
+    field. Any decode failure, field mismatch, or count mismatch raises
+    ``BUILD_SELF_CHECK_FAILED`` (a structurally invalid FASTA surfaces as its
+    native InputError instead — either way the build stops before indexing).
+    """
+    fasta_stream = iter_fasta(sequences_path)
+    for record in read_records(records_path):
+        fasta = next(fasta_stream, None)
+        if fasta is None:
+            raise _self_check_failed(record.gene, "missing_fasta_record")
+        try:
+            header = decode_seqid(fasta.id, default_db=name)
+        except DatabaseError as exc:
+            raise _self_check_failed(record.gene, f"decode:{exc.code}") from exc
+        expected = DbHeader(
+            database=record.db or name,
+            gene=record.gene,
+            accession=record.accession,
+            function=";".join(record.function),
+        )
+        if header != expected:
+            raise _self_check_failed(record.gene, "header_mismatch")
+        if fasta.sequence != record.sequence:
+            raise _self_check_failed(record.gene, "sequence_mismatch")
+    if next(fasta_stream, None) is not None:
+        raise _self_check_failed("", "extra_fasta_record")
+
+
+def _sha256(path: Path) -> str:
+    """Streaming SHA256 of a file's bytes (1 MiB chunks)."""
+    hasher = hashlib.sha256()
+    with path.open("rb") as handle:
+        while chunk := handle.read(_CHUNK_BYTES):
+            hasher.update(chunk)
+    return hasher.hexdigest()
+
+
+def _version_line(argv: list[str]) -> str:
+    """First line of a version command's stdout ('' when it printed nothing)."""
+    stdout = run_tool(argv).stdout
+    return stdout.splitlines()[0].strip() if stdout else ""
+
+
+def build_database(
+    db_dir: Path,
+    *,
+    name: str,
+    dbtype: Literal["nucl", "prot"],
+    source_urls: Sequence[str],
+    fetched_at: str,
+    upstream_version: str = "",
+    quiet: bool = True,
+    debug: bool = False,
+) -> Manifest:
+    """Build every gapit-native artifact in ``db_dir`` from its records.jsonl.
+
+    Pipeline — each step runs only after the previous one succeeded, and the
+    manifest is written LAST (it certifies the artifacts):
+
+    1. ``generate_sequences`` -> ``db_dir/sequences``
+    2. self-check: every header decodes back (``BUILD_SELF_CHECK_FAILED``)
+    3. streaming SHA256 of ``sequences``
+    4. ``makeblastdb`` with the EXPLICIT ``dbtype`` — the manifest declares
+       the type, so the mol_type heuristic is skipped
+    5. count records and capture ``blastn -version`` / ``minimap2 --version``
+    6. write ``db_dir/gapit-manifest.json`` and return the Manifest
+
+    No ``.mmi`` is built for either dbtype: reads mode indexes the FASTA in
+    memory with the invocation preset's own parameters (minimap2 is
+    nucleotide-only, which is why prot databases never participated anyway).
+    """
+    records_path = db_dir / "records.jsonl"
+    sequences_path = db_dir / "sequences"
+    generate_sequences(records_path, sequences_path)
+    note(quiet, f"generated {sequences_path}")
+    verify_sequences(sequences_path, records_path, name)
+    note(quiet, f"self-check passed for {name}")
+    sha256 = _sha256(sequences_path)
+    make_blast_db(sequences_path, name, dbtype=dbtype, debug=debug)
+    note(quiet, f"BLAST index built ({dbtype})")
+    manifest = Manifest(
+        name=name,
+        source_urls=tuple(source_urls),
+        fetched_at=fetched_at,
+        sha256=sha256,
+        n_records=count_records(records_path),
+        dbtype=dbtype,
+        upstream_version=upstream_version,
+        makeblastdb_version=_version_line(["blastn", "-version"]),
+        # Kept although no .mmi is built: environment provenance for the
+        # machine that produced the artifacts (spec'd in gapit.manifest/1).
+        minimap2_version=_version_line(["minimap2", "--version"]),
+    )
+    write_manifest(manifest, db_dir / "gapit-manifest.json")
+    return manifest

gapit/dbcodec.py

@@ -0,0 +1,117 @@
+"""gapit/v1 database header codec: tagged, percent-encoded sequence ids.
+
+Native header format (Phase 7): ``gapit|db=<v>|gene=<v>|acc=<v>|func=<v>`` with
+fixed key order and lowercase-hex percent escapes inside values. Encode is
+total for any input; decode is strict (``DatabaseError HEADER_MALFORMED``).
+Seqids without the ``gapit|`` prefix delegate to the legacy ``~~~`` rules in
+``gapit.db`` (SPEC.md §4 step 5).
+
+The fourth slot is named ``func`` because gapit answers presence/absence — the
+values it carries are functional categories (AMR classes, virulence, O-antigen,
+replicon, biocide), not only resistance (Wave F1 rename; outputs keep the
+frozen ``RESISTANCE``/``resistance`` names).
+"""
+
+import re
+from collections.abc import Sequence
+
+from gapit.db import DbHeader, parse_db_header
+from gapit.errors import DatabaseError
+
+PREFIX = "gapit|"
+
+# Values-only escape alphabet (probe-verified byte-identical through
+# makeblastdb/blastn/minimap2); every other char travels verbatim.
+_ENCODE_TABLE = str.maketrans(
+    {
+        "%": "%25",
+        "|": "%7C",
+        "=": "%3D",
+        ";": "%3B",
+        " ": "%20",
+        "\t": "%09",
+        "\n": "%0A",
+        "\r": "%0D",
+    }
+)
+
+_ESCAPE_RE = re.compile("%([0-9a-fA-F]{2})")
+_BAD_ESCAPE_RE = re.compile(r"%(?![0-9a-fA-F]{2})")
+
+
+def is_gapit_header(seqid: str) -> bool:
+    """True iff the seqid carries the native ``gapit|`` tagged prefix."""
+    return seqid.startswith(PREFIX)
+
+
+def encode_seqid(db: str, gene: str, accession: str, function: Sequence[str]) -> str:
+    """Render one record as a native gapit/v1 seqid (total: never raises).
+
+    The function list is joined with ``;`` first, then encoded as one value,
+    so it decodes back to the abricate display string ``a;b``.
+    """
+    return "|".join(
+        (
+            "gapit",
+            f"db={db.translate(_ENCODE_TABLE)}",
+            f"gene={gene.translate(_ENCODE_TABLE)}",
+            f"acc={accession.translate(_ENCODE_TABLE)}",
+            f"func={';'.join(function).translate(_ENCODE_TABLE)}",
+        )
+    )
+
+
+def decode_seqid(seqid: str, default_db: str) -> DbHeader:
+    """Parse a seqid into a ``DbHeader``.
+
+    ``gapit|``-prefixed seqids use the strict native format: bad escapes,
+    missing/empty gene, a missing ``db``/``acc`` KEY, and duplicate keys raise
+    ``HEADER_MALFORMED``; unknown keys are skipped (forward compat); an empty
+    ``db`` value falls back to ``default_db`` (symmetric with legacy) and an
+    empty ``acc`` value is allowed (unpublished sequences); the ``func`` key is
+    intentionally optional and defaults to ``""``. Anything else delegates to
+    ``gapit.db.parse_db_header``.
+    """
+    if not is_gapit_header(seqid):
+        return parse_db_header(seqid, default_db)
+    fields: dict[str, str] = {}
+    for segment in seqid.split("|")[1:]:
+        key, sep, raw_value = segment.partition("=")
+        if not sep:
+            raise _malformed(seqid, "segment_without_key")
+        if not key:
+            raise _malformed(seqid, "empty_key")
+        if key in fields:
+            raise _malformed(seqid, f"duplicate_key:{key}")
+        fields[key] = _decode_value(raw_value, key=key, seqid=seqid)
+    gene = fields.get("gene", "")
+    if not gene:
+        raise _malformed(seqid, "missing_gene")
+    if "db" not in fields:
+        raise _malformed(seqid, "missing_db")
+    if "acc" not in fields:
+        raise _malformed(seqid, "missing_acc")
+    return DbHeader(
+        database=fields["db"] or default_db,
+        gene=gene,
+        accession=fields["acc"],
+        # func was encoded as one ;-joined value; the decoded string already is
+        # the abricate display form "a;b".
+        function=fields.get("func", ""),
+    )
+
+
+def _decode_value(raw: str, *, key: str, seqid: str) -> str:
+    """Percent-decode one raw value; every ``%`` must precede two hex digits."""
+    if _BAD_ESCAPE_RE.search(raw):
+        raise _malformed(seqid, f"invalid_percent_escape:{key}")
+    return _ESCAPE_RE.sub(lambda match: chr(int(match.group(1), 16)), raw)
+
+
+def _malformed(seqid: str, reason: str) -> DatabaseError:
+    """Build the strict-decode failure (exit 4) with seqid + reason context."""
+    return DatabaseError(
+        f"malformed gapit/v1 sequence header: {reason}",
+        code="HEADER_MALFORMED",
+        context={"seqid": seqid, "reason": reason},
+    )

gapit/dispatch.py

@@ -0,0 +1,31 @@
+"""Shared CLI plumbing: error-envelope dispatch and the ``--datadir`` option.
+
+Imported by every command module (cli.py, cmd_*.py); imports nothing from
+gapit except :mod:`gapit.errors`, so it can never participate in a cycle.
+"""
+
+from collections.abc import Callable
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from gapit.errors import GapitError, render_error
+
+Datadir = Annotated[
+    Path | None,
+    typer.Option(
+        "--datadir",
+        help="Database directory (default: $GAPIT_DATADIR, then ~/.local/share/gapit/db).",
+    ),
+]
+
+
+def dispatch(action: Callable[[], None]) -> None:
+    """Run a command body; any failure renders the gapit.error/1 envelope on
+    stderr and exits with the documented code (UNEXPECTED/1 for non-GapitError)."""
+    try:
+        action()
+    except Exception as exc:
+        typer.echo(render_error(exc), err=True)
+        raise typer.Exit(code=exc.exit_code if isinstance(exc, GapitError) else 1) from exc

gapit/errors.py

@@ -0,0 +1,87 @@
+"""Typed errors, shared raise-helpers, and the JSON error envelope
+(gapit.error/1).
+
+Exit-code contract (AGENTS.md §5): 2 usage, 3 missing dependency, 4 db error,
+5 input error, 1 unexpected. Failures render as a one-line JSON envelope on
+stderr via ``render_error``.
+"""
+
+from pathlib import Path
+from typing import Literal, NoReturn
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class GapitError(Exception):
+    """Base class for all typed gapit errors."""
+
+    code: str
+    exit_code: int = 1
+
+    def __init__(self, message: str, *, code: str, context: dict[str, str] | None = None) -> None:
+        super().__init__(message)
+        self.code = code
+        self.context: dict[str, str] = context if context is not None else {}
+
+
+class UsageError(GapitError):
+    """Invalid command-line usage."""
+
+    exit_code = 2
+
+
+class DependencyError(GapitError):
+    """An external binary (BLAST+, minimap2) is missing from PATH."""
+
+    exit_code = 3
+
+
+class DatabaseError(GapitError):
+    """A datadir/database is missing, unindexed, or failed to build."""
+
+    exit_code = 4
+
+
+class InputError(GapitError):
+    """User-supplied input is malformed."""
+
+    exit_code = 5
+
+
+def usage_fail(message: str) -> NoReturn:
+    """Raise a usage error (gapit.error/1 envelope, exit 2)."""
+    raise UsageError(message, code="USAGE_ERROR")
+
+
+def ensure_input_file(path: Path, what: str = "input file") -> None:
+    """Raise INPUT_NOT_FOUND for a missing/unreadable input path; ``what``
+    names the kind in the message (e.g. "reads file")."""
+    if not path.is_file():
+        raise InputError(
+            f"{what} not found or unreadable: {path}",
+            code="INPUT_NOT_FOUND",
+            context={"file": str(path)},
+        )
+
+
+class ErrorEnvelope(BaseModel, frozen=True):
+    """The gapit.error/1 stderr envelope."""
+
+    model_config = ConfigDict(populate_by_name=True)
+
+    schema_name: Literal["gapit.error/1"] = Field(default="gapit.error/1", alias="schema")
+    code: str
+    message: str
+    context: dict[str, str]
+
+
+def render_error(exc: BaseException) -> str:
+    """One-line compact JSON envelope for any failure (non-GapitError becomes
+    UNEXPECTED, exit 1)."""
+    if isinstance(exc, GapitError):
+        envelope = ErrorEnvelope(code=exc.code, message=str(exc), context=exc.context)
+    else:
+        envelope = ErrorEnvelope(
+            code="UNEXPECTED", message=f"{type(exc).__name__}: {exc}", context={}
+        )
+    return envelope.model_dump_json(by_alias=True)

gapit/fasta.py

@@ -0,0 +1,123 @@
+"""Streaming FASTA reader (plain, gzip, bzip2)."""
+
+import bz2
+import gzip
+from collections.abc import Iterator
+from pathlib import Path
+from typing import IO
+
+from pydantic import BaseModel
+
+from gapit.errors import InputError
+
+
+class FastaRecord(BaseModel, frozen=True):
+    """One FASTA record: ``id`` is the first whitespace-delimited header token."""
+
+    id: str
+    description: str
+    sequence: str
+
+
+def open_text(path: Path) -> IO[str]:
+    """Open a sequence file as UTF-8 text, transparently decompressing .gz/.bz2.
+
+    Shared by the FASTA readers and seqconvert's normalizer; the suffix
+    (not content) picks the decompressor, matching the any2fasta CLI contract.
+    """
+    if path.name.endswith(".gz"):
+        return gzip.open(path, "rt", encoding="utf-8")
+    if path.name.endswith(".bz2"):
+        return bz2.open(path, "rt", encoding="utf-8")
+    return path.open("rt", encoding="utf-8")
+
+
+def _finalize(path: Path, pending: tuple[str, str, list[str]]) -> FastaRecord:
+    """Turn accumulated (id, description, sequence lines) into a FastaRecord."""
+    seqid, description, chunks = pending
+    if not chunks:
+        raise InputError(
+            f"{path}: record {seqid!r} has an empty sequence",
+            code="INVALID_FASTA",
+            context={"file": str(path)},
+        )
+    return FastaRecord(id=seqid, description=description, sequence="".join(chunks))
+
+
+def iter_fasta(path: Path) -> Iterator[FastaRecord]:
+    """Stream FastaRecords from a FASTA file (plain/.gz/.bz2, UTF-8).
+
+    Raises InputError on content before the first ``>`` header, on a record
+    with an empty sequence, or on an unreadable/truncated stream (a .gz/.bz2
+    cut short raises EOFError mid-read, not an OSError); an empty file yields
+    zero records.
+    """
+    try:
+        with open_text(path) as handle:
+            # pending = (id, description, sequence lines) of the record being read
+            pending: tuple[str, str, list[str]] | None = None
+            for lineno, raw_line in enumerate(handle, start=1):
+                line = raw_line.strip()
+                if not line:
+                    continue
+                if not line.startswith(">"):
+                    if pending is None:
+                        raise InputError(
+                            f"{path}: content before first '>' header at line {lineno}",
+                            code="INVALID_FASTA",
+                            context={"file": str(path)},
+                        )
+                    pending[2].append(line)
+                    continue
+                if pending is not None:
+                    yield _finalize(path, pending)
+                parts = line[1:].strip().split(maxsplit=1)
+                pending = (
+                    parts[0] if parts else "",
+                    parts[1] if len(parts) > 1 else "",
+                    [],
+                )
+            if pending is not None:
+                yield _finalize(path, pending)
+    except (OSError, UnicodeDecodeError, EOFError) as exc:
+        raise InputError(
+            f"{path}: could not read input: {exc}",
+            code="INVALID_FASTA",
+            context={"file": str(path)},
+        ) from exc
+
+
+def iter_fasta_headers(path: Path) -> Iterator[tuple[str, str]]:
+    """Stream (id, description) pairs from FASTA headers only.
+
+    Sequence lines are skipped, not accumulated — the cheap iterator for
+    header-only consumers (e.g. product lookups on a large db). Unlike
+    iter_fasta, the empty-sequence InputError does NOT apply: no sequence
+    check is performed because sequences are never read. Content before the
+    first ``>`` header and an unreadable/truncated stream still raise
+    InputError; header parsing (id is the first whitespace token, description
+    the rest) matches iter_fasta; an empty file yields nothing.
+    """
+    try:
+        with open_text(path) as handle:
+            seen_header = False
+            for lineno, raw_line in enumerate(handle, start=1):
+                line = raw_line.strip()
+                if line.startswith(">"):
+                    seen_header = True
+                    parts = line[1:].strip().split(maxsplit=1)
+                    yield (parts[0] if parts else "", parts[1] if len(parts) > 1 else "")
+                elif not line or seen_header:
+                    continue
+                else:
+                    raise InputError(
+                        f"{path}: content before first '>' header at line {lineno}",
+                        code="INVALID_FASTA",
+                        context={"file": str(path)},
+                    )
+    except (OSError, UnicodeDecodeError, EOFError) as exc:
+        raise InputError(
+            f"{path}: could not read input: {exc}",
+            code="INVALID_FASTA",
+            context={"file": str(path)},
+        ) from exc

gapit/formats/__init__.py

@@ -0,0 +1 @@
+"""Output formatters for gapit reports (TSV/CSV today; JSON and Markdown in Phase 4)."""