gapit 0.2.2__py3-none-any.whl

gapit/providers/resfinder.py

@@ -0,0 +1,123 @@
+"""resfinder database provider — CGE ResFinder acquired resistance genes.
+
+Upstream ``get_resfinder`` (abricate-get_db 1.4.0) git-clones the resfinder_db
+repo; gapit downloads the same tree as the bitbucket ``HEAD.zip`` archive (no
+git dependency) whose members sit under an arbitrary top-level directory.
+``phenotypes.txt`` maps each gene to its function categories; every ``*.fsa``
+carries ids like ``demoA_1_FAKE0001`` — gene prefix, copy number, accession.
+
+Issue #62 repair: resfinder .fsa files can glue a record header onto the end
+of the previous sequence line (a letter immediately followed by ``>``); a
+newline is inserted there in the raw text before parsing, exactly like
+upstream's ``sed s/([A-Z])>/\\1\\n>/gi`` (the /i makes it both cases).
+
+Perl semantics kept: Class-cell pieces are filtered of unknown/notes/none
+markers BEFORE stripping (upstream greps, then trims), the LAST phenotypes
+row for a gene wins (plain hash assign), and the product is always the gene
+prefix (upstream's ``$anno{$id}{DESC}`` is never assigned). Deviation: a
+record whose id has no ``_<digits>_<accession>`` structure is SKIPPED —
+upstream would emit it with undef fields (same policy as vfdb).
+"""
+
+import re
+import zipfile
+from collections.abc import Iterator
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from gapit.errors import DatabaseError
+from gapit.fasta import iter_fasta
+from gapit.providers.common import Provider
+from gapit.records import Record
+
+NAME = "resfinder"
+ARCHIVE = "HEAD.zip"
+
+_PHENOTYPES = "phenotypes.txt"
+_GENE = re.compile(r"^(.*?)_\w+$")  # col0 minus its final _<word> chunk
+_CLASS_SPLIT = re.compile(r",\s*")
+_CLASS_FILTER = re.compile(r"(unknown|notes|^none)", re.IGNORECASE)
+# abricate issue #62: a letter directly followed by '>' is a glued header.
+_INLINE_HEADER = re.compile(r"([A-Za-z])>")
+_ID = re.compile(r"^(.*?)_(\d+)_(\S+)$")  # base _ copy _ accession
+
+
+def _read_classes(root: Path) -> dict[str, tuple[str, ...]]:
+    """phenotypes.txt under the extracted archive: {gene prefix: classes}.
+
+    Comment lines start with '#'; rows split on tabs; the gene prefix is
+    col0 minus its final ``_<word>`` chunk; classes are the col2 pieces
+    (split on comma + whitespace) that do not match unknown/notes/none —
+    filtered BEFORE stripping, like upstream — each stripped. A later row
+    for the same gene overwrites the earlier one (plain hash assign), and a
+    missing Class cell means no classes (upstream splits undef into an
+    empty list).
+    """
+    found = sorted(root.glob(f"**/{_PHENOTYPES}"))
+    if not found:
+        raise DatabaseError(
+            f"{ARCHIVE} contains no {_PHENOTYPES}",
+            code="PROVIDER_INVALID",
+            context={"db": NAME},
+        )
+    classes: dict[str, tuple[str, ...]] = {}
+    for line in found[0].read_text(encoding="utf-8").splitlines():
+        if line.startswith("#"):
+            continue
+        row = line.split("\t")
+        gene = _GENE.match(row[0])
+        if gene is None:
+            continue
+        cell = row[2] if len(row) > 2 else ""
+        classes[gene.group(1)] = tuple(
+            piece.strip()
+            for piece in (_CLASS_SPLIT.split(cell) if cell else [])
+            if not _CLASS_FILTER.search(piece)
+        )
+    return classes
+
+
+def transform(workdir: Path) -> Iterator[Record]:
+    """Yield Records from ``workdir/HEAD.zip`` (the bitbucket archive).
+
+    The archive is extracted into a scratch directory (members sit under an
+    arbitrary top-level directory); each ``*.fsa`` is repaired for glued
+    headers into a scratch copy before FASTA parsing, so the download is
+    never modified. Files are visited in sorted order for determinism.
+    """
+    with (
+        TemporaryDirectory(prefix=".resfinder-extract.") as extract_name,
+        TemporaryDirectory(prefix=".resfinder-repair.") as repair_name,
+    ):
+        extract_dir = Path(extract_name)
+        with zipfile.ZipFile(workdir / ARCHIVE) as archive:
+            archive.extractall(extract_dir)
+        classes = _read_classes(extract_dir)
+        for index, fsa in enumerate(sorted(extract_dir.glob("**/*.fsa"))):
+            repaired = _INLINE_HEADER.sub(r"\1\n>", fsa.read_text(encoding="utf-8"))
+            temp = Path(repair_name) / f"{index:06d}.fsa"
+            temp.write_text(repaired, encoding="utf-8")
+            for fasta in iter_fasta(temp):
+                id_match = _ID.match(fasta.id)
+                if id_match is None:
+                    continue
+                base = id_match.group(1)
+                yield Record(
+                    db=NAME,
+                    gene=f"{base}_{id_match.group(2)}",
+                    sequence=fasta.sequence,
+                    accession=id_match.group(3),
+                    function=classes.get(base, ()),
+                    product=base,
+                    source_id=fasta.id,
+                )
+
+
+PROVIDER = Provider(
+    name=NAME,
+    description="CGE ResFinder acquired resistance genes",
+    source_urls=("https://bitbucket.org/genomicepidemiology/resfinder_db/get/HEAD.zip",),
+    dbtype="nucl",
+    transform=transform,
+    snapshot=None,
+)

gapit/providers/snapshots.py

@@ -0,0 +1,119 @@
+"""Bundled database snapshots (Wave G): deterministic archives + extraction.
+
+A snapshot is ``<name>.tar.gz`` containing EXACTLY ``records.jsonl`` and
+``gapit-manifest.json`` from an installed gapit-native database directory.
+card and vfdb ship inside the wheel (``src/gapit/data/snapshots/``) so the
+two default databases install with zero network — snapshots eliminate the
+flaky-source failure modes for exactly those two (Wave E/F3: mgc.ac.cn UA
+blocks, card re-tarring, Wayback interstitials). Every other provider keeps
+the upstream fetch; a future DB is bundled by dropping a ``<name>.tar.gz``
+into the snapshots dir and setting ``snapshot=`` on its Provider (one line).
+
+``records.jsonl`` (post-normalize records) is what gets snapshotted — NOT
+the BLAST index: index bytes are BLAST-version-sensitive while a
+local rebuild from records is deterministic and fast. The archived manifest
+contributes only ``upstream_version``; fetched_at, sha256 and tool versions
+are rebuilt locally by :func:`gapit.dbbuild.build_database`.
+
+Determinism (frozen layout contract): entries added in sorted name order,
+PAX format, uid/gid 0, mtime 0, and a gzip stream with mtime 0 and no
+embedded filename — two builds from the same inputs are byte-identical.
+"""
+
+import gzip
+import io
+import os
+import tarfile
+from pathlib import Path
+from tempfile import NamedTemporaryFile
+
+from gapit.errors import DatabaseError
+from gapit.records import Manifest
+
+_MEMBERS = ("gapit-manifest.json", "records.jsonl")  # sorted: manifest < records
+
+
+def _snapshot_error(db_dir: Path, missing: str) -> DatabaseError:
+    """The uniform archive-content failure: located, machine-stable."""
+    return DatabaseError(
+        f"cannot snapshot {db_dir}: missing {missing}",
+        code="SNAPSHOT_INVALID",
+        context={"db_dir": str(db_dir), "missing": missing},
+    )
+
+
+def make_snapshot(db_dir: Path, dest: Path) -> None:
+    """Write the deterministic snapshot archive of ``db_dir`` to ``dest``.
+
+    Both members are required (a snapshot of a half-built db dir refuses
+    loudly instead of shipping an uninstallable archive).
+    """
+    with (
+        dest.open("wb") as raw,
+        gzip.GzipFile(
+            # filename="" keeps the dest path OUT of the gzip header (the
+            # fileobj's .name would otherwise be embedded — path-dependent bytes).
+            filename="",
+            mode="wb",
+            fileobj=raw,
+            mtime=0,
+        ) as gz,
+        tarfile.open(fileobj=gz, mode="w", format=tarfile.PAX_FORMAT) as tar,
+    ):
+        for name in _MEMBERS:
+            source = db_dir / name
+            if not source.is_file():
+                raise _snapshot_error(db_dir, name)
+            data = source.read_bytes()
+            info = tarfile.TarInfo(name)
+            info.size = len(data)
+            info.mtime = 0
+            tar.addfile(info, io.BytesIO(data))
+
+
+def _member(tar: tarfile.TarFile, name: str, archive: Path) -> bytes:
+    """Read one regular-file member; typed SNAPSHOT_INVALID when absent
+    (no untyped KeyError escape — the Wave E card.py lesson)."""
+    member = next((m for m in tar.getmembers() if m.name == name and m.isfile()), None)
+    if member is None:
+        raise DatabaseError(
+            f"snapshot {archive} has no {name} member",
+            code="SNAPSHOT_INVALID",
+            context={"archive": str(archive), "missing": name},
+        )
+    extracted = tar.extractfile(member)
+    if extracted is None:
+        raise DatabaseError(
+            f"{name} in {archive} is not a regular file",
+            code="SNAPSHOT_INVALID",
+            context={"archive": str(archive), "missing": name},
+        )
+    return extracted.read()
+
+
+def read_snapshot_manifest(archive: Path) -> Manifest:
+    """The archived gapit-manifest.json parsed in-memory (no extraction,
+    nothing written) — read-only queries such as `db outdated`."""
+    with tarfile.open(archive, "r:gz") as tar:
+        return Manifest.model_validate_json(_member(tar, "gapit-manifest.json", archive))
+
+
+def extract_snapshot(archive: Path, db_dir: Path) -> Manifest:
+    """Install ``records.jsonl`` from ``archive`` into ``db_dir`` atomically;
+    return the ARCHIVED manifest (its upstream_version seeds the fresh build)."""
+    with tarfile.open(archive, "r:gz") as tar:
+        records = _member(tar, "records.jsonl", archive)
+        archived = Manifest.model_validate_json(_member(tar, "gapit-manifest.json", archive))
+    temp_path: Path | None = None
+    try:
+        with NamedTemporaryFile(dir=db_dir, prefix=".records.", suffix=".jsonl", delete=False) as (
+            temp
+        ):
+            temp_path = Path(temp.name)
+            temp.write(records)
+        assert temp_path is not None
+        os.replace(temp_path, db_dir / "records.jsonl")
+    finally:
+        if temp_path is not None:
+            temp_path.unlink(missing_ok=True)
+    return archived

gapit/providers/upec_expec_vf.py

@@ -0,0 +1,85 @@
+"""upec_expec_vf provider — UPEC/ExPEC virulence genes (FordeGenomics).
+
+Transform-only module (Wave B12): turns the downloaded
+``UPEC_ExPEC_VF.tsv`` into typed ``Record``s exactly like upstream
+``abricate-get_db get_upec_expec_vf`` — ``load_tabular`` keys every row by
+its ``Gene name`` column with first occurrence winning (perl ``||=``), then
+ID=Gene name, ACC=``Accession/Source:Begin-End``, DESC=Description,
+SEQ=Sequence. Function is the locked ``virulence`` constant: the TSV Class
+column is source metadata, NOT the function slot (Wave F2c). Sequences
+stay RAW here: normalization, sequence dedupe, and sorting belong to the
+generic :func:`gapit.providers.common.fetch_provider` pipeline.
+"""
+
+import csv
+from collections.abc import Iterator
+from pathlib import Path
+
+from gapit.errors import DatabaseError
+from gapit.providers.common import Provider
+from gapit.records import Record
+
+NAME = "upec_expec_vf"
+
+_FUNCTION = ("virulence",)  # locked gapit/v1 func vocabulary (Wave F2c)
+
+_FILENAME = "UPEC_ExPEC_VF.tsv"
+_REQUIRED_COLUMNS = (
+    "Gene name",
+    "Accession/Source",
+    "Begin",
+    "End",
+    "Description",
+    "Class",
+    "Sequence",
+)
+
+
+def transform(workdir: Path) -> Iterator[Record]:
+    """Yield one Record per well-formed row of ``workdir/UPEC_ExPEC_VF.tsv``.
+
+    First row is the header; every column is looked up by NAME. Rows whose
+    Gene name or Sequence is empty are skipped (upstream would splice undef
+    pieces into the record — deviation noted in the Phase 7 notepad). A
+    header missing required columns raises DatabaseError PROVIDER_MALFORMED.
+    """
+    path = workdir / _FILENAME
+    with path.open(encoding="utf-8", newline="") as handle:
+        rows = csv.DictReader(handle, delimiter="\t", restval="")
+        missing = [
+            column for column in _REQUIRED_COLUMNS if column not in set(rows.fieldnames or ())
+        ]
+        if missing:
+            raise DatabaseError(
+                f"upec_expec_vf source is missing column(s): {', '.join(missing)}",
+                code="PROVIDER_MALFORMED",
+                context={"file": str(path), "missing_columns": ",".join(missing)},
+            )
+        seen: set[str] = set()
+        for row in rows:
+            gene = row["Gene name"]
+            sequence = row["Sequence"]
+            if not gene or not sequence or gene in seen:
+                continue
+            seen.add(gene)
+            yield Record(
+                db=NAME,
+                gene=gene,
+                accession=f"{row['Accession/Source']}:{row['Begin']}-{row['End']}",
+                function=_FUNCTION,
+                product=row["Description"],
+                sequence=sequence,
+                source_id=gene,
+            )
+
+
+PROVIDER = Provider(
+    name=NAME,
+    description="UPEC/ExPEC virulence genes (FordeGenomics)",
+    source_urls=(
+        "https://raw.githubusercontent.com/FordeGenomics/ST167_Code/refs/heads/main/UPEC-ExPEC_VF/UPEC_ExPEC_VF.tsv",
+    ),
+    dbtype="nucl",
+    transform=transform,
+    snapshot=None,
+)

gapit/providers/vfdb.py

@@ -0,0 +1,92 @@
+"""VFDB provider (set A, nucleotide) — transform only.
+
+Upstream ``get_vfdb`` (abricate-get_db 1.4.0) decompresses
+``VFDB_setA_nt.fas.gz`` and, per record, pulls the accession from the
+``<gene>(<db>|<acc>.<version>)`` id suffix and renames the gene to the
+leading paren group of the description::
+
+    >VFG000676(gb|AAD32411) (lef) anthrax toxin lethal factor precursor ...
+
+Perl quirk: a record failing either regex keeps the PREVIOUS record's $1/$2
+(stale fields). We skip such records instead — see the phase 7 notepad.
+
+Encoding quirk (Wave E): the real ``VFDB_setA_nt.fas.gz`` is not UTF-8-clean
+(a latin-1 0xA0 nbsp crashes ``gapit.fasta``'s strict decoder), so the
+decompressed bytes are decoded as latin-1 before parsing — see the comment
+in :func:`transform`.
+"""
+
+import gzip
+import re
+from collections.abc import Iterable
+from pathlib import Path
+from tempfile import NamedTemporaryFile
+
+from gapit.fasta import iter_fasta
+from gapit.providers.common import Provider
+from gapit.records import Record
+
+_NAME = "vfdb"
+
+# Upstream regexes verbatim (abricate issue #64 comment by @VGalata):
+# accession = group 2; group 3 (".2" version suffix) is consumed, not used.
+_ID = re.compile(r"^(\w+)\(\w+\|(\w+)(\.\d+)?\)$")
+_DESC = re.compile(r"^\((.*?)\)")
+
+
+def transform(workdir: Path) -> Iterable[Record]:
+    """Yield Records from ``workdir/VFDB_setA_nt.fas.gz``.
+
+    The gunzipped text is decoded as latin-1 into a scratch UTF-8 file for
+    :func:`gapit.fasta.iter_fasta` (it takes a path); the scratch file is
+    removed on completion or early generator close (argannot precedent).
+
+    A record failing EITHER regex is skipped; product keeps the full
+    description (upstream leaves DESC untouched) and source_id keeps the
+    original id.
+    """
+    # Byte-preserving decode: upstream perl is byte-oriented here, and
+    # latin-1 maps every byte 1:1 — descriptions keep the character and
+    # sequence junk flows on to fetch_provider's N-normalization. Our
+    # JSONL/FASTA outputs are UTF-8 by contract, so writing the scratch
+    # file as UTF-8 re-encodes the character deterministically. Never
+    # errors="replace": that would silently rewrite the data.
+    text = gzip.decompress((workdir / "VFDB_setA_nt.fas.gz").read_bytes()).decode("latin-1")
+    with NamedTemporaryFile(
+        mode="w",
+        encoding="utf-8",
+        newline="\n",
+        prefix=".vfdb.",
+        suffix=".fas",
+        dir=workdir,
+        delete=False,
+    ) as handle:
+        handle.write(text)
+        scratch = Path(handle.name)
+    try:
+        for fasta in iter_fasta(scratch):
+            id_match = _ID.match(fasta.id)
+            desc_match = _DESC.match(fasta.description)
+            if id_match is None or desc_match is None:
+                continue
+            yield Record(
+                db=_NAME,
+                gene=desc_match.group(1),
+                sequence=fasta.sequence,
+                accession=id_match.group(2),
+                function=("virulence",),
+                product=fasta.description,
+                source_id=fasta.id,
+            )
+    finally:
+        scratch.unlink(missing_ok=True)
+
+
+PROVIDER = Provider(
+    name=_NAME,
+    description="VFDB virulence factors (set A, nucleotide)",
+    source_urls=("http://www.mgc.ac.cn/VFs/Down/VFDB_setA_nt.fas.gz",),
+    dbtype="nucl",
+    transform=transform,
+    snapshot="vfdb.tar.gz",
+)

gapit/providers/victors.py

@@ -0,0 +1,109 @@
+"""Victors provider (virulence factors, nucleotide) — transform only.
+
+Upstream ``get_victors`` (abricate-get_db 1.4.0) cross-references two
+phidias.us downloads, both served under ``.php`` URLs:
+
+- ``gen_downloads.php`` — nucleotide CDS content (.ffn), ids like
+  ``gi|115534241:2616-3152`` carrying the source GI and coordinates;
+- ``gen_downloads_protein.php`` — protein content (.faa), headers like
+  ``>gi|115534244|ref|YP_783826.1| hypothetical protein pCJ01p4
+  [Campylobacter jejuni]``.
+
+Protein headers are keyed by gi; each .ffn record looks its gi up in that
+map for accession and product, falling back to ``gi|<gi>:<start>-<stop>``
+and ``hypothetical protein``. The gene stays the ORIGINAL .ffn id —
+upstream never renames it here.
+
+Perl quirks (phase 7 notepad): upstream's ``.`` in both regexes matches
+the literal ``|`` (escaped here); a protein header failing the regex is
+skipped (``next unless``); an .ffn id failing its regex would reuse the
+PREVIOUS match's stale capture variables — such records are skipped, the
+same policy as the vfdb provider. Upstream's product line
+(``$s->{DESC} =~ ... || 'hypothetical protein'``) is a void-context match
+whose result is discarded; the notepad-locked digest semantics (map
+product or ``hypothetical protein``) are implemented instead.
+"""
+
+import re
+from collections.abc import Iterable
+from pathlib import Path
+
+from gapit.fasta import iter_fasta
+from gapit.providers.common import Provider
+from gapit.records import Record
+
+_NAME = "victors"
+_FFN_NAME = "gen_downloads.php"  # nucleotide CDS content
+_FAA_NAME = "gen_downloads_protein.php"  # protein content
+_HYPOTHETICAL = "hypothetical protein"
+_FUNCTION = ("virulence",)  # locked gapit/v1 func vocabulary (Wave F2c)
+
+# Upstream m"^>gi.(\d+).ref.([^|]+). ([^[]+)" — its dots matched the pipes.
+# \s+ eats the id/product separator; the product runs to end of line.
+_FAA_HEADER = re.compile(r">gi\|(\d+)\|ref\|([^|]+)\|\s+(.+)")
+# Upstream m/gi.(\d+):(\d+)-(\d+)/ searched against the .ffn record id.
+_FFN_ID = re.compile(r"gi\|(\d+):(\d+)-(\d+)")
+
+
+def _read_protein_map(path: Path) -> dict[str, tuple[str, str]]:
+    """gi -> (accession, product) from .faa header lines.
+
+    Headers failing the pattern are skipped (upstream ``next unless``);
+    the product stops at the first `` [`` strain bracket (upstream
+    ``([^[]+)``). A later duplicate gi overwrites an earlier one, exactly
+    like upstream's hash assignment.
+    """
+    gi_map: dict[str, tuple[str, str]] = {}
+    with path.open(encoding="utf-8") as handle:
+        for line in handle:
+            header = _FAA_HEADER.match(line)
+            if header is None:
+                continue
+            gi, accession, product = (
+                header.group(1),
+                header.group(2),
+                header.group(3).split(" [", 1)[0],
+            )
+            gi_map[gi] = (accession, product)
+    return gi_map
+
+
+def transform(workdir: Path) -> Iterable[Record]:
+    """Yield Records from the two phidias.us downloads in ``workdir``.
+
+    Each .ffn record keeps its ORIGINAL id as the gene; the gi carved out
+    of that id looks accession/product up in the protein map, falling back
+    to ``gi|<gi>:<start>-<stop>`` / ``hypothetical protein`` for an unknown
+    gi. Records whose id carries no gi coordinates are skipped (upstream
+    would build them from the previous match's stale captures).
+    """
+    gi_map = _read_protein_map(workdir / _FAA_NAME)
+    for fasta in iter_fasta(workdir / _FFN_NAME):
+        coords = _FFN_ID.search(fasta.id)
+        if coords is None:
+            continue
+        gi = coords.group(1)
+        fallback = f"gi|{gi}:{coords.group(2)}-{coords.group(3)}"
+        accession, product = gi_map.get(gi, (fallback, _HYPOTHETICAL))
+        yield Record(
+            db=_NAME,
+            gene=fasta.id,
+            sequence=fasta.sequence,
+            accession=accession,
+            function=_FUNCTION,
+            product=product,
+            source_id=gi,
+        )
+
+
+PROVIDER = Provider(
+    name=_NAME,
+    description="Victors virulence factors",
+    source_urls=(
+        "http://phidias.us/victors/downloads/gen_downloads.php",
+        "http://phidias.us/victors/downloads/gen_downloads_protein.php",
+    ),
+    dbtype="nucl",
+    transform=transform,
+    snapshot=None,
+)