gapit 0.2.2__py3-none-any.whl

gapit/reads.py

@@ -0,0 +1,221 @@
+"""FASTQ/FASTA read screening via minimap2 (SPEC.md §10 — gapit extension)."""
+
+import enum
+import gzip
+from collections.abc import Iterable, Mapping
+from pathlib import Path
+from typing import Protocol
+
+from pydantic import BaseModel, Field
+
+from gapit.db import Database
+from gapit.dbcodec import decode_seqid
+from gapit.errors import InputError
+from gapit.fasta import iter_fasta_headers
+from gapit.minimap2_run import run_minimap2
+from gapit.paf import (
+    PafRecord,
+    ReadType,
+    alignment_identity,
+    filter_alignments,
+    union_length,
+)
+
+_GZIP_MAGIC = b"\x1f\x8b"
+
+
+class _ByteStream(Protocol):
+    """The byte-read surface shared by raw and gzip-decompressed handles."""
+
+    def read(self, size: int = -1, /) -> bytes: ...
+
+
+class ReadTypeEnum(enum.Enum):
+    """CLI-facing read-type preset choices."""
+
+    sr = "sr"
+    map_ont = "map-ont"
+    map_hifi = "map-hifi"
+
+
+class ReadFileKind(enum.Enum):
+    """Input file kind detected from content (minimap2 takes FASTA and FASTQ
+    queries natively; gapit detects to resolve the preset)."""
+
+    fasta = "fasta"
+    fastq = "fastq"
+
+
+def _peek_read_kind(handle: _ByteStream, path: Path) -> ReadFileKind:
+    """First non-whitespace byte: '>' = FASTA, '@' = FASTQ; anything else (or
+    EOF) is a typed input error."""
+    while True:
+        byte = handle.read(1)
+        if not byte:
+            raise InputError(
+                f"reads file is empty: {path}",
+                code="INVALID_READS_FORMAT",
+                context={"file": str(path)},
+            )
+        if byte.isspace():
+            continue
+        if byte == b">":
+            return ReadFileKind.fasta
+        if byte == b"@":
+            return ReadFileKind.fastq
+        raise InputError(
+            f"reads file is neither FASTA nor FASTQ: {path}",
+            code="INVALID_READS_FORMAT",
+            context={"file": str(path)},
+        )
+
+
+def detect_read_kind(path: Path) -> ReadFileKind:
+    """Detect a --r1/--r2 file's kind from content. Gzip-wrapped files
+    (magic 1f 8b) are peeked through the decompressor: minimap2 reads them
+    natively, so detection must not reject them. A truncated or corrupt
+    gzip stream raises the same typed INVALID_READS_FORMAT error as any
+    other unreadable reads file, never a raw EOFError."""
+    with path.open("rb") as raw:
+        compressed = raw.read(2) == _GZIP_MAGIC
+    if compressed:
+        with gzip.open(path, "rb") as handle:
+            try:
+                return _peek_read_kind(handle, path)
+            except (OSError, EOFError, UnicodeDecodeError) as exc:
+                raise InputError(
+                    f"cannot decompress reads file: {path}",
+                    code="INVALID_READS_FORMAT",
+                    context={"file": str(path)},
+                ) from exc
+    with path.open("rb") as handle:
+        return _peek_read_kind(handle, path)
+
+
+class ReadsParams(BaseModel, frozen=True):
+    """Parameters for one read-screening run (SPEC.md §10). The reads/2
+    thresholds default off; gapit.reads/1 rendering ignores them entirely."""
+
+    db: str
+    read_type: ReadType = "sr"
+    min_breadth: float = Field(default=90.0, ge=0.0, le=100.0)
+    threads: int = Field(default=1, ge=1)
+    min_identity: float = Field(default=0.0, ge=0.0, le=100.0)
+    min_mapq: int = Field(default=0, ge=0)
+
+
+class GeneCoverage(BaseModel, frozen=True):
+    """Per-gene presence call over primary alignments. mean_identity_pct is
+    the alen-weighted mean of per-alignment identity over the aggregated
+    (kept) rows — the weighting favors long alignments; computed in reads/1
+    mode too but rendered only by gapit.reads/2."""
+
+    database: str
+    gene: str
+    accession: str
+    function: str
+    product: str
+    tlen: int
+    breadth_pct: float
+    mean_depth: float
+    reads_mapped: int
+    present: bool
+    mean_identity_pct: float
+
+
+class ReadsReport(BaseModel, frozen=True):
+    """Result of screening one set of read files."""
+
+    reads: tuple[str, ...]
+    genes: tuple[GeneCoverage, ...]
+
+
+def aggregate_coverage(
+    rows: Iterable[PafRecord],
+    *,
+    default_db: str,
+    min_breadth: float,
+    products: Mapping[str, str] | None = None,
+) -> list[GeneCoverage]:
+    """Aggregate primary alignments per target: breadth = union of
+    tstart..tend intervals, depth = summed interval lengths, reads = distinct
+    qnames. tlen is the FIRST-SEEN row's tlen per target (denominator
+    contract). Zero-read genes are omitted; output sorts by (-breadth, gene)."""
+    intervals: dict[str, list[tuple[int, int]]] = {}
+    tlens: dict[str, int] = {}
+    qnames: dict[str, set[str]] = {}
+    weights: dict[str, list[tuple[float, int]]] = {}
+    for row in rows:
+        if not row.is_primary:
+            continue
+        if row.tname not in intervals:
+            intervals[row.tname] = []
+            tlens[row.tname] = row.tlen
+            qnames[row.tname] = set()
+            weights[row.tname] = []
+        intervals[row.tname].append((row.tstart, row.tend))
+        qnames[row.tname].add(row.qname)
+        weights[row.tname].append((alignment_identity(row), row.alen))
+    descriptions = products or {}
+    coverages: list[GeneCoverage] = []
+    for tname, spans in intervals.items():
+        tlen = tlens[tname]
+        covered = union_length(spans)
+        breadth_pct = 100.0 * covered / tlen
+        header = decode_seqid(tname, default_db)
+        pairs = weights[tname]
+        weight_sum = sum(weight for _, weight in pairs)
+        coverages.append(
+            GeneCoverage(
+                database=header.database,
+                gene=header.gene,
+                accession=header.accession,
+                function=header.function,
+                product=descriptions.get(tname, ""),
+                tlen=tlen,
+                breadth_pct=breadth_pct,
+                mean_depth=sum(end - start for start, end in spans) / tlen,
+                reads_mapped=len(qnames[tname]),
+                present=breadth_pct >= min_breadth,
+                mean_identity_pct=(
+                    sum(identity * weight for identity, weight in pairs) / weight_sum
+                    if weight_sum
+                    else 0.0
+                ),
+            )
+        )
+    return sorted(coverages, key=lambda entry: (-entry.breadth_pct, entry.gene))
+
+
+def screen_reads(
+    lanes: list[tuple[Path, Path | None]],
+    database: Database,
+    *,
+    read_type: ReadType,
+    min_breadth: float,
+    threads: int,
+    debug: bool = False,
+    min_identity: float = 0.0,
+    min_mapq: int = 0,
+) -> ReadsReport:
+    """Screen one sample's lanes against one database into a sample-level
+    ReadsReport (union of all lanes' primary alignments). With a nonzero
+    min_identity/min_mapq (gapit.reads/2), alignments are filtered BEFORE
+    aggregation and the minimap2 run emits NM tags for the identity rule."""
+    reads2 = min_identity > 0.0 or min_mapq > 0
+    rows = run_minimap2(
+        lanes,
+        database,
+        read_type=read_type,
+        threads=threads,
+        debug=debug,
+        nm_tags=reads2,
+    )
+    if reads2:
+        rows = filter_alignments(rows, min_identity=min_identity, min_mapq=min_mapq)
+    products = dict(iter_fasta_headers(database.sequences_path))
+    genes = aggregate_coverage(
+        rows, default_db=database.name, min_breadth=min_breadth, products=products
+    )
+    files = [str(r1) for r1, _ in lanes] + [str(r2) for _, r2 in lanes if r2 is not None]
+    return ReadsReport(reads=tuple(files), genes=tuple(genes))

gapit/records.py

@@ -0,0 +1,152 @@
+"""The per-database truth source: Record JSONL store and gapit.manifest/1.
+
+``records.jsonl`` is the editable truth a database directory is built from
+(the ``sequences`` FASTA and its BLAST index are generated projections, built
+later in Wave A3); ``gapit-manifest.json`` is the provenance sidecar describing
+how that truth was obtained. Both are FILE contracts — they never appear on
+stdout and are not registered in ``gapit schema``.
+"""
+
+import re
+from collections.abc import Iterable, Iterator
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator
+
+from gapit.errors import InputError
+from gapit.formats.json import ToolDocument
+
+_SHA256_SHAPE = re.compile(r"[0-9a-fA-F]{64}")
+
+
+class Record(BaseModel, frozen=True):
+    """One known gene — the unit of truth a database projection is built from.
+
+    ``function`` carries functional categories (AMR classes, virulence,
+    O-antigen, replicon, biocide) — gapit answers presence/absence, so the
+    slot's old ``resistance`` name was a misnomer (Wave F1).
+    """
+
+    db: str
+    gene: str
+    sequence: str
+    accession: str = ""
+    function: tuple[str, ...] = ()
+    product: str = "n/a"
+    source_id: str = ""
+
+
+def write_records(records: Iterable[Record], path: Path) -> None:
+    """Write records as deterministic JSONL: given order, one JSON object per
+    line, LF endings, UTF-8."""
+    with path.open("w", encoding="utf-8", newline="\n") as handle:
+        for record in records:
+            handle.write(record.model_dump_json())
+            handle.write("\n")
+
+
+def read_records(path: Path) -> Iterator[Record]:
+    """Stream records back line by line (nothing is materialized).
+
+    Blank lines are skipped; a non-parsing line raises InputError
+    RECORDS_MALFORMED with the physical line number in context. The path is
+    checked eagerly; iteration itself is lazy.
+    """
+    if not path.is_file():
+        raise InputError(
+            f"records file not found or unreadable: {path}",
+            code="INPUT_NOT_FOUND",
+            context={"file": str(path)},
+        )
+    return _iter_records(path)
+
+
+def _iter_records(path: Path) -> Iterator[Record]:
+    with path.open(encoding="utf-8") as handle:
+        for line_number, line in enumerate(handle, start=1):
+            if not line.strip():
+                continue
+            try:
+                yield Record.model_validate_json(line)
+            except ValidationError as exc:
+                raise InputError(
+                    f"malformed record at {path}:{line_number}: {_reason(exc)}",
+                    code="RECORDS_MALFORMED",
+                    context={"file": str(path), "line": str(line_number)},
+                ) from exc
+
+
+def count_records(path: Path) -> int:
+    """Count record lines by streaming (blank lines excluded, no list built)."""
+    return sum(1 for _ in read_records(path))
+
+
+class Manifest(BaseModel, frozen=True):
+    """gapit.manifest/1 — provenance sidecar for one database directory."""
+
+    model_config = ConfigDict(populate_by_name=True)
+
+    schema_name: Literal["gapit.manifest/1"] = Field(default="gapit.manifest/1", alias="schema")
+    name: str
+    source_urls: tuple[str, ...]
+    fetched_at: str
+    sha256: str
+    n_records: int = Field(ge=0)
+    dbtype: Literal["nucl", "prot"]
+    header_format: Literal["gapit/v1"] = "gapit/v1"
+    upstream_version: str = ""
+    tool: ToolDocument = ToolDocument()
+    makeblastdb_version: str = ""
+    minimap2_version: str = ""
+
+    @field_validator("sha256")
+    @classmethod
+    def _sha256_hex(cls, value: str) -> str:
+        """Exactly 64 hex chars; case-insensitive on input, stored lowercase."""
+        if not _SHA256_SHAPE.fullmatch(value):
+            raise ValueError("sha256 must be exactly 64 hex characters")
+        return value.lower()
+
+
+def _reason(exc: ValidationError) -> str:
+    """The validation failure rendered as one line (str(exc) is multiline)."""
+    return " ".join(str(exc).split())
+
+
+def write_manifest(manifest: Manifest, path: Path) -> None:
+    """Write the manifest as indented JSON (by alias, ``schema`` first) with a
+    trailing newline, LF endings, UTF-8."""
+    with path.open("w", encoding="utf-8", newline="\n") as handle:
+        handle.write(manifest.model_dump_json(indent=2, by_alias=True))
+        handle.write("\n")
+
+
+def read_manifest(path: Path) -> Manifest:
+    """Parse and validate a gapit.manifest/1 file."""
+    if not path.is_file():
+        raise InputError(
+            f"manifest file not found or unreadable: {path}",
+            code="INPUT_NOT_FOUND",
+            context={"file": str(path)},
+        )
+    try:
+        return Manifest.model_validate_json(path.read_text(encoding="utf-8"))
+    except ValidationError as exc:
+        raise InputError(
+            f"malformed manifest {path}: {_reason(exc)}",
+            code="MANIFEST_MALFORMED",
+            context={"file": str(path)},
+        ) from exc
+
+
+def installed_db_dirs(root: Path) -> list[Path]:
+    """Datadir subdirectories holding a gapit-manifest.json, sorted by name."""
+    return sorted(
+        (
+            child
+            for child in root.iterdir()
+            if child.is_dir() and (child / "gapit-manifest.json").is_file()
+        ),
+        key=lambda child: child.name,
+    )

gapit/report.py

@@ -0,0 +1,25 @@
+"""Screening parameters and the canonical Report model."""
+
+from pydantic import BaseModel, Field
+
+from gapit.hits import Hit
+
+
+class ScreeningParams(BaseModel, frozen=True):
+    """Parameters for one screening run (SPEC.md §1 defaults and bounds)."""
+
+    db: str
+    minid: float = Field(default=80.0, gt=0.0, le=100.0)
+    mincov: float = Field(default=80.0, ge=0.0, le=100.0)
+    threads: int = Field(default=1, ge=1)
+
+
+class Report(BaseModel, frozen=True):
+    """Canonical in-memory result of screening one input file.
+
+    Hits are sorted (sequence, start) before construction; the tuple is the
+    final, stable order.
+    """
+
+    file: str
+    hits: tuple[Hit, ...]

gapit/screening.py

@@ -0,0 +1,145 @@
+"""The blastn contig-screening use-case plus helpers shared by both engines.
+
+The minimap2 engine use-cases (--r1/--r2 reads, --aligner minimap2 assemblies)
+live in screening_reads.py; this module keeps the abricate-parity contig
+pipeline and the shared OutputFormat / AlignerEnum / database lookup.
+"""
+
+import enum
+import os
+from concurrent.futures import ThreadPoolExecutor
+from datetime import UTC, datetime
+from pathlib import Path
+
+import typer
+
+from gapit import config, db
+from gapit.blast import ensure_blast, screen_file
+from gapit.errors import DatabaseError, InputError, ensure_input_file, usage_fail
+from gapit.formats.json import render_json
+from gapit.formats.md import render_markdown
+from gapit.formats.tsv import format_tsv
+from gapit.report import Report, ScreeningParams
+
+
+class OutputFormat(enum.Enum):
+    """Screen output formats."""
+
+    tsv = "tsv"
+    csv = "csv"
+    json = "json"
+    md = "md"
+
+
+class AlignerEnum(enum.Enum):
+    """Alignment engines for screen (SPEC.md §1/§10)."""
+
+    blastn = "blastn"
+    minimap2 = "minimap2"
+
+
+def _resolve_inputs(files: list[Path] | None, fofn: Path | None) -> list[Path]:
+    """Input files: --fofn (lines stripped, empties dropped) REPLACES positionals."""
+    if fofn is not None:
+        if not fofn.is_file():
+            raise InputError(
+                f"--fofn file not found: {fofn}",
+                code="INPUT_NOT_FOUND",
+                context={"file": str(fofn)},
+            )
+        inputs = [
+            Path(line.strip())
+            for line in fofn.read_text(encoding="utf-8").splitlines()
+            if line.strip()
+        ]
+    elif files:
+        inputs = list(files)
+    else:
+        usage_fail("no input files given (positional FILEs or --fofn)")
+    for path in inputs:
+        ensure_input_file(path)
+    return inputs
+
+
+def find_database(datadir: Path, name: str) -> db.Database:
+    """Look up a database by name under the datadir; unknown names list what exists."""
+    databases = db.discover_databases(datadir)
+    for database in databases:
+        if database.name == name:
+            return database
+    available = ", ".join(entry.name for entry in databases) or "(none)"
+    raise DatabaseError(
+        f"Database {name} is not in {datadir}. Available: {available}",
+        code="DATABASE_NOT_FOUND",
+        context={"db": name, "datadir": str(datadir)},
+    )
+
+
+def run_screen(
+    files: list[Path] | None,
+    db_name: str,
+    datadir: Path | None,
+    minid: float,
+    mincov: float,
+    threads: int,
+    jobs: int,
+    fofn: Path | None,
+    quiet: bool,
+    noheader: bool,
+    nopath: bool,
+    debug: bool,
+    output_format: OutputFormat,
+) -> str:
+    """Screen each input file in order; buffer reports; render once at the
+    end and return the output for the caller to echo.
+
+    The per-run gates (blastn presence via ``ensure_blast``, dbtype via one
+    ``blastdbcmd -info``) fire once up front, so MISSING_DEPENDENCY and
+    DATABASE_NOT_INDEXED errors precede any "Processing:" stderr lines.
+    """
+    if not 0.0 < minid <= 100.0:
+        usage_fail(f"--minid must be in (0, 100]: got {minid}")
+    if not 0.0 <= mincov <= 100.0:
+        usage_fail(f"--mincov must be in [0, 100]: got {mincov}")
+    if threads < 1:
+        usage_fail(f"--threads must be >= 1: got {threads}")
+    if jobs < 1:
+        usage_fail(f"--jobs must be >= 1: got {jobs}")
+    inputs = _resolve_inputs(files, fofn)
+    params = ScreeningParams(db=db_name, minid=minid, mincov=mincov, threads=threads)
+    database = find_database(config.resolve_datadir(datadir), db_name)
+    ensure_blast()
+    dbtype = db.blast_db_info(database.sequences_path).dbtype
+    cpu_count = os.cpu_count()
+    if not quiet and cpu_count is not None and jobs * threads > cpu_count:
+        typer.echo(f"--jobs {jobs} --threads {threads} oversubscribes {cpu_count} cpus", err=True)
+    reports: list[Report] = []
+    if jobs == 1:
+        for path in inputs:
+            if not quiet:
+                typer.echo(f"Processing: {path}", err=True)
+            report = screen_file(path, database, params, dbtype=dbtype, debug=debug)
+            if not quiet:
+                typer.echo(f"Found {len(report.hits)} genes in {path}", err=True)
+            reports.append(report)
+    else:
+
+        def screen_one(path: Path) -> Report:
+            if not quiet:
+                typer.echo(f"Processing: {path}", err=True)
+            report = screen_file(path, database, params, dbtype=dbtype, debug=debug)
+            if not quiet:
+                typer.echo(f"Found {len(report.hits)} genes in {path}", err=True)
+            return report
+
+        # Subprocess-bound work (GIL irrelevant); executor.map collects
+        # positionally, so reports stay in input order (SPEC.md §4) and a
+        # failing file raises at its position, like the sequential loop.
+        with ThreadPoolExecutor(max_workers=jobs) as executor:
+            reports = list(executor.map(screen_one, inputs))
+    if output_format is OutputFormat.json:
+        return render_json(reports, params, now=datetime.now(UTC))
+    if output_format is OutputFormat.md:
+        return render_markdown(reports, params, now=datetime.now(UTC))
+    as_csv = output_format is OutputFormat.csv
+    return format_tsv(reports, csv=as_csv, noheader=noheader, nopath=nopath)