fetchm2 0.1.0__py3-none-any.whl

fetchm2/__init__.py

@@ -0,0 +1,6 @@
+"""FetchM2 standalone package."""
+
+from __future__ import annotations
+
+__version__ = "0.1.0"
+

fetchm2/audit.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+from collections import Counter
+from pathlib import Path
+from typing import Any
+
+from .standardization import load_rules, normalize_lookup
+from .utils import write_csv, write_text
+
+ALLOWED_MARINE_REGIONS = {
+    "Arctic Ocean",
+    "Atlantic Ocean",
+    "Indian Ocean",
+    "Pacific Ocean",
+    "Southern Ocean",
+    "Mediterranean Sea",
+    "North Sea",
+    "Baltic Sea",
+}
+
+
+def value_present(value: Any) -> bool:
+    return bool(str(value or "").strip())
+
+
+def summarize_rows(rows: list[dict[str, Any]]) -> dict[str, Any]:
+    total = len(rows)
+    host_taxid = sum(1 for row in rows if value_present(row.get("Host_TaxID")))
+    host_review = sum(1 for row in rows if row.get("Host_Review_Status") == "review_needed")
+    country = sum(1 for row in rows if value_present(row.get("Country")))
+    collection_year = sum(1 for row in rows if value_present(row.get("Collection_Year")))
+    sample_type = sum(1 for row in rows if value_present(row.get("Sample_Type_SD")))
+    isolation_source = sum(1 for row in rows if value_present(row.get("Isolation_Source_SD")))
+    isolation_site = sum(1 for row in rows if value_present(row.get("Isolation_Site_SD")))
+    environment_medium = sum(1 for row in rows if value_present(row.get("Environment_Medium_SD")))
+    host_disease = sum(1 for row in rows if value_present(row.get("Host_Disease_SD")))
+    host_health = sum(1 for row in rows if value_present(row.get("Host_Health_State_SD")))
+    invalid_sample = [
+        row
+        for row in rows
+        if normalize_lookup(row.get("Sample_Type_SD")) in {"human", "patient", "animal", "poultry", "cattle", "pig", "plant", "bacteria"}
+    ]
+    non_country = [
+        row
+        for row in rows
+        if value_present(row.get("Country"))
+        and row.get("Country") not in load_rules().country_mapping
+        and row.get("Country") not in ALLOWED_MARINE_REGIONS
+    ]
+    broad_values = Counter(str(row.get("Isolation_Source_SD_Broad") or "").strip() for row in rows if value_present(row.get("Isolation_Source_SD_Broad")))
+    approved_broad = load_rules().approved_broad.get("Isolation_Source_SD_Broad", set())
+    unapproved_broad = {
+        value: count
+        for value, count in broad_values.items()
+        if value and value not in approved_broad
+    }
+    return {
+        "rows": total,
+        "host_taxid_mapped": host_taxid,
+        "host_taxid_percent": round((host_taxid / total) * 100, 2) if total else 0,
+        "host_review_needed": host_review,
+        "country_present": country,
+        "country_percent": round((country / total) * 100, 2) if total else 0,
+        "collection_year_present": collection_year,
+        "collection_year_percent": round((collection_year / total) * 100, 2) if total else 0,
+        "sample_type_present": sample_type,
+        "isolation_source_present": isolation_source,
+        "isolation_site_present": isolation_site,
+        "environment_medium_present": environment_medium,
+        "host_disease_present": host_disease,
+        "host_health_state_present": host_health,
+        "invalid_host_like_sample_type_rows": len(invalid_sample),
+        "non_country_values_in_country_rows": len(non_country),
+        "unapproved_isolation_source_broad_rows": sum(unapproved_broad.values()),
+        "unique_isolation_source_broad_values": len(broad_values),
+    }
+
+
+def write_audit_outputs(rows: list[dict[str, Any]], output_dir: Path) -> dict[str, Any]:
+    summary = summarize_rows(rows)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    write_csv(output_dir / "standardization_summary.csv", [summary])
+
+    top_host_review = Counter(
+        str(row.get("Host_Original") or "").strip()
+        for row in rows
+        if row.get("Host_Review_Status") == "review_needed"
+    )
+    write_csv(
+        output_dir / "top_host_review_needed.csv",
+        [{"host_original": key, "count": count} for key, count in top_host_review.most_common(200)],
+    )
+
+    markdown = [
+        "# FetchM2 Metadata Standardization Audit",
+        "",
+        f"Rows scanned: {summary['rows']}",
+        f"Host TaxID mapped: {summary['host_taxid_mapped']} ({summary['host_taxid_percent']}%)",
+        f"Host review needed: {summary['host_review_needed']}",
+        f"Country present: {summary['country_present']} ({summary['country_percent']}%)",
+        f"Collection year present: {summary['collection_year_present']} ({summary['collection_year_percent']}%)",
+        f"Sample_Type_SD present: {summary['sample_type_present']}",
+        f"Isolation_Source_SD present: {summary['isolation_source_present']}",
+        f"Isolation_Site_SD present: {summary['isolation_site_present']}",
+        f"Environment_Medium_SD present: {summary['environment_medium_present']}",
+        f"Invalid host-like Sample_Type_SD rows: {summary['invalid_host_like_sample_type_rows']}",
+        f"Non-country values in Country rows: {summary['non_country_values_in_country_rows']}",
+        f"Unapproved Isolation_Source_SD_Broad rows: {summary['unapproved_isolation_source_broad_rows']}",
+    ]
+    write_text(output_dir / "standardization_audit.md", "\n".join(markdown) + "\n")
+    return summary
+
+
+def production_gate(summary: dict[str, Any]) -> tuple[bool, list[str], list[str]]:
+    hard_failures: list[str] = []
+    warnings: list[str] = []
+    for key in [
+        "invalid_host_like_sample_type_rows",
+        "non_country_values_in_country_rows",
+        "unapproved_isolation_source_broad_rows",
+    ]:
+        if int(summary.get(key) or 0) > 0:
+            hard_failures.append(f"{key}={summary[key]}")
+    if int(summary.get("host_review_needed") or 0) > 1000:
+        warnings.append(f"host_review_needed={summary['host_review_needed']}")
+    return not hard_failures, hard_failures, warnings

fetchm2/cli.py

@@ -0,0 +1,175 @@
+from __future__ import annotations
+
+import argparse
+import os
+from pathlib import Path
+
+from . import __version__
+from .audit import production_gate, summarize_rows, write_audit_outputs
+from .metadata import run_metadata
+from .sequence import run_sequence_downloads
+
+
+def add_filter_args(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--host", nargs="+", help="Filter sequence downloads by Host_SD.")
+    parser.add_argument("--host-rank", nargs="+", help="Filter sequence downloads by Host_Rank.")
+    parser.add_argument("--country", nargs="+", help="Filter sequence downloads by standardized Country.")
+    parser.add_argument("--continent", nargs="+", help="Filter sequence downloads by Continent.")
+    parser.add_argument("--subcontinent", nargs="+", help="Filter sequence downloads by Subcontinent.")
+    parser.add_argument("--sample-type", nargs="+", help="Filter by Sample_Type_SD.")
+    parser.add_argument("--isolation-source", nargs="+", help="Filter by Isolation_Source_SD.")
+    parser.add_argument("--environment-medium", nargs="+", help="Filter by Environment_Medium_SD.")
+    parser.add_argument("--year-from", type=int, help="Minimum Collection_Year.")
+    parser.add_argument("--year-to", type=int, help="Maximum Collection_Year.")
+    parser.add_argument("--max-genomes", type=int, help="Maximum selected genomes for sequence download.")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="fetchm2",
+        description="Comprehensive standalone metadata standardization and sequence download toolkit.",
+    )
+    parser.add_argument("--version", action="version", version=f"fetchm2 {__version__}")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    metadata = subparsers.add_parser("metadata", help="Fetch/standardize metadata and write audit outputs.")
+    metadata.add_argument("--input", required=True, type=Path, help="NCBI Datasets TSV/CSV input.")
+    metadata.add_argument("--outdir", required=True, type=Path, help="Output directory.")
+    metadata.add_argument("--ani", nargs="+", default=["all"], help="ANI Check status filter.")
+    metadata.add_argument("--checkm", type=float, help="Minimum CheckM completeness.")
+    metadata.add_argument("--api-key", default=os.environ.get("NCBI_API_KEY"), help="NCBI API key or NCBI_API_KEY env var.")
+    metadata.add_argument("--email", default=os.environ.get("NCBI_EMAIL"), help="Optional NCBI contact email.")
+    metadata.add_argument("--workers", type=int, default=3, help="Metadata fetch workers.")
+    metadata.add_argument("--sleep", type=float, default=0.34, help="Delay before BioSample requests.")
+    metadata.add_argument("--offline", action="store_true", help="Do not fetch BioSample metadata; standardize existing columns only.")
+    metadata.set_defaults(func=run_metadata_command)
+
+    run = subparsers.add_parser("run", help="Run metadata standardization and optionally download sequences.")
+    run.add_argument("--input", required=True, type=Path, help="NCBI Datasets TSV/CSV input.")
+    run.add_argument("--outdir", required=True, type=Path, help="Output directory.")
+    run.add_argument("--ani", nargs="+", default=["all"], help="ANI Check status filter.")
+    run.add_argument("--checkm", type=float, help="Minimum CheckM completeness.")
+    run.add_argument("--api-key", default=os.environ.get("NCBI_API_KEY"), help="NCBI API key or NCBI_API_KEY env var.")
+    run.add_argument("--email", default=os.environ.get("NCBI_EMAIL"), help="Optional NCBI contact email.")
+    run.add_argument("--workers", type=int, default=3, help="Metadata fetch workers.")
+    run.add_argument("--sleep", type=float, default=0.34, help="Delay before BioSample requests.")
+    run.add_argument("--offline", action="store_true", help="Do not fetch BioSample metadata; standardize existing columns only.")
+    run.add_argument("--download", action="store_true", help="Download sequences after metadata standardization.")
+    run.add_argument("--download-workers", type=int, default=4, help="Sequence download workers.")
+    run.add_argument("--retries", type=int, default=3, help="Download retries.")
+    run.add_argument("--retry-delay", type=float, default=5.0, help="Download retry delay.")
+    run.add_argument("--keep-gz", action="store_true", help="Keep compressed FASTA files instead of decompressing.")
+    add_filter_args(run)
+    run.set_defaults(func=run_all_command)
+
+    seq = subparsers.add_parser("seq", help="Download sequences from fetchm2_clean.csv.")
+    seq.add_argument("--input", required=True, type=Path, help="Path to fetchm2_clean.csv.")
+    seq.add_argument("--outdir", required=True, type=Path, help="Sequence output directory.")
+    seq.add_argument("--download-workers", type=int, default=4, help="Sequence download workers.")
+    seq.add_argument("--retries", type=int, default=3, help="Download retries.")
+    seq.add_argument("--retry-delay", type=float, default=5.0, help="Download retry delay.")
+    seq.add_argument("--check-only", action="store_true", help="Audit sequence directory without downloading.")
+    seq.add_argument("--keep-gz", action="store_true", help="Keep compressed FASTA files instead of decompressing.")
+    add_filter_args(seq)
+    seq.set_defaults(func=run_seq_command)
+
+    audit = subparsers.add_parser("audit", help="Audit an existing standardized CSV.")
+    audit.add_argument("--input", required=True, type=Path, help="Path to fetchm2_clean.csv.")
+    audit.add_argument("--outdir", required=True, type=Path, help="Audit output directory.")
+    audit.set_defaults(func=run_audit_command)
+    return parser
+
+
+def filter_dict(args: argparse.Namespace) -> dict[str, object]:
+    return {
+        "host": args.host,
+        "host_rank": args.host_rank,
+        "country": args.country,
+        "continent": args.continent,
+        "subcontinent": args.subcontinent,
+        "sample_type": args.sample_type,
+        "isolation_source": args.isolation_source,
+        "environment_medium": args.environment_medium,
+        "year_from": args.year_from,
+        "year_to": args.year_to,
+    }
+
+
+def run_metadata_command(args: argparse.Namespace) -> None:
+    result = run_metadata(
+        input_path=args.input,
+        outdir=args.outdir,
+        ani=args.ani,
+        checkm=args.checkm,
+        api_key=args.api_key,
+        email=args.email,
+        workers=args.workers,
+        sleep=args.sleep,
+        offline=args.offline,
+    )
+    print(f"Wrote clean metadata: {result['clean_path']}")
+    print(f"Production gate: {'PASS' if result['production_ready'] else 'FAIL'}")
+
+
+def run_all_command(args: argparse.Namespace) -> None:
+    result = run_metadata(
+        input_path=args.input,
+        outdir=args.outdir,
+        ani=args.ani,
+        checkm=args.checkm,
+        api_key=args.api_key,
+        email=args.email,
+        workers=args.workers,
+        sleep=args.sleep,
+        offline=args.offline,
+    )
+    print(f"Wrote clean metadata: {result['clean_path']}")
+    if args.download:
+        summary = run_sequence_downloads(
+            input_path=Path(result["clean_path"]),
+            outdir=args.outdir / "sequence",
+            filters=filter_dict(args),
+            retries=args.retries,
+            retry_delay=args.retry_delay,
+            workers=args.download_workers,
+            max_genomes=args.max_genomes,
+            keep_gz=args.keep_gz,
+        )
+        print(f"Sequence summary: {summary}")
+
+
+def run_seq_command(args: argparse.Namespace) -> None:
+    summary = run_sequence_downloads(
+        input_path=args.input,
+        outdir=args.outdir,
+        filters=filter_dict(args),
+        retries=args.retries,
+        retry_delay=args.retry_delay,
+        workers=args.download_workers,
+        check_only=args.check_only,
+        max_genomes=args.max_genomes,
+        keep_gz=args.keep_gz,
+    )
+    print(f"Sequence summary: {summary}")
+
+
+def run_audit_command(args: argparse.Namespace) -> None:
+    import pandas as pd
+
+    rows = pd.read_csv(args.input).fillna("").to_dict(orient="records")
+    summary = write_audit_outputs(rows, args.outdir)
+    ready, failures, warnings = production_gate(summary)
+    print(f"Production gate: {'PASS' if ready else 'FAIL'}")
+    if failures:
+        print(f"Hard failures: {failures}")
+    if warnings:
+        print(f"Warnings: {warnings}")
+
+
+def main() -> None:
+    args = build_parser().parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

fetchm2/data/__init__.py

@@ -0,0 +1,2 @@
+"""Packaged FetchM2 standardization data."""
+

fetchm2/data/approved_broad_categories.csv

@@ -0,0 +1,51 @@
+field,approved_value,description,examples
+Isolation_Source_SD_Broad,clinical/host-associated material,Clinical or host-associated biological material,blood; sputum; tissue
+Isolation_Source_SD_Broad,host-associated context,Host-associated context where a specific specimen/source is not available,whole organism; wildlife
+Isolation_Source_SD_Broad,feces/stool,Fecal or stool material,feces; stool; faeces
+Isolation_Source_SD_Broad,food,Food or food product,ready-to-eat food; fermented food
+Isolation_Source_SD_Broad,food/meat,Meat or animal-origin food product,chicken meat; beef; pork
+Isolation_Source_SD_Broad,food/dairy,Dairy food or dairy product,milk; cheese; dairy product
+Isolation_Source_SD_Broad,food/produce,Produce or plant-origin food,spinach; papaya; vegetable
+Isolation_Source_SD_Broad,food/plant product,Processed plant-origin food,peanut butter
+Isolation_Source_SD_Broad,food/processing environment,Food-processing or food-contact environment,food-contact surface
+Isolation_Source_SD_Broad,water,Aquatic or water-associated source,river water; lake water; seawater
+Isolation_Source_SD_Broad,wastewater/sewage,Wastewater sewage or sludge source,wastewater; sewage; activated sludge
+Isolation_Source_SD_Broad,soil,Soil-associated material,soil; rhizosphere soil
+Isolation_Source_SD_Broad,sediment,Sediment-associated material,marine sediment; pond sediment
+Isolation_Source_SD_Broad,environmental material,Generic environmental material,environmental sample
+Isolation_Source_SD_Broad,environmental/geologic material,Geologic or extreme environmental material,rock; hydrothermal vent
+Isolation_Source_SD_Broad,healthcare-associated environment,Healthcare or hospital environment,hospital surface; ICU environment
+Isolation_Source_SD_Broad,agricultural environment,Agricultural or farm environment,farm; dairy farm
+Isolation_Source_SD_Broad,agricultural fecal material,Agricultural fecal material,manure
+Isolation_Source_SD_Broad,animal-associated environment,Animal production or animal-associated environment,poultry house; animal facility
+Isolation_Source_SD_Broad,plant-associated material,Plant-associated non-food material,root; rhizosphere; leaves
+Isolation_Source_SD_Broad,culture,Culture material or culture condition,pure culture; mixed culture
+Isolation_Source_SD_Broad,culture/assembly,Assembly or metagenomic culture descriptor,metagenomic assembly
+Isolation_Source_SD_Broad,culture/isolate,Microbial isolate or culture isolate,microbial isolate
+Isolation_Source_SD_Broad,culture medium,Culture medium,blood agar culture medium
+Isolation_Source_SD_Broad,laboratory environment,Laboratory source or context,laboratory
+Isolation_Source_SD_Broad,built environment,Built environment or built surface,sink; drain; cleanroom floor
+Isolation_Source_SD_Broad,surface sample,Surface sample or surface context,surface
+Isolation_Source_SD_Broad,biofilm,Biofilm source,biofilm; wall biofilm
+Isolation_Source_SD_Broad,respiratory sample,Respiratory sample category,tracheal aspirate; bronchoalveolar lavage
+Isolation_Source_SD_Broad,upper respiratory tract,Upper respiratory tract site,nasopharynx/oropharynx
+Isolation_Source_SD_Broad,upper respiratory site,Upper respiratory site,nasal site
+Isolation_Source_SD_Broad,oral cavity,Oral cavity site,dental plaque; oral cavity
+Isolation_Source_SD_Broad,urogenital site,Urogenital site,urethra; cervix
+Isolation_Source_SD_Broad,gastrointestinal site,Gastrointestinal site,colon; ileum
+Isolation_Source_SD_Broad,gut content,Gut or intestinal content,intestine; stomach; rumen
+Isolation_Source_SD_Broad,tissue,Tissue specimen,FFPE tissue
+Isolation_Source_SD_Broad,swab,Swab specimen category,rectal swab; nasal swab
+Isolation_Source_SD_Broad,clinical fluid/material,Clinical fluid or material,pus; abscess; pleural fluid
+Isolation_Source_SD_Broad,medical device,Medical device or catheter context,catheter
+Isolation_Source_SD_Broad,aquatic food product,Aquatic food product,seafood; fish product
+Isolation_Source_SD_Broad,biological/clinical product,Biological or clinical product,biological product
+Isolation_Source_SD_Broad,molecular extract,Molecular extract,DNA extract
+Isolation_Source_SD_Broad,single cell,Single-cell source,single cell
+Isolation_Source_SD_Broad,sample,Generic sample label,sample
+Isolation_Source_SD_Broad,cloacal sample,Cloacal specimen,cloacal sample
+Isolation_Source_SD_Broad,fermented food,Fermented food,kimchi; fermented food
+Isolation_Source_SD_Broad,surface/sample collection material,Collection material or surface sampling tool,sponge
+Isolation_Source_SD_Broad,gut/host-associated material,Gut-associated host material,intestinal epithelial cells
+Isolation_Source_SD_Broad,wastewater/organic waste,Wastewater or organic waste digestion source,anaerobic digester
+Isolation_Source_SD_Broad,metadata descriptor / non-source,Metadata descriptor rather than biological source,metagenome