omix 0.1.0__py3-none-any.whl

omix/__init__.py

@@ -0,0 +1,13 @@
+"""
+omix: A modular Python package for fetching, enriching, and analyzing
+omics metadata and publications.
+
+Usage:
+    from omix import Config
+    config = Config(email="you@example.com")
+"""
+
+__version__ = "0.1.0"
+
+from .config import Config, load_config
+from .logging_utils import setup_logging, get_logger

omix/cli.py

@@ -0,0 +1,531 @@
+"""
+Command‑line interface for omix.
+
+Provides subcommands:
+- build-primer-db : download and build a probeBase primer database.
+- fetch-metadata  : enrich a metadata file with ENA data.
+- fetch-publications : search and analyse publications linked to accessions.
+- run-pipeline    : run the full MetadataManager pipeline on a file.
+"""
+
+import asyncio
+from pathlib import Path
+from typing import List, Optional
+
+import click
+
+from omix import __version__
+from omix.config import Config, load_config
+from omix.logging_utils import setup_logging
+from omix.metadata.file_workflow import enrich_metadata_from_path
+from omix.metadata.manager import MetadataManager
+from omix.publications.apis.ncbi import PMIDSource
+from omix.publications.fetcher import PublicationFetcher
+from omix.publications.apis import (
+    CrossrefAPI,
+    EuropePMCAPI,
+    NCBIAPI,
+    SemanticScholarAPI,
+    ArxivAPI,
+    BioarxivAPI,
+    CoreAPI,
+    DataciteAPI,
+    DOAJAPI,
+    PLOSAPI,
+    UnpaywallAPI,
+    ZenodoAPI,
+)
+from omix.publications.extractors.omics import SixteenSExtractor
+from omix.publications.cache import PublicationCache
+from omix.validators.primer_db import ProbeBaseDatabase
+
+
+# --------------------------------------------------------------------------- #
+#  Common options
+# --------------------------------------------------------------------------- #
+
+def _config_option(f):
+    """Decorator that adds --config / --email / --cache-dir options to a command."""
+    f = click.option(
+        "--config", "-c",
+        type=click.Path(exists=True, path_type=Path),
+        help="Path to YAML configuration file.",
+    )(f)
+    f = click.option(
+        "--email", "-e",
+        default=None,
+        help="Email address for polite API requests (overrides config).",
+    )(f)
+    f = click.option(
+        "--cache-dir",
+        type=click.Path(path_type=Path),
+        help="Override the cache directory.",
+    )(f)
+    return f
+
+
+def _build_config(config_path: Optional[Path], email: Optional[str], cache_dir: Optional[Path]) -> Config:
+    """Build a Config object from CLI arguments and/or a YAML file."""
+    if config_path:
+        config = load_config(config_path)
+    else:
+        config = Config()
+
+    if email:
+        config.credentials.email = email
+    if cache_dir:
+        config.paths.cache_dir = cache_dir
+        config.cache_dir = cache_dir
+    return config
+
+
+# --------------------------------------------------------------------------- #
+#  Main entry point
+# --------------------------------------------------------------------------- #
+
+@click.group()
+@click.version_option(version=__version__)
+def main():
+    """omix – a modular toolkit for omics metadata & publication analysis."""
+    pass
+
+
+# --------------------------------------------------------------------------- #
+#  build-primer-db
+# --------------------------------------------------------------------------- #
+
+@main.command()
+@click.option("--csv", type=click.Path(path_type=Path), default=Path("data/probe_data.csv"),
+              help="Path to downloaded probeBase CSV (will be downloaded if missing).")
+@click.option("--db", type=click.Path(path_type=Path), default=Path("data/primer_data.db"),
+              help="Output SQLite database path.")
+@_config_option
+def build_primer_db(
+    csv: Path,
+    db: Path,
+    config: Optional[Path],
+    email: Optional[str],
+    cache_dir: Optional[Path],
+):
+    """Download the probeBase primer list and build a searchable SQLite database."""
+    cfg = _build_config(config, email, cache_dir)
+    setup_logging(cfg.logs_dir)
+
+    from omix.validators.probebase_builder import import_and_save_database
+    success = import_and_save_database(csv, db)
+    if not success:
+        click.echo("❌ Primer database build failed.", err=True)
+        raise click.Abort()
+    click.echo(f"✅ Primer database built at {db}")
+
+
+# --------------------------------------------------------------------------- #
+#  fetch-metadata
+# --------------------------------------------------------------------------- #
+
+@main.command()
+@click.argument("input_path", type=click.Path(exists=True, path_type=Path))
+@click.option("--output", "-o", type=click.Path(path_type=Path), help="Output file path.")
+@click.option("--sample-id-col", default=None, help="Override sample ID column name.")
+@click.option("--no-ena", is_flag=True, help="Disable ENA enrichment.")
+@click.option("--no-geocode", is_flag=True, help="Disable reverse geocoding.")
+@click.option("--preserve-rows", is_flag=True, default=True, help="Keep rows without coordinates.")
+@click.option("--omics-type", default=None, help="Filter by omics type (amplicon, metagenomics, transcriptomics, etc.).")
+@click.option("--amplicon-gene", default=None, help="Filter by amplicon gene (16S, 18S, ITS, COI, etc.).")
+@click.option("--primer-set", default=None, help="Filter by primer set name.")
+@click.option("--subfragment", default=None, help="Filter by subfragment/variable region.")
+@click.option("--report", "-r", type=click.Path(path_type=Path), help="Save composition report as JSON to this path.")
+@_config_option
+def fetch_metadata(
+    input_path: Path,
+    output: Optional[Path],
+    sample_id_col: Optional[str],
+    no_ena: bool,
+    no_geocode: bool,
+    preserve_rows: bool,
+    omics_type: Optional[str],
+    amplicon_gene: Optional[str],
+    primer_set: Optional[str],
+    subfragment: Optional[str],
+    report: Optional[Path],
+    config: Optional[Path],
+    email: Optional[str],
+    cache_dir: Optional[Path],
+):
+    """Enrich a metadata file with ENA data and generate an omics profile report."""
+    cfg = _build_config(config, email, cache_dir)
+    if no_geocode:
+        cfg.metadata.enable_geocoding = False
+    setup_logging(cfg.logs_dir)
+
+    click.echo(f"📁 Loading {input_path} ...")
+    result = asyncio.run(
+        enrich_metadata_from_path(
+            input_path=input_path,
+            output_path=output,
+            config=cfg,
+            sample_id_column=sample_id_col,
+            enable_ena_lookup=not no_ena,
+            preserve_all_rows=preserve_rows,
+            omics_type=omics_type,
+            amplicon_gene=amplicon_gene,
+            primer_set=primer_set,
+            subfragment=subfragment,
+            report_output=report,
+        )
+    )
+    click.echo(f"✅ Done. Output has {len(result)} rows and {len(result.columns)} columns.")
+    if report:
+        click.echo(f"📊 Composition report saved to {report}")
+
+
+# --------------------------------------------------------------------------- #
+#  fetch-publications
+# --------------------------------------------------------------------------- #
+
+@main.command()
+@click.argument("accessions", nargs=-1)
+@click.option("--omics", default="16S", help="Omics type (currently only '16S').")
+@click.option("--pmid", type=str, default=None, help="PubMed ID for direct lookup.")
+@click.option("--api-key", envvar="OMIX_LLM_API_KEY", help="LLM API key for methodology extraction.")
+@click.option("--no-llm", is_flag=True, help="Skip LLM extraction (regex only).")
+@click.option("--builtin", is_flag=True, help="Use the built‑in primer database (no probeBase needed).")
+@click.option("--primer-db", type=click.Path(exists=True, path_type=Path),
+              help="Path to a probeBase SQLite primer database.")
+@click.option("--max-rounds", type=int, default=3, help="Maximum citation chasing rounds.")
+@click.option("--output", "-o", type=click.Path(path_type=Path), help="Save results as JSON.")
+@_config_option
+def fetch_publications(
+    accessions: List[str],
+    omics: str,
+    pmid: Optional[str],
+    api_key: Optional[str],
+    no_llm: bool,
+    builtin: bool,
+    primer_db: Optional[Path],
+    max_rounds: int,
+    output: Optional[Path],
+    config: Optional[Path],
+    email: Optional[str],
+    cache_dir: Optional[Path],
+):
+    """Search and analyse publications for one or more accessions."""
+    cfg = _build_config(config, email, cache_dir)
+    setup_logging(cfg.logs_dir)
+
+    if not accessions:
+        click.echo("❌ You must provide at least one accession.", err=True)
+        raise click.Abort()
+
+    # ---- Build publication sources (free and reliable ones) ----
+    # FEATURE 3: Pass retry config from publication config
+    retry_config = {
+        'max_retries': cfg.publication.max_retries,
+        'base_delay': cfg.publication.base_delay_seconds,
+        'max_delay': cfg.publication.max_delay_seconds,
+    }
+    sources = [
+        CrossrefAPI(cfg.credentials.email, **retry_config),
+        EuropePMCAPI(cfg.credentials.email, **retry_config),
+        NCBIAPI(cfg.credentials.email, cfg.credentials.ncbi_api_key, **retry_config),
+        SemanticScholarAPI(cfg.credentials.email, **retry_config),
+        ArxivAPI(cfg.credentials.email, **retry_config),
+        BioarxivAPI(cfg.credentials.email, **retry_config),
+        CoreAPI(cfg.credentials.email, **retry_config),
+        DataciteAPI(cfg.credentials.email, **retry_config),
+        DOAJAPI(cfg.credentials.email, **retry_config),
+        PLOSAPI(cfg.credentials.email, **retry_config),
+        UnpaywallAPI(cfg.credentials.email, **retry_config),
+        ZenodoAPI(cfg.credentials.email, **retry_config),
+    ]
+
+    # ---- Build omics extractor ----
+    llm_key = api_key or cfg.credentials.llm_api_key
+
+    # Primer database: prefer builtin, then external file, else None
+    primer_database = None
+    if builtin:
+        primer_database = ProbeBaseDatabase(use_builtin=True)
+    elif primer_db:
+        primer_database = ProbeBaseDatabase(db_path=primer_db)
+
+    if pmid:
+        sources = [PMIDSource(cfg.credentials.email, pmid, **retry_config)]
+    
+    if omics.lower() == "16s":
+        if no_llm:
+            extractor = SixteenSExtractor(api_key="", primer_db=primer_database)
+        else:
+            extractor = SixteenSExtractor(api_key=llm_key or "", primer_db=primer_database)
+    else:
+        click.echo(f"❌ Unknown omics type: {omics}", err=True)
+        raise click.Abort()
+
+    # ---- Run fetcher ----
+    cache = PublicationCache(cfg.paths.cache_dir / "publications.db")
+    fetcher = PublicationFetcher(cfg, sources, extractor, cache)
+    fetcher.MAX_PUBLICATION_ROUNDS = max_rounds
+
+    click.echo(f"🔍 Searching publications for {len(accessions)} accessions...")
+    results = fetcher.fetch_and_analyze_sync(accessions)
+
+    # ---- Output ----
+    if output:
+        import json
+        output.parent.mkdir(parents=True, exist_ok=True)
+        with open(output, "w") as f:
+            json.dump(results, f, indent=2, default=str)
+        click.echo(f"📄 Results written to {output}")
+    else:
+        for acc, pubs in results.items():
+            click.echo(f"\n📌 {acc} – {len(pubs)} publications")
+            for pub in pubs[:5]:
+                title = (pub.get('publication_title') or 'N/A')[:80]
+                status = pub.get('status', '?')
+                click.echo(f"   [{status}] {title}")
+            if len(pubs) > 5:
+                click.echo(f"   … and {len(pubs) - 5} more.")
+
+
+# --------------------------------------------------------------------------- #
+#  run-pipeline
+# --------------------------------------------------------------------------- #
+
+@main.command()
+@click.argument("input_path", type=click.Path(exists=True, path_type=Path))
+@click.option("--output", "-o", type=click.Path(path_type=Path), help="Output file path.")
+@click.option("--sample-id-col", default=None, help="Override sample ID column name.")
+@click.option("--no-geocode", is_flag=True, help="Disable reverse geocoding.")
+@_config_option
+def run_pipeline(
+    input_path: Path,
+    output: Optional[Path],
+    sample_id_col: Optional[str],
+    no_geocode: bool,
+    config: Optional[Path],
+    email: Optional[str],
+    cache_dir: Optional[Path],
+):
+    """Run the full MetadataManager pipeline on a file (cleaning + enrichment)."""
+    cfg = _build_config(config, email, cache_dir)
+    if no_geocode:
+        cfg.metadata.enable_geocoding = False
+    setup_logging(cfg.logs_dir)
+
+    import pandas as pd
+    from omix.metadata.file_workflow import _load_table, _save_table, _ensure_sample_id_column
+
+    df = _load_table(input_path)
+    if sample_id_col:
+        df = _ensure_sample_id_column(df, sample_id_col)
+    elif cfg.metadata.sample_id_column not in df.columns:
+        df = _ensure_sample_id_column(df, cfg.metadata.sample_id_column)
+
+    manager = MetadataManager(df, cfg, sample_id_column=sample_id_col)
+    enriched = asyncio.run(manager.run_pipeline())
+
+    if output:
+        _save_table(enriched, output)
+        click.echo(f"✅ Enriched metadata written to {output}")
+    else:
+        click.echo(f"✅ Pipeline complete – {len(enriched)} rows, {len(enriched.columns)} columns.")
+        click.echo(enriched.head())
+
+
+# --------------------------------------------------------------------------- #
+#  enrich-with-publications
+# --------------------------------------------------------------------------- #
+
+@main.command()
+@click.argument("input_path", type=click.Path(exists=True, path_type=Path))
+@click.option("--output", "-o", type=click.Path(path_type=Path), help="Output file path.")
+@click.option("--no-validate", is_flag=True, help="Skip publication validation (keep all publications).")
+@click.option("--api-key", envvar="OMIX_LLM_API_KEY", help="LLM API key for methodology extraction.")
+@click.option("--no-llm", is_flag=True, help="Skip LLM extraction (regex only).")
+@click.option("--builtin", is_flag=True, help="Use the built‑in primer database.")
+@click.option("--primer-db", type=click.Path(exists=True, path_type=Path),
+              help="Path to a probeBase SQLite primer database.")
+@click.option("--max-rounds", type=int, default=3, help="Maximum citation chasing rounds.")
+@_config_option
+def enrich_with_publications(
+    input_path: Path,
+    output: Optional[Path],
+    no_validate: bool,
+    api_key: Optional[str],
+    no_llm: bool,
+    builtin: bool,
+    primer_db: Optional[Path],
+    max_rounds: int,
+    config: Optional[Path],
+    email: Optional[str],
+    cache_dir: Optional[Path],
+):
+    """
+    Enrich metadata with ENA data AND publication information in one pipeline.
+    
+    This unified command:
+    1. Fetches and enriches metadata from ENA
+    2. Discovers publications from 12+ sources
+    3. Validates publications for accession relevance
+    4. Integrates publication counts and DOIs into metadata
+    
+    Output includes: all ENA metadata fields + publication_count + publication_dois
+    """
+    cfg = _build_config(config, email, cache_dir)
+    setup_logging(cfg.logs_dir)
+    
+    # Import the unified pipeline
+    from omix.metadata.file_workflow import _load_table, _save_table
+    import pandas as pd
+    
+    async def async_enrich():
+        """Run async enrichment pipeline."""
+        from omix.metadata.file_workflow import enrich_metadata_from_path
+        from omix.publications.fetcher import PublicationFetcher
+        from omix.publications.extractors.omics import SixteenSExtractor
+        from omix.publications.cache import PublicationCache
+        from omix.publications.apis import (
+            CrossrefAPI, EuropePMCAPI, NCBIAPI, SemanticScholarAPI,
+            ArxivAPI, BioarxivAPI, CoreAPI, DataciteAPI, DOAJAPI, PLOSAPI,
+            UnpaywallAPI, ZenodoAPI,
+        )
+        
+        # ---- Phase 1: Metadata ----
+        click.echo("📊 Phase 1: Metadata enrichment...")
+        metadata = await enrich_metadata_from_path(
+            input_path=input_path,
+            output_path=None,
+            config=cfg,
+            enable_ena_lookup=True,
+            preserve_all_rows=True,
+        )
+        click.echo(f"   ✓ {len(metadata)} rows × {len(metadata.columns)} columns")
+        
+        # Extract study accessions
+        if 'study_accession' not in metadata.columns:
+            click.echo("❌ study_accession column missing; cannot fetch publications", err=True)
+            if output:
+                _save_table(metadata, output)
+            return metadata
+        
+        study_accessions = sorted(set(metadata['study_accession'].dropna().unique()))
+        
+        if not study_accessions:
+            click.echo("⚠️  No study accessions found; skipping publication fetch")
+            if output:
+                _save_table(metadata, output)
+            return metadata
+        
+        # ---- Phase 2: Publications ----
+        click.echo(f"📚 Phase 2: Publication discovery ({len(study_accessions)} studies)...")
+        
+        retry_config = {
+            'max_retries': cfg.publication.max_retries,
+            'base_delay': cfg.publication.base_delay_seconds,
+            'max_delay': cfg.publication.max_delay_seconds,
+        }
+        sources = [
+            CrossrefAPI(cfg.credentials.email, **retry_config),
+            EuropePMCAPI(cfg.credentials.email, **retry_config),
+            NCBIAPI(cfg.credentials.email, cfg.credentials.ncbi_api_key, **retry_config),
+            SemanticScholarAPI(cfg.credentials.email, **retry_config),
+            ArxivAPI(cfg.credentials.email, **retry_config),
+            BioarxivAPI(cfg.credentials.email, **retry_config),
+            CoreAPI(cfg.credentials.email, **retry_config),
+            DataciteAPI(cfg.credentials.email, **retry_config),
+            DOAJAPI(cfg.credentials.email, **retry_config),
+            PLOSAPI(cfg.credentials.email, **retry_config),
+            UnpaywallAPI(cfg.credentials.email, **retry_config),
+            ZenodoAPI(cfg.credentials.email, **retry_config),
+        ]
+        
+        llm_key = api_key or cfg.credentials.llm_api_key
+        primer_database = None
+        if builtin:
+            primer_database = ProbeBaseDatabase(use_builtin=True)
+        elif primer_db:
+            primer_database = ProbeBaseDatabase(db_path=primer_db)
+        
+        extractor = SixteenSExtractor(api_key=llm_key if not no_llm else "", primer_db=primer_database)
+        cache = PublicationCache(cfg.paths.cache_dir / "publications.db")
+        fetcher = PublicationFetcher(cfg, sources, extractor, cache)
+        fetcher.MAX_PUBLICATION_ROUNDS = max_rounds
+        
+        publications = fetcher.fetch_and_analyze_sync(study_accessions)
+        total_pubs = sum(len(p) for p in publications.values())
+        click.echo(f"   ✓ {total_pubs} publications found")
+        
+        # ---- Phase 3: Validation ----
+        if not no_validate:
+            click.echo("🔍 Phase 3: Publication validation...")
+            
+            # Inline validation
+            filtered = {}
+            for study_accession, pubs in publications.items():
+                filtered[study_accession] = []
+                for pub in pubs:
+                    if pub.get('status') != '✓ Extraction complete.':
+                        continue
+                    matched_queries = pub.get('matched_queries', [])
+                    accession_in_text = pub.get('accession_mentions_in_text', 0) > 0
+                    direct_match = any(
+                        q == study_accession or (study_accession in q and not q.startswith('DATA:'))
+                        for q in matched_queries
+                    )
+                    if direct_match or accession_in_text:
+                        filtered[study_accession].append(pub)
+            
+            publications = filtered
+            valid_count = sum(len(p) for p in publications.values())
+            click.echo(f"   ✓ {valid_count} publications with direct accession matches")
+        
+        # ---- Phase 4: Integration ----
+        click.echo("🔗 Phase 4: Integration...")
+        
+        if 'publication_count' not in metadata.columns:
+            metadata['publication_count'] = None
+        if 'publication_dois' not in metadata.columns:
+            metadata['publication_dois'] = None
+        
+        filled_rows = 0
+        studies_with_dois = 0
+        
+        for study_accession, pubs in publications.items():
+            dois = []
+            seen = set()
+            for pub in pubs:
+                if not isinstance(pub, dict):
+                    continue
+                doi = pub.get('doi')
+                if doi and doi not in seen:
+                    dois.append(doi)
+                    seen.add(doi)
+            
+            mask = metadata['study_accession'] == study_accession
+            filled_rows += int(mask.sum())
+            metadata.loc[mask, 'publication_count'] = len(dois)
+            metadata.loc[mask, 'publication_dois'] = '; '.join(dois) if dois else ''
+            
+            if dois:
+                studies_with_dois += 1
+        
+        click.echo(f"   ✓ {filled_rows} rows updated, {studies_with_dois} studies with DOIs")
+        
+        return metadata
+    
+    try:
+        enriched_metadata = asyncio.run(async_enrich())
+        
+        if output:
+            output.parent.mkdir(parents=True, exist_ok=True)
+            _save_table(enriched_metadata, output)
+            click.echo(f"\n✅ Output saved to {output}")
+        
+        click.echo(f"✅ Complete! {len(enriched_metadata)} rows × {len(enriched_metadata.columns)} columns")
+    except Exception as e:
+        click.echo(f"\n❌ Error: {e}", err=True)
+        raise click.Abort()
+
+
+if __name__ == "__main__":
+    main()

omix/config.py

@@ -0,0 +1,150 @@
+"""
+Unified configuration for omix, loadable from YAML files and environment variables.
+"""
+
+import os
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+import yaml
+
+
+class Credentials:
+    """Holds all API credentials, loaded from config or environment."""
+
+    def __init__(self, cred_dict: Optional[Dict[str, Any]] = None):
+        cred = cred_dict or {}
+        self.email = cred.get("email") or os.getenv("OMIX_EMAIL", "")
+        self.ena_email = cred.get("ena_email") or os.getenv("OMIX_ENA_EMAIL", self.email)
+        self.ncbi_api_key = cred.get("ncbi_api_key") or os.getenv("OMIX_NCBI_API_KEY")
+        self.llm_api_key = cred.get("llm_api_key") or os.getenv("OMIX_LLM_API_KEY")
+        self.dimensions_api_key = cred.get("dimensions_api_key") or os.getenv("OMIX_DIMENSIONS_API_KEY")
+        self.ieee_api_key = cred.get("ieee_api_key") or os.getenv("OMIX_IEEE_API_KEY")
+        self.mendeley_api_key = cred.get("mendeley_api_key") or os.getenv("OMIX_MENDELEY_API_KEY")
+        self.springer_api_key = cred.get("springer_api_key") or os.getenv("OMIX_SPRINGER_API_KEY")
+        self.lens_api_key = cred.get("lens_api_key") or os.getenv("OMIX_LENS_API_KEY")
+
+
+class MetadataConfig:
+    """Metadata normalization and enrichment settings."""
+
+    def __init__(self, cfg: Optional[Dict[str, Any]] = None):
+        cfg = cfg or {}
+        self.columns_to_drop = cfg.get("columns_to_drop", [])
+        self.force_numeric_columns = cfg.get("force_numeric_columns", ["lat", "lon", "depth", "altitude"])
+        self.mappings = cfg.get("mappings", {})
+        self.sample_id_column = cfg.get("sample_id_column", "#sampleid")
+        self.suffixes_to_collapse = cfg.get("suffixes_to_collapse", [])
+        self.exclude_host = cfg.get("exclude_host", False)
+        self.enable_geocoding = cfg.get("enable_geocoding", True)
+
+
+class ENAApiConfig:
+    """ENA API specific options."""
+
+    def __init__(self, cfg: Dict[str, Any]):
+        self.enabled = cfg.get("enabled", True)
+        self.max_concurrent = cfg.get("max_concurrent", 5)
+        self.batch_size = cfg.get("batch_size", 100)
+        self.cache_ttl_days = cfg.get("cache_ttl_days", 30)
+        self.fetch_phases = cfg.get("fetch_phases", True)
+        self.phase2_async = cfg.get("phase2_async", True)
+        self.cache_write = CacheWriteConfig(cfg.get("cache_write", {}))
+
+
+class CacheWriteConfig:
+    """Batched write configuration for SQLite cache."""
+
+    def __init__(self, cfg: Dict[str, Any]):
+        self.batch_size = cfg.get("batch_size", 100)
+        self.flush_interval_seconds = cfg.get("flush_interval_seconds", 5.0)
+
+
+class ApisConfig:
+    """Toggle which external APIs are enabled."""
+
+    def __init__(self, cfg: Optional[Dict[str, Any]] = None):
+        cfg = cfg or {}
+        self.enabled = cfg.get("enabled", True)
+        seq = cfg.get("sequence", {})
+        self.ena = ENAApiConfig(seq.get("ena", {}))
+
+
+class PublicationConfig:
+    """Publication search and analysis settings."""
+
+    def __init__(self, cfg: Optional[Dict[str, Any]] = None):
+        cfg = cfg or {}
+        self.max_concurrent_apis = cfg.get("max_concurrent_apis", 5)
+        self.rounds = cfg.get("rounds", 3)
+        self.max_pdf_pages = cfg.get("max_pdf_pages", 10)
+        self.max_file_size = cfg.get("max_file_size", 15_000_000)
+        
+        # FEATURE 3: Retry/backoff configuration per API source
+        retry_cfg = cfg.get("retry", {})
+        self.max_retries = retry_cfg.get("max_retries", 5)
+        self.base_delay_seconds = retry_cfg.get("base_delay_seconds", 1.0)
+        self.max_delay_seconds = retry_cfg.get("max_delay_seconds", 32.0)
+
+
+class PathsConfig:
+    """Project and dependency paths."""
+
+    def __init__(self, cfg: Optional[Dict[str, Any]] = None):
+        cfg = cfg or {}
+        self.project = Path(cfg.get("project", "."))
+        self.cache_dir = Path(cfg.get("cache_dir", self.project / ".cache"))
+        self.logs_dir = Path(cfg.get("logs_dir", self.project / "logs"))
+        self.primer_db = cfg.get("primer_db")  # optional
+
+
+class OmicsType:
+    """Enumeration of supported omics types."""
+
+    _16S = "16S"
+    METAGENOMICS = "metagenomics"
+
+    def __init__(self, name: str):
+        self.name = name
+
+
+class Config:
+    """Top‑level configuration object, aggregating all sub‑configs."""
+
+    def __init__(self, config_path: Optional[Path] = None, **overrides):
+        self._raw: Dict[str, Any] = {}
+        if config_path:
+            with open(config_path, "r") as f:
+                self._raw = yaml.safe_load(f) or {}
+
+        # Move known overrides into the correct sub-dicts BEFORE building sub‑configs
+        cred_overrides = {}
+        for key in ("email", "ena_email", "llm_api_key", "ncbi_api_key",
+                    "dimensions_api_key", "ieee_api_key", "mendeley_api_key",
+                    "springer_api_key", "lens_api_key"):
+            if key in overrides:
+                cred_overrides[key] = overrides.pop(key)
+
+        self._raw.update(overrides)                     # remaining top-level overrides
+        credentials_raw = self._raw.setdefault("credentials", {})
+        credentials_raw.update(cred_overrides)          # apply credential overrides
+
+        # Build sub‑configs
+        self.credentials = Credentials(self._raw.get("credentials"))
+        self.paths = PathsConfig(self._raw.get("paths"))
+        self.metadata = MetadataConfig(self._raw.get("metadata"))
+        self.apis = ApisConfig(self._raw.get("apis"))
+        self.publication = PublicationConfig(self._raw.get("publication"))
+        self.omics_type = OmicsType(self._raw.get("omics_type", "16S"))
+        self.cache_dir = self.paths.cache_dir
+        self.logs_dir = self.paths.logs_dir
+
+    def save(self, path: Path) -> None:
+        """Write current configuration to a YAML file."""
+        with open(path, "w") as f:
+            yaml.dump(self._raw, f)
+
+
+def load_config(config_path: Path) -> Config:
+    """Load configuration from a YAML file and return a Config instance."""
+    return Config(config_path)