genelastic 0.8.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

genelastic/common/types.py

@@ -4,20 +4,7 @@ import typing
 Bucket: typing.TypeAlias = dict[str, dict[typing.Any, typing.Any]]
 BundleDict: typing.TypeAlias = dict[str, typing.Any]
 
-AnalysisMetaData: typing.TypeAlias = dict[str, str | int]
-WetProcessesData: typing.TypeAlias = dict[str, str | int | float]
-BioInfoProcessData: typing.TypeAlias = dict[str, str | list[str]]
-
-AnalysisDocument: typing.TypeAlias = dict[str, str | None | AnalysisMetaData]
-MetadataDocument: typing.TypeAlias = dict[
-    str, int | str | list[typing.Any | None]
-]
-ProcessDocument: typing.TypeAlias = (
-    dict[str, str] | WetProcessesData | BioInfoProcessData
-)
-BulkItems: typing.TypeAlias = list[
-    dict[str, str | MetadataDocument | AnalysisDocument | ProcessDocument]
-]
+Metadata: typing.TypeAlias = dict[str, str | int]
 
 # Types related to random bundle generation.
 RandomBiProcessData: typing.TypeAlias = dict[str, str | list[dict[str, str]]]

genelastic/import_data/__init__.py

@@ -1,27 +0,0 @@
-"""Genelastic package for importing Genomic data into Elasticsearch."""
-
-from .analysis import Analysis
-from .import_bundle import ImportBundle
-from .import_bundle_factory import (
-    load_import_bundle_file,
-    make_import_bundle_from_files,
-)
-from .random_bundle import (
-    RandomAnalysis,
-    RandomBiProcess,
-    RandomBundle,
-    RandomWetProcess,
-)
-from .tags import Tags
-
-__all__ = [
-    "Analysis",
-    "ImportBundle",
-    "RandomAnalysis",
-    "RandomBiProcess",
-    "RandomBundle",
-    "RandomWetProcess",
-    "Tags",
-    "load_import_bundle_file",
-    "make_import_bundle_from_files",
-]

genelastic/import_data/checker.py

@@ -0,0 +1,99 @@
+import logging
+
+from genelastic.common.elastic import ElasticQueryConn
+from genelastic.import_data.checker_observer import CheckerObserver
+from genelastic.import_data.models.analyses import Analyses
+from genelastic.import_data.models.processes import Processes
+
+logger = logging.getLogger("genelastic")
+
+
+class Checker:
+    """Validate coherence between YAML metadata and Elasticsearch,
+    using a project-specific observer mechanism.
+    """
+
+    def __init__(self, es: ElasticQueryConn, *, strict: bool = False) -> None:
+        """Initialize the Checker.
+
+        Args:
+            es: Elasticsearch connection instance.
+            strict: Treat ES-only entries as errors when True.
+        """
+        self.es = es
+        self.strict = strict
+        self.errors_detected = False
+        self._observers: list[CheckerObserver] = []
+
+    def attach(self, observer: CheckerObserver) -> None:
+        """Register an observer to receive Checker notifications."""
+        self._observers.append(observer)
+
+    def detach(self, observer: CheckerObserver) -> None:
+        """Unregister an observer so it no longer receives notifications."""
+        self._observers.remove(observer)
+
+    def _notify_missing(self, label: str, missing: list[str]) -> None:
+        """Notify observers about missing IDs."""
+        self.errors_detected = True
+        for obs in self._observers:
+            obs.notify_missing(label, missing)
+
+    def _notify_extra(self, label: str, extra: list[str]) -> None:
+        """Notify observers about extra IDs."""
+        self.errors_detected = True
+        for obs in self._observers:
+            obs.notify_extra(label, extra)
+
+    def _check_generic(
+        self, label: str, ids_yaml: set[str], ids_es: set[str]
+    ) -> None:
+        """Compare YAML IDs vs Elasticsearch IDs for a given entity type."""
+        logger.info("Checking %s...", label)
+
+        missing = sorted(ids_yaml - ids_es)
+        extra = sorted(ids_es - ids_yaml)
+
+        if missing:
+            logger.error("Missing %s in ES: %s", label, missing)
+            self._notify_missing(label, missing)
+
+        if extra:
+            if self.strict:
+                logger.error(
+                    "%s in ES but missing from YAML: %s",
+                    label.capitalize(),
+                    extra,
+                )
+                self._notify_extra(label, extra)
+            else:
+                logger.info("Extra %s ignored (non-strict mode).", label)
+
+        if not missing and (self.strict and not extra):
+            logger.info("OK ✓ All %s match exactly.", label)
+        elif not missing and not self.strict:
+            logger.info("OK ✓ YAML %s present (extra ignored).", label)
+
+    def check_analyses(self, analyses: Analyses) -> None:
+        """Check analysis IDs between YAML and Elasticsearch."""
+        ids_yaml = {a.id for a in analyses}
+        ids_es = set(
+            self.es.get_field_values(self.es.data_files_index, "analysis_id")
+        )
+        self._check_generic("analyses", ids_yaml, ids_es)
+
+    def check_wet_processes(self, processes: Processes) -> None:
+        """Check wet process IDs between YAML and Elasticsearch."""
+        ids_yaml = set(processes.keys())
+        ids_es = set(
+            self.es.get_field_values(self.es.wet_processes_index, "proc_id")
+        )
+        self._check_generic("wet processes", ids_yaml, ids_es)
+
+    def check_bi_processes(self, processes: Processes) -> None:
+        """Check biological process IDs between YAML and Elasticsearch."""
+        ids_yaml = set(processes.keys())
+        ids_es = set(
+            self.es.get_field_values(self.es.bi_processes_index, "proc_id")
+        )
+        self._check_generic("bi processes", ids_yaml, ids_es)

genelastic/import_data/checker_observer.py

@@ -0,0 +1,13 @@
+from typing import Protocol
+
+
+class CheckerObserver(Protocol):
+    """Protocol for classes observing Checker events."""
+
+    def notify_missing(self, label: str, missing: list[str]) -> None:
+        """Called when expected IDs are missing in Elasticsearch."""
+        ...
+
+    def notify_extra(self, label: str, extra: list[str]) -> None:
+        """Called when unexpected IDs exist in Elasticsearch."""
+        ...

genelastic/import_data/cli/cli_check.py

@@ -0,0 +1,136 @@
+import argparse
+import logging
+import sys
+from pathlib import Path
+
+from genelastic.common.cli import (
+    add_es_connection_args,
+    add_verbose_control_args,
+    add_version_arg,
+)
+from genelastic.common.elastic import ElasticQueryConn
+from genelastic.import_data.checker import Checker
+from genelastic.import_data.import_bundle_factory import (
+    make_import_bundle_from_files,
+)
+from genelastic.import_data.logger import configure_logging
+
+logger = logging.getLogger("genelastic")
+logging.getLogger("elastic_transport").setLevel(logging.WARNING)
+
+
+class CLICheckObserver:
+    """Observer used by the CLI to log Checker errors."""
+
+    def __init__(self) -> None:
+        self._logger = logger
+
+    def notify_missing(self, label: str, missing: list[str]) -> None:
+        """Handle missing IDs by logging an error."""
+        self._logger.error("[CHECKER] Missing %s in ES: %s", label, missing)
+
+    def notify_extra(self, label: str, extra: list[str]) -> None:
+        """Handle extra IDs by logging an error."""
+        self._logger.error("[CHECKER] Extra %s in ES: %s", label, extra)
+
+
+def read_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Check database coherency against one or more YAML bundles.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+
+    add_version_arg(parser)
+    add_verbose_control_args(parser)
+    add_es_connection_args(parser)
+
+    parser.add_argument(
+        "files",
+        type=Path,
+        nargs="+",
+        help="Paths to YAML bundle files to validate.",
+    )
+
+    parser.add_argument(
+        "--strict",
+        action="store_true",
+        help=(
+            "Enable strict mode: also report entries present in Elasticsearch "
+            "but missing from YAML bundles."
+        ),
+    )
+
+    parser.add_argument(
+        "-A",
+        "--check-analyses",
+        action="store_true",
+        help="Check only analyses coherence.",
+    )
+
+    parser.add_argument(
+        "-W",
+        "--check-wet",
+        action="store_true",
+        help="Check only wet processes coherence.",
+    )
+
+    parser.add_argument(
+        "-B",
+        "--check-bi",
+        action="store_true",
+        help="Check only biological processes coherence.",
+    )
+
+    parser.add_argument(
+        "-X",
+        "--all",
+        action="store_true",
+        help="Check all entities (analyses, wet processes and bi processes).",
+    )
+
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = read_args()
+    configure_logging(args.verbose)
+
+    logger.info(
+        "Connecting to Elasticsearch at https://%s:%s ...",
+        args.es_host,
+        args.es_port,
+    )
+
+    es = ElasticQueryConn(
+        f"https://{args.es_host}:{args.es_port}",
+        args.es_cert_fp,
+        args.es_index_prefix,
+        basic_auth=(args.es_usr, args.es_pwd),
+    )
+
+    import_bundle = make_import_bundle_from_files(args.files)
+
+    checker = Checker(es, strict=args.strict)
+    checker.attach(CLICheckObserver())
+
+    run_all = args.all or not (
+        args.check_analyses or args.check_wet or args.check_bi
+    )
+
+    if args.check_analyses or run_all:
+        checker.check_analyses(import_bundle.analyses)
+
+    if args.check_wet or run_all:
+        checker.check_wet_processes(import_bundle.wet_processes)
+
+    if args.check_bi or run_all:
+        checker.check_bi_processes(import_bundle.bi_processes)
+
+    if checker.errors_detected:
+        sys.exit(1)
+
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

genelastic/import_data/{cli_gen_data.py → cli/gen_data.py}

@@ -4,10 +4,9 @@ from pathlib import Path
 
 from biophony import DEFAULT_RATE, MutSimParams
 
-from genelastic.common import add_verbose_control_args
-
-from .logger import configure_logging
-from .random_bundle import (
+from genelastic.common.cli import add_verbose_control_args, add_version_arg
+from genelastic.import_data.logger import configure_logging
+from genelastic.import_data.random_bundle import (
     RandomBundle,
 )
 
@@ -26,6 +25,7 @@ def read_args() -> argparse.Namespace:
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
         allow_abbrev=False,
     )
+    add_version_arg(parser)
     add_verbose_control_args(parser)
     parser.add_argument(
         "output_dir",

genelastic/import_data/cli/import_data.py

@@ -0,0 +1,346 @@
+# vi: se tw=80
+
+# Elasticsearch Python API:
+# https://www.elastic.co/guide/en/elasticsearch/client/python-api/current/overview.html
+# https://elasticsearch-py.readthedocs.io/en/latest/api.html
+
+import argparse
+import logging
+import sys
+from datetime import UTC, datetime
+from pathlib import Path
+
+from genelastic.common.cli import (
+    add_es_connection_args,
+    add_verbose_control_args,
+    add_version_arg,
+    log_item,
+    log_section,
+    log_subsection,
+    positive_int,
+)
+from genelastic.common.elastic import ElasticImportConn
+from genelastic.import_data.import_bundle_factory import (
+    make_import_bundle_from_files,
+)
+from genelastic.import_data.importers.importer_base import ImporterError
+from genelastic.import_data.importers.importer_factory import ImporterFactory
+from genelastic.import_data.logger import configure_logging
+from genelastic.import_data.models.analysis import Analysis
+from genelastic.import_data.models.data_file import DataFile
+from genelastic.import_data.models.processes import Processes
+
+logger = logging.getLogger("genelastic")
+logging.getLogger("elastic_transport").setLevel(
+    logging.WARNING
+)  # Disable excessive logging
+logging.getLogger("urllib3").setLevel(
+    logging.WARNING
+)  # Disable excessive logging
+
+
+def read_args() -> argparse.Namespace:
+    """Read arguments from command line."""
+    parser = argparse.ArgumentParser(
+        description="Genetics data importer.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        allow_abbrev=False,
+    )
+    add_version_arg(parser)
+    add_verbose_control_args(parser)
+    add_es_connection_args(parser)
+    parser.add_argument(
+        "-D",
+        "--dry-run",
+        dest="dryrun",
+        action="count",
+        default=0,
+        help=(
+            "Dry-run level. -D for data files loading (VCF, coverage, etc) "
+            "without connecting or importing to database. "
+            "-DD for metadata YAML files loading only (no loading of data files)."
+        ),
+    )
+    parser.add_argument(
+        "--log-file", dest="log_file", help="Path to a log file."
+    )
+    parser.add_argument(
+        "--no-list",
+        dest="no_list",
+        action="store_true",
+        help="Do not print list of files to be imported.",
+    )
+    parser.add_argument(
+        "--no-confirm",
+        dest="no_confirm",
+        action="store_true",
+        help="Do not ask confirmation before importing.",
+    )
+    parser.add_argument(
+        "-t",
+        "--threads",
+        dest="thread_count",
+        type=positive_int,
+        default=4,
+        help="Number of threads to use for parallel data files import.",
+    )
+    parser.add_argument(
+        "--multi-match",
+        dest="multi_match",
+        action="store_true",
+        help=(
+            "Enable grouping of files from the same 'data_path' into multiple "
+            "analyses by extracting variable metadata fields directly from "
+            "filenames using the file prefix. If some metadata fields (e.g., "
+            "sample_name, wet_process, bi_process) are not defined in the YAML "
+            "bundle, the importer detects all analyses sharing the same "
+            "defined metadata, but differing by the undefined fields. This "
+            "allows importing and filtering several analyses at once from a "
+            "single directory, based on the metadata present in filenames. "
+            "When disabled (default), only files matching the fixed filename "
+            "pattern (where all metadata fields are defined in the YAML) are "
+            "grouped into a single analysis; other files are ignored."
+        ),
+    )
+    parser.add_argument(
+        "files",
+        type=Path,
+        nargs="+",
+        default=None,
+        help="Data files that describe what to import.",
+    )
+    return parser.parse_args()
+
+
+def import_analysis(
+    es_import_conn: ElasticImportConn,
+    analysis: Analysis,
+) -> None:
+    """Import analysis into a dedicated index."""
+    logger.info(
+        " -> Importing analysis '%s' metadata into index '%s'...",
+        analysis.id,
+        es_import_conn.analyses_index,
+    )
+
+    documents = [
+        {
+            "_index": es_import_conn.analyses_index,
+            "_source": {
+                "created_at": datetime.now(UTC).isoformat(),
+                "analysis_id": analysis.id,
+                "bundle_file": str(analysis.bundle_file),
+                "data_path": str(analysis.data_path),
+                "metadata": analysis.metadata,
+            },
+        }
+    ]
+
+    es_import_conn.bulk_import(documents)
+
+
+def import_data_file(
+    es_import_conn: ElasticImportConn,
+    data_file: DataFile,
+) -> None:
+    """Import data files into a dedicated index."""
+    logger.info(
+        " -> Importing metadata into index '%s'...",
+        es_import_conn.data_files_index,
+    )
+
+    documents = [
+        {
+            "_index": es_import_conn.data_files_index,
+            "_source": {
+                "created_at": datetime.now(UTC).isoformat(),
+                "analysis_id": data_file.analysis_id,
+                "path": str(data_file.path),
+                "bundle_file": str(data_file.bundle_file),
+                "metadata": data_file.metadata,
+                "metrics": data_file.metrics,
+            },
+        }
+    ]
+
+    es_import_conn.bulk_import(documents)
+
+
+def import_data_file_content(
+    es_import_conn: ElasticImportConn,
+    data_file: DataFile,
+    thread_count: int,
+    dry_run: int,
+) -> None:
+    """Import data file content into a dedicated index,
+    based on their extension and type.
+    """
+    # -DD: no file processing, no import.
+    if dry_run > 1:
+        logger.info("[Dryrun] Data file neither processed nor imported.")
+        return
+
+    try:
+        logger.info(
+            " -> Processing file content for import...",
+        )
+        importer = ImporterFactory.get_importer(
+            data_file, es_import_conn, thread_count
+        )
+
+        # -D: only process files, no import.
+        if dry_run == 1:
+            logger.info("[Dryrun] Data file processed but not imported.")
+            return
+
+        logger.info(
+            " -> Importing file content into index '%s'...",
+            importer.target_index,
+        )
+        importer.import_docs()
+    except ImporterError as e:
+        logger.error(e)
+
+
+def import_processes(
+    es_import_conn: ElasticImportConn,
+    index: str,
+    processes: Processes,
+) -> None:
+    """Import processes into a dedicated index, based on their type."""
+    documents = [
+        {
+            "_index": index,
+            "_source": {
+                "proc_id": process.id,
+                "type": process.type,
+                "metadata": process.data,
+            },
+        }
+        for process in processes.values()
+    ]
+
+    es_import_conn.bulk_import(documents)
+
+
+def main() -> None:
+    """Entry point of the import script."""
+    # Read command line arguments
+    args = read_args()
+
+    # Configure logging
+    configure_logging(args.verbose, log_file=args.log_file)
+    logger.debug("Arguments: %s", args)
+    logger.debug("LOGGERS: %s", logging.root.manager.loggerDict)
+
+    # Open connection to ES
+    addr = f"https://{args.es_host}:{args.es_port}"
+    logger.info("Connecting to Elasticsearch at %s...", addr)
+    es_import_conn = ElasticImportConn(
+        addr,
+        args.es_cert_fp,
+        args.es_index_prefix,
+        args.dryrun,
+        basic_auth=(args.es_usr, args.es_pwd),
+    )
+
+    log_section("LOAD DATA")
+    logger.info("")
+    import_bundle = make_import_bundle_from_files(
+        args.files, multi_match=args.multi_match, check=True
+    )
+    all_bundled_files = import_bundle.analyses.get_data_files()
+
+    if not all_bundled_files:
+        logger.warning("No matching data files found from import bundle(s) !")
+
+    log_section("IMPORT DATA")
+    # List files before importing.
+    if not args.no_list:
+        logger.info("")
+        logger.info(
+            "The following %s file(s) will be imported:", len(all_bundled_files)
+        )
+
+        for data_file in all_bundled_files:
+            logger.info("- '%s'", data_file.path)
+    else:
+        logger.debug(
+            "'--no-list' argument provided: "
+            "not listing files about to be imported."
+        )
+
+    # Ask confirmation for importing
+    if not args.no_confirm:
+        answer: str = "maybe"
+        while answer not in ["", "n", "y"]:
+            answer = input("Import (y/N)? ").lower()
+        if answer != "y":
+            logger.info("Import canceled.")
+            sys.exit(0)
+    else:
+        logger.debug(
+            "'--no-confirm' argument provided: "
+            "not asking for confirmation before importing files."
+        )
+
+    # Start import.
+    log_subsection("Importing wet processes...")
+    logger.info(
+        "-> Importing %s wet process(es) into index '%s': %s.",
+        len(import_bundle.wet_processes),
+        es_import_conn.wet_processes_index,
+        ", ".join(import_bundle.wet_processes.keys()),
+    )
+    import_processes(
+        es_import_conn,
+        es_import_conn.wet_processes_index,
+        import_bundle.wet_processes,
+    )
+    log_subsection("Importing bioinformatics processes...")
+    logger.info(
+        "-> Importing %s bioinformatics process(es) into index '%s': %s.",
+        len(import_bundle.bi_processes),
+        es_import_conn.bi_processes_index,
+        ", ".join(import_bundle.bi_processes.keys()),
+    )
+    import_processes(
+        es_import_conn,
+        es_import_conn.bi_processes_index,
+        import_bundle.bi_processes,
+    )
+
+    log_subsection("Importing analysis metadata...")
+    for i, analysis in enumerate(sorted(import_bundle.analyses)):
+        log_item(
+            "Analysis",
+            i + 1,
+            len(import_bundle.analyses),
+        )
+        import_analysis(es_import_conn, analysis)
+
+    log_subsection("Importing data files...")
+    counter = 1
+    for ext in sorted(import_bundle.analyses.extensions):
+        data_files = import_bundle.analyses.get_data_files(ext)
+        logger.info("[ %s data files ]", ext.upper())
+
+        for data_file in data_files:
+            logger.info(
+                " -> Processing data file #%s/%s: '%s'...",
+                counter,
+                len(import_bundle.analyses.get_data_files()),
+                data_file.path.name,
+            )
+            import_data_file(es_import_conn, data_file)
+            import_data_file_content(
+                es_import_conn, data_file, args.thread_count, args.dryrun
+            )
+            logger.info("")
+            counter += 1
+
+    logger.info("=> Done.")
+
+
+if __name__ == "__main__":
+    main()