genelastic 0.7.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

genelastic/common/server.py

@@ -0,0 +1,59 @@
+import argparse
+import subprocess
+import sys
+
+import uvicorn
+
+
+def start_dev_server(
+    app_module: str,
+    args: argparse.Namespace,
+    reload_includes: list[str] | None = None,
+) -> None:
+    """Start the development server using Uvicorn.
+    :args app_module: The module containing the Flask server to start.
+    :args argparse.Namespace: The parsed arguments.
+    """
+    if reload_includes is None:
+        reload_includes = []
+
+    uvicorn.run(
+        app_module,
+        host=args.host,
+        port=args.port,
+        log_level=args.log_level,
+        reload=True,
+        reload_includes=reload_includes,
+    )
+
+
+def start_prod_server(app_module: str, args: argparse.Namespace) -> None:
+    """Start the production server using Gunicorn.
+    It will spawn one primary process and workers
+    :args app_module: The module containing the Flask server to start.
+    :args argparse.Namespace: The parsed arguments.
+    :raises subprocess.CalledProcessError: If gunicorn exits with a non-zero status code.
+    """
+    cmd = [
+        sys.executable,
+        "-m",
+        "gunicorn",
+        "-k",
+        "uvicorn.workers.UvicornWorker",
+        "--workers",
+        str(args.workers),
+        "--log-level",
+        args.log_level,
+        "-b",
+        f"{args.host}:{args.port}",
+        "--capture-output",
+        app_module,
+    ]
+
+    if args.log_file:
+        cmd.extend(["--log-file", args.log_file])
+
+    if args.access_logfile:
+        cmd.extend(["--access-logfile", args.access_logfile])
+
+    subprocess.run(cmd, check=True)  # noqa: S603

genelastic/common/types.py

@@ -4,20 +4,7 @@ import typing
 Bucket: typing.TypeAlias = dict[str, dict[typing.Any, typing.Any]]
 BundleDict: typing.TypeAlias = dict[str, typing.Any]
 
-AnalysisMetaData: typing.TypeAlias = dict[str, str | int]
-WetProcessesData: typing.TypeAlias = dict[str, str | int | float]
-BioInfoProcessData: typing.TypeAlias = dict[str, str | list[str]]
-
-AnalysisDocument: typing.TypeAlias = dict[str, str | None | AnalysisMetaData]
-MetadataDocument: typing.TypeAlias = dict[
-    str, int | str | list[typing.Any | None]
-]
-ProcessDocument: typing.TypeAlias = (
-    dict[str, str] | WetProcessesData | BioInfoProcessData
-)
-BulkItems: typing.TypeAlias = list[
-    dict[str, str | MetadataDocument | AnalysisDocument | ProcessDocument]
-]
+Metadata: typing.TypeAlias = dict[str, str | int]
 
 # Types related to random bundle generation.
 RandomBiProcessData: typing.TypeAlias = dict[str, str | list[dict[str, str]]]

genelastic/import_data/__init__.py

@@ -1,27 +0,0 @@
-"""Genelastic package for importing Genomic data into Elasticsearch."""
-
-from .analysis import Analysis
-from .import_bundle import ImportBundle
-from .import_bundle_factory import (
-    load_import_bundle_file,
-    make_import_bundle_from_files,
-)
-from .random_bundle import (
-    RandomAnalysis,
-    RandomBiProcess,
-    RandomBundle,
-    RandomWetProcess,
-)
-from .tags import Tags
-
-__all__ = [
-    "Analysis",
-    "ImportBundle",
-    "RandomAnalysis",
-    "RandomBiProcess",
-    "RandomBundle",
-    "RandomWetProcess",
-    "Tags",
-    "load_import_bundle_file",
-    "make_import_bundle_from_files",
-]

genelastic/import_data/checker.py

@@ -0,0 +1,99 @@
+import logging
+
+from genelastic.common.elastic import ElasticQueryConn
+from genelastic.import_data.checker_observer import CheckerObserver
+from genelastic.import_data.models.analyses import Analyses
+from genelastic.import_data.models.processes import Processes
+
+logger = logging.getLogger("genelastic")
+
+
+class Checker:
+    """Validate coherence between YAML metadata and Elasticsearch,
+    using a project-specific observer mechanism.
+    """
+
+    def __init__(self, es: ElasticQueryConn, *, strict: bool = False) -> None:
+        """Initialize the Checker.
+
+        Args:
+            es: Elasticsearch connection instance.
+            strict: Treat ES-only entries as errors when True.
+        """
+        self.es = es
+        self.strict = strict
+        self.errors_detected = False
+        self._observers: list[CheckerObserver] = []
+
+    def attach(self, observer: CheckerObserver) -> None:
+        """Register an observer to receive Checker notifications."""
+        self._observers.append(observer)
+
+    def detach(self, observer: CheckerObserver) -> None:
+        """Unregister an observer so it no longer receives notifications."""
+        self._observers.remove(observer)
+
+    def _notify_missing(self, label: str, missing: list[str]) -> None:
+        """Notify observers about missing IDs."""
+        self.errors_detected = True
+        for obs in self._observers:
+            obs.notify_missing(label, missing)
+
+    def _notify_extra(self, label: str, extra: list[str]) -> None:
+        """Notify observers about extra IDs."""
+        self.errors_detected = True
+        for obs in self._observers:
+            obs.notify_extra(label, extra)
+
+    def _check_generic(
+        self, label: str, ids_yaml: set[str], ids_es: set[str]
+    ) -> None:
+        """Compare YAML IDs vs Elasticsearch IDs for a given entity type."""
+        logger.info("Checking %s...", label)
+
+        missing = sorted(ids_yaml - ids_es)
+        extra = sorted(ids_es - ids_yaml)
+
+        if missing:
+            logger.error("Missing %s in ES: %s", label, missing)
+            self._notify_missing(label, missing)
+
+        if extra:
+            if self.strict:
+                logger.error(
+                    "%s in ES but missing from YAML: %s",
+                    label.capitalize(),
+                    extra,
+                )
+                self._notify_extra(label, extra)
+            else:
+                logger.info("Extra %s ignored (non-strict mode).", label)
+
+        if not missing and (self.strict and not extra):
+            logger.info("OK ✓ All %s match exactly.", label)
+        elif not missing and not self.strict:
+            logger.info("OK ✓ YAML %s present (extra ignored).", label)
+
+    def check_analyses(self, analyses: Analyses) -> None:
+        """Check analysis IDs between YAML and Elasticsearch."""
+        ids_yaml = {a.id for a in analyses}
+        ids_es = set(
+            self.es.get_field_values(self.es.data_files_index, "analysis_id")
+        )
+        self._check_generic("analyses", ids_yaml, ids_es)
+
+    def check_wet_processes(self, processes: Processes) -> None:
+        """Check wet process IDs between YAML and Elasticsearch."""
+        ids_yaml = set(processes.keys())
+        ids_es = set(
+            self.es.get_field_values(self.es.wet_processes_index, "proc_id")
+        )
+        self._check_generic("wet processes", ids_yaml, ids_es)
+
+    def check_bi_processes(self, processes: Processes) -> None:
+        """Check biological process IDs between YAML and Elasticsearch."""
+        ids_yaml = set(processes.keys())
+        ids_es = set(
+            self.es.get_field_values(self.es.bi_processes_index, "proc_id")
+        )
+        self._check_generic("bi processes", ids_yaml, ids_es)

genelastic/import_data/checker_observer.py

@@ -0,0 +1,13 @@
+from typing import Protocol
+
+
+class CheckerObserver(Protocol):
+    """Protocol for classes observing Checker events."""
+
+    def notify_missing(self, label: str, missing: list[str]) -> None:
+        """Called when expected IDs are missing in Elasticsearch."""
+        ...
+
+    def notify_extra(self, label: str, extra: list[str]) -> None:
+        """Called when unexpected IDs exist in Elasticsearch."""
+        ...

genelastic/import_data/cli/cli_check.py

@@ -0,0 +1,136 @@
+import argparse
+import logging
+import sys
+from pathlib import Path
+
+from genelastic.common.cli import (
+    add_es_connection_args,
+    add_verbose_control_args,
+    add_version_arg,
+)
+from genelastic.common.elastic import ElasticQueryConn
+from genelastic.import_data.checker import Checker
+from genelastic.import_data.import_bundle_factory import (
+    make_import_bundle_from_files,
+)
+from genelastic.import_data.logger import configure_logging
+
+logger = logging.getLogger("genelastic")
+logging.getLogger("elastic_transport").setLevel(logging.WARNING)
+
+
+class CLICheckObserver:
+    """Observer used by the CLI to log Checker errors."""
+
+    def __init__(self) -> None:
+        self._logger = logger
+
+    def notify_missing(self, label: str, missing: list[str]) -> None:
+        """Handle missing IDs by logging an error."""
+        self._logger.error("[CHECKER] Missing %s in ES: %s", label, missing)
+
+    def notify_extra(self, label: str, extra: list[str]) -> None:
+        """Handle extra IDs by logging an error."""
+        self._logger.error("[CHECKER] Extra %s in ES: %s", label, extra)
+
+
+def read_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Check database coherency against one or more YAML bundles.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+
+    add_version_arg(parser)
+    add_verbose_control_args(parser)
+    add_es_connection_args(parser)
+
+    parser.add_argument(
+        "files",
+        type=Path,
+        nargs="+",
+        help="Paths to YAML bundle files to validate.",
+    )
+
+    parser.add_argument(
+        "--strict",
+        action="store_true",
+        help=(
+            "Enable strict mode: also report entries present in Elasticsearch "
+            "but missing from YAML bundles."
+        ),
+    )
+
+    parser.add_argument(
+        "-A",
+        "--check-analyses",
+        action="store_true",
+        help="Check only analyses coherence.",
+    )
+
+    parser.add_argument(
+        "-W",
+        "--check-wet",
+        action="store_true",
+        help="Check only wet processes coherence.",
+    )
+
+    parser.add_argument(
+        "-B",
+        "--check-bi",
+        action="store_true",
+        help="Check only biological processes coherence.",
+    )
+
+    parser.add_argument(
+        "-X",
+        "--all",
+        action="store_true",
+        help="Check all entities (analyses, wet processes and bi processes).",
+    )
+
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = read_args()
+    configure_logging(args.verbose)
+
+    logger.info(
+        "Connecting to Elasticsearch at https://%s:%s ...",
+        args.es_host,
+        args.es_port,
+    )
+
+    es = ElasticQueryConn(
+        f"https://{args.es_host}:{args.es_port}",
+        args.es_cert_fp,
+        args.es_index_prefix,
+        basic_auth=(args.es_usr, args.es_pwd),
+    )
+
+    import_bundle = make_import_bundle_from_files(args.files)
+
+    checker = Checker(es, strict=args.strict)
+    checker.attach(CLICheckObserver())
+
+    run_all = args.all or not (
+        args.check_analyses or args.check_wet or args.check_bi
+    )
+
+    if args.check_analyses or run_all:
+        checker.check_analyses(import_bundle.analyses)
+
+    if args.check_wet or run_all:
+        checker.check_wet_processes(import_bundle.wet_processes)
+
+    if args.check_bi or run_all:
+        checker.check_bi_processes(import_bundle.bi_processes)
+
+    if checker.errors_detected:
+        sys.exit(1)
+
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

genelastic/import_data/cli/gen_data.py

@@ -0,0 +1,143 @@
+import argparse
+import logging
+from pathlib import Path
+
+from biophony import DEFAULT_RATE, MutSimParams
+
+from genelastic.common.cli import add_verbose_control_args, add_version_arg
+from genelastic.import_data.logger import configure_logging
+from genelastic.import_data.random_bundle import (
+    RandomBundle,
+)
+
+logger = logging.getLogger("genelastic")
+
+
+def read_args() -> argparse.Namespace:
+    """Read arguments from the command line."""
+    parser = argparse.ArgumentParser(
+        description="Random bundle generator. "
+        "A bundle is a YAML file format used to import genetic data into an Elasticsearch database. "
+        "It can contain one or more analyses; "
+        "each analysis including metadata, references to "
+        "a wet lab and bioinformatics process "
+        "and paths to a VCF file and optionally to a coverage file.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        allow_abbrev=False,
+    )
+    add_version_arg(parser)
+    add_verbose_control_args(parser)
+    parser.add_argument(
+        "output_dir",
+        help="Path where analyses VCF and coverage files will be generated.",
+        type=Path,
+    )
+    parser.add_argument("--log-file", help="Path to a log file.")
+    parser.add_argument(
+        "-n",
+        "--chrom-nb",
+        type=int,
+        default=5,
+        help="Number of chromosomes to include in the generated VCF file.",
+    )
+    parser.add_argument(
+        "-o",
+        "--output-bundle",
+        default=None,
+        help="Path where the YAML bundle file will be written. "
+        "If no path is provided, the bundle is written to stdout.",
+        type=Path,
+    )
+    parser.add_argument(
+        "-l",
+        "--sequence-length",
+        type=int,
+        default=2000,
+        help="Sequence length (number of nucleotides) generated for each chromosome.",
+    )
+    parser.add_argument(
+        "-c",
+        "--coverage",
+        action="store_true",
+        help="Generate a coverage file for each analysis.",
+    )
+    parser.add_argument(
+        "-a",
+        "--analyses",
+        help="Number of analyses to generate. "
+        "Each analysis will reference a wet lab and bioinformatics process, "
+        "a VCF file and optionally a coverage file.",
+        default=1,
+        type=int,
+    )
+    parser.add_argument(
+        "-p",
+        "--processes",
+        help="Number of wet lab and bioinformatics processes to generate.",
+        default=1,
+        type=int,
+    )
+    parser.add_argument(
+        "-s",
+        "--snp-rate",
+        help="Generated VCF SNP rate.",
+        type=float,
+        default=DEFAULT_RATE,
+    )
+    parser.add_argument(
+        "-i",
+        "--ins-rate",
+        help="Generated VCF insertion rate.",
+        type=float,
+        default=DEFAULT_RATE,
+    )
+    parser.add_argument(
+        "-d",
+        "--del-rate",
+        help="Generated VCF deletion rate.",
+        type=float,
+        default=DEFAULT_RATE,
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    """Entry point of the gen-data script."""
+    # Read command line arguments
+    args = read_args()
+    output_dir = args.output_dir.resolve()
+
+    if not output_dir.is_dir():
+        msg = f"ERROR: '{output_dir}' does not exist or is not a directory."
+        raise SystemExit(msg)
+
+    if args.analyses < 1:
+        msg = "Analyses count must be at least 1."
+        raise SystemExit(msg)
+
+    if args.processes < 1:
+        msg = "Processes count must be at least 1."
+        raise SystemExit(msg)
+
+    # Configure logging
+    configure_logging(args.verbose, log_file=args.log_file)
+    logger.debug("Arguments: %s", args)
+
+    # Write to stdout or file
+    RandomBundle(
+        output_dir,
+        args.analyses,
+        args.processes,
+        args.chrom_nb,
+        args.sequence_length,
+        MutSimParams(
+            snp_rate=args.snp_rate,
+            ins_rate=args.ins_rate,
+            del_rate=args.del_rate,
+        ),
+        do_gen_coverage=args.coverage,
+    ).to_yaml(args.output_bundle)
+
+
+if __name__ == "__main__":
+    main()