genelastic 0.6.1__py3-none-any.whl → 0.8.0__py3-none-any.whl

genelastic/import_data/cli_integrity.py

@@ -0,0 +1,384 @@
+import argparse
+import logging
+
+from elasticsearch import NotFoundError
+
+from genelastic.common import (
+    Bucket,
+    DBIntegrityError,
+    ElasticQueryConn,
+    add_es_connection_args,
+    add_verbose_control_args,
+)
+
+from .logger import configure_logging
+
+logger = logging.getLogger("genelastic")
+logging.getLogger("elastic_transport").setLevel(
+    logging.WARNING
+)  # Disable excessive logging
+
+
+def read_args() -> argparse.Namespace:
+    """Read arguments from command line."""
+    parser = argparse.ArgumentParser(
+        description="Utility to check the integrity "
+        "of the genelastic ElasticSearch database.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        allow_abbrev=False,
+    )
+    add_verbose_control_args(parser)
+    add_es_connection_args(parser)
+    return parser.parse_args()
+
+
+def check_for_undefined_file_indices(
+    es_query_conn: ElasticQueryConn, analyses_index: str
+) -> None:
+    """Check for potentially undefined files indices in the analyses index.
+
+    :param es_query_conn: Elasticsearch database instance.
+    :param analyses_index: Name of the index where analyses are stored.
+    :raises genelastic.common.DBIntegrityError:
+        Some files indices are used in the analyses index but are undefined.
+    """
+    logger.info(
+        "Checking for references to undefined file indices in the index '%s'...",
+        analyses_index,
+    )
+
+    undefined_indices = set()
+
+    query = {
+        "size": 0,
+        "aggs": {
+            "get_file_indices": {
+                "composite": {
+                    "sources": {
+                        "file_index": {"terms": {"field": "file_index.keyword"}}
+                    },
+                    "size": 1000,
+                }
+            }
+        },
+    }
+
+    buckets: list[Bucket] = es_query_conn.run_composite_aggregation(
+        analyses_index, query
+    )
+
+    for bucket in buckets:
+        file_index = bucket["key"]["file_index"]
+
+        try:
+            es_query_conn.client.indices.get(index=file_index)
+            logger.debug(
+                "File index %s used in index '%s' is defined.",
+                file_index,
+                analyses_index,
+            )
+        except NotFoundError:
+            logger.debug(
+                "File index %s used in '%s' is undefined.",
+                file_index,
+                analyses_index,
+            )
+            undefined_indices.add(file_index)
+
+    if len(undefined_indices) > 0:
+        msg = (
+            f"Found the following undefined file indices defined in the index '{analyses_index}': "
+            f"{', '.join(undefined_indices)}"
+        )
+        raise DBIntegrityError(msg)
+
+    logger.info("All defined file indices are referenced.")
+
+
+def get_undefined_processes(
+    es_query_conn: ElasticQueryConn,
+    analyses_index: str,
+    process_index: str,
+    field: str,
+) -> set[str]:
+    """Return a set of undefined processes IDs in an index.
+
+    :param es_query_conn: Elasticsearch database instance.
+    :param analyses_index: Name of the index where analyses are stored.
+    :param process_index: Name of the index to check for undefined processes.
+    :param field: Field name used to retrieve the process ID.
+    :returns: A set of undefined processes IDs.
+    """
+    query = {
+        "size": 0,
+        "aggs": {
+            "get_analyses_processes": {
+                "composite": {
+                    "sources": {
+                        "process": {"terms": {"field": f"{field}.keyword"}}
+                    },
+                    "size": 1000,
+                }
+            }
+        },
+    }
+
+    buckets: list[Bucket] = es_query_conn.run_composite_aggregation(
+        analyses_index, query
+    )
+
+    used_processes = {bucket["key"]["process"] for bucket in buckets}
+    logger.debug(
+        "Used values for field '%s' in index '%s': %s",
+        field,
+        analyses_index,
+        used_processes,
+    )
+
+    defined_processes = es_query_conn.get_field_values(process_index, "proc_id")
+    logger.debug(
+        "Defined values in index '%s': %s", process_index, defined_processes
+    )
+
+    return used_processes.difference(defined_processes)
+
+
+def check_for_undefined_wet_processes(
+    es_query_conn: ElasticQueryConn, analyses_index: str, wet_process_index: str
+) -> None:
+    """Check that each wet process used in the analyses index is defined.
+
+    :param es_query_conn: Elasticsearch database instance.
+    :param analyses_index: Name of the index where analyses are stored.
+    :param wet_process_index: Name of the index where wet processes are stored.
+    :raises genelastic.common.DBIntegrityError:
+        Some wet processes used in the analyses index are undefined.
+    """
+    logger.info(
+        "Checking for undefined wet processes used in index '%s'...",
+        analyses_index,
+    )
+    undefined_wet_processes = get_undefined_processes(
+        es_query_conn, analyses_index, wet_process_index, "metadata.wet_process"
+    )
+
+    if len(undefined_wet_processes) > 0:
+        msg = (
+            f"Index '{analyses_index}' uses the following undefined wet processes: "
+            f"{', '.join(undefined_wet_processes)}."
+        )
+        raise DBIntegrityError(msg)
+
+    logger.info(
+        "All wet processes used in index '%s' are defined.", wet_process_index
+    )
+
+
+def check_for_undefined_bi_processes(
+    es_query_conn: ElasticQueryConn, analyses_index: str, bi_process_index: str
+) -> None:
+    """Check that each bio info process used in the analyses index is defined.
+
+    :param es_query_conn: Elasticsearch database instance.
+    :param analyses_index: Name of the index where analyses are stored.
+    :param bi_process_index: Name of the index where bio info processes are stored.
+    :raises genelastic.common.DBIntegrityError:
+        Some bio info processes used in the analyses index are undefined.
+    """
+    logger.info(
+        "Checking for undefined bio info processes used in index '%s'...",
+        analyses_index,
+    )
+    undefined_bi_processes = get_undefined_processes(
+        es_query_conn, analyses_index, bi_process_index, "metadata.bi_process"
+    )
+
+    if len(undefined_bi_processes) > 0:
+        msg = (
+            f"Index '{analyses_index}' uses the following undefined bio info processes: "
+            f"{', '.join(undefined_bi_processes)}."
+        )
+        raise DBIntegrityError(msg)
+
+    logger.info(
+        "All bio info processes used in index '%s' are defined.",
+        bi_process_index,
+    )
+
+
+def check_for_unused_file_indices(
+    es_query_conn: ElasticQueryConn, analyses_index: str, index_prefix: str
+) -> int:
+    """Check that each of the file indices are used in at least one analysis.
+
+    :param es_query_conn: Elasticsearch database instance.
+    :param analyses_index: Name of the index where analyses are stored.
+    :param index_prefix: Prefix given to all the indices of the ElasticSearch database.
+    :returns: 1 if some file indices exists but are unused in the analyses index,
+        and 0 otherwise.
+    """
+    json_indices = es_query_conn.client.cat.indices(
+        index=f"{index_prefix}-file-*", format="json"
+    ).body
+
+    found_file_indices = set()
+    for x in json_indices:
+        if isinstance(x, dict):
+            found_file_indices.add(x["index"])
+
+    query = {
+        "size": 0,
+        "aggs": {
+            "get_file_indices": {
+                "composite": {
+                    "sources": {
+                        "file_index": {"terms": {"field": "file_index.keyword"}}
+                    },
+                    "size": 1000,
+                }
+            }
+        },
+    }
+
+    buckets: list[Bucket] = es_query_conn.run_composite_aggregation(
+        analyses_index, query
+    )
+
+    used_files_indices = {bucket["key"]["file_index"] for bucket in buckets}
+    unused_files_indices = found_file_indices.difference(used_files_indices)
+
+    if len(unused_files_indices) > 0:
+        logger.warning(
+            "Found the following unused files indices: %s",
+            ", ".join(unused_files_indices),
+        )
+        return 1
+
+    logger.info("All files indices are used.")
+    return 0
+
+
+def check_for_unused_wet_processes(
+    es_query_conn: ElasticQueryConn, analyses_index: str, wet_proc_index: str
+) -> int:
+    """Check for defined wet processes that are not used in the analyses index.
+
+    :param es_query_conn: Elasticsearch database instance.
+    :param analyses_index: Name of the index where analyses are stored.
+    :param wet_proc_index: Name of the index where wet processes are stored.
+    :returns: 1 if some wet process are defined but unused in the analyses index,
+        and 0 otherwise.
+    """
+    logger.info(
+        "Checking for unused wet processes in the index '%s'...", wet_proc_index
+    )
+
+    defined_wet_procs = es_query_conn.get_field_values(
+        wet_proc_index, "proc_id"
+    )
+    logger.debug(
+        "Found the following defined wet processes: %s", defined_wet_procs
+    )
+
+    used_wet_procs = es_query_conn.get_field_values(
+        analyses_index, "metadata.wet_process"
+    )
+    logger.debug(
+        "Following processes are used in the index '%s': %s",
+        analyses_index,
+        used_wet_procs,
+    )
+
+    unused_wet_procs = defined_wet_procs - used_wet_procs
+    if len(unused_wet_procs) > 0:
+        logger.warning("Found unused wet processes: %s", unused_wet_procs)
+        return 1
+
+    logger.info("No unused wet processes found.")
+    return 0
+
+
+def check_for_unused_bi_processes(
+    es_query_conn: ElasticQueryConn, analyses_index: str, bi_proc_index: str
+) -> int:
+    """Check for defined bio info processes that are not used in the analyses index.
+
+    :param es_query_conn: Elasticsearch database instance.
+    :param analyses_index: Name of the index where analyses are stored.
+    :param bi_proc_index: Name of the index where bio info processes are stored.
+    :returns: 1 if some wet process are defined but unused in the analyses index,
+        and 0 otherwise.
+    """
+    logger.info(
+        "Checking for unused bio info processes in the index '%s'...",
+        bi_proc_index,
+    )
+
+    defined_bi_procs = es_query_conn.get_field_values(bi_proc_index, "proc_id")
+    logger.debug(
+        "Found the following defined bio info processes: %s", defined_bi_procs
+    )
+
+    used_bi_procs = es_query_conn.get_field_values(
+        analyses_index, "metadata.bi_process"
+    )
+    logger.debug(
+        "Following processes are used in the index '%s': %s",
+        analyses_index,
+        used_bi_procs,
+    )
+
+    unused_bi_procs = defined_bi_procs - used_bi_procs
+    if len(unused_bi_procs) > 0:
+        logger.warning("Found unused bio info processes: %s", unused_bi_procs)
+        return 1
+
+    logger.info("No unused bio info processes found.")
+    return 0
+
+
+def main() -> None:
+    """Entry point of the integrity script."""
+    args = read_args()
+
+    configure_logging(args.verbose)
+    logger.debug("Arguments: %s", args)
+
+    analyses_index = f"{args.es_index_prefix}-analyses"
+    wet_processes_index = f"{args.es_index_prefix}-wet_processes"
+    bi_processes_index = f"{args.es_index_prefix}-bi_processes"
+
+    addr = f"https://{args.es_host}:{args.es_port}"
+    logger.info("Trying to connect to Elasticsearch at %s...", addr)
+    es_query_conn = ElasticQueryConn(
+        addr, args.es_cert_fp, basic_auth=(args.es_usr, args.es_pwd)
+    )
+
+    # Fatal errors
+    try:
+        es_query_conn.ensure_unique(wet_processes_index, "proc_id")
+        es_query_conn.ensure_unique(bi_processes_index, "proc_id")
+        check_for_undefined_file_indices(es_query_conn, analyses_index)
+        check_for_undefined_wet_processes(
+            es_query_conn, analyses_index, wet_processes_index
+        )
+        check_for_undefined_bi_processes(
+            es_query_conn, analyses_index, bi_processes_index
+        )
+    except DBIntegrityError as e:
+        raise SystemExit(e) from e
+
+    # Warnings
+    check_for_unused_wet_processes(
+        es_query_conn, analyses_index, wet_processes_index
+    )
+    check_for_unused_bi_processes(
+        es_query_conn, analyses_index, bi_processes_index
+    )
+    check_for_unused_file_indices(
+        es_query_conn, analyses_index, args.es_index_prefix
+    )
+
+
+if __name__ == "__main__":
+    main()

genelastic/import_data/cli_validate.py

@@ -0,0 +1,54 @@
+import argparse
+import logging
+from pathlib import Path
+
+from schema import SchemaError
+
+from genelastic.common import add_verbose_control_args
+
+from .import_bundle_factory import make_import_bundle_from_files
+from .logger import configure_logging
+
+logger = logging.getLogger("genelastic")
+
+
+def read_args() -> argparse.Namespace:
+    """Read arguments from command line."""
+    parser = argparse.ArgumentParser(
+        description="Ensure that YAML files "
+        "follow the genelastic YAML bundle schema.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        allow_abbrev=False,
+    )
+    add_verbose_control_args(parser)
+    parser.add_argument(
+        "files",
+        type=Path,
+        nargs="+",
+        default=None,
+        help="YAML files to validate.",
+    )
+    parser.add_argument(
+        "-c",
+        "--check",
+        action="store_true",
+        help="In addition to validating the schema, "
+        "check for undefined referenced processes.",
+    )
+    return parser.parse_args()
+
+
+def main() -> int:
+    """Entry point of the validate script."""
+    args = read_args()
+    configure_logging(args.verbose)
+
+    try:
+        make_import_bundle_from_files(args.files, check=args.check)
+    except (ValueError, RuntimeError, TypeError, SchemaError) as e:
+        # Catch any exception that can be raised by 'make_import_bundle_from_files'.
+        logger.error(e)
+        return 1
+
+    logger.info("All YAML files respect the genelastic YAML bundle format.")
+    return 0

genelastic/import_data/constants.py

@@ -1,11 +1,11 @@
-"""
-Module: constants
+"""Module: constants
 
 This module contains genelastic constants.
 """
+
 import typing
 
-ALLOWED_CATEGORIES: typing.Final[typing.List[str]] = ['vcf', 'cov']
+ALLOWED_CATEGORIES: typing.Final[list[str]] = ["vcf", "cov"]
 
 BUNDLE_CURRENT_VERSION = 3
 
@@ -13,33 +13,12 @@ DEFAULT_TAG_REGEX = "[^_-]+"
 DEFAULT_TAG_PREFIX = "%"
 DEFAULT_TAG_SUFFIX = ""
 
-DEFAULT_TAG2FIELD: typing.Final[typing.Dict[str, typing.Dict[str, str]]] = {
-    '%S': {
-        "field": 'sample_name',
-        "regex": DEFAULT_TAG_REGEX
-    },
-    '%F': {
-        "field": 'source',
-        "regex": DEFAULT_TAG_REGEX
-    },
-    '%W': {
-        "field": 'wet_process',
-        "regex": DEFAULT_TAG_REGEX
-    },
-    '%B': {
-        "field": 'bi_process',
-        "regex": DEFAULT_TAG_REGEX
-    },
-    '%D': {
-        "field": 'cov_depth',
-        "regex": DEFAULT_TAG_REGEX
-    },
-    '%A': {
-        "field": 'barcode',
-        "regex": DEFAULT_TAG_REGEX
-    },
-    '%R':  {
-        "field": 'reference_genome',
-        "regex": DEFAULT_TAG_REGEX
-    }
+DEFAULT_TAG2FIELD: typing.Final[dict[str, dict[str, str]]] = {
+    "%S": {"field": "sample_name", "regex": DEFAULT_TAG_REGEX},
+    "%F": {"field": "source", "regex": DEFAULT_TAG_REGEX},
+    "%W": {"field": "wet_process", "regex": DEFAULT_TAG_REGEX},
+    "%B": {"field": "bi_process", "regex": DEFAULT_TAG_REGEX},
+    "%D": {"field": "cov_depth", "regex": DEFAULT_TAG_REGEX},
+    "%A": {"field": "barcode", "regex": DEFAULT_TAG_REGEX},
+    "%R": {"field": "reference_genome", "regex": DEFAULT_TAG_REGEX},
 }

genelastic/import_data/data_file.py

@@ -1,5 +1,4 @@
-"""
-This module defines the DataFile class, which handles the representation,
+"""This module defines the DataFile class, which handles the representation,
 management, and extraction of metadata for a data file within a data bundle.
 
 It includes functionality to construct DataFile instances from paths and
@@ -8,45 +7,47 @@ for extracting metadata from filenames using specified patterns.
 """
 
 import logging
-import os
 import pathlib
-import typing
+from pathlib import Path
 
 from genelastic.common import AnalysisMetaData
 
 from .filename_pattern import FilenamePattern
 
-logger = logging.getLogger('genelastic')
+logger = logging.getLogger("genelastic")
 
 
 class DataFile:
     """Class for handling a data file and its metadata."""
 
     # Initializer
-    def __init__(self, path: str, bundle_path: str | None = None,
-                 metadata: typing.Optional[AnalysisMetaData] = None) -> None:
+    def __init__(
+        self,
+        path: Path,
+        bundle_path: Path | None = None,
+        metadata: AnalysisMetaData | None = None,
+    ) -> None:
         self._path = path
         self._bundle_path = bundle_path  # The bundle YAML file in which this
         # file was listed.
         self._metadata = {} if metadata is None else metadata
 
     def __repr__(self) -> str:
-        return (f"File {self._path}, from bundle {self._bundle_path}"
-                + f", with metadata {self._metadata}")
+        return f"File {self._path}, from bundle {self._bundle_path}, with metadata {self._metadata}"
 
     # Get path
     @property
-    def path(self) -> str:
+    def path(self) -> Path:
         """Retrieve the data file path."""
         return self._path
 
     def exists(self) -> bool:
         """Tests if the associated file exists on disk."""
-        return os.path.isfile(self._path)
+        return self._path.is_file()
 
     # Get bundle path
     @property
-    def bundle_path(self) -> str | None:
+    def bundle_path(self) -> Path | None:
         """Retrieve the path to the associated data bundle file."""
         return self._bundle_path
 
@@ -59,20 +60,22 @@ class DataFile:
     # Factory
     @classmethod
     def make_from_bundle(
-            cls,
-            path: str,
-            bundle_path: str | None,
-            pattern: typing.Optional[FilenamePattern] = None) -> 'DataFile':
+        cls,
+        path: Path,
+        bundle_path: Path | None,
+        pattern: FilenamePattern | None = None,
+    ) -> "DataFile":
         """Construct a DataFile instance from a bundle path, file path,
-        and optional filename pattern."""
+        and optional filename pattern.
+        """
         # Make absolute path
-        if not os.path.isabs(path) and not bundle_path is None:
-            path = os.path.join(os.path.dirname(bundle_path), path)
+        if not path.is_absolute() and bundle_path is not None:
+            path = bundle_path.parent / path
 
         # Extract filename metadata
         metadata = None
         if pattern is not None:
-            metadata = pattern.extract_metadata(os.path.basename(path))
+            metadata = pattern.extract_metadata(path.name)
 
         if metadata:
             if "ext" not in metadata:

genelastic/import_data/filename_pattern.py

@@ -1,14 +1,11 @@
-"""
-This module defines the FilenamePattern class, used to define a filename pattern
+"""This module defines the FilenamePattern class, used to define a filename pattern
 and extract metadata from file names using this pattern.
 """
 
-import logging
 import re
 
 from genelastic.common import AnalysisMetaData
 
-logger = logging.getLogger('genelastic')
 
 class FilenamePattern:
     """Class for defining a filename pattern.
@@ -18,46 +15,43 @@ class FilenamePattern:
 
     # Initializer
     def __init__(self, pattern: str) -> None:
-        """
-                Initializes a FilenamePattern instance.
+        """Initializes a FilenamePattern instance.
 
-                Args:
-                    pattern (str): The pattern string used for defining
-                     the filename pattern.
-                """
+        Args:
+            pattern (str): The pattern string used for defining
+             the filename pattern.
+        """
         self._re = re.compile(pattern)
 
     def extract_metadata(self, filename: str) -> AnalysisMetaData:
-        """
-                Extracts metadata from the given filename based
-                on the defined pattern.
+        """Extracts metadata from the given filename based
+        on the defined pattern.
 
-                Args:
-                    filename (str): The filename from which metadata
-                    needs to be extracted.
+        Args:
+            filename (str): The filename from which metadata
+            needs to be extracted.
 
-                Returns:
-                    dict: A dictionary containing the extracted metadata.
+        Returns:
+            dict: A dictionary containing the extracted metadata.
 
-                Raises:
-                    RuntimeError: If parsing of filename fails
-                    with the defined pattern.
-                """
+        Raises:
+            RuntimeError: If parsing of filename fails
+            with the defined pattern.
+        """
         m = self._re.search(filename)
         if not m:
-            raise RuntimeError(f'Failed parsing filename "{filename}"' +
-                               f'with pattern "{self._re.pattern}".')
+            msg = f'Failed parsing filename "{filename}" with pattern "{self._re.pattern}".'
+            raise RuntimeError(msg)
         return m.groupdict()
 
     def matches_pattern(self, filename: str) -> bool:
-        """
-                Checks if the given filename matches the defined pattern.
+        """Checks if the given filename matches the defined pattern.
 
-                Args:
-                    filename (str): The filename to be checked.
+        Args:
+            filename (str): The filename to be checked.
 
-                Returns:
-                    bool: True if the filename matches the pattern,
-                    False otherwise.
-                """
+        Returns:
+            bool: True if the filename matches the pattern,
+            False otherwise.
+        """
         return bool(self._re.match(filename))