genelastic 0.8.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

genelastic/import_data/cli/info.py

@@ -0,0 +1,247 @@
+import argparse
+import logging
+from datetime import datetime
+
+from genelastic.common.cli import (
+    add_es_connection_args,
+    add_verbose_control_args,
+    add_version_arg,
+)
+from genelastic.common.elastic import ElasticQueryConn
+from genelastic.import_data.logger import configure_logging
+
+logger = logging.getLogger("genelastic")
+logging.getLogger("elastic_transport").setLevel(
+    logging.WARNING
+)  # Disable excessive logging
+
+
+def read_args() -> argparse.Namespace:
+    """Read arguments from the command line."""
+    parser = argparse.ArgumentParser(
+        description="ElasticSearch database info.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        allow_abbrev=False,
+    )
+    add_version_arg(parser)
+    add_verbose_control_args(parser)
+    add_es_connection_args(parser)
+    parser.add_argument(
+        "-a",
+        "--list-analyses",
+        action="store_true",
+        help="List all imported analyses.",
+    )
+    parser.add_argument(
+        "-w",
+        "--list-wet-processes",
+        action="store_true",
+        help="List all imported wet processes.",
+    )
+    parser.add_argument(
+        "-b",
+        "--list-bi-processes",
+        action="store_true",
+        help="List all imported bio info processes.",
+    )
+    parser.add_argument(
+        "-B",
+        "--list-bundles",
+        action="store_true",
+        help="List YAML bundles and associated analyses.",
+    )
+
+    return parser.parse_args()
+
+
+def list_processes(es_query_conn: ElasticQueryConn, index: str) -> None:
+    """List all processes."""
+    process_ids = es_query_conn.get_field_values(index, "proc_id")
+
+    if len(process_ids) == 0:
+        logger.info("Empty response.")
+        return
+
+    for process_id in process_ids:
+        logger.info("- %s", process_id)
+
+
+def list_wet_processes(es_query_conn: ElasticQueryConn) -> None:
+    """List all wet processes."""
+    logger.info("Imported wet processes")
+    logger.info("======================")
+    list_processes(es_query_conn, es_query_conn.wet_processes_index)
+
+
+def list_bi_processes(es_query_conn: ElasticQueryConn) -> None:
+    """List all bio info processes."""
+    logger.info("Imported bi processes")
+    logger.info("=====================")
+    list_processes(es_query_conn, es_query_conn.bi_processes_index)
+
+
+def list_analyses(es_query_conn: ElasticQueryConn) -> None:
+    """List all imported analyses and their associated data files."""
+    query = {
+        "size": 0,
+        "aggs": {
+            "by_analysis": {
+                "composite": {
+                    "size": 1000,
+                    "sources": [
+                        {
+                            "analysis_id": {
+                                "terms": {"field": "analysis_id.keyword"}
+                            }
+                        }
+                    ],
+                },
+                "aggs": {
+                    "data_files": {
+                        "top_hits": {
+                            "size": 100,
+                        }
+                    }
+                },
+            }
+        },
+    }
+
+    buckets = es_query_conn.run_composite_aggregation(
+        es_query_conn.data_files_index, query
+    )
+
+    if not buckets:
+        logger.info("No data files found.")
+        return
+
+    logger.info("Data files per YAML bundle")
+    logger.info("=" * 80)
+
+    for i, bucket in enumerate(buckets):
+        analysis_id = bucket["key"]["analysis_id"]
+        hits = bucket["data_files"]["hits"]["hits"]
+        doc_count = len(hits)
+
+        logger.info(
+            "[%d] Analysis ID: %s (%d file%s)",
+            i + 1,
+            analysis_id,
+            doc_count,
+            "s" if doc_count > 1 else "",
+        )
+        logger.info("-" * 80)
+
+        for j, hit in enumerate(hits):
+            source = hit["_source"]
+
+            created_at = datetime.fromisoformat(source["created_at"])
+            created_at_formatted = created_at.strftime("%Y-%m-%d")
+
+            logger.info(" File %d of %d:", j + 1, doc_count)
+            logger.info(" created_at : %s", created_at_formatted)
+            logger.info(" bundle_file : %s", source["bundle_file"])
+            logger.info(" path : %s", source["path"])
+
+
+def list_bundles(es_query_conn: ElasticQueryConn) -> None:
+    """List bundle_file → associated analysis_id (clean visual CLI output)."""
+    query = {
+        "size": 0,
+        "aggs": {
+            "by_bundle": {
+                "composite": {
+                    "size": 2000,
+                    "sources": [
+                        {
+                            "bundle_file": {
+                                "terms": {"field": "bundle_file.keyword"}
+                            }
+                        }
+                    ],
+                },
+                "aggs": {
+                    "analyses": {
+                        "terms": {
+                            "field": "analysis_id.keyword",
+                            "size": 2000,
+                        }
+                    }
+                },
+            }
+        },
+    }
+
+    buckets = es_query_conn.run_composite_aggregation(
+        es_query_conn.data_files_index, query
+    )
+
+    if not buckets:
+        logger.info("No bundles found.")
+        return
+
+    # Sort bundles by bundle_file path
+    buckets = sorted(buckets, key=lambda b: b["key"]["bundle_file"])
+
+    logger.info("========================================")
+    logger.info(" BUNDLES AND ASSOCIATED ANALYSES")
+    logger.info("========================================")
+    logger.info("")
+
+    for idx, bucket in enumerate(buckets, start=1):
+        bundle = bucket["key"]["bundle_file"]
+        analyses = bucket["analyses"]["buckets"]
+
+        logger.info("#%d %s", idx, bundle)
+        if not analyses:
+            logger.info("   (no analyses)")
+        else:
+            for a in analyses:
+                logger.info("   • %s", a["key"])
+
+        logger.info("----------------------------------------")
+
+
+def main() -> None:
+    """Entry point of the info script."""
+    args = read_args()
+
+    configure_logging(args.verbose)
+    logger.debug("Arguments: %s", args)
+
+    addr = f"https://{args.es_host}:{args.es_port}"
+    logger.info("Connecting to Elasticsearch at %s...", addr)
+    es_query_conn = ElasticQueryConn(
+        addr,
+        args.es_cert_fp,
+        args.es_index_prefix,
+        basic_auth=(args.es_usr, args.es_pwd),
+    )
+
+    list_call_count = 0
+
+    if args.list_bundles:
+        list_bundles(es_query_conn)
+        list_call_count += 1
+
+    if args.list_analyses:
+        list_analyses(es_query_conn)
+        list_call_count += 1
+
+    if args.list_wet_processes:
+        list_wet_processes(es_query_conn)
+        list_call_count += 1
+
+    if args.list_bi_processes:
+        list_bi_processes(es_query_conn)
+        list_call_count += 1
+
+    if list_call_count == 0:
+        logger.debug("No list option specified, listing everything.")
+        list_analyses(es_query_conn)
+        list_wet_processes(es_query_conn)
+        list_bi_processes(es_query_conn)
+
+
+if __name__ == "__main__":
+    main()

genelastic/import_data/{cli_integrity.py → cli/integrity.py}

@@ -1,17 +1,20 @@
 import argparse
 import logging
+import typing
 
 from elasticsearch import NotFoundError
 
-from genelastic.common import (
-    Bucket,
-    DBIntegrityError,
-    ElasticQueryConn,
+from genelastic.common.cli import (
     add_es_connection_args,
     add_verbose_control_args,
+    add_version_arg,
 )
+from genelastic.common.elastic import ElasticQueryConn
+from genelastic.common.exceptions import DBIntegrityError
+from genelastic.import_data.logger import configure_logging
 
-from .logger import configure_logging
+if typing.TYPE_CHECKING:
+    from genelastic.common.types import Bucket
 
 logger = logging.getLogger("genelastic")
 logging.getLogger("elastic_transport").setLevel(
@@ -27,6 +30,7 @@ def read_args() -> argparse.Namespace:
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
         allow_abbrev=False,
     )
+    add_version_arg(parser)
     add_verbose_control_args(parser)
     add_es_connection_args(parser)
     return parser.parse_args()
@@ -42,6 +46,13 @@ def check_for_undefined_file_indices(
     :raises genelastic.common.DBIntegrityError:
         Some files indices are used in the analyses index but are undefined.
     """
+    if not es_query_conn.client:
+        logger.info(
+            "[Dryrun] check_for_undefined_file_indices: "
+            "no Elasticsearch client."
+        )
+        return
+
     logger.info(
         "Checking for references to undefined file indices in the index '%s'...",
         analyses_index,
@@ -217,6 +228,13 @@ def check_for_unused_file_indices(
     :returns: 1 if some file indices exists but are unused in the analyses index,
         and 0 otherwise.
     """
+    if not es_query_conn.client:
+        logger.info(
+            "[Dryrun] check_for_unused_file_indices: "
+            "no Elasticsearch client."
+        )
+        return -1
+
     json_indices = es_query_conn.client.cat.indices(
         index=f"{index_prefix}-file-*", format="json"
     ).body
@@ -349,9 +367,13 @@ def main() -> None:
     bi_processes_index = f"{args.es_index_prefix}-bi_processes"
 
     addr = f"https://{args.es_host}:{args.es_port}"
-    logger.info("Trying to connect to Elasticsearch at %s...", addr)
+    logger.info("Connecting to Elasticsearch at %s...", addr)
     es_query_conn = ElasticQueryConn(
-        addr, args.es_cert_fp, basic_auth=(args.es_usr, args.es_pwd)
+        addr,
+        args.es_cert_fp,
+        args.es_index_prefix,
+        args.dryrun,
+        basic_auth=(args.es_usr, args.es_pwd),
     )
 
     # Fatal errors

genelastic/import_data/cli/validate.py

@@ -0,0 +1,146 @@
+import argparse
+import logging
+from pathlib import Path
+
+from genelastic.common.cli import add_verbose_control_args, add_version_arg
+from genelastic.common.exceptions import (
+    ValidationError,
+    YAMLFileReadError,
+)
+from genelastic.import_data.import_bundle_factory import (
+    load_yaml_file,
+    validate_doc,
+)
+from genelastic.import_data.logger import configure_logging
+from genelastic.import_data.models.validate import ValidationIssue
+
+logger = logging.getLogger("genelastic")
+
+
+def read_args() -> argparse.Namespace:
+    """Read arguments from command line."""
+    parser = argparse.ArgumentParser(
+        description="Statically validates YAML bundles: "
+        "ensure they comply to the bundle schema.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        allow_abbrev=False,
+    )
+    add_version_arg(parser)
+    add_verbose_control_args(parser)
+    parser.add_argument(
+        "files",
+        type=Path,
+        nargs="+",
+        default=None,
+        help="Paths to YAML files containing bundles to validate.",
+    )
+    parser.add_argument(
+        "-x",
+        "--fail-fast",
+        dest="fail_fast",
+        action="store_true",
+        help="Stop validating files after the first error is encountered.",
+    )
+    return parser.parse_args()
+
+
+def main() -> int:
+    """Entry point of the validate script."""
+    args = read_args()
+    configure_logging(args.verbose)
+
+    validation_issues = []
+    file_count = len(args.files)
+
+    for file_index, file_path in enumerate(args.files):
+        resolved_file_path = file_path.resolve()
+
+        logger.info(
+            "[%s/%s] Validating bundle(s) from file '%s'.",
+            file_index + 1,
+            file_count,
+            resolved_file_path,
+        )
+        logger.info("Loading YAML file...")
+
+        try:
+            docs = load_yaml_file(resolved_file_path)
+        except YAMLFileReadError as e:
+            logger.error(e)
+
+            if args.fail_fast:
+                raise SystemExit(1) from None
+
+            validation_issues.append(
+                ValidationIssue(
+                    exc_type=type(e).__name__,
+                    file_path=resolved_file_path,
+                    file_index=file_index + 1,
+                    file_count=file_count,
+                )
+            )
+            continue
+
+        logger.info("-> YAML file successfully loaded.")
+
+        doc_count = len(docs)
+        logger.info("Found %s document(s) in the YAML file.", doc_count)
+
+        for doc_index, doc in enumerate(docs):
+            logger.info(
+                "  Validating bundle format for document #%s/%s...",
+                doc_index + 1,
+                doc_count,
+            )
+
+            try:
+                validate_doc(doc)
+            except ValidationError as e:
+                logger.error(e)
+
+                if args.fail_fast:
+                    raise SystemExit(1) from None
+
+                validation_issues.append(
+                    ValidationIssue(
+                        exc_type=type(e).__name__,
+                        file_path=resolved_file_path,
+                        file_index=file_index + 1,
+                        file_count=file_count,
+                        doc_index=doc_index + 1,
+                        doc_count=doc_count,
+                    )
+                )
+                continue
+
+            logger.info("  -> Bundle format is valid.")
+
+        logger.info("")
+
+    if len(validation_issues) > 0:
+        logger.error("Some files raised exceptions:")
+        for issue in validation_issues:
+            logger.error("  - %s", issue)
+
+        ret_code = 1
+    else:
+        logger.info("All bundles respect the genelastic YAML bundle format.")
+        ret_code = 0
+
+    files_failing_validation = len(
+        {issue.file_path for issue in validation_issues}
+    )
+    files_passing_validation = file_count - files_failing_validation
+
+    logger.info(
+        "Out of %s file(s), validation passed for %s and failed for %s.",
+        file_count,
+        files_passing_validation,
+        files_failing_validation,
+    )
+
+    return ret_code
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

genelastic/import_data/collect.py

@@ -0,0 +1,185 @@
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+
+from genelastic.common.exceptions import DataFileCollectorError
+from genelastic.common.types import Metadata
+from genelastic.import_data.models.data_file import DataFile
+from genelastic.import_data.models.tags import Tags
+from genelastic.import_data.patterns import FilenamePattern
+from genelastic.import_data.resolve import resolve_analysis_id
+
+logger = logging.getLogger("genelastic")
+
+
+def collect_files(data_path: Path) -> set[Path]:
+    """Collect files for a given analysis.
+    All files directly under ``data_path`` are returned.
+
+    :param data_path: Directory containing the files.
+    :raises DataFileCollectorError: If ``data_path`` is not an existing
+        directory.
+    :return: Set of absolute paths to collected files.
+    """
+    try:
+        collected_files = {x for x in data_path.iterdir() if x.is_file()}
+    except OSError as e:
+        msg = f"Error collecting files: data directory is invalid. {e}."
+        raise DataFileCollectorError(msg) from e
+    return collected_files
+
+
+def match_files(
+    files: set[Path],
+    filename_pattern: FilenamePattern,
+) -> tuple[set[Path], set[Path]]:
+    """Splits a set of files into those that match a given filename pattern and
+    those that don't.
+
+    This function applies the provided ``filename_pattern`` to each file name
+    in ``files``, and returns two sets: one containing files that match the
+    pattern, and one containing those that do not.
+
+    :param files: A set of file paths to check.
+    :param filename_pattern: The filename pattern used for matching.
+
+    :returns: A tuple containing in first position a set of files that match
+      the pattern, and in second position a set of files that do not match the
+      pattern.
+    """
+    matched_files = {
+        f for f in files if filename_pattern.matches_pattern(f.name)
+    }
+    return matched_files, files - matched_files
+
+
+def extract_analysis_metadata(
+    data_path: Path,
+    file_prefix: str,
+    tags: Tags,
+    filename_pattern: FilenamePattern,
+) -> dict[str, Metadata]:
+    analysis = {}
+
+    for file in collect_files(data_path):
+        if not filename_pattern.matches_pattern(file.name):
+            logger.debug("File '%s' was not matched.", file.name)
+            continue
+
+        filename_metadata = filename_pattern.extract_metadata(file.name)
+        analysis_id = resolve_analysis_id(file_prefix, tags, filename_metadata)
+        analysis[analysis_id] = filename_metadata
+
+    return analysis
+
+
+def init_data_files(
+    analysis_id: str,
+    files: set[Path],
+    filename_pattern: FilenamePattern,
+    bundle_file: Path,
+) -> set[DataFile]:
+    """Instantiate ``DataFile`` objects from a set of file paths associated
+    with an analysis.
+
+    :param analysis_id: ID of the analysis, shared by all created ``DataFile``
+      instances.
+    :param files: Set of file paths associated with the analysis.
+    :param filename_pattern: Pattern used to extract metadata from filenames.
+      The extracted metadata is included in each ``DataFile``.
+    :param bundle_file: Path to the YAML bundle file from which the analysis is
+      defined.
+    :raises DataFileCollectorError: If metadata extraction or instantiation
+        of a data file objet fails for a given file.
+    :return: A set of successfully instantiated ``DataFile`` objects.
+    """
+    data_files = set()
+    for file in files:
+        try:
+            metadata = filename_pattern.extract_metadata(file.name)
+            data_file = DataFile(
+                analysis_id=analysis_id,
+                path=file,
+                bundle_file=bundle_file,
+                metadata=metadata,
+            )
+            data_files.add(data_file)
+        except RuntimeError as e:
+            msg = f"Error instantiating data files: {e}"
+            raise DataFileCollectorError(msg) from None
+    return data_files
+
+
+@dataclass(frozen=True)
+class DataFileCollectorResult:
+    """Result of a data file collection."""
+
+    matched_files: set[Path]
+    unmatched_files: set[Path]
+    data_files: set[DataFile]
+
+
+class DataFileCollector:
+    """Collect all data files belonging to an analysis."""
+
+    def __init__(
+        self,
+        analysis_id: str,
+        bundle_file: Path,
+        data_path: Path,
+        filename_pattern: FilenamePattern,
+        *,
+        multi_match: bool = False,
+    ) -> None:
+        self._analysis_id = analysis_id
+        self._bundle_file = bundle_file
+        self._data_path = data_path
+        self._filename_pattern = filename_pattern
+        self._multi_match = multi_match
+
+    def run(self) -> DataFileCollectorResult:
+        """Collects files from the analysis data path, matches them against the
+        analysis filename pattern, and instantiates ``DataFile`` objects for
+        each matched file.
+
+        :raises DataFileCollectorError: If the ``data_path`` is not an existing
+            directory or if metadata extraction or instantiation of a data file
+            objet fails for a given file.
+        :return: A ``DataFileCollectorResult`` containing the sets of matched
+            and unmatched files, as well as a set of instantiated ``DataFile``
+            objects.
+        """
+        files = collect_files(self._data_path)
+        logger.debug(
+            " -> Collected %s file(s):",
+            len(files),
+        )
+        for path in sorted(files):
+            logger.debug("  - '%s'", path.name)
+
+        matched_files, unmatched_files = match_files(
+            files, self._filename_pattern
+        )
+        logger.info(" -> Found %s matching file(s):", len(matched_files))
+        for path in sorted(matched_files):
+            logger.info("  - '%s'", path.name)
+
+        logger.info(
+            " -> Found %s non-matching file(s):",
+            len(unmatched_files),
+        )
+        for path in sorted(unmatched_files):
+            logger.info("  - '%s'", path.name)
+
+        data_files = init_data_files(
+            self._analysis_id,
+            matched_files,
+            self._filename_pattern,
+            self._bundle_file,
+        )
+
+        return DataFileCollectorResult(
+            matched_files=matched_files,
+            unmatched_files=unmatched_files,
+            data_files=data_files,
+        )