genelastic 0.7.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

genelastic/import_data/cli/validate.py

@@ -0,0 +1,146 @@
+import argparse
+import logging
+from pathlib import Path
+
+from genelastic.common.cli import add_verbose_control_args, add_version_arg
+from genelastic.common.exceptions import (
+    ValidationError,
+    YAMLFileReadError,
+)
+from genelastic.import_data.import_bundle_factory import (
+    load_yaml_file,
+    validate_doc,
+)
+from genelastic.import_data.logger import configure_logging
+from genelastic.import_data.models.validate import ValidationIssue
+
+logger = logging.getLogger("genelastic")
+
+
+def read_args() -> argparse.Namespace:
+    """Read arguments from command line."""
+    parser = argparse.ArgumentParser(
+        description="Statically validates YAML bundles: "
+        "ensure they comply to the bundle schema.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        allow_abbrev=False,
+    )
+    add_version_arg(parser)
+    add_verbose_control_args(parser)
+    parser.add_argument(
+        "files",
+        type=Path,
+        nargs="+",
+        default=None,
+        help="Paths to YAML files containing bundles to validate.",
+    )
+    parser.add_argument(
+        "-x",
+        "--fail-fast",
+        dest="fail_fast",
+        action="store_true",
+        help="Stop validating files after the first error is encountered.",
+    )
+    return parser.parse_args()
+
+
+def main() -> int:
+    """Entry point of the validate script."""
+    args = read_args()
+    configure_logging(args.verbose)
+
+    validation_issues = []
+    file_count = len(args.files)
+
+    for file_index, file_path in enumerate(args.files):
+        resolved_file_path = file_path.resolve()
+
+        logger.info(
+            "[%s/%s] Validating bundle(s) from file '%s'.",
+            file_index + 1,
+            file_count,
+            resolved_file_path,
+        )
+        logger.info("Loading YAML file...")
+
+        try:
+            docs = load_yaml_file(resolved_file_path)
+        except YAMLFileReadError as e:
+            logger.error(e)
+
+            if args.fail_fast:
+                raise SystemExit(1) from None
+
+            validation_issues.append(
+                ValidationIssue(
+                    exc_type=type(e).__name__,
+                    file_path=resolved_file_path,
+                    file_index=file_index + 1,
+                    file_count=file_count,
+                )
+            )
+            continue
+
+        logger.info("-> YAML file successfully loaded.")
+
+        doc_count = len(docs)
+        logger.info("Found %s document(s) in the YAML file.", doc_count)
+
+        for doc_index, doc in enumerate(docs):
+            logger.info(
+                "  Validating bundle format for document #%s/%s...",
+                doc_index + 1,
+                doc_count,
+            )
+
+            try:
+                validate_doc(doc)
+            except ValidationError as e:
+                logger.error(e)
+
+                if args.fail_fast:
+                    raise SystemExit(1) from None
+
+                validation_issues.append(
+                    ValidationIssue(
+                        exc_type=type(e).__name__,
+                        file_path=resolved_file_path,
+                        file_index=file_index + 1,
+                        file_count=file_count,
+                        doc_index=doc_index + 1,
+                        doc_count=doc_count,
+                    )
+                )
+                continue
+
+            logger.info("  -> Bundle format is valid.")
+
+        logger.info("")
+
+    if len(validation_issues) > 0:
+        logger.error("Some files raised exceptions:")
+        for issue in validation_issues:
+            logger.error("  - %s", issue)
+
+        ret_code = 1
+    else:
+        logger.info("All bundles respect the genelastic YAML bundle format.")
+        ret_code = 0
+
+    files_failing_validation = len(
+        {issue.file_path for issue in validation_issues}
+    )
+    files_passing_validation = file_count - files_failing_validation
+
+    logger.info(
+        "Out of %s file(s), validation passed for %s and failed for %s.",
+        file_count,
+        files_passing_validation,
+        files_failing_validation,
+    )
+
+    return ret_code
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

genelastic/import_data/collect.py

@@ -0,0 +1,185 @@
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+
+from genelastic.common.exceptions import DataFileCollectorError
+from genelastic.common.types import Metadata
+from genelastic.import_data.models.data_file import DataFile
+from genelastic.import_data.models.tags import Tags
+from genelastic.import_data.patterns import FilenamePattern
+from genelastic.import_data.resolve import resolve_analysis_id
+
+logger = logging.getLogger("genelastic")
+
+
+def collect_files(data_path: Path) -> set[Path]:
+    """Collect files for a given analysis.
+    All files directly under ``data_path`` are returned.
+
+    :param data_path: Directory containing the files.
+    :raises DataFileCollectorError: If ``data_path`` is not an existing
+        directory.
+    :return: Set of absolute paths to collected files.
+    """
+    try:
+        collected_files = {x for x in data_path.iterdir() if x.is_file()}
+    except OSError as e:
+        msg = f"Error collecting files: data directory is invalid. {e}."
+        raise DataFileCollectorError(msg) from e
+    return collected_files
+
+
+def match_files(
+    files: set[Path],
+    filename_pattern: FilenamePattern,
+) -> tuple[set[Path], set[Path]]:
+    """Splits a set of files into those that match a given filename pattern and
+    those that don't.
+
+    This function applies the provided ``filename_pattern`` to each file name
+    in ``files``, and returns two sets: one containing files that match the
+    pattern, and one containing those that do not.
+
+    :param files: A set of file paths to check.
+    :param filename_pattern: The filename pattern used for matching.
+
+    :returns: A tuple containing in first position a set of files that match
+      the pattern, and in second position a set of files that do not match the
+      pattern.
+    """
+    matched_files = {
+        f for f in files if filename_pattern.matches_pattern(f.name)
+    }
+    return matched_files, files - matched_files
+
+
+def extract_analysis_metadata(
+    data_path: Path,
+    file_prefix: str,
+    tags: Tags,
+    filename_pattern: FilenamePattern,
+) -> dict[str, Metadata]:
+    analysis = {}
+
+    for file in collect_files(data_path):
+        if not filename_pattern.matches_pattern(file.name):
+            logger.debug("File '%s' was not matched.", file.name)
+            continue
+
+        filename_metadata = filename_pattern.extract_metadata(file.name)
+        analysis_id = resolve_analysis_id(file_prefix, tags, filename_metadata)
+        analysis[analysis_id] = filename_metadata
+
+    return analysis
+
+
+def init_data_files(
+    analysis_id: str,
+    files: set[Path],
+    filename_pattern: FilenamePattern,
+    bundle_file: Path,
+) -> set[DataFile]:
+    """Instantiate ``DataFile`` objects from a set of file paths associated
+    with an analysis.
+
+    :param analysis_id: ID of the analysis, shared by all created ``DataFile``
+      instances.
+    :param files: Set of file paths associated with the analysis.
+    :param filename_pattern: Pattern used to extract metadata from filenames.
+      The extracted metadata is included in each ``DataFile``.
+    :param bundle_file: Path to the YAML bundle file from which the analysis is
+      defined.
+    :raises DataFileCollectorError: If metadata extraction or instantiation
+        of a data file objet fails for a given file.
+    :return: A set of successfully instantiated ``DataFile`` objects.
+    """
+    data_files = set()
+    for file in files:
+        try:
+            metadata = filename_pattern.extract_metadata(file.name)
+            data_file = DataFile(
+                analysis_id=analysis_id,
+                path=file,
+                bundle_file=bundle_file,
+                metadata=metadata,
+            )
+            data_files.add(data_file)
+        except RuntimeError as e:
+            msg = f"Error instantiating data files: {e}"
+            raise DataFileCollectorError(msg) from None
+    return data_files
+
+
+@dataclass(frozen=True)
+class DataFileCollectorResult:
+    """Result of a data file collection."""
+
+    matched_files: set[Path]
+    unmatched_files: set[Path]
+    data_files: set[DataFile]
+
+
+class DataFileCollector:
+    """Collect all data files belonging to an analysis."""
+
+    def __init__(
+        self,
+        analysis_id: str,
+        bundle_file: Path,
+        data_path: Path,
+        filename_pattern: FilenamePattern,
+        *,
+        multi_match: bool = False,
+    ) -> None:
+        self._analysis_id = analysis_id
+        self._bundle_file = bundle_file
+        self._data_path = data_path
+        self._filename_pattern = filename_pattern
+        self._multi_match = multi_match
+
+    def run(self) -> DataFileCollectorResult:
+        """Collects files from the analysis data path, matches them against the
+        analysis filename pattern, and instantiates ``DataFile`` objects for
+        each matched file.
+
+        :raises DataFileCollectorError: If the ``data_path`` is not an existing
+            directory or if metadata extraction or instantiation of a data file
+            objet fails for a given file.
+        :return: A ``DataFileCollectorResult`` containing the sets of matched
+            and unmatched files, as well as a set of instantiated ``DataFile``
+            objects.
+        """
+        files = collect_files(self._data_path)
+        logger.debug(
+            " -> Collected %s file(s):",
+            len(files),
+        )
+        for path in sorted(files):
+            logger.debug("  - '%s'", path.name)
+
+        matched_files, unmatched_files = match_files(
+            files, self._filename_pattern
+        )
+        logger.info(" -> Found %s matching file(s):", len(matched_files))
+        for path in sorted(matched_files):
+            logger.info("  - '%s'", path.name)
+
+        logger.info(
+            " -> Found %s non-matching file(s):",
+            len(unmatched_files),
+        )
+        for path in sorted(unmatched_files):
+            logger.info("  - '%s'", path.name)
+
+        data_files = init_data_files(
+            self._analysis_id,
+            matched_files,
+            self._filename_pattern,
+            self._bundle_file,
+        )
+
+        return DataFileCollectorResult(
+            matched_files=matched_files,
+            unmatched_files=unmatched_files,
+            data_files=data_files,
+        )

genelastic/import_data/constants.py

@@ -5,20 +5,145 @@ This module contains genelastic constants.
 
 import typing
 
-ALLOWED_CATEGORIES: typing.Final[list[str]] = ["vcf", "cov"]
+import schema
+
+ALLOWED_EXTENSIONS: typing.Final[list[str]] = [
+    "vcf",
+    "cov",
+    "json",
+    "yml",
+    "yaml",
+]
 
 BUNDLE_CURRENT_VERSION = 3
 
-DEFAULT_TAG_REGEX = "[^_-]+"
-DEFAULT_TAG_PREFIX = "%"
-DEFAULT_TAG_SUFFIX = ""
+DEFAULT_TAG_REGEX = "[^_]+"
+DEFAULT_TAG_DELIMITER_START = "%"
+DEFAULT_TAG_DELIMITER_END = ""
 
 DEFAULT_TAG2FIELD: typing.Final[dict[str, dict[str, str]]] = {
-    "%S": {"field": "sample_name", "regex": DEFAULT_TAG_REGEX},
-    "%F": {"field": "source", "regex": DEFAULT_TAG_REGEX},
-    "%W": {"field": "wet_process", "regex": DEFAULT_TAG_REGEX},
-    "%B": {"field": "bi_process", "regex": DEFAULT_TAG_REGEX},
-    "%D": {"field": "cov_depth", "regex": DEFAULT_TAG_REGEX},
-    "%A": {"field": "barcode", "regex": DEFAULT_TAG_REGEX},
-    "%R": {"field": "reference_genome", "regex": DEFAULT_TAG_REGEX},
+    "S": {"field": "sample_name", "regex": DEFAULT_TAG_REGEX},
+    "F": {"field": "source", "regex": DEFAULT_TAG_REGEX},
+    "W": {"field": "wet_process", "regex": DEFAULT_TAG_REGEX},
+    "B": {"field": "bi_process", "regex": DEFAULT_TAG_REGEX},
+    "D": {"field": "cov_depth", "regex": DEFAULT_TAG_REGEX},
+    "A": {"field": "barcode", "regex": DEFAULT_TAG_REGEX},
+    "R": {"field": "reference_genome", "regex": DEFAULT_TAG_REGEX},
 }
+
+TOOLS_SUFFIX_RE = r"_(?P<tool>[a-zA-Z0-9]+)-(?P<version>\d+(?:-\d+){0,2})(?!-)"
+"""
+Regular expression to extract individual tool-version metadata pairs from a
+validated ``.metrics`` suffix in filenames.
+
+- Captures exactly one tool-version pair, where:
+
+  - ``tool`` is an alphanumeric identifier (letters and digits),
+  - ``version`` consists of 1 to 3 numeric components separated by hyphens
+    (e.g., '1', '1-0', '1-0-0'),
+- Uses named capture groups (``tool`` and ``version``) to extract data,
+- The negative lookahead ``(?!-)`` ensures the version does not end with a
+  hyphen,
+- Intended for extracting all matching pairs after the ``.metrics`` prefix has
+  been validated.
+"""
+
+_METRICS_SUFFIX_RE = r"(?:\.metrics(?:_[a-zA-Z0-9]+-\d+(?:-\d+){0,2}(?!-))*)?"
+"""
+Regular expression to match and validate the entire optional ``.metrics``
+suffix in filenames.
+
+- Matches zero or one occurrence of:
+
+  - A literal ``.metrics`` prefix, which must be the first suffix in the
+    filename,
+  - Followed optionally by zero or more tool-version pairs, each starting with
+    an underscore ``_`` and matching the same format as ``TOOLS_SUFFIX_RE``,
+- Validates that the whole suffix structure is correct (including optional
+  presence),
+- Ensures that when present, the suffix starts with ``.metrics`` and is
+  correctly formatted,
+- Does not extract individual tool-version pairs; its role is to validate the
+  suffix as a whole.
+"""
+
+_EXTENSIONS_SUFFIX_RE = rf"\.(?P<ext>{'|'.join(ALLOWED_EXTENSIONS)})(\.gz)?"
+"""
+Regular expression for matching allowed file extensions with optional gzip
+compression.
+
+This regex matches the file extension suffixes for files belonging to
+a set of predefined allowed extensions, specified in the ``ALLOWED_EXTENSIONS``
+list.
+
+The pattern matches:
+
+- a dot (``.``) followed by one of the allowed extensions,
+- optionally, a second extension ``.gz`` indicating gzip compression.
+
+Examples of matched suffixes: ``.vcf``, ``.cov``, ``.json``, ``.vcf.gz``,
+``.json.gz``.
+"""
+
+FILE_SUFFIXES_RE = rf"{_METRICS_SUFFIX_RE}{_EXTENSIONS_SUFFIX_RE}"
+"""Regex used to validate the suffix part of a filename.
+
+It matches an optional metrics suffix (containing tool-version metadata),
+immediately followed by a required allowed file extension suffix
+(possibly compressed with .gz).
+
+This regex is the combination of ``_METRICS_SUFFIX_RE`` and
+``_EXTENSIONS_SUFFIX_RE``.
+"""
+
+QC_METRICS_SCHEMA = schema.Schema(
+    {
+        "id": str,
+        "genome_coverage_size": float,
+        "genome_coverage_percent": float,
+        "n50": int,
+        "larger_contig": int,
+        "iqr": int,
+        "outlier_percent": float,
+        "mean_depth": float,
+        "mean_duplicat_percent": float,
+        "fold_regions_percents": {
+            "5": float,
+            "10": float,
+            "20": float,
+            "30": float,
+            "40": float,
+        },
+    }
+)
+
+
+SV_METRICS_SCHEMA = schema.Schema(
+    {
+        "metadata_mandatory": [{str: schema.Or(str, int, float, bool)}],
+        schema.Optional("metadata_optional"): [
+            {str: schema.Or(str, int, float, bool)}
+        ],
+        "regions": [
+            {
+                "name": str,
+                "bed": str,
+                "results": [
+                    {
+                        "svtype": str,
+                        "size": str,
+                        "FP_query": int,
+                        "TP_truth": int,
+                        "TP_query": int,
+                        "FN_truth": int,
+                        "total_truth": int,
+                        "total_query": int,
+                        "precision": schema.Or(int, float),
+                        "recall": schema.Or(int, float),
+                        "f1": schema.Or(int, float),
+                    }
+                ],
+            }
+        ],
+    }
+)