genelastic 0.8.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

genelastic/import_data/importers/importer_base.py

@@ -0,0 +1,131 @@
+import gzip
+import json
+import logging
+from abc import ABC, abstractmethod
+from collections.abc import Iterable
+from json import JSONDecodeError
+from typing import Any, Generic, TypeVar
+
+import yaml
+
+from genelastic.common.elastic import ElasticImportConn
+from genelastic.import_data.models.data_file import DataFile
+
+logger = logging.getLogger("genelastic")
+
+T = TypeVar("T")
+
+
+class ImporterError(Exception):
+    """An error occurred while loading, validating or transforming a data file
+    into JSON documents.
+    """
+
+
+class BaseImporter(ABC, Generic[T]):
+    """Abstract base class for all importers."""
+
+    def __init__(
+        self,
+        data_file: DataFile,
+        es_import_conn: ElasticImportConn,
+        thread_count: int = 4,
+    ) -> None:
+        self._data_file = data_file
+        self._es_import_conn = es_import_conn
+        self._thread_count = thread_count
+
+        self._cls_name = self.__class__.__name__
+        self._process_file()
+
+    def _process_file(self) -> None:
+        """Process the file before import: load, validate and transform the
+        data into JSON documents.
+        :raises ImporterError: If an error occurs while processing the file.
+        """
+        logger.debug("%s: Loading data...", self._cls_name)
+        data = self._load()
+        logger.debug("%s: Validating data...", self._cls_name)
+        self._validate(data)
+        logger.debug("%s: Transforming data...", self._cls_name)
+        self._documents = self._transform(data)
+
+    def import_docs(self) -> None:
+        """Import the JSON documents into Elasticsearch."""
+        logger.debug("%s: Indexing documents...", self._cls_name)
+        self._es_import_conn.parallel_bulk_import(
+            self._documents, self._thread_count
+        )
+
+    @property
+    @abstractmethod
+    def target_index(self) -> str:
+        """Returns the import target index name."""
+
+    @property
+    def documents(self) -> Iterable[dict[str, Any]]:
+        """Return the documents about to be indexed."""
+        return self._documents
+
+    @abstractmethod
+    def _load(self) -> T:
+        """Load and parse raw data from the file."""
+
+    def _validate(self, data: T) -> None:
+        """Validate the data structure (optional)."""
+
+    @abstractmethod
+    def _transform(self, data: T) -> Iterable[dict[str, Any]]:
+        """Transform raw data into Elasticsearch-ready documents."""
+
+
+class JSONBaseImporter(BaseImporter[Any], ABC):
+    """Base importer to load JSON and gzipped JSON data files."""
+
+    def _load(self) -> Any:  # noqa: ANN401
+        try:
+            if self._data_file.path.suffix == ".gz":
+                logger.debug(
+                    "Opening gzip-compressed file in text mode and "
+                    "reading content...",
+                )
+                with gzip.open(
+                    self._data_file.path, "rt", encoding="utf-8"
+                ) as f:
+                    content = f.read()
+            else:
+                logger.debug(
+                    "Opening uncompressed file in text mode and "
+                    "reading content...",
+                )
+                content = self._data_file.path.read_text(encoding="utf-8")
+        except (OSError, UnicodeDecodeError) as e:
+            raise ImporterError(e) from None
+
+        if not content.strip():
+            msg = f"JSON file '{self._data_file.path}' is empty."
+            raise ImporterError(msg) from None
+
+        try:
+            data = json.loads(content)
+        except JSONDecodeError as e:
+            raise ImporterError(e) from None
+
+        return data
+
+
+class YAMLBaseImporter(BaseImporter[Any], ABC):
+    """Base importer to load YAML data files."""
+
+    def _load(self) -> dict[str, Any]:
+        try:
+            with self._data_file.path.open(encoding="utf-8") as f:
+                doc: dict[str, Any] = yaml.safe_load(f)
+        except (yaml.YAMLError, OSError, UnicodeDecodeError) as e:
+            raise ImporterError(e) from None
+
+        if doc is None:
+            msg = f"YAML file '{self._data_file.path}' is empty."
+            raise ImporterError(msg) from None
+
+        return doc

genelastic/import_data/importers/importer_factory.py

@@ -0,0 +1,85 @@
+import logging
+import typing
+from typing import ClassVar, TypedDict
+
+from genelastic.common.elastic import ElasticImportConn
+from genelastic.import_data.importers.importer_base import (
+    BaseImporter,
+    ImporterError,
+)
+from genelastic.import_data.importers.importer_types import (
+    CoverageImporter,
+    QCImporter,
+    SmallvarImporter,
+    SVImporter,
+    VCFImporter,
+)
+from genelastic.import_data.models.data_file import DataFile
+
+logger = logging.getLogger("genelastic")
+
+
+class _ImporterConfig(TypedDict):
+    """Internal configuration mapping an importer class to its supported file
+    extensions.
+    """
+
+    cls: type[BaseImporter[typing.Any]]
+    extensions: set[str]
+
+
+class ImporterFactory:
+    """Factory to create a BaseImporter instance based on the file's
+    extension and type.
+    """
+
+    _importers: ClassVar[dict[str, _ImporterConfig]] = {
+        "vcf": _ImporterConfig(cls=VCFImporter, extensions={"vcf"}),
+        "cov": _ImporterConfig(cls=CoverageImporter, extensions={"cov"}),
+        "qc": _ImporterConfig(cls=QCImporter, extensions={"yaml", "yml"}),
+        "smallvar": _ImporterConfig(cls=SmallvarImporter, extensions={"json"}),
+        "sv": _ImporterConfig(cls=SVImporter, extensions={"json"}),
+    }
+
+    @staticmethod
+    def get_importer(
+        data_file: DataFile,
+        es_import_conn: ElasticImportConn,
+        thread_count: int = 4,
+    ) -> BaseImporter[typing.Any]:
+        """Create an appropriate BaseImporter instance based on the data
+        file's extension and type.
+
+        :param data_file: Data file to process and import.
+        :param es_import_conn: Elasticsearch import connector instance.
+        :param thread_count: Number of threads to use for parallel data file
+            import.
+        :return: An instance of the appropriate BaseImporter subclass.
+        :raises ImporterError: If the data file extension or type is invalid.
+        """
+        try:
+            importer = ImporterFactory._importers[data_file.type]
+        except KeyError:
+            supported_types = sorted(
+                [f"'{i_type}'" for i_type in ImporterFactory._importers]
+            )
+            msg = (
+                f"Data file '{data_file.path.name}': no importer for type "
+                f"'{data_file.type}'. Supported types are: "
+                f"{', '.join(supported_types)}."
+            )
+            raise ImporterError(msg) from None
+
+        if data_file.ext not in importer["extensions"]:
+            supported_exts = sorted(
+                [f"'{ext}'" for ext in importer["extensions"]]
+            )
+            msg = (
+                f"Data file '{data_file.path.name}': extension "
+                f"'{data_file.ext}' not supported by importer "
+                f"{importer['cls'].__name__}. Supported extensions are: "
+                f"{', '.join(supported_exts)}."
+            )
+            raise ImporterError(msg)
+
+        return importer["cls"](data_file, es_import_conn, thread_count)

genelastic/import_data/importers/importer_types.py

@@ -0,0 +1,223 @@
+import csv
+import logging
+from collections.abc import Iterable
+from datetime import UTC, datetime
+from typing import Any
+
+import schema
+import vcf
+from vcf.model import _Record
+
+from genelastic.import_data.constants import (
+    QC_METRICS_SCHEMA,
+    SV_METRICS_SCHEMA,
+)
+from genelastic.import_data.importers.importer_base import (
+    BaseImporter,
+    ImporterError,
+    JSONBaseImporter,
+    YAMLBaseImporter,
+)
+
+logger = logging.getLogger("genelastic")
+
+
+class CoverageImporter(BaseImporter[Iterable[list[str]]]):
+    """Importer for coverage files."""
+
+    @property
+    def target_index(self) -> str:
+        """Returns the import target index name."""
+        return self._es_import_conn.coverage_index
+
+    def _load(self) -> Iterable[list[str]]:
+        """Load a TSV formatted coverage file.
+        :raises ImporterError: If the file could not be opened or decoded.
+        """
+        try:
+            with self._data_file.path.open(newline="", encoding="utf-8") as f:
+                reader = csv.reader(f, delimiter="\t")
+                try:
+                    first_row = next(reader)
+                except StopIteration:
+                    msg = f"Coverage file '{self._data_file.path}' is empty."
+                    raise ImporterError(msg) from None
+                yield first_row
+                yield from reader
+        except (OSError, UnicodeDecodeError) as e:
+            raise ImporterError(e) from None
+
+    def _transform(self, data: Iterable[list[str]]) -> Iterable[dict[str, Any]]:
+        """Transform each coverage file row into a JSON document."""
+        for row in data:
+            yield {
+                "_index": self.target_index,
+                "_source": {
+                    "analysis_id": self._data_file.analysis_id,
+                    "created_at": datetime.now(UTC).isoformat(),
+                    "row": {
+                        "chr": row[0],
+                        "pos": int(row[1]) + 1,
+                        "depth": int(row[2]),
+                    },
+                },
+            }
+
+
+class VCFImporter(BaseImporter[Iterable[_Record]]):
+    """Importer for VCF files."""
+
+    @property
+    def target_index(self) -> str:
+        """Returns the import target index name."""
+        return self._es_import_conn.vcf_variants_index
+
+    def _load(self) -> Iterable[_Record]:
+        """Load a VCF file. GZ compressed VCF files are supported.
+        :raises ImporterError: If the file could not be opened, decoded or is empty.
+        """
+        try:
+            yield from vcf.Reader(
+                filename=str(self._data_file.path), encoding="utf-8"
+            )
+        except StopIteration:
+            msg = f"VCF file '{self._data_file.path}' is empty."
+            raise ImporterError(msg) from None
+        except (OSError, UnicodeDecodeError) as e:
+            raise ImporterError(e) from None
+
+    def _transform(self, data: Iterable[_Record]) -> Iterable[dict[str, Any]]:
+        """Transform each VCF file record into a JSON document."""
+        for record in data:
+            # Fix values
+            if not record.CHROM.startswith("chr"):
+                if record.CHROM.lower().startswith("chr"):
+                    record.CHROM = "chr" + record.CHROM[3:]
+                else:
+                    record.CHROM = "chr" + record.CHROM
+
+            # Build document
+            alt = [x if x is None else x.type for x in record.ALT]
+
+            yield {
+                "_index": self.target_index,
+                "_source": {
+                    "created_at": datetime.now(UTC).isoformat(),
+                    "analysis_id": self._data_file.analysis_id,
+                    "record": {
+                        "type": "vcf",
+                        "chr": record.CHROM,
+                        "pos": record.POS,
+                        "alt": alt,
+                        "info": record.INFO,
+                    },
+                },
+            }
+
+
+class QCImporter(YAMLBaseImporter):
+    """Importer for QC YAML metrics files."""
+
+    @property
+    def target_index(self) -> str:
+        """Returns the import target index name."""
+        return self._es_import_conn.qc_metrics_index
+
+    def _validate(self, data: dict[str, Any]) -> None:
+        """Validate the YAML document against the expected schema.
+
+        :raises ImporterError: If the file format is invalid.
+        """
+        try:
+            QC_METRICS_SCHEMA.validate(data)
+        except schema.SchemaError as e:
+            raise ImporterError(e) from None
+
+    def _transform(self, data: dict[str, Any]) -> Iterable[dict[str, Any]]:
+        """Transform a QC YAML metrics file into a JSON document."""
+        yield {
+            "_index": self.target_index,
+            "_source": {
+                "created_at": datetime.now(UTC).isoformat(),
+                "analysis_id": self._data_file.analysis_id,
+                "metrics": data,
+            },
+        }
+
+
+class SmallvarImporter(JSONBaseImporter):
+    """Importer for SmallVar JSON metrics files."""
+
+    @property
+    def target_index(self) -> str:
+        """Returns the import target index name."""
+        return self._es_import_conn.smallvar_metrics_index
+
+    def _transform(self, data: dict[str, Any]) -> Iterable[dict[str, Any]]:
+        """Transform a SmallVar metrics file into JSON documents."""
+        try:
+            for metric in data["metrics"]:
+                values_count = len(metric["data"][0]["values"])
+
+                metric_id = metric["id"].replace(".", "_").lower()
+
+                for i in range(values_count):
+                    doc = {}
+                    for item in metric["data"]:
+                        # Attribute name should not use '.' as it refers
+                        # to nested objects.
+                        label = item["label"].replace(".", "_")
+                        doc[label] = item["values"][i]
+
+                    yield {
+                        "_index": self.target_index,
+                        "_source": {
+                            "created_at": datetime.now(UTC).isoformat(),
+                            "analysis_id": self._data_file.analysis_id,
+                            "metric_id": metric_id,
+                            "metrics": doc,
+                        },
+                    }
+        except KeyError as e:
+            msg = (
+                f"Smallvar metrics file '{self._data_file.path}' "
+                f"is invalid: missing key {e}."
+            )
+            raise ImporterError(msg) from None
+
+
+class SVImporter(JSONBaseImporter):
+    """Importer for SV JSON metrics files."""
+
+    @property
+    def target_index(self) -> str:
+        """Returns the import target index name."""
+        return self._es_import_conn.sv_metrics_index
+
+    def _validate(self, data: dict[str, Any]) -> None:
+        """Validate the YAML document against the expected schema.
+
+        :raises ImporterError: If the file format is invalid.
+        """
+        try:
+            SV_METRICS_SCHEMA.validate(data)
+        except schema.SchemaError as e:
+            raise ImporterError(e) from None
+
+    def _transform(self, data: dict[str, Any]) -> Iterable[dict[str, Any]]:
+        """Transform a SV metrics file into a JSON document."""
+        for region in data["regions"]:
+            for result in region["results"]:
+                # Convert all values to float to avoid mapping issues.
+                result["precision"] = float(result["precision"])
+                result["recall"] = float(result["recall"])
+                result["f1"] = float(result["f1"])
+
+        yield {
+            "_index": self.target_index,
+            "_source": {
+                "created_at": datetime.now(UTC).isoformat(),
+                "analysis_id": self._data_file.analysis_id,
+                "metrics": data,
+            },
+        }

genelastic/import_data/logger.py

@@ -56,5 +56,6 @@ def configure_logging(verbose: int, log_file: str | None = None) -> None:
         1: logging.INFO,  # default
         2: logging.DEBUG,  # verbose mode
     }
+    level = level_map.get(verbose)
     # If verbose is greater than 2, set level to TRACE.
-    root.setLevel(level_map.get(verbose, logging.TRACE))  # type: ignore[attr-defined]
+    root.setLevel(level if level else logging.TRACE)  # type: ignore[attr-defined]

genelastic/import_data/models/analyses.py

@@ -0,0 +1,178 @@
+import logging
+import typing
+from pathlib import Path
+
+from genelastic.common.types import BundleDict
+from genelastic.import_data.collect import (
+    extract_analysis_metadata,
+)
+from genelastic.import_data.models.analysis import Analysis
+from genelastic.import_data.models.data_file import DataFile
+from genelastic.import_data.models.unique_list import UniqueList
+from genelastic.import_data.resolve import (
+    resolve_filename_pattern,
+    validate_file_prefix,
+)
+
+logger = logging.getLogger("genelastic")
+
+
+class Analyses(UniqueList[Analysis]):
+    """Container of Analysis objects."""
+
+    def get_data_files(self, ext: str | None = None) -> list[DataFile]:
+        """Returns matched files as DataFile objects across all analyses.
+
+        :param ext: Filter the list of matched files by their extension
+            (case-sensitive).
+        """
+        return [df for a in self for df in a.get_data_files(ext=ext)]
+
+    @property
+    def extensions(self) -> set[str]:
+        """Returns all matched files extensions across all analyses."""
+        return {ext for a in self for ext in a.extensions}
+
+    @property
+    def matched_files(self) -> set[Path]:
+        """Returns the number of files that matched the pattern across all
+        analyses.
+        """
+        return {f for a in self for f in a.matched_files}
+
+    @property
+    def unmatched_files(self) -> set[Path]:
+        """Return the set of files that were not matched by any analysis.
+
+        The behavior differs depending on whether analyses share the same
+        ``data_path``:
+
+        - Within the same directory: a file is considered unmatched only if
+            **all** analyses in that directory failed to match it. This is
+            computed as the intersection of their respective ``unmatched_files``
+            sets.
+
+        - Across different directories: unmatched files are simply aggregated
+            (union of sets), since each directory is independent.
+
+        :return: A set of paths corresponding to unmatched files across all
+            analyses.
+        """
+        unmatched_per_dir: dict[Path, set[Path]] = {}
+
+        for a in self:
+            try:
+                unmatched_per_dir[a.data_path] = set.intersection(
+                    unmatched_per_dir[a.data_path], a.unmatched_files
+                )
+            except KeyError:
+                unmatched_per_dir[a.data_path] = a.unmatched_files
+
+        if not unmatched_per_dir.values():
+            return set()
+        return set.union(*unmatched_per_dir.values())
+
+    @classmethod
+    def from_dict(cls, bundle: BundleDict) -> typing.Self:
+        """Initialize an ``Analyses`` container from a single bundle dictionary.
+
+        Expected bundle keys:
+
+        - Mandatory: ``file_prefix``, ``tags``, ``bundle_file``, ``data_path``.
+        - Optional: ``multi_match`` (default: ``False``), ``suffix`` (default: ``None``).
+
+        :param bundle: A dictionary describing one analysis configuration.
+        :raises InvalidFilePrefixError: If the ``file_prefix`` is invalid.
+        :raises FilenamePatternResolveError: If ``multi_match=False`` and some
+            tag fields are missing from the bundle metadata.
+        :raises UniqueListDuplicateError: If two ``Analysis`` objects happens
+            to share the same ID inside the ``Analyses`` instance.
+        :raises DataFileCollectorError: If the ``data_path`` is not an existing
+            directory or if metadata extraction or instantiation of a data file
+            objet fails for a given file.
+        :return: An ``Analyses`` instance containing one or several
+            ``Analysis`` objects.
+        """
+        analyses = cls()
+
+        # Validate file prefix structure.
+        logger.info("- Validating file prefix '%s'...", bundle["file_prefix"])
+        validate_file_prefix(
+            file_prefix=bundle["file_prefix"], tags=bundle["tags"]
+        )
+
+        # Resolve the filename pattern. In multi-match mode, tags without
+        # metadata values are accepted. They will be resolved later from
+        # filename-extracted metadata. In single-match mode, a
+        # FilenamePatternResolveError exception will be raised.
+        strict_mode = not bool(bundle.get("multi_match"))
+        logger.info(
+            "- Resolving filename pattern in %s mode...",
+            "strict" if strict_mode else "non-strict",
+        )
+        filename_pattern = resolve_filename_pattern(
+            file_prefix=bundle["file_prefix"],
+            tags=bundle["tags"],
+            metadata=bundle,
+            suffix=bundle.get("suffix"),
+            strict=strict_mode,
+        )
+
+        # Scan the data path to extract metadata from filenames.
+        logger.info(
+            "- Collecting files to extract metadata from using the resolved "
+            "filename pattern."
+        )
+        extracted_metadata = extract_analysis_metadata(
+            data_path=bundle["data_path"],
+            file_prefix=bundle["file_prefix"],
+            tags=bundle["tags"],
+            filename_pattern=filename_pattern,
+        )
+
+        logger.info(
+            "- Extracted metadata from %d analysis(es): %s",
+            len(extracted_metadata.keys()),
+            ", ".join(extracted_metadata.keys()),
+        )
+
+        for analysis_id, metadata in extracted_metadata.items():
+            # For each file match, merge filename-extracted metadata with the
+            # original bundle to describe one analysis.
+            full_metadata = {**bundle, **metadata}
+            full_metadata["analysis_id"] = analysis_id
+
+            # Re-resolve filename pattern in strict mode to let the analysis
+            # collect its own files (all tags should now be defined).
+            full_metadata["filename_pattern"] = resolve_filename_pattern(
+                file_prefix=full_metadata["file_prefix"],
+                tags=full_metadata["tags"],
+                metadata=full_metadata,
+                suffix=full_metadata.get("suffix"),
+                strict=True,
+            )
+
+            # Instantiate the Analysis and add it to the container.
+            analyses.append(Analysis(**full_metadata))
+            logger.info("")
+
+        return analyses
+
+    @classmethod
+    def from_dicts(cls, arr: typing.Sequence[BundleDict]) -> typing.Self:
+        """Initialize an ``Analyses`` container from multiple bundle
+        dictionaries.
+
+        This is a convenience wrapper that calls ``from_dict`` for each
+        bundle in the sequence and concatenates the results.
+
+        :param arr: A sequence of bundle dictionaries.
+        :return: An ``Analyses`` instance containing all analyses from the
+            input bundles.
+        """
+        analyses = cls()
+
+        for bundle in arr:
+            analyses.extend(analyses.from_dict(bundle))
+
+        return analyses