genelastic 0.8.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

genelastic/import_data/models/analysis.py

@@ -0,0 +1,144 @@
+import contextlib
+import copy
+import logging
+from collections import defaultdict
+from pathlib import Path
+from types import NotImplementedType
+
+from genelastic.common.types import Metadata
+from genelastic.import_data.collect import (
+    DataFileCollector,
+)
+from genelastic.import_data.constants import (
+    ALLOWED_EXTENSIONS,
+)
+from genelastic.import_data.models.data_file import DataFile
+from genelastic.import_data.patterns import FilenamePattern
+
+logger = logging.getLogger("genelastic")
+
+
+class Analysis:
+    """Class Analysis that represents an analysis."""
+
+    METADATA_INTERNAL_KEYS = frozenset(
+        ["tags", "multi_match", "ext", "file_prefix"]
+    )
+
+    def __init__(
+        self,
+        analysis_id: str,
+        bundle_file: Path,
+        data_path: Path,
+        filename_pattern: FilenamePattern,
+        **metadata: str | int,
+    ) -> None:
+        self._analysis_id = analysis_id
+        self._bundle_file = bundle_file
+        self._data_path = data_path
+        self._metadata = self._remove_internal_keys(metadata)
+        self._data_files_by_ext: dict[str, set[DataFile]] = defaultdict(set)
+
+        logger.info("")
+        logger.info("[ Analysis ID %s ]", self._analysis_id)
+
+        self._collected_files = DataFileCollector(
+            analysis_id,
+            bundle_file,
+            data_path,
+            filename_pattern,
+        ).run()
+
+        for data_file in self._collected_files.data_files:
+            self._data_files_by_ext[data_file.ext].add(data_file)
+
+        logger.info(
+            " -> Extracted %s file extension(s): %s.",
+            len(self._data_files_by_ext.keys()),
+            ", ".join(ext.upper() for ext in self._data_files_by_ext),
+        )
+
+    def __eq__(self, other: object) -> bool | NotImplementedType:
+        """Defines equality comparison for Analysis instances based on their
+        ID.
+        """
+        if isinstance(other, Analysis):
+            return self._analysis_id == other._analysis_id
+        return NotImplemented
+
+    def __lt__(self, other: object) -> bool | NotImplementedType:
+        """Defines sort order for Analysis instances based on their ID."""
+        if isinstance(other, Analysis):
+            return self._analysis_id < other._analysis_id
+        return NotImplemented
+
+    def __str__(self) -> str:
+        return (
+            f"Analysis(id='{self._analysis_id}', "
+            f"bundle_file='{self._bundle_file}', "
+            f"data_path='{self._data_path}', "
+            f"metadata={self._metadata})"
+        )
+
+    @staticmethod
+    def _remove_internal_keys(
+        metadata: Metadata,
+    ) -> Metadata:
+        updated_metadata = metadata.copy()
+
+        for key in Analysis.METADATA_INTERNAL_KEYS:
+            with contextlib.suppress(KeyError):
+                del updated_metadata[key]
+
+        return updated_metadata
+
+    @property
+    def metadata(self) -> Metadata:
+        """Get metadata."""
+        return copy.deepcopy(self._metadata)
+
+    @property
+    def bundle_file(self) -> Path:
+        """Get the bundle file."""
+        return self._bundle_file
+
+    @property
+    def data_path(self) -> Path:
+        """Get the data path specified in the bundle file."""
+        return self._data_path
+
+    @property
+    def id(self) -> str:
+        """Get the analysis ID."""
+        return self._analysis_id
+
+    @property
+    def matched_files(self) -> set[Path]:
+        """Returns the list of files that matched the filename pattern."""
+        return self._collected_files.matched_files
+
+    @property
+    def unmatched_files(self) -> set[Path]:
+        """Returns the list of files that did not match the filename pattern."""
+        return self._collected_files.unmatched_files
+
+    @property
+    def extensions(self) -> set[str]:
+        """Returns all the matched files extensions."""
+        return set(self._data_files_by_ext.keys())
+
+    def get_data_files(self, ext: str | None = None) -> set[DataFile]:
+        """Returns the list of matched files as DataFile objects.
+
+        :param ext: Filter the list of matched files by their extension
+            (case-sensitive).
+        """
+        if ext:
+            if ext not in ALLOWED_EXTENSIONS:
+                msg = f"Unsupported extension {ext}."
+                raise ValueError(msg)
+
+            if ext in self._data_files_by_ext:
+                return self._data_files_by_ext[ext]
+            return set()
+        return {f for value in self._data_files_by_ext.values() for f in value}

genelastic/import_data/models/data_file.py

@@ -0,0 +1,110 @@
+"""This module defines the DataFile class, which handles the representation,
+management, and extraction of metadata for a data file within a data bundle.
+
+It includes functionality to construct DataFile instances from paths and
+optional filename patterns, retrieve file paths and metadata, and support
+for extracting metadata from filenames using specified patterns.
+"""
+
+import logging
+from pathlib import Path
+from types import NotImplementedType
+
+from genelastic.common.types import Metadata
+from genelastic.import_data.patterns import MetricsPattern
+
+logger = logging.getLogger("genelastic")
+
+
+class DataFile:
+    """Class for handling a data file and its metadata."""
+
+    def __init__(
+        self,
+        analysis_id: str,
+        path: Path,
+        bundle_file: Path,
+        metadata: Metadata,
+    ) -> None:
+        self._analysis_id = analysis_id
+        self._path = path
+        self._bundle_file = bundle_file
+        self._metadata = metadata
+        self._metrics = MetricsPattern.extract_metadata(path)
+        self._validate_params()
+
+        self._ext = str(self._metadata["ext"]).lower()
+
+        key = "type" if self._metrics is not None else "ext"
+        self._type = str(self._metadata[key]).lower()
+
+    def __eq__(self, other: object) -> bool | NotImplementedType:
+        """Defines equality comparison for DataFile instances based on their
+        file path.
+        """
+        if isinstance(other, DataFile):
+            return self._path == other._path
+        return NotImplemented
+
+    def __hash__(self) -> int:
+        """Defines hash behavior for DataFile to allow use in sets and as dict keys."""
+        return hash(self._path)
+
+    def _validate_params(self) -> None:
+        """Validate values of some ``DataFile`` constructor parameters.
+
+        :raises RuntimeError: One of the parameters value is invalid.
+        """
+        if "ext" not in self._metadata:
+            msg = (
+                f"Data file '{self._path}' "
+                f"is missing the required metadata key 'ext'."
+            )
+            raise RuntimeError(msg)
+
+        if self._metrics is not None and "type" not in self._metadata:
+            msg = (
+                f"Metrics data file '{self._path}' "
+                f"is missing the required metadata key 'type'."
+            )
+            raise RuntimeError(msg)
+
+    @property
+    def analysis_id(self) -> str:
+        """Get the analysis ID."""
+        return self._analysis_id
+
+    @property
+    def path(self) -> Path:
+        """Retrieve the data file path."""
+        return self._path
+
+    @property
+    def ext(self) -> str:
+        """Retrieve the data file extension."""
+        return self._ext
+
+    @property
+    def type(self) -> str:
+        """Retrieve the data file type.
+
+        Normally, the type is the file's extension.
+        If the file is a metrics file, its type is taken from the metadata key
+        'type'.
+        """
+        return self._type
+
+    @property
+    def bundle_file(self) -> Path:
+        """Retrieve the path to the associated data bundle file."""
+        return self._bundle_file
+
+    @property
+    def metadata(self) -> Metadata:
+        """Retrieve a copy of the metadata associated with the data file."""
+        return self._metadata.copy()
+
+    @property
+    def metrics(self) -> list[dict[str, str]] | None:
+        """Retrieve a copy of the metrics associated with the data file."""
+        return self._metrics

genelastic/import_data/models/process.py

@@ -0,0 +1,45 @@
+import copy
+from abc import ABC
+from typing import Any
+
+
+class Process(ABC):  # noqa: B024
+    """Abstract base class for a Process.
+
+    It is not intended to be instantiated directly. Instead, use one of its
+    subclasses, ``WetProcess`` or ``BioInfoProcess``.
+    """
+
+    def __init__(
+        self,
+        proc_id: str,
+        bundle_file: str | None = None,
+        **data: Any,  # noqa: ANN401
+    ) -> None:
+        self._proc_id = proc_id
+        self._bundle_file = bundle_file
+        self._data = data
+        self._type = self.__class__.__name__
+
+    @property
+    def id(self) -> str:
+        """Unique identifier of the process."""
+        return self._proc_id
+
+    @property
+    def data(self) -> dict[str, Any]:
+        """Return a copy of the associated data."""
+        return copy.deepcopy(self._data)
+
+    @property
+    def type(self) -> str:
+        """Type of the process."""
+        return self._type
+
+
+class WetProcess(Process):
+    """Concrete wet lab process."""
+
+
+class BioInfoProcess(Process):
+    """Concrete bioinformatics process."""

genelastic/import_data/models/processes.py

@@ -0,0 +1,84 @@
+import logging
+import typing
+from collections import UserDict
+from typing import Self
+
+from genelastic.common.types import BundleDict
+from genelastic.import_data.models.process import (
+    Process,
+)
+
+logger = logging.getLogger("genelastic")
+
+
+class Processes(UserDict[str, Process]):
+    """Container for homogeneous Process objects.
+
+    Unlike a standard dict:
+      - Only subclasses of ``Process`` are allowed as values.
+      - All items must be of the same concrete subclass of ``Process``.
+      - Duplicate keys are not allowed and will raise an exception.
+
+    :ivar _item_type: Internal attribute storing the concrete subclass
+        of ``Process`` enforced in this container.
+    """
+
+    _item_type: type | None = None
+
+    def __setitem__(self, key: str, value: Process) -> None:
+        if not isinstance(value, Process):
+            msg = (
+                "Object type not supported. "
+                "Container only supports 'Process' subclasses as items."
+            )
+            raise TypeError(msg)
+
+        if self._item_type is None:
+            self._item_type = type(value)
+        elif not isinstance(value, self._item_type):
+            msg = (
+                f"Cannot mix types. Container already holds "
+                f"{self._item_type.__name__} items."
+            )
+            raise TypeError(msg)
+
+        if key in self:
+            msg = (
+                f"Duplicate key. "
+                f"Container already holds an item with key '{key}'."
+            )
+            raise ValueError(msg)
+
+        super().__setitem__(key, value)
+
+    def add(self, item: Process) -> None:
+        """Add one process item to the container.
+
+        :raises TypeError: If ``item`` is not a subclass of ``Process``,
+            or if it does not match the subclass type of items already in the
+            container.
+        :raises ValueError: If an item with the same key (``item.id``) already
+            exists in the container.
+        """
+        self[item.id] = item
+
+    @classmethod
+    def from_dicts(
+        cls, arr: typing.Sequence[BundleDict], process_cls: type[Process]
+    ) -> Self:
+        """Build a Processes container instance from a sequence of dictionaries.
+
+        :param arr: Sequence of dictionaries representing process data.
+        :param process_cls: The subclass of ``Process`` to instantiate for each
+            dict.
+        :raises TypeError: If instantiating ``process_cls`` fails due to invalid
+                dictionary arguments, or if the resulting object type does not
+                match the container's enforced type.
+        :raises ValueError: If two or more dictionaries yield items with the
+            same key (``id``), leading to duplicate entries in the container.
+        :return: A Processes container instance populated with process objects.
+        """
+        instance = cls()
+        for d in arr:
+            instance.add(process_cls(**d))
+        return instance

genelastic/import_data/models/tags.py

@@ -0,0 +1,170 @@
+import logging
+import re
+import typing
+from collections import UserDict
+
+from genelastic.common.exceptions import TagsDefinitionError
+from genelastic.common.types import BundleDict
+from genelastic.import_data.constants import (
+    DEFAULT_TAG2FIELD,
+    DEFAULT_TAG_DELIMITER_END,
+    DEFAULT_TAG_DELIMITER_START,
+)
+
+logger = logging.getLogger("genelastic")
+
+
+class Tags(UserDict[str, dict[str, str]]):
+    """Represents a set of tags used to extract metadata from filenames.
+
+    Each tag maps a name to a metadata field and a regex pattern, supporting
+    custom delimiters. This class combines default tags (``DEFAULT_TAG2FIELD``)
+    with optional user-defined tags, and provides utilities for searching,
+    accessing, and resolving tags in filename patterns.
+    """
+
+    def __init__(
+        self,
+        delimiter_start: str | None = None,
+        delimiter_end: str | None = None,
+        match: dict[str, dict[str, str]] | None = None,
+    ) -> None:
+        """Initialize a Tags instance.
+
+        :param delimiter_start: Optional character prepended to all tag names.
+            Defaults to ``DEFAULT_TAG_DELIMITER_START``.
+        :param delimiter_end: Optional character appended to all tag names.
+            Defaults to ``DEFAULT_TAG_DELIMITER_END``.
+        :param match: Optional dictionary of user-defined tags. Overrides
+            ``DEFAULT_TAG2FIELD`` if keys overlap.
+        """
+        super().__init__()
+
+        if delimiter_start is None:
+            self._delimiter_start = DEFAULT_TAG_DELIMITER_START
+        else:
+            if not self.validate_tag_delimiter(delimiter_start):
+                msg = (
+                    "A tag delimiter start should contain only one special "
+                    "character, excluding the following: (, ), ?, <, >."
+                )
+                raise TagsDefinitionError(msg)
+            self._delimiter_start = delimiter_start
+
+        if delimiter_end is None:
+            self._delimiter_end = DEFAULT_TAG_DELIMITER_END
+        else:
+            if not self.validate_tag_delimiter(delimiter_end):
+                msg = (
+                    "A tag delimiter end should contain only one special "
+                    "character, excluding the following: (, ), ?, <, >."
+                )
+                raise TagsDefinitionError(msg)
+            self._delimiter_end = delimiter_end
+
+        # Combine default tags with user-provided tags. User-defined ones takes
+        # precedence.
+        effective_match = DEFAULT_TAG2FIELD | (match or {})
+
+        # Store each tag in the dictionary using the full name
+        # (delimiter start + tag name + delimiter end).
+        for tag_name, tag_attrs in effective_match.items():
+            if not self.validate_tag_name(tag_name):
+                msg = (
+                    f"Invalid tag '{tag_name}': its name should contain at "
+                    f"least one alphanumeric character: a-z, A-Z and 0-9."
+                )
+                raise TagsDefinitionError(msg)
+
+            for mandatory_key in ("field", "regex"):
+                if mandatory_key not in tag_attrs:
+                    msg = (
+                        f"Invalid tag '{tag_name}': mandatory key "
+                        f"'{mandatory_key}' missing."
+                    )
+                    raise TagsDefinitionError(msg)
+
+            tag = f"{self._delimiter_start}{tag_name}{self._delimiter_end}"
+            self[tag] = tag_attrs
+
+        logger.info(
+            "The following tags will be used "
+            "to extract metadata from filenames : %s",
+            self,
+        )
+
+    @property
+    def delimiter_start(self) -> str:
+        """Return the tag delimiter start. Defaults to
+        ``DEFAULT_TAG_DELIMITER_START``.
+        """
+        return self._delimiter_start
+
+    @property
+    def delimiter_end(self) -> str:
+        """Return the tag delimiter end. Defaults to
+        ``DEFAULT_TAG_DELIMITER_END``.
+        """
+        return self._delimiter_end
+
+    @property
+    def search_regex(self) -> str:
+        """Return a regex pattern to search for tags inside a string.
+
+        This regex matches any tag using the current start and end delimiters.
+        Used for filename prefixes validation or resolving tags into regex
+        patterns.
+        """
+        return (
+            re.escape(self._delimiter_start)
+            + r"[a-zA-Z0-9]+"
+            + re.escape(self._delimiter_end)
+        )
+
+    @classmethod
+    def from_dict(cls, bundle: BundleDict) -> typing.Self:
+        """Create tags from a bundle dict."""
+        delimiter_start, delimiter_end = None, None
+
+        if "tags" not in bundle:
+            msg = (
+                "Could not create a Tags object: bundle does not define tags "
+                "(root key 'tags' missing)."
+            )
+            raise TagsDefinitionError(msg)
+
+        tags = bundle["tags"]
+        match = tags.get("match")
+        tag_delimiter = tags.get("delimiter")
+
+        if tag_delimiter:
+            delimiter_start = tag_delimiter.get("start")
+            delimiter_end = tag_delimiter.get("end")
+
+        return cls(
+            delimiter_start=delimiter_start,
+            delimiter_end=delimiter_end,
+            match=match,
+        )
+
+    @staticmethod
+    def validate_tag_delimiter(s: str) -> bool:
+        """A tag delimiter should only contain one special character,
+        excluding the following: (, ), ?, <, >.
+        """
+        if len(s) != 1:
+            return False
+
+        return not re.match(r"^[\w()<>?]$", s)
+
+    @staticmethod
+    def validate_tag_name(s: str) -> bool:
+        """A tag name should contain at least one alphanumeric character:
+        ``a-z``, ``A-Z`` and ``0-9``.
+
+        :return: True if the tag name is valid, False otherwise.
+        """
+        if len(s) < 1:
+            return False
+
+        return bool(re.match(r"^[^_\W]+$", s))

genelastic/import_data/models/unique_list.py

@@ -0,0 +1,109 @@
+import typing
+from collections import UserList
+from typing import SupportsIndex
+
+from genelastic.common.exceptions import UniqueListDuplicateError
+
+T = typing.TypeVar("T")
+
+
+class UniqueList(UserList[T]):
+    """A list that only allows unique elements.
+
+    :param init_list: Optional iterable to initialize the list.
+    """
+
+    def __init__(self, init_list: typing.Iterable[T] | None = None) -> None:
+        super().__init__()
+
+        if init_list:
+            for item in init_list:
+                self._ensure_unique(item)
+                super().append(item)
+
+    def __setitem__(
+        self, i: SupportsIndex | slice, item: T | typing.Iterable[T]
+    ) -> None:
+        if isinstance(i, slice):
+            if not isinstance(item, typing.Iterable):
+                msg = "Expected iterable for slice assignment."
+                raise TypeError(msg)
+
+            slice_dupes = self._find_dupes(item)
+            if slice_dupes:
+                formatted_dupes = [str(dupe) for dupe in slice_dupes]
+                msg = (
+                    f"Duplicate item(s) in slice assignment: "
+                    f"{', '.join(formatted_dupes)}."
+                )
+                raise UniqueListDuplicateError(msg)
+            for x in item:
+                if x in self and x not in self[i]:
+                    msg = f"Duplicate item: {x}."
+                    raise UniqueListDuplicateError(msg)
+            super().__setitem__(i, item)
+        else:
+            self._ensure_unique(typing.cast(T, item))
+            super().__setitem__(i, typing.cast(T, item))
+
+    def __add__(self, other: typing.Iterable[T]) -> "UniqueList[T]":
+        for item in other:
+            self._ensure_unique(item)
+        return UniqueList(super().__add__(other))
+
+    def __iadd__(self, other: typing.Iterable[T]) -> typing.Self:
+        for item in other:
+            self._ensure_unique(item)
+        return super().__iadd__(other)
+
+    def __mul__(self, n: int) -> typing.Self:
+        raise NotImplementedError
+
+    def __imul__(self, n: int) -> typing.Self:
+        raise NotImplementedError
+
+    @staticmethod
+    def _find_dupes(a: typing.Iterable[T]) -> list[T]:
+        seen = set()
+        dupes = []
+        for x in a:
+            if x in seen:
+                dupes.append(x)
+            else:
+                seen.add(x)
+        return dupes
+
+    def _ensure_unique(self, item: T) -> None:
+        if item in self:
+            msg = f"Duplicate item: {item}."
+            raise UniqueListDuplicateError(msg)
+
+    def append(self, item: T) -> None:
+        """Appends a unique item to the end of the list.
+
+        :param item: Element to append.
+        :raises UniqueListError: If the item already exists in the list.
+        """
+        self._ensure_unique(item)
+        super().append(item)
+
+    def insert(self, i: int, item: T) -> None:
+        """Inserts a unique item at a specified position.
+
+        :param i: Index where the item should be inserted.
+        :param item: Element to insert.
+        :raises UniqueListError: If the item already exists in the list.
+        """
+        self._ensure_unique(item)
+        super().insert(i, item)
+
+    def extend(self, other: typing.Iterable[T]) -> None:
+        """Extends the list with unique elements from another iterable.
+
+        :param other: Iterable of elements to add.
+        :raises UniqueListError: If any element in the iterable already exists in
+            the list.
+        """
+        for item in other:
+            self._ensure_unique(item)
+        super().extend(other)

genelastic/import_data/models/validate.py

@@ -0,0 +1,26 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class ValidationIssue:
+    """Contains context about a bundle validation issue."""
+
+    exc_type: str
+    file_path: Path
+    file_index: int
+    file_count: int
+    doc_index: int | None = None
+    doc_count: int | None = None
+
+    def __str__(self) -> str:
+        if not self.doc_index:
+            return (
+                f"[{self.exc_type}] "
+                f"File {self.file_index}/{self.file_count}: {self.file_path}"
+            )
+        return (
+            f"[{self.exc_type}] "
+            f"File {self.file_index}/{self.file_count}: {self.file_path} "
+            f"(in doc #{self.doc_index}/{self.doc_count})"
+        )