genelastic 0.8.0__py3-none-any.whl → 0.10.0__py3-none-any.whl

genelastic/import_data/cli_info.py

@@ -1,256 +0,0 @@
-import argparse
-import logging
-
-from genelastic.common import (
-    Bucket,
-    ElasticQueryConn,
-    add_es_connection_args,
-    add_verbose_control_args,
-)
-
-from .logger import configure_logging
-
-logger = logging.getLogger("genelastic")
-logging.getLogger("elastic_transport").setLevel(
-    logging.WARNING
-)  # Disable excessive logging
-
-
-def read_args() -> argparse.Namespace:
-    """Read arguments from command line."""
-    parser = argparse.ArgumentParser(
-        description="ElasticSearch database info.",
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-        allow_abbrev=False,
-    )
-    add_verbose_control_args(parser)
-    add_es_connection_args(parser)
-    parser.add_argument(
-        "-y",
-        "--list-bundles",
-        action="store_true",
-        help="List all imported YAML bundles.",
-    )
-    parser.add_argument(
-        "-f",
-        "--list-data-files",
-        action="store_true",
-        help="List all imported data files.",
-    )
-    parser.add_argument(
-        "-w",
-        "--list-wet-processes",
-        action="store_true",
-        help="List all imported wet processes.",
-    )
-    parser.add_argument(
-        "-b",
-        "--list-bi-processes",
-        action="store_true",
-        help="List all imported bio info processes.",
-    )
-    parser.add_argument(
-        "-Y",
-        "--list-data-files-per-bundle",
-        action="store_true",
-        help="For each imported YAML bundle, "
-        "display some info and list its data files.",
-    )
-    return parser.parse_args()
-
-
-def list_bundles(es_query_conn: ElasticQueryConn, index: str) -> None:
-    """List all imported YAML bundles."""
-    query = {
-        "size": 0,
-        "aggs": {
-            "get_bundle_paths": {
-                "composite": {
-                    "sources": {
-                        "bundle_path": {
-                            "terms": {"field": "bundle_path.keyword"}
-                        }
-                    },
-                    "size": 1000,
-                }
-            }
-        },
-    }
-
-    buckets: list[Bucket] = es_query_conn.run_composite_aggregation(
-        index, query
-    )
-
-    logger.info("Imported YAML files")
-    logger.info("===================")
-
-    if len(buckets) == 0:
-        logger.info("Empty response.")
-        return
-
-    for bucket in buckets:
-        bundle_path = bucket["key"]["bundle_path"]
-        logger.info("- %s", bundle_path)
-
-
-def list_data_files(es_query_conn: ElasticQueryConn, index: str) -> None:
-    """List all imported data files."""
-    query = {
-        "size": 0,
-        "aggs": {
-            "get_paths": {
-                "composite": {
-                    "sources": {"path": {"terms": {"field": "path.keyword"}}},
-                    "size": 1000,
-                }
-            }
-        },
-    }
-
-    buckets: list[Bucket] = es_query_conn.run_composite_aggregation(
-        index, query
-    )
-
-    logger.info("Imported data files")
-    logger.info("===================")
-
-    if len(buckets) == 0:
-        logger.info("Empty response.")
-        return
-
-    for bucket in buckets:
-        bundle_path = bucket["key"]["path"]
-        logger.info("- %s", bundle_path)
-
-
-def list_processes(es_query_conn: ElasticQueryConn, index: str) -> None:
-    """List all processes."""
-    process_ids = es_query_conn.get_field_values(index, "proc_id")
-
-    if len(process_ids) == 0:
-        logger.info("Empty response.")
-        return
-
-    for process_id in process_ids:
-        logger.info("- %s", process_id)
-
-
-def list_wet_processes(es_query_conn: ElasticQueryConn, index: str) -> None:
-    """List all wet processes."""
-    logger.info("Imported wet processes")
-    logger.info("======================")
-    list_processes(es_query_conn, index)
-
-
-def list_bi_processes(es_query_conn: ElasticQueryConn, index: str) -> None:
-    """List all bio info processes."""
-    logger.info("Imported bi processes")
-    logger.info("=====================")
-    list_processes(es_query_conn, index)
-
-
-def list_data_files_per_bundle(
-    es_query_conn: ElasticQueryConn, index: str
-) -> None:
-    """For each imported YAML bundle, display some info and list its data files."""
-    query = {
-        "size": 0,
-        "aggs": {
-            "data_files": {
-                "composite": {
-                    "sources": [
-                        {
-                            "bundle_path": {
-                                "terms": {"field": "bundle_path.keyword"}
-                            }
-                        }
-                    ],
-                    "size": 100,
-                },
-                "aggs": {"docs": {"top_hits": {"size": 100}}},
-            }
-        },
-    }
-
-    buckets: list[Bucket] = es_query_conn.run_composite_aggregation(
-        index, query
-    )
-
-    logger.info("Data files per YAML bundle")
-    logger.info("==========================")
-
-    if len(buckets) == 0:
-        logger.info("Empty response.")
-        return
-
-    for bucket in buckets:
-        documents = bucket["docs"]["hits"]["hits"]
-        if len(documents) == 0:
-            continue
-
-        logger.info("- Bundle Path: %s", bucket["key"]["bundle_path"])
-        logger.info(
-            "    -> Wet process: %s",
-            documents[0]["_source"]["metadata"]["wet_process"],
-        )
-        logger.info(
-            "    -> Bio info process: %s",
-            documents[0]["_source"]["metadata"]["bi_process"],
-        )
-        logger.info("    -> Data files:")
-
-        for doc in documents:
-            logger.info("        - Index: %s", doc["_source"]["file_index"])
-            logger.info("          Path: %s", doc["_source"]["path"])
-
-
-def main() -> None:
-    """Entry point of the info script."""
-    args = read_args()
-
-    configure_logging(args.verbose)
-    logger.debug("Arguments: %s", args)
-
-    addr = f"https://{args.es_host}:{args.es_port}"
-    logger.info("Trying to connect to Elasticsearch at %s...", addr)
-    es_query_conn = ElasticQueryConn(
-        addr, args.es_cert_fp, basic_auth=(args.es_usr, args.es_pwd)
-    )
-
-    analysis_index = f"{args.es_index_prefix}-analyses"
-    wet_processes_index = f"{args.es_index_prefix}-wet_processes"
-    bi_processes_index = f"{args.es_index_prefix}-bi_processes"
-
-    list_call_count = 0
-
-    if args.list_bundles:
-        list_bundles(es_query_conn, analysis_index)
-        list_call_count += 1
-
-    if args.list_data_files:
-        list_data_files(es_query_conn, analysis_index)
-        list_call_count += 1
-
-    if args.list_wet_processes:
-        list_wet_processes(es_query_conn, wet_processes_index)
-        list_call_count += 1
-
-    if args.list_bi_processes:
-        list_bi_processes(es_query_conn, bi_processes_index)
-        list_call_count += 1
-
-    if args.list_data_files_per_bundle:
-        list_data_files_per_bundle(es_query_conn, analysis_index)
-        list_call_count += 1
-
-    if list_call_count == 0:
-        logger.debug("No list option specified, listing everything.")
-        list_bundles(es_query_conn, analysis_index)
-        list_data_files(es_query_conn, analysis_index)
-        list_wet_processes(es_query_conn, wet_processes_index)
-        list_bi_processes(es_query_conn, bi_processes_index)
-        list_data_files_per_bundle(es_query_conn, analysis_index)
-
-
-if __name__ == "__main__":
-    main()

genelastic/import_data/cli_validate.py

@@ -1,54 +0,0 @@
-import argparse
-import logging
-from pathlib import Path
-
-from schema import SchemaError
-
-from genelastic.common import add_verbose_control_args
-
-from .import_bundle_factory import make_import_bundle_from_files
-from .logger import configure_logging
-
-logger = logging.getLogger("genelastic")
-
-
-def read_args() -> argparse.Namespace:
-    """Read arguments from command line."""
-    parser = argparse.ArgumentParser(
-        description="Ensure that YAML files "
-        "follow the genelastic YAML bundle schema.",
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-        allow_abbrev=False,
-    )
-    add_verbose_control_args(parser)
-    parser.add_argument(
-        "files",
-        type=Path,
-        nargs="+",
-        default=None,
-        help="YAML files to validate.",
-    )
-    parser.add_argument(
-        "-c",
-        "--check",
-        action="store_true",
-        help="In addition to validating the schema, "
-        "check for undefined referenced processes.",
-    )
-    return parser.parse_args()
-
-
-def main() -> int:
-    """Entry point of the validate script."""
-    args = read_args()
-    configure_logging(args.verbose)
-
-    try:
-        make_import_bundle_from_files(args.files, check=args.check)
-    except (ValueError, RuntimeError, TypeError, SchemaError) as e:
-        # Catch any exception that can be raised by 'make_import_bundle_from_files'.
-        logger.error(e)
-        return 1
-
-    logger.info("All YAML files respect the genelastic YAML bundle format.")
-    return 0

genelastic/import_data/data_file.py

@@ -1,87 +0,0 @@
-"""This module defines the DataFile class, which handles the representation,
-management, and extraction of metadata for a data file within a data bundle.
-
-It includes functionality to construct DataFile instances from paths and
-optional filename patterns, retrieve file paths and metadata, and support
-for extracting metadata from filenames using specified patterns.
-"""
-
-import logging
-import pathlib
-from pathlib import Path
-
-from genelastic.common import AnalysisMetaData
-
-from .filename_pattern import FilenamePattern
-
-logger = logging.getLogger("genelastic")
-
-
-class DataFile:
-    """Class for handling a data file and its metadata."""
-
-    # Initializer
-    def __init__(
-        self,
-        path: Path,
-        bundle_path: Path | None = None,
-        metadata: AnalysisMetaData | None = None,
-    ) -> None:
-        self._path = path
-        self._bundle_path = bundle_path  # The bundle YAML file in which this
-        # file was listed.
-        self._metadata = {} if metadata is None else metadata
-
-    def __repr__(self) -> str:
-        return f"File {self._path}, from bundle {self._bundle_path}, with metadata {self._metadata}"
-
-    # Get path
-    @property
-    def path(self) -> Path:
-        """Retrieve the data file path."""
-        return self._path
-
-    def exists(self) -> bool:
-        """Tests if the associated file exists on disk."""
-        return self._path.is_file()
-
-    # Get bundle path
-    @property
-    def bundle_path(self) -> Path | None:
-        """Retrieve the path to the associated data bundle file."""
-        return self._bundle_path
-
-    # Get metadata
-    @property
-    def metadata(self) -> AnalysisMetaData:
-        """Retrieve a copy of the metadata associated with the data file."""
-        return self._metadata.copy()
-
-    # Factory
-    @classmethod
-    def make_from_bundle(
-        cls,
-        path: Path,
-        bundle_path: Path | None,
-        pattern: FilenamePattern | None = None,
-    ) -> "DataFile":
-        """Construct a DataFile instance from a bundle path, file path,
-        and optional filename pattern.
-        """
-        # Make absolute path
-        if not path.is_absolute() and bundle_path is not None:
-            path = bundle_path.parent / path
-
-        # Extract filename metadata
-        metadata = None
-        if pattern is not None:
-            metadata = pattern.extract_metadata(path.name)
-
-        if metadata:
-            if "ext" not in metadata:
-                metadata["ext"] = pathlib.Path(path).suffixes[0][1:]
-
-            if "cov_depth" in metadata:
-                metadata["cov_depth"] = int(metadata["cov_depth"])
-
-        return cls(path, bundle_path, metadata)

genelastic/import_data/filename_pattern.py

@@ -1,57 +0,0 @@
-"""This module defines the FilenamePattern class, used to define a filename pattern
-and extract metadata from file names using this pattern.
-"""
-
-import re
-
-from genelastic.common import AnalysisMetaData
-
-
-class FilenamePattern:
-    """Class for defining a filename pattern.
-    The pattern is used to extract metadata from filenames
-    and verify filename conformity.
-    """
-
-    # Initializer
-    def __init__(self, pattern: str) -> None:
-        """Initializes a FilenamePattern instance.
-
-        Args:
-            pattern (str): The pattern string used for defining
-             the filename pattern.
-        """
-        self._re = re.compile(pattern)
-
-    def extract_metadata(self, filename: str) -> AnalysisMetaData:
-        """Extracts metadata from the given filename based
-        on the defined pattern.
-
-        Args:
-            filename (str): The filename from which metadata
-            needs to be extracted.
-
-        Returns:
-            dict: A dictionary containing the extracted metadata.
-
-        Raises:
-            RuntimeError: If parsing of filename fails
-            with the defined pattern.
-        """
-        m = self._re.search(filename)
-        if not m:
-            msg = f'Failed parsing filename "{filename}" with pattern "{self._re.pattern}".'
-            raise RuntimeError(msg)
-        return m.groupdict()
-
-    def matches_pattern(self, filename: str) -> bool:
-        """Checks if the given filename matches the defined pattern.
-
-        Args:
-            filename (str): The filename to be checked.
-
-        Returns:
-            bool: True if the filename matches the pattern,
-            False otherwise.
-        """
-        return bool(self._re.match(filename))

genelastic/import_data/tags.py

@@ -1,123 +0,0 @@
-import logging
-import re
-import typing
-
-from genelastic.common import BundleDict
-
-from .constants import DEFAULT_TAG2FIELD, DEFAULT_TAG_PREFIX, DEFAULT_TAG_SUFFIX
-
-logger = logging.getLogger("genelastic")
-
-TagsDefinition: typing.TypeAlias = dict[str, dict[str, str | dict[str, str]]]
-
-
-class Tags:
-    """This class handles the definition of default and custom tags.
-    Tags are used to extract custom metadata from files belonging to an analysis.
-    """
-
-    def __init__(self, documents: typing.Sequence[BundleDict] | None) -> None:
-        """Create a Tag instance."""
-        self._tags: dict[str, dict[str, str]] = DEFAULT_TAG2FIELD
-        self._tag_prefix: str = DEFAULT_TAG_PREFIX
-        self._tag_suffix: str = DEFAULT_TAG_SUFFIX
-
-        redefined_tags = None
-
-        if documents:
-            # Search for tags definition across loaded YAML documents.
-            redefined_tags = self._search_redefined_tags(documents)
-
-        if redefined_tags:
-            self._build_tags(redefined_tags)
-            logger.info(
-                "The following tags will be used to extract metadata from filenames : %s",
-                self._tags,
-            )
-        else:
-            logger.info(
-                "Using the default tags to extract metadata from filenames : %s",
-                self._tags,
-            )
-
-    def _build_tags(self, redefined_tags: TagsDefinition) -> None:
-        # Erase the tags defined by defaults.
-        self._tags = {}
-
-        if "format" in redefined_tags:
-            tag_format = redefined_tags["format"]
-
-            # extra type check for mypy
-            if "prefix" in tag_format and isinstance(tag_format["prefix"], str):
-                self._tag_prefix = tag_format["prefix"]
-
-            # extra type check for mypy
-            if "suffix" in tag_format and isinstance(tag_format["suffix"], str):
-                self._tag_suffix = tag_format["suffix"]
-
-        for tag_name, tag_attrs in redefined_tags["match"].items():
-            if isinstance(tag_attrs, dict):  # extra type check for mypy
-                self._tags[
-                    f"{self._tag_prefix}{tag_name}{self._tag_suffix}"
-                ] = tag_attrs
-
-    @staticmethod
-    def _search_redefined_tags(
-        documents: typing.Sequence[BundleDict],
-    ) -> TagsDefinition | None:
-        documents_with_redefined_tags: list[BundleDict] = [
-            d for d in documents if "tags" in d
-        ]
-        bundle_paths = [d["bundle_file"] for d in documents_with_redefined_tags]
-
-        # If there are more than one 'tags' redefinition across the documents, raise an error.
-        if len(documents_with_redefined_tags) > 1:
-            msg = (
-                f"Only one 'tags' key should be defined across all documents, "
-                f"but multiple were found : {', '.join(bundle_paths)}"
-            )
-            raise RuntimeError(msg)
-
-        if len(documents_with_redefined_tags) == 1:
-            redefined_tags: TagsDefinition = documents_with_redefined_tags[0][
-                "tags"
-            ]
-            return redefined_tags
-
-        return None
-
-    @property
-    def tag_prefix(self) -> str:
-        """Return the tag prefix. Default prefix is '%'."""
-        return self._tag_prefix
-
-    @property
-    def tag_suffix(self) -> str:
-        """Return the tag suffix. There is no suffix by default."""
-        return self._tag_suffix
-
-    @property
-    def items(self) -> typing.ItemsView[str, dict[str, str]]:
-        """Returns the tag items : the key is the tag name,
-        and the value is the tag attributes (a dict containing the 'field' and 'regex' keys).
-        """
-        return self._tags.items()
-
-    @property
-    def search_regex(self) -> str:
-        """Returns a regex to search for a tag inside a string."""
-        return (
-            r"("
-            + re.escape(self._tag_prefix)
-            + r"\w+"
-            + re.escape(self._tag_suffix)
-            + r")"
-        )
-
-    def __len__(self) -> int:
-        """Return the number of registered tags."""
-        return len(self._tags)
-
-    def __getitem__(self, key: str) -> dict[str, str]:
-        """Return a tag by its key."""
-        return self._tags[key]

genelastic/import_data/wet_process.py

@@ -1,28 +0,0 @@
-import copy
-
-from genelastic.common import WetProcessesData
-
-
-class WetProcess:
-    """Class WetProcess that represents a wet process."""
-
-    def __init__(
-        self,
-        proc_id: str,
-        bundle_file: str | None = None,
-        **data: str | float,
-    ) -> None:
-        """Create a WetProcess instance."""
-        self._proc_id = proc_id
-        self._bundle_file = bundle_file
-        self._data: WetProcessesData = data
-
-    @property
-    def id(self) -> str:
-        """Get the wet process ID."""
-        return self._proc_id
-
-    @property
-    def data(self) -> WetProcessesData:
-        """Get data associated to the wet process."""
-        return copy.deepcopy(self._data)

genelastic/import_data/wet_processes.py

@@ -1,53 +0,0 @@
-import logging
-import typing
-
-from genelastic.common import BundleDict
-
-from .wet_process import WetProcess
-
-logger = logging.getLogger("genelastic")
-
-
-class WetProcesses:
-    """Class WetProcesses is a container of WetProces objects."""
-
-    def __init__(self) -> None:
-        """Create an empty container."""
-        self._dict: dict[str, WetProcess] = {}
-
-    def __len__(self) -> int:
-        """Return the number of WetProcess objects inside the container."""
-        return len(self._dict)
-
-    def __getitem__(self, key: str) -> WetProcess:
-        """Return a WetProcess present in the container by its key."""
-        return self._dict[key]
-
-    def add(self, process: WetProcess) -> None:
-        """Add one WetProces object.
-        If a WetProces object with the same ID already exists in the container, the program exits.
-        """
-        if process.id in self._dict:
-            msg = (
-                f"A wet process with the id '{process.id}' is already present."
-            )
-            raise ValueError(msg)
-
-        # Add one WetProcess object.
-        self._dict[process.id] = process
-
-    def get_process_ids(self) -> set[str]:
-        """Get a list of the wet processes IDs."""
-        return set(self._dict.keys())
-
-    @classmethod
-    def from_array_of_dicts(
-        cls, arr: typing.Sequence[BundleDict]
-    ) -> typing.Self:
-        """Build a WetProcesses instance."""
-        wet_processes = cls()
-
-        for d in arr:
-            wet_processes.add(WetProcess(**d))
-
-        return wet_processes