genelastic 0.6.0__py3-none-any.whl → 0.7.0__py3-none-any.whl

genelastic/{import_bundle.py → import_data/import_bundle.py}

@@ -1,5 +1,4 @@
-"""
-Module: import_bundle
+"""Module: import_bundle
 
 This module provides functionality for importing data bundles.
 """
@@ -8,64 +7,70 @@ import logging
 import sys
 import typing
 
+from genelastic.common import BundleDict
+
+from .analyses import Analyses
 from .bi_processes import BioInfoProcesses
-from .data_file import DataFile
-from .common import BundleDict
 from .constants import BUNDLE_CURRENT_VERSION
-from .analyses import Analyses
+from .data_file import DataFile
 from .tags import Tags
 from .wet_processes import WetProcesses
 
-logger = logging.getLogger('genelastic')
+logger = logging.getLogger("genelastic")
 
 
 class ImportBundle:
     """Class for handling an import bundle description."""
 
-    def __init__(self, x: typing.Sequence[BundleDict],
-                 check: bool = False) -> None:
-
-        analyses: typing.List[BundleDict] = []
-        wet_processes: typing.List[BundleDict] = []
-        bi_processes: typing.List[BundleDict] = []
+    def __init__(  # noqa: C901
+        self, x: typing.Sequence[BundleDict], *, check: bool = False
+    ) -> None:
+        analyses: list[BundleDict] = []
+        wet_processes: list[BundleDict] = []
+        bi_processes: list[BundleDict] = []
         tags = Tags(x)
 
         # Loop on dicts
         for d in x:
             # Check version
-            if 'version' not in d:
-                raise RuntimeError("No version inside YAML document.")
-            if int(d['version']) != BUNDLE_CURRENT_VERSION:
-                raise RuntimeError("")
+            if "version" not in d:
+                msg = "No version inside YAML document."
+                raise RuntimeError(msg)
+            if int(d["version"]) != BUNDLE_CURRENT_VERSION:
+                raise RuntimeError
 
             # Gather all analyses
-            if 'analyses' in d and d['analyses'] is not None:
+            if "analyses" in d and d["analyses"] is not None:
                 # Copy some bundle properties into each analysis
-                for analysis in d['analyses']:
-                    for key in ['bundle_file', 'root_dir']:
+                for analysis in d["analyses"]:
+                    for key in ["bundle_file", "root_dir"]:
                         if key in d:
                             analysis[key] = d[key]
 
                     # Add the tags to use.
-                    analysis['tags'] = tags
+                    analysis["tags"] = tags
 
-                analyses.extend(d['analyses'])
+                analyses.extend(d["analyses"])
 
             # If some wet processes are defined, copy the bundle file path into each of them.
-            if 'wet_processes' in d and d['wet_processes'] is not None:
-                for wet_process in d['wet_processes']:
-                    wet_process['bundle_file'] = d['bundle_file']
-                wet_processes.extend(d['wet_processes'])
+            if "wet_processes" in d and d["wet_processes"] is not None:
+                for wet_process in d["wet_processes"]:
+                    wet_process["bundle_file"] = d["bundle_file"]
+                wet_processes.extend(d["wet_processes"])
 
             # If some bio processes are defined, copy the bundle file path into each of them.
-            if 'bi_processes' in d and d['bi_processes'] is not None:
-                for bi_process in d['bi_processes']:
-                    bi_process['bundle_file'] = d['bundle_file']
-                bi_processes.extend(d['bi_processes'])
+            if "bi_processes" in d and d["bi_processes"] is not None:
+                for bi_process in d["bi_processes"]:
+                    bi_process["bundle_file"] = d["bundle_file"]
+                bi_processes.extend(d["bi_processes"])
 
         # Instantiate all objects
-        self._wet_processes: WetProcesses = WetProcesses.from_array_of_dicts(wet_processes)
-        self._bi_processes: BioInfoProcesses = BioInfoProcesses.from_array_of_dicts(bi_processes)
+        self._wet_processes: WetProcesses = WetProcesses.from_array_of_dicts(
+            wet_processes
+        )
+        self._bi_processes: BioInfoProcesses = (
+            BioInfoProcesses.from_array_of_dicts(bi_processes)
+        )
         self._analyses: Analyses = Analyses.from_array_of_dicts(analyses)
 
         if check:
@@ -78,17 +83,27 @@ class ImportBundle:
         for index, analysis in enumerate(self._analyses):
             analysis_wet_process = analysis.metadata.get("wet_process")
 
-            if (analysis_wet_process and
-                    analysis_wet_process not in self._wet_processes.get_process_ids()):
-                sys.exit(f"Analysis at index {index} in file {analysis.bundle_file} "
-                         f"is referencing an undefined wet process: {analysis_wet_process}")
+            if (
+                analysis_wet_process
+                and analysis_wet_process
+                not in self._wet_processes.get_process_ids()
+            ):
+                sys.exit(
+                    f"Analysis at index {index} in file {analysis.bundle_file} "
+                    f"is referencing an undefined wet process: {analysis_wet_process}"
+                )
 
             analysis_bi_process = analysis.metadata.get("bi_process")
 
-            if (analysis_bi_process and
-                    analysis_bi_process not in self._bi_processes.get_process_ids()):
-                sys.exit(f"Analysis at index {index} in file {analysis.bundle_file} "
-                         f"is referencing an undefined bi process: {analysis_bi_process}")
+            if (
+                analysis_bi_process
+                and analysis_bi_process
+                not in self._bi_processes.get_process_ids()
+            ):
+                sys.exit(
+                    f"Analysis at index {index} in file {analysis.bundle_file} "
+                    f"is referencing an undefined bi process: {analysis_bi_process}"
+                )
 
     @property
     def analyses(self) -> Analyses:
@@ -110,10 +125,9 @@ class ImportBundle:
         files = self.get_files(cat)
         return len(files)
 
-    def get_files(self, cat: str | None = None) -> typing.List[DataFile]:
+    def get_files(self, cat: str | None = None) -> list[DataFile]:
         """Returns all files of a category."""
-
-        files: typing.List[DataFile] = []
+        files: list[DataFile] = []
 
         # Loop on all analyses
         for analysis in self.analyses:
@@ -123,12 +137,8 @@ class ImportBundle:
 
     def get_nb_matched_files(self) -> int:
         """Get the number of files that match the pattern."""
-
-        return sum(a.get_nb_files()
-                   for a in self.analyses)
+        return sum(a.get_nb_files() for a in self.analyses)
 
     def get_nb_unmatched_files(self) -> int:
         """Get the number of files that do not match."""
-
-        return sum(len(a.get_unmatched_file_paths())
-                   for a in self.analyses)
+        return sum(len(a.get_unmatched_file_paths()) for a in self.analyses)

genelastic/import_data/import_bundle_factory.py

@@ -0,0 +1,298 @@
+"""ImportBundle factory module."""
+
+import logging
+import re
+import sys
+from pathlib import Path
+
+import schema
+import yaml
+from yaml.parser import ParserError
+from yaml.scanner import ScannerError
+
+from genelastic.common import BundleDict
+
+from .constants import BUNDLE_CURRENT_VERSION
+from .import_bundle import ImportBundle
+
+logger = logging.getLogger("genelastic")
+
+
+def validate_tag_char(s: str) -> bool:
+    """A tag should only contain one special character, excluding the following : (, ), ?, <, >."""
+    if len(s) > 1:
+        return False
+
+    return re.match(r"^[^\w()<>?]$", s) is not None
+
+
+def validate_field_chars(s: str) -> bool:
+    """Fields should only contain word characters.
+    A word character is a character a-z, A-Z, 0-9, including _ (underscore).
+    """
+    return re.match(r"^\w+$", s) is not None
+
+
+_SCHEMA_V1 = schema.Schema(
+    {"version": 1, schema.Optional("vcf_files"): schema.Or(None, [str])}
+)
+
+_SCHEMA_V2 = schema.Schema(
+    {
+        "version": 2,
+        schema.Optional("vcf"): {
+            schema.Optional("filename_pattern"): str,
+            "files": [str],
+        },
+    }
+)
+
+_SCHEMA_V3 = schema.Schema(
+    {
+        "version": 3,
+        schema.Optional("analyses"): schema.Or(
+            None,
+            [
+                {
+                    schema.Optional("file_prefix"): str,
+                    schema.Optional("files"): [str],
+                    schema.Optional("sample_name"): str,
+                    schema.Optional("source"): str,
+                    schema.Optional("barcode"): str,
+                    schema.Optional("wet_process"): str,
+                    schema.Optional("bi_process"): str,
+                    schema.Optional("reference_genome"): str,
+                    schema.Optional("flowcell"): str,
+                    schema.Optional("lanes"): [int],
+                    schema.Optional("seq_indices"): [str],
+                    schema.Optional("cov_depth"): int,
+                    schema.Optional("qc_comment"): str,
+                    schema.Optional("data_path"): str,
+                }
+            ],
+        ),
+        schema.Optional("wet_processes"): schema.Or(
+            None,
+            [
+                {
+                    "proc_id": str,
+                    "manufacturer": str,
+                    "sequencer": str,
+                    "generic_kit": str,
+                    "fragmentation": int,
+                    "reads_size": int,
+                    "input_type": str,
+                    "amplification": str,
+                    "flowcell_type": str,
+                    "sequencing_type": str,
+                    schema.Optional("desc"): str,
+                    schema.Optional("library_kit"): str,
+                    schema.Optional("sequencing_kit"): str,
+                    schema.Optional("error_rate_expected"): float,
+                }
+            ],
+        ),
+        schema.Optional("bi_processes"): schema.Or(
+            None,
+            [
+                {
+                    "proc_id": str,
+                    "name": str,
+                    "pipeline_version": str,
+                    schema.Optional("steps"): [
+                        {
+                            "name": str,
+                            "cmd": str,
+                            schema.Optional("version"): str,
+                            schema.Optional("output"): str,
+                        }
+                    ],
+                    "sequencing_type": str,
+                    schema.Optional("desc"): str,
+                }
+            ],
+        ),
+        schema.Optional("tags"): {
+            schema.Optional("format"): {
+                schema.Optional("prefix"): schema.And(
+                    str,
+                    validate_tag_char,
+                    error="Key 'prefix' should only contain one special character, "
+                    "excluding the following : (, ), ?, <, >.",
+                ),
+                schema.Optional("suffix"): schema.And(
+                    str,
+                    validate_tag_char,
+                    error="Key 'suffix' should only contain one special character, "
+                    "excluding the following : (, ), ?, <, >.",
+                ),
+            },
+            "match": {
+                schema.And(
+                    str,
+                    validate_field_chars,
+                    error="Tags listed under the 'match' key should only contain "
+                    "word characters. A word character is a character "
+                    "a-z, A-Z, 0-9, including _ (underscore).",
+                ): {"field": str, "regex": str}
+            },
+        },
+    }
+)
+
+
+def make_import_bundle_from_files(
+    files: list[Path], *, check: bool = False
+) -> ImportBundle:
+    """Create an ImportBundle instance from a list of YAML files."""
+    all_documents = []
+    for file in files:
+        # Load documents stored in each file.
+        new_documents = load_import_bundle_file(file)
+
+        for i, new_document in enumerate(new_documents):
+            # Upgrade each new document to the latest/current version.
+            if new_document["version"] != BUNDLE_CURRENT_VERSION:
+                new_documents[i] = upgrade_bundle_version(
+                    new_document, BUNDLE_CURRENT_VERSION
+                )
+            # Set the root directory path in each new document.
+            new_documents[i]["root_dir"] = str(file.parent)
+            # Set the original bundle YAML file path in each new document.
+            new_documents[i]["bundle_file"] = str(file)
+
+        all_documents.extend(new_documents)
+
+    # Create bundle instance.
+    return ImportBundle(all_documents, check=check)
+
+
+def set_version(x: BundleDict) -> None:
+    """Set version number.
+
+    Deduce the version number from the keys present inside the dictionary.
+    """
+    # Empty doc
+    if len(x) == 0:
+        x["version"] = BUNDLE_CURRENT_VERSION
+
+    # Wrong content in version field
+    elif "version" in x:
+        if not isinstance(x["version"], int):
+            msg = "Version must be an integer."
+            raise ValueError(msg)
+
+    # Version 1
+    elif "vcf_files" in x or "cov_files" in x:
+        x["version"] = 1
+
+    # Version 2
+    elif "vcf" in x and "filename_pattern" in x["vcf"]:
+        x["version"] = 2
+
+    # Latest version
+    else:
+        x["version"] = BUNDLE_CURRENT_VERSION
+
+
+def validate_doc(x: BundleDict) -> None:
+    """Validate the dictionary using its corresponding schema."""
+    # Get schema
+    bundle_schema = globals().get("_SCHEMA_V" + str(x["version"]))
+    if bundle_schema is None:
+        raise ValueError(
+            f"Unknown version \"{x['version']}\" for import " + "bundle file."
+        )
+
+    # Validate
+    bundle_schema.validate(x)
+
+
+def load_import_bundle_file(file: Path) -> list[BundleDict]:
+    """Loads a YAML import bundle file."""
+    # Load YAML
+    logger.info('Load YAML data import file "%s".', file)
+    docs: list[BundleDict] = []
+
+    try:
+        with file.open(encoding="utf-8") as f:
+            docs = list(yaml.safe_load_all(f))
+    except (IsADirectoryError, FileNotFoundError) as e:
+        logger.error(e)
+        sys.exit(1)
+    except ScannerError as e:
+        logger.error("YAML file lexical analysis failed : %s", e)
+        sys.exit(1)
+    except ParserError as e:
+        logger.error("YAML file syntactic analysis failed : %s", e)
+        sys.exit(1)
+
+    # Guess/set version
+    if docs is None:
+        docs = [{"version": BUNDLE_CURRENT_VERSION}]
+    else:
+        for i, x in enumerate(docs):
+            if x is None:
+                docs[i] = {"version": BUNDLE_CURRENT_VERSION}
+            else:
+                set_version(x)
+
+    # Find schema and validate document
+    for x in docs:
+        validate_doc(x)
+
+    return docs
+
+
+def upgrade_bundle_version(x: BundleDict, to_version: int) -> BundleDict:
+    """Upgrade a loaded import bundle dictionary.
+
+    :raises ValueError: Raised if the input bundle lacks a version key or if the target version is invalid.
+    :raises TypeError: Raised if the version value in the input bundle is not an integer.
+    """
+    # Check version
+    if "version" not in x:
+        msg = "No version in input bundle dictionary."
+        raise ValueError(msg)
+    if not isinstance(x["version"], int):
+        msg = "Version of input bundle is not an integer."
+        raise TypeError(msg)
+    if x["version"] >= to_version:
+        msg = f"Original version ({x['version']}) is greater or equal to target version ({to_version})."
+        raise ValueError(msg)
+
+    # Loop on upgrades to run
+    y = x.copy()
+    for v in range(x["version"], to_version):
+        upgrade_fct = globals().get(f"_upgrade_from_v{v}_to_v{v + 1}")
+        y = upgrade_fct(y)  # type: ignore[misc]
+
+    return y
+
+
+def _upgrade_from_v1_to_v2(x: BundleDict) -> BundleDict:
+    # Upgrade
+    y = {"version": 2, "vcf": {"files": []}}
+    if "vcf_files" in x and x["vcf_files"] is not None:
+        y["vcf"]["files"] = x["vcf_files"]  # type: ignore[index]
+
+    # Validate schema
+    _SCHEMA_V2.validate(y)
+
+    return y
+
+
+def _upgrade_from_v2_to_v3(x: BundleDict) -> BundleDict:
+    # Upgrade
+    y: BundleDict = {"version": 3, "analyses": []}
+    if "vcf" in x:
+        analysis_entry = {}
+        if "files" in x["vcf"]:
+            analysis_entry["files"] = x["vcf"]["files"]
+        if "filename_pattern" in x["vcf"]:
+            analysis_entry["file_prefix"] = x["vcf"]["filename_pattern"]
+        y["analyses"].append(analysis_entry)
+
+    _SCHEMA_V3.validate(y)
+
+    return y

genelastic/{logger.py → import_data/logger.py}

@@ -1,4 +1,3 @@
-# pylint: disable=missing-module-docstring
 import logging
 import typing
 
@@ -8,32 +7,37 @@ import colorlog
 def configure_logging(verbose: int, log_file: str | None = None) -> None:
     """Configure logging for both import and gen-data scripts."""
     # Define TRACE level
-    logging.TRACE = 5  # type: ignore
-    logging.addLevelName(logging.TRACE, "TRACE")  # type: ignore
+    logging.TRACE = 5  # type: ignore[attr-defined]
+    logging.addLevelName(logging.TRACE, "TRACE")  # type: ignore[attr-defined]
 
-    def trace(self: logging.Logger, message: object, *args: typing.Any, **kws: typing.Any) -> None:
-        if self.isEnabledFor(logging.TRACE):  # type: ignore
-            self._log(logging.TRACE, message, args, **kws)  # type: ignore # pylint: disable=protected-access
+    def trace(
+        self: logging.Logger,
+        message: object,
+        *args: typing.Any,  # noqa: ANN401
+        **kws: typing.Any,  # noqa: ANN401
+    ) -> None:
+        if self.isEnabledFor(logging.TRACE):  # type: ignore[attr-defined]
+            self._log(logging.TRACE, message, args, **kws)  # type: ignore[attr-defined]
 
-    logging.Logger.trace = trace  # type: ignore
+    logging.Logger.trace = trace  # type: ignore[attr-defined]
 
     # Get root logger
     root = logging.getLogger()
 
     # Define formatter for file logging.
-    fmt = logging.Formatter('%(asctime)s %(levelname)-8s %(message)s')
+    fmt = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
 
     # Define formatter for colored console logging.
     color_fmt = colorlog.ColoredFormatter(
-        '%(log_color)s%(asctime)s %(levelname)-8s %(message)s',
+        "%(log_color)s%(asctime)s %(levelname)-8s %(message)s",
         log_colors={
-            'TRACE': 'light_cyan',
-            'DEBUG': 'light_yellow',
-            'INFO': 'light_green',
-            'WARNING': 'light_purple',
-            'ERROR': 'light_red',
-            'CRITICAL': 'light_red'
-        }
+            "TRACE": "light_cyan",
+            "DEBUG": "light_yellow",
+            "INFO": "light_green",
+            "WARNING": "light_purple",
+            "ERROR": "light_red",
+            "CRITICAL": "light_red",
+        },
     )
 
     # Define console handler
@@ -50,7 +54,7 @@ def configure_logging(verbose: int, log_file: str | None = None) -> None:
     level_map = {
         0: logging.WARNING,  # quiet mode
         1: logging.INFO,  # default
-        2: logging.DEBUG  # verbose mode
+        2: logging.DEBUG,  # verbose mode
     }
     # If verbose is greater than 2, set level to TRACE.
-    root.setLevel(level_map.get(verbose, logging.TRACE))  # type: ignore
+    root.setLevel(level_map.get(verbose, logging.TRACE))  # type: ignore[attr-defined]