genelastic 0.6.0__py3-none-any.whl → 0.7.0__py3-none-any.whl

genelastic/import_data/__init__.py

@@ -0,0 +1,27 @@
+"""Genelastic package for importing Genomic data into Elasticsearch."""
+
+from .analysis import Analysis
+from .import_bundle import ImportBundle
+from .import_bundle_factory import (
+    load_import_bundle_file,
+    make_import_bundle_from_files,
+)
+from .random_bundle import (
+    RandomAnalysis,
+    RandomBiProcess,
+    RandomBundle,
+    RandomWetProcess,
+)
+from .tags import Tags
+
+__all__ = [
+    "Analysis",
+    "ImportBundle",
+    "RandomAnalysis",
+    "RandomBiProcess",
+    "RandomBundle",
+    "RandomWetProcess",
+    "Tags",
+    "load_import_bundle_file",
+    "make_import_bundle_from_files",
+]

genelastic/{analyses.py → import_data/analyses.py}

@@ -1,21 +1,22 @@
-# pylint: disable=missing-module-docstring
 import typing
+
+from genelastic.common import BundleDict
+
 from .analysis import Analysis
-from .common import BundleDict
 from .data_file import DataFile
 
+
 class Analyses:
-    """Class Analyses is a container of Analysis objects.
-    """
+    """Class Analyses is a container of Analysis objects."""
 
     def __init__(self) -> None:
-        self._arr: typing.List[Analysis] = []
+        self._arr: list[Analysis] = []
         self._iter_index: int = 0
 
     def __len__(self) -> int:
         return len(self._arr)
 
-    def __iter__(self) -> typing.Generator[Analysis, typing.Any, None]:
+    def __iter__(self) -> typing.Iterator[Analysis]:
         yield from self._arr
 
     @typing.overload
@@ -23,13 +24,13 @@ class Analyses:
         pass
 
     @typing.overload
-    def __getitem__(self, k: slice) -> typing.List[Analysis]:
+    def __getitem__(self, k: slice) -> list[Analysis]:
         pass
 
-    def __getitem__(self, k): # type: ignore
+    def __getitem__(self, k):  # type: ignore[no-untyped-def]
         if isinstance(k, int):
             return self._arr[k]
-        return self._arr[k.start:k.stop]
+        return self._arr[k.start : k.stop]
 
     def add(self, a: Analysis) -> None:
         """Add one Analysis object."""
@@ -37,20 +38,18 @@ class Analyses:
 
     def get_nb_files(self, cat: str | None = None) -> int:
         """Get the total number of files as paths."""
-        return len(self.get_data_files(cat = cat))
+        return len(self.get_data_files(cat=cat))
 
-    def get_data_files(self, cat: str | None = None) -> typing.List[DataFile]:
-        """Get the total number of files as DataFile objects.
-        """
-
-        data_files: typing.List[DataFile] = []
+    def get_data_files(self, cat: str | None = None) -> list[DataFile]:
+        """Get the total number of files as DataFile objects."""
+        data_files: list[DataFile] = []
 
         for a in self._arr:
-            data_files.extend(a.get_data_files(cat = cat))
+            data_files.extend(a.get_data_files(cat=cat))
 
         return data_files
 
-    def get_all_categories(self) -> typing.Set[str]:
+    def get_all_categories(self) -> set[str]:
         """Return all the categories of the analyses."""
         categories = set()
         for a in self._arr:
@@ -58,10 +57,10 @@ class Analyses:
         return categories
 
     @classmethod
-    def from_array_of_dicts(cls, arr: typing.Sequence[BundleDict]
-                            ) -> typing.Self:
+    def from_array_of_dicts(
+        cls, arr: typing.Sequence[BundleDict]
+    ) -> typing.Self:
         """Build an Analyses instance."""
-
         analyses = cls()
 
         for d in arr:

genelastic/{analysis.py → import_data/analysis.py}

@@ -1,40 +1,41 @@
-# pylint: disable=missing-module-docstring
 import copy
-import glob
 import logging
-import os
 import re
 import typing
 from pathlib import Path
 
-from .common import AnalysisMetaData
+from genelastic.common import AnalysisMetaData
+
 from .constants import ALLOWED_CATEGORIES
 from .data_file import DataFile
 from .filename_pattern import FilenamePattern
 from .tags import Tags
 
-logger = logging.getLogger('genelastic')
+logger = logging.getLogger("genelastic")
 
 
 class Analysis:
     """Class Analysis that represents an analysis."""
 
-    # pylint: disable-next=too-many-arguments
-    def __init__(self,
-                 tags: Tags,
-                 root_dir: str = '.',
-                 bundle_file: str | None = None,
-                 file_prefix: str | None = None,
-                 files: typing.Sequence[str] | None = None,
-                 data_path: str | None = None,
-                 **metadata: str | int) -> None:
-        self._bundle_file = bundle_file
+    def __init__(  # noqa: PLR0913
+        self,
+        tags: Tags,
+        root_dir: str = ".",
+        bundle_file: str | None = None,
+        file_prefix: str | None = None,
+        files: typing.Sequence[str] | None = None,
+        data_path: str | None = None,
+        **metadata: str | int,
+    ) -> None:
+        self._bundle_file = Path(bundle_file) if bundle_file else None
         self._file_prefix = file_prefix
         self._files = files
-        self._data_path = Analysis._resolve_data_path(root_dir, data_path)
+        self._data_path = Analysis._resolve_data_path(
+            Path(root_dir), Path(data_path) if data_path else None
+        )
         self._tags = tags
         self._metadata: AnalysisMetaData = metadata
-        self._categories: typing.Set[str] = set()
+        self._categories: set[str] = set()
 
     @property
     def metadata(self) -> AnalysisMetaData:
@@ -42,17 +43,15 @@ class Analysis:
         return copy.deepcopy(self._metadata)
 
     @property
-    def bundle_file(self) -> str | None:
+    def bundle_file(self) -> Path | None:
         """Get the bundle file."""
         return self._bundle_file
 
     @property
     def filename_regex(self) -> str:
-        """
-        Resolve placeholders in a file prefix using metadata
+        """Resolve placeholders in a file prefix using metadata
         and unresolved placeholders are converted to regex groups
         """
-
         x: str = r"^.+\.(?P<ext>vcf|cov)(\.gz)?$"
 
         # Use existing generic prefix
@@ -64,84 +63,87 @@ class Analysis:
                 regex = tag_attrs["regex"]
 
                 # Build field regex
-                field_regex = (f"(?P<{field}>{self._metadata.get(field)})"
-                               if field in self._metadata else
-                               f"(?P<{field}>{regex})")
+                field_regex = (
+                    f"(?P<{field}>{self._metadata.get(field)})"
+                    if field in self._metadata
+                    else f"(?P<{field}>{regex})"
+                )
                 # Replace tag with field regex
                 x = x.replace(tag_name, field_regex)
 
             # Check for tags that were not replaced.
             groups = re.findall(self._tags.search_regex, x)
             for match in groups:
-                logger.warning("String '%s' in key 'file_prefix' looks like an undefined tag. "
-                               "If this string is not a tag, you can ignore this warning.",
-                               match)
+                logger.warning(
+                    "String '%s' in key 'file_prefix' looks like an undefined tag. "
+                    "If this string is not a tag, you can ignore this warning.",
+                    match,
+                )
 
             # Add missing start and end markers
             if not x.startswith("^"):
                 x = "^" + x
             if not x.endswith("$"):
-                x += (r"\.(?P<ext>" + '|'.join(ALLOWED_CATEGORIES)
-                      + r")(\.gz)?$")
+                x += r"\.(?P<ext>" + "|".join(ALLOWED_CATEGORIES) + r")(\.gz)?$"
             logger.debug("File regex for %s: %s", self._bundle_file, x)
 
         return x
 
     def get_nb_files(self, cat: str | None = None) -> int:
-        """Returns the total number of files.
-        """
+        """Returns the total number of files."""
         return len(self.get_file_paths(cat=cat))
 
-    def get_data_files(self, cat: str | None = None) -> typing.List[DataFile]:
-        """Returns the list of matched files as DataFile objects.
-        """
-
+    def get_data_files(self, cat: str | None = None) -> list[DataFile]:
+        """Returns the list of matched files as DataFile objects."""
         files = self.get_file_paths(cat=cat)
         filename_pattern = FilenamePattern(self.filename_regex)
 
-        data_files: typing.List[DataFile] = []
+        data_files: list[DataFile] = []
 
         for f in files:
             try:
-                data_files.append(DataFile.make_from_bundle(
-                    path=f, bundle_path=self._bundle_file,
-                    pattern=filename_pattern))
-            except (IOError, ValueError) as e:
+                data_files.append(
+                    DataFile.make_from_bundle(
+                        path=f,
+                        bundle_path=self._bundle_file,
+                        pattern=filename_pattern,
+                    )
+                )
+            except (OSError, ValueError) as e:
                 logger.error("Error processing file %s: %s", f, str(e))
 
         return data_files
 
-    def get_file_paths(self, cat: str | None = None) -> typing.Sequence[str]:
-        """Returns the list of matched files.
-        """
+    def get_file_paths(self, cat: str | None = None) -> typing.Sequence[Path]:
+        """Returns the list of matched files."""
         files, _, _ = self._do_get_file_paths(cat=cat)
         return files
 
-    def get_unmatched_file_paths(self, cat: str | None = None
-                                 ) -> typing.Sequence[str]:
-        """Returns the list of unmatched files.
-        """
+    def get_unmatched_file_paths(
+        self, cat: str | None = None
+    ) -> typing.Sequence[Path]:
+        """Returns the list of unmatched files."""
         _, files, _ = self._do_get_file_paths(cat=cat)
         return files
 
-    def get_all_categories(self) -> typing.Set[str]:
+    def get_all_categories(self) -> set[str]:
         """Returns all categories of the analysis."""
         _, _, categories = self._do_get_file_paths()
         return categories
 
     @staticmethod
-    def _resolve_data_path(root_dir: str, data_path: str | None) -> str:
-        resolved_data_path = '' if data_path is None else data_path
+    def _resolve_data_path(root_dir: Path, data_path: Path | None) -> Path:
+        resolved_data_path = Path() if data_path is None else data_path
 
-        if not os.path.isabs(resolved_data_path):
-            resolved_data_path = os.path.abspath(os.path.join(root_dir, resolved_data_path))
+        if not resolved_data_path.is_absolute():
+            resolved_data_path = (root_dir / resolved_data_path).absolute()
 
         return resolved_data_path
 
-    def _get_files_with_allowed_categories(self) -> typing.Dict[str, str]:
+    def _get_files_with_allowed_categories(self) -> dict[Path, str]:
         # Create a dict to store allowed files. Keys are the filepaths,
         # and values are their corresponding category.
-        allowed_files: typing.Dict[str, str] = {}
+        allowed_files: dict[Path, str] = {}
         # If files are listed explicitly in the YAML in the 'files' attribute, process them.
         if self._files is not None:
             abs_filepaths = [Path(self._data_path) / f for f in self._files]
@@ -150,14 +152,14 @@ class Analysis:
                 cat = file.suffixes[0][1:]
                 # Add each matching file and its category to the dict.
                 if cat in ALLOWED_CATEGORIES:
-                    allowed_files[str(file)] = cat
+                    allowed_files[file] = cat
         # Else, look for files on disk using the YAML 'data_path' attribute.
         else:
             # Try to retrieve files matching allowed categories using glob.
             for cat in ALLOWED_CATEGORIES:
-                glob_res = []
-                glob_res.extend(glob.glob(os.path.join(self._data_path, f"*.{cat}")))
-                glob_res.extend(glob.glob(os.path.join(self._data_path, f"*.{cat}.gz")))
+                glob_res: list[Path] = []
+                glob_res.extend(self._data_path.glob(f"*.{cat}"))
+                glob_res.extend(self._data_path.glob(f"*.{cat}.gz"))
 
                 # Add each globed file and its category to the dict.
                 for g_file in glob_res:
@@ -165,12 +167,13 @@ class Analysis:
 
         return allowed_files
 
-    def _do_get_file_paths(self, cat: str | None = None) \
-            -> tuple[typing.Sequence[str], typing.Sequence[str], typing.Set[str]]:
-
+    def _do_get_file_paths(
+        self, cat: str | None = None
+    ) -> tuple[typing.Sequence[Path], typing.Sequence[Path], set[str]]:
         # Raise an error if the category given as a parameter is not part of the allowed categories.
         if cat is not None and cat not in ALLOWED_CATEGORIES:
-            raise ValueError(f"Unknown category {cat}.")
+            msg = f"Unknown category {cat}."
+            raise ValueError(msg)
 
         # Obtain a dict of all files matching the allowed categories.
         allowed_files = self._get_files_with_allowed_categories()
@@ -180,16 +183,18 @@ class Analysis:
             files_to_match = allowed_files
         else:
             # A category was given as a parameter, so we match only this specific category.
-            files_to_match = dict((k, v) for (k, v) in allowed_files.items() if v == cat)
+            files_to_match = {
+                k: v for k, v in allowed_files.items() if v == cat
+            }
 
         filename_pattern = FilenamePattern(self.filename_regex)
-        matching_files: typing.List[str] = []
-        non_matching_files: typing.List[str] = []
+        matching_files: list[Path] = []
+        non_matching_files: list[Path] = []
         categories = set()
 
         # We filter files by ensuring that they match the filename pattern defined in the analysis.
         for file, category in sorted(files_to_match.items()):
-            if filename_pattern.matches_pattern(os.path.basename(file)):
+            if filename_pattern.matches_pattern(file.name):
                 matching_files.append(file)
                 logger.info("MATCHED file %s.", file)
                 # Add the file category to the categories set.

genelastic/{bi_process.py → import_data/bi_process.py}

@@ -1,15 +1,17 @@
-# pylint: disable=missing-module-docstring
 import copy
-import typing
 
-from .common import BioInfoProcessData
+from genelastic.common import BioInfoProcessData
 
 
 class BioInfoProcess:
     """Class representing a bio process."""
-    def __init__(self, proc_id: str,
-                 bundle_file: str | None = None,
-                 **data: str | typing.List[str]) -> None:
+
+    def __init__(
+        self,
+        proc_id: str,
+        bundle_file: str | None = None,
+        **data: str | list[str],
+    ) -> None:
         self._proc_id = proc_id
         self._bundle_file = bundle_file
         self._data: BioInfoProcessData = data

genelastic/{bi_processes.py → import_data/bi_processes.py}

@@ -1,18 +1,18 @@
-# pylint: disable=missing-module-docstring
 import logging
 import typing
 
+from genelastic.common import BundleDict
+
 from .bi_process import BioInfoProcess
-from .common import BundleDict
 
-logger = logging.getLogger('genelastic')
+logger = logging.getLogger("genelastic")
 
 
 class BioInfoProcesses:
     """Class BioInfoProcesses is a container of BioInfoProcess objects."""
 
     def __init__(self) -> None:
-        self._dict: typing.Dict[str, BioInfoProcess] = {}
+        self._dict: dict[str, BioInfoProcess] = {}
 
     def __len__(self) -> int:
         return len(self._dict)
@@ -26,20 +26,21 @@ class BioInfoProcesses:
         the program exits.
         """
         if process.id in self._dict:
-            raise ValueError(f"A bi process with the id '{process.id}' is already present.")
+            msg = f"A bi process with the id '{process.id}' is already present."
+            raise ValueError(msg)
 
         # Add one WetProcess object.
         self._dict[process.id] = process
 
-    def get_process_ids(self) -> typing.Set[str]:
+    def get_process_ids(self) -> set[str]:
         """Get a list of the bio processes IDs."""
         return set(self._dict.keys())
 
     @classmethod
-    def from_array_of_dicts(cls, arr: typing.Sequence[BundleDict]
-                            ) -> typing.Self:
+    def from_array_of_dicts(
+        cls, arr: typing.Sequence[BundleDict]
+    ) -> typing.Self:
         """Build a BioInfoProcesses instance."""
-
         bi_processes = cls()
 
         for d in arr:

genelastic/import_data/cli_gen_data.py

@@ -0,0 +1,116 @@
+import argparse
+import logging
+from pathlib import Path
+
+from genelastic.common import add_verbose_control_args
+
+from .logger import configure_logging
+from .random_bundle import (
+    RandomBundle,
+)
+
+logger = logging.getLogger("genelastic")
+
+
+def read_args() -> argparse.Namespace:
+    """Read arguments from command line."""
+    parser = argparse.ArgumentParser(
+        description="Genetics data random generator.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        allow_abbrev=False,
+    )
+    add_verbose_control_args(parser)
+    parser.add_argument(
+        "-d",
+        "--data-folder",
+        dest="data_folder",
+        required=True,
+        help="Data destination folder.",
+        type=Path,
+    )
+    parser.add_argument(
+        "--log-file", dest="log_file", help="Path to a log file."
+    )
+    parser.add_argument(
+        "-n",
+        "--chrom-nb",
+        dest="chrom_nb",
+        type=int,
+        default=5,
+        help="Number of chromosomes to include in the generated VCF file.",
+    )
+    parser.add_argument(
+        "-o",
+        "--output-yaml-file",
+        dest="output_file",
+        default=None,
+        help="Output YAML file.",
+        type=Path,
+    )
+    parser.add_argument(
+        "-s",
+        "--sequence-size",
+        type=int,
+        default=2000,
+        help="Sequence size (number of nucleotides) generated for each chromosome.",
+    )
+    parser.add_argument(
+        "-c",
+        "--coverage",
+        action="store_true",
+        help="Generate a coverage file for each analysis.",
+    )
+    parser.add_argument(
+        "-a",
+        "--analyses",
+        help="Number of analyses to generate. "
+        "Each analysis is composed of a YAML bundle file declaring its wet lab and bioinformatics processes, "
+        "a VCF file and optionally a coverage file.",
+        default=1,
+        type=int,
+    )
+    parser.add_argument(
+        "-p",
+        "--processes",
+        help="Number of Wet Lab and Bioinformatics processes to generate.",
+        default=1,
+        type=int,
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    """Entry point of the gen-data script."""
+    # Read command line arguments
+    args = read_args()
+    folder = args.data_folder.resolve()
+
+    if not folder.is_dir():
+        msg = f"ERROR: '{folder}' does not exist or is not a directory."
+        raise SystemExit(msg)
+
+    if args.analyses < 1:
+        msg = "Analyses count must be at least 1."
+        raise SystemExit(msg)
+
+    if args.processes < 1:
+        msg = "Processes count must be at least 1."
+        raise SystemExit(msg)
+
+    # Configure logging
+    configure_logging(args.verbose, log_file=args.log_file)
+    logger.debug("Arguments: %s", args)
+
+    # Write to stdout or file
+    RandomBundle(
+        folder,
+        args.analyses,
+        args.processes,
+        args.chrom_nb,
+        args.sequence_size,
+        do_gen_coverage=args.coverage,
+    ).to_yaml(args.output_file)
+
+
+if __name__ == "__main__":
+    main()