genelastic 0.8.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

genelastic/import_data/analysis.py

@@ -1,205 +0,0 @@
-import copy
-import logging
-import re
-import typing
-from pathlib import Path
-
-from genelastic.common import AnalysisMetaData
-
-from .constants import ALLOWED_CATEGORIES
-from .data_file import DataFile
-from .filename_pattern import FilenamePattern
-from .tags import Tags
-
-logger = logging.getLogger("genelastic")
-
-
-class Analysis:
-    """Class Analysis that represents an analysis."""
-
-    def __init__(  # noqa: PLR0913
-        self,
-        tags: Tags,
-        root_dir: str = ".",
-        bundle_file: str | None = None,
-        file_prefix: str | None = None,
-        files: typing.Sequence[str] | None = None,
-        data_path: str | None = None,
-        **metadata: str | int,
-    ) -> None:
-        self._bundle_file = Path(bundle_file) if bundle_file else None
-        self._file_prefix = file_prefix
-        self._files = files
-        self._data_path = Analysis._resolve_data_path(
-            Path(root_dir), Path(data_path) if data_path else None
-        )
-        self._tags = tags
-        self._metadata: AnalysisMetaData = metadata
-        self._categories: set[str] = set()
-
-    @property
-    def metadata(self) -> AnalysisMetaData:
-        """Get metadata."""
-        return copy.deepcopy(self._metadata)
-
-    @property
-    def bundle_file(self) -> Path | None:
-        """Get the bundle file."""
-        return self._bundle_file
-
-    @property
-    def filename_regex(self) -> str:
-        """Resolve placeholders in a file prefix using metadata
-        and unresolved placeholders are converted to regex groups
-        """
-        x: str = r"^.+\.(?P<ext>vcf|cov)(\.gz)?$"
-
-        # Use existing generic prefix
-        if self._file_prefix:
-            x = self._file_prefix
-            # Replace %* tags
-            for tag_name, tag_attrs in self._tags.items:
-                field = tag_attrs["field"]
-                regex = tag_attrs["regex"]
-
-                # Build field regex
-                field_regex = (
-                    f"(?P<{field}>{self._metadata.get(field)})"
-                    if field in self._metadata
-                    else f"(?P<{field}>{regex})"
-                )
-                # Replace tag with field regex
-                x = x.replace(tag_name, field_regex)
-
-            # Check for tags that were not replaced.
-            groups = re.findall(self._tags.search_regex, x)
-            for match in groups:
-                logger.warning(
-                    "String '%s' in key 'file_prefix' looks like an undefined tag. "
-                    "If this string is not a tag, you can ignore this warning.",
-                    match,
-                )
-
-            # Add missing start and end markers
-            if not x.startswith("^"):
-                x = "^" + x
-            if not x.endswith("$"):
-                x += r"\.(?P<ext>" + "|".join(ALLOWED_CATEGORIES) + r")(\.gz)?$"
-            logger.debug("File regex for %s: %s", self._bundle_file, x)
-
-        return x
-
-    def get_nb_files(self, cat: str | None = None) -> int:
-        """Returns the total number of files."""
-        return len(self.get_file_paths(cat=cat))
-
-    def get_data_files(self, cat: str | None = None) -> list[DataFile]:
-        """Returns the list of matched files as DataFile objects."""
-        files = self.get_file_paths(cat=cat)
-        filename_pattern = FilenamePattern(self.filename_regex)
-
-        data_files: list[DataFile] = []
-
-        for f in files:
-            try:
-                data_files.append(
-                    DataFile.make_from_bundle(
-                        path=f,
-                        bundle_path=self._bundle_file,
-                        pattern=filename_pattern,
-                    )
-                )
-            except (OSError, ValueError) as e:
-                logger.error("Error processing file %s: %s", f, str(e))
-
-        return data_files
-
-    def get_file_paths(self, cat: str | None = None) -> typing.Sequence[Path]:
-        """Returns the list of matched files."""
-        files, _, _ = self._do_get_file_paths(cat=cat)
-        return files
-
-    def get_unmatched_file_paths(
-        self, cat: str | None = None
-    ) -> typing.Sequence[Path]:
-        """Returns the list of unmatched files."""
-        _, files, _ = self._do_get_file_paths(cat=cat)
-        return files
-
-    def get_all_categories(self) -> set[str]:
-        """Returns all categories of the analysis."""
-        _, _, categories = self._do_get_file_paths()
-        return categories
-
-    @staticmethod
-    def _resolve_data_path(root_dir: Path, data_path: Path | None) -> Path:
-        resolved_data_path = Path() if data_path is None else data_path
-
-        if not resolved_data_path.is_absolute():
-            resolved_data_path = (root_dir / resolved_data_path).absolute()
-
-        return resolved_data_path
-
-    def _get_files_with_allowed_categories(self) -> dict[Path, str]:
-        # Create a dict to store allowed files. Keys are the filepaths,
-        # and values are their corresponding category.
-        allowed_files: dict[Path, str] = {}
-        # If files are listed explicitly in the YAML in the 'files' attribute, process them.
-        if self._files is not None:
-            abs_filepaths = [Path(self._data_path) / f for f in self._files]
-            # Try to retrieve files matching allowed categories by checking their first suffix.
-            for file in abs_filepaths:
-                cat = file.suffixes[0][1:]
-                # Add each matching file and its category to the dict.
-                if cat in ALLOWED_CATEGORIES:
-                    allowed_files[file] = cat
-        # Else, look for files on disk using the YAML 'data_path' attribute.
-        else:
-            # Try to retrieve files matching allowed categories using glob.
-            for cat in ALLOWED_CATEGORIES:
-                glob_res: list[Path] = []
-                glob_res.extend(self._data_path.glob(f"*.{cat}"))
-                glob_res.extend(self._data_path.glob(f"*.{cat}.gz"))
-
-                # Add each globed file and its category to the dict.
-                for g_file in glob_res:
-                    allowed_files[g_file] = cat
-
-        return allowed_files
-
-    def _do_get_file_paths(
-        self, cat: str | None = None
-    ) -> tuple[typing.Sequence[Path], typing.Sequence[Path], set[str]]:
-        # Raise an error if the category given as a parameter is not part of the allowed categories.
-        if cat is not None and cat not in ALLOWED_CATEGORIES:
-            msg = f"Unknown category {cat}."
-            raise ValueError(msg)
-
-        # Obtain a dict of all files matching the allowed categories.
-        allowed_files = self._get_files_with_allowed_categories()
-
-        if cat is None:
-            # No category was given as a parameter, so we match all categories.
-            files_to_match = allowed_files
-        else:
-            # A category was given as a parameter, so we match only this specific category.
-            files_to_match = {
-                k: v for k, v in allowed_files.items() if v == cat
-            }
-
-        filename_pattern = FilenamePattern(self.filename_regex)
-        matching_files: list[Path] = []
-        non_matching_files: list[Path] = []
-        categories = set()
-
-        # We filter files by ensuring that they match the filename pattern defined in the analysis.
-        for file, category in sorted(files_to_match.items()):
-            if filename_pattern.matches_pattern(file.name):
-                matching_files.append(file)
-                logger.info("MATCHED file %s.", file)
-                # Add the file category to the categories set.
-                categories.add(category)
-            else:
-                logger.warning("UNMATCHED file %s.", file)
-                non_matching_files.append(file)
-        return matching_files, non_matching_files, categories

genelastic/import_data/bi_process.py

@@ -1,27 +0,0 @@
-import copy
-
-from genelastic.common import BioInfoProcessData
-
-
-class BioInfoProcess:
-    """Class representing a bio process."""
-
-    def __init__(
-        self,
-        proc_id: str,
-        bundle_file: str | None = None,
-        **data: str | list[str],
-    ) -> None:
-        self._proc_id = proc_id
-        self._bundle_file = bundle_file
-        self._data: BioInfoProcessData = data
-
-    @property
-    def id(self) -> str:
-        """Get the bio process ID."""
-        return self._proc_id
-
-    @property
-    def data(self) -> BioInfoProcessData:
-        """Get data associated to the bio process."""
-        return copy.deepcopy(self._data)

genelastic/import_data/bi_processes.py

@@ -1,49 +0,0 @@
-import logging
-import typing
-
-from genelastic.common import BundleDict
-
-from .bi_process import BioInfoProcess
-
-logger = logging.getLogger("genelastic")
-
-
-class BioInfoProcesses:
-    """Class BioInfoProcesses is a container of BioInfoProcess objects."""
-
-    def __init__(self) -> None:
-        self._dict: dict[str, BioInfoProcess] = {}
-
-    def __len__(self) -> int:
-        return len(self._dict)
-
-    def __getitem__(self, key: str) -> BioInfoProcess:
-        return self._dict[key]
-
-    def add(self, process: BioInfoProcess) -> None:
-        """Add one BioInfoProcess object.
-        If a BioInfoProcess object with the same ID already exists in the container,
-        the program exits.
-        """
-        if process.id in self._dict:
-            msg = f"A bi process with the id '{process.id}' is already present."
-            raise ValueError(msg)
-
-        # Add one WetProcess object.
-        self._dict[process.id] = process
-
-    def get_process_ids(self) -> set[str]:
-        """Get a list of the bio processes IDs."""
-        return set(self._dict.keys())
-
-    @classmethod
-    def from_array_of_dicts(
-        cls, arr: typing.Sequence[BundleDict]
-    ) -> typing.Self:
-        """Build a BioInfoProcesses instance."""
-        bi_processes = cls()
-
-        for d in arr:
-            bi_processes.add(BioInfoProcess(**d))
-
-        return bi_processes

genelastic/import_data/cli_import.py

@@ -1,379 +0,0 @@
-# vi: se tw=80
-
-# Elasticsearch Python API:
-# https://www.elastic.co/guide/en/elasticsearch/client/python-api/current/overview.html
-# https://elasticsearch-py.readthedocs.io/en/latest/api.html
-
-import argparse
-import csv
-import datetime
-import hashlib
-import logging
-import sys
-import time
-from pathlib import Path
-
-import vcf
-
-from genelastic.common import (
-    AnalysisDocument,
-    BulkItems,
-    ElasticImportConn,
-    MetadataDocument,
-    ProcessDocument,
-    add_es_connection_args,
-    add_verbose_control_args,
-)
-
-from .bi_processes import BioInfoProcesses
-from .data_file import DataFile
-from .import_bundle_factory import make_import_bundle_from_files
-from .logger import configure_logging
-from .wet_processes import WetProcesses
-
-logger = logging.getLogger("genelastic")
-logging.getLogger("elastic_transport").setLevel(
-    logging.WARNING
-)  # Disable excessive logging
-logging.getLogger("urllib3").setLevel(
-    logging.WARNING
-)  # Disable excessive logging
-
-
-def read_args() -> argparse.Namespace:
-    """Read arguments from command line."""
-    parser = argparse.ArgumentParser(
-        description="Genetics data importer.",
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-        allow_abbrev=False,
-    )
-    add_verbose_control_args(parser)
-    add_es_connection_args(parser)
-    parser.add_argument(
-        "-D",
-        "--dry-run",
-        dest="dryrun",
-        action="count",
-        default=0,
-        help=(
-            "Dry-run level. -D for data files loading (VCF, coverage, etc) "
-            "without connecting or importing to database. "
-            "-DD for metadata YAML files loading only (no loading of data files)."
-        ),
-    )
-    parser.add_argument(
-        "--log-file", dest="log_file", help="Path to a log file."
-    )
-    parser.add_argument(
-        "--no-list",
-        dest="no_list",
-        action="store_true",
-        help="Do not print list of files to be imported.",
-    )
-    parser.add_argument(
-        "--no-confirm",
-        dest="no_confirm",
-        action="store_true",
-        help="Do not ask confirmation before importing.",
-    )
-    parser.add_argument(
-        "files",
-        type=Path,
-        nargs="+",
-        default=None,
-        help="Data files that describe what to import.",
-    )
-    return parser.parse_args()
-
-
-def import_cov_file(
-    es_import_conn: ElasticImportConn | None,
-    file_index: str,
-    file: Path,
-    dryrun: int = 0,
-) -> None:
-    """Import a coverage file to the Elasticsearch database."""
-    # Set field types
-    if dryrun == 0 and es_import_conn:
-        es_import_conn.client.indices.put_mapping(
-            index=file_index,
-            body={
-                "properties": {
-                    "pos": {"type": "integer"},
-                    "depth": {"type": "byte"},
-                }
-            },
-        )
-
-    # Open file
-    if dryrun > 1:
-        logger.info(
-            "Would load and import Coverage file %s " "into index %s.",
-            file,
-            file_index,
-        )
-    else:
-        logger.info("Load Coverage file %s.", file)
-        if dryrun == 1:
-            logger.info(
-                "Would import Coverage file %s into index %s.", file, file_index
-            )
-        else:
-            logger.info(
-                "Import Coverage file %s into index %s.", file, file_index
-            )
-        with file.open(newline="", encoding="utf-8") as f:
-            # Read file as CSV
-            reader = csv.reader(f, delimiter="\t", quotechar='"')
-
-            # Loop on al lines
-            for row in reader:
-                # Build document
-                # Position starts at 0 inside coverage file
-                doc: MetadataDocument = {
-                    "type": "coverage",
-                    "chr": row[0],
-                    "pos": int(row[1]) + 1,
-                    "depth": int(row[2]),
-                }
-
-                # Insert document
-                if dryrun == 0 and es_import_conn:
-                    es_import_conn.client.index(index=file_index, document=doc)
-
-
-def import_analysis_metadata(  # noqa: PLR0913
-    es_import_conn: ElasticImportConn | None,
-    index_prefix: str,
-    file_index: str,
-    file: DataFile,
-    analysis_type: str,
-    dryrun: int = 0,
-) -> None:
-    """Import analysis metadata into a dedicated index."""
-    doc: AnalysisDocument = {
-        "path": str(file.path.resolve()),
-        "bundle_path": str(file.bundle_path.resolve())
-        if file.bundle_path
-        else None,
-        "metadata": file.metadata,
-        "file_index": file_index,
-        "type": analysis_type,
-    }
-
-    bulk_items: BulkItems = [
-        {"_index": f"{index_prefix}-analyses", "_source": doc}
-    ]
-
-    if dryrun == 0 and es_import_conn:
-        es_import_conn.import_items(
-            bulk_items,
-            start_time=time.perf_counter(),
-            total_items=len(bulk_items),
-        )
-
-
-def import_vcf_file(
-    es_import_conn: ElasticImportConn | None,
-    file_index: str,
-    file: DataFile,
-    dryrun: int = 0,
-) -> None:
-    """Import a VCF file to the Elasticsearch database."""
-    logger.info('Import VCF file "%s".', file)
-
-    if dryrun > 1:
-        logger.info(
-            "Would load and import VCF file %s " "into index %s.",
-            file.path,
-            file_index,
-        )
-    else:
-        logger.info("Load VCF file %s.", file.path)
-        if dryrun == 1:
-            logger.info(
-                "Would import VCF file %s into index %s.", file.path, file_index
-            )
-        else:
-            logger.info(
-                "Importing VCF file %s into index %s...", file.path, file_index
-            )
-
-        try:
-            vcf_reader = vcf.Reader(filename=str(file.path))
-            n = 0
-            start = time.perf_counter()
-            bulk_sz = 256  # Bulk size
-            bulk_items: BulkItems = []
-            for record in vcf_reader:
-                # Correct values
-                if not record.CHROM.startswith("chr"):
-                    if record.CHROM.lower().startswith("chr"):
-                        record.CHROM = "chr" + record.CHROM[3:]
-                    else:
-                        record.CHROM = "chr" + record.CHROM
-
-                # Build document
-                alt = [x if x is None else x.type for x in record.ALT]
-                doc: MetadataDocument = {
-                    "type": "vcf",
-                    "chr": record.CHROM,
-                    "pos": record.POS,
-                    "alt": alt,
-                    "info": record.INFO,
-                }
-
-                if dryrun == 0:
-                    # Append item to bulk
-                    bulk_items.append({"_index": file_index, "_source": doc})
-                    n += 1
-
-                    # Insert bulk of items
-                    if len(bulk_items) >= bulk_sz and es_import_conn:
-                        es_import_conn.import_items(
-                            bulk_items, start_time=start, total_items=n
-                        )
-                        bulk_items = []
-
-            # Insert remaining items
-            if dryrun == 0 and es_import_conn:
-                es_import_conn.import_items(
-                    bulk_items, start_time=start, total_items=n
-                )
-
-        except StopIteration:
-            logger.error("Skipping empty file : %s.", file.path)
-
-
-def import_processes(
-    es_import_conn: ElasticImportConn | None,
-    index: str,
-    processes: WetProcesses | BioInfoProcesses,
-    dryrun: int = 0,
-) -> None:
-    """Import processes into their own index."""
-    bulk_items: BulkItems = []
-
-    for proc_id in processes.get_process_ids():
-        process = processes[proc_id]
-        process_type = process.__class__.__name__
-        doc: ProcessDocument = process.data | {
-            "proc_id": proc_id,
-            "type": process_type,
-        }
-        bulk_items.append({"_index": index, "_source": doc})
-
-    if dryrun == 0 and es_import_conn:
-        es_import_conn.import_items(
-            bulk_items,
-            start_time=time.perf_counter(),
-            total_items=len(bulk_items),
-        )
-
-
-def generate_unique_index(index_prefix: str, filepath: Path) -> str:
-    """Generate a unique index with the following format:
-    <index_prefix>_<current_date>_<md5_hashed_filepath>
-    """
-    current_date = datetime.datetime.now(tz=datetime.UTC).strftime("%Y%m%d")
-    hashed_filepath = hashlib.md5(
-        str(filepath).encode("utf-8"), usedforsecurity=False
-    ).hexdigest()
-    return f"{index_prefix}-file-{current_date}-{hashed_filepath}"
-
-
-def main() -> None:  # noqa: C901
-    """Entry point of the import script."""
-    # Read command line arguments
-    args = read_args()
-
-    # Configure logging
-    configure_logging(args.verbose, log_file=args.log_file)
-    logger.debug("Arguments: %s", args)
-    logger.debug("LOGGERS: %s", logging.root.manager.loggerDict)
-
-    # Open connection to ES
-    if args.dryrun == 0:
-        addr = f"https://{args.es_host}:{args.es_port}"
-        logger.info("Trying to connect to Elasticsearch at %s...", addr)
-        es_import_conn = ElasticImportConn(
-            addr, args.es_cert_fp, basic_auth=(args.es_usr, args.es_pwd)
-        )
-    else:
-        es_import_conn = None
-
-    # Load YAML import bundle
-    import_bundle = make_import_bundle_from_files(args.files, check=True)
-    all_bundled_files = import_bundle.get_files()
-
-    # CHECK
-    for f in all_bundled_files:
-        if not f.exists():
-            msg = f"Path {f.path} does not point to a valid file."
-            raise RuntimeError(msg)
-
-    # LIST
-    if not args.no_list:
-        for f in all_bundled_files:
-            logger.info("Will import %s.", f.path)
-
-    # Ask confirmation for importing
-    if not args.no_confirm:
-        answer: str = "maybe"
-        while answer not in ["", "n", "y"]:
-            answer = input("Import (y/N)? ").lower()
-        if answer != "y":
-            logger.info("Import canceled.")
-            sys.exit(0)
-
-    # IMPORT
-    # Loop on file categories
-    for cat in import_bundle.analyses.get_all_categories():
-        # Import all files in this category.
-        for f in import_bundle.get_files(cat):
-            logger.info("Import %s files from %s.", cat, f.path)
-            # First, generate a unique index name for each file.
-            file_index = generate_unique_index(args.es_index_prefix, f.path)
-            # Then, import the analysis metadata into a dedicated index.
-            import_analysis_metadata(
-                es_import_conn,
-                args.es_index_prefix,
-                file_index,
-                f,
-                cat,
-                args.dryrun,
-            )
-            # Finally, import the file in its own index.
-            globals()[f"import_{cat}_file"](
-                es_import_conn=es_import_conn,
-                file_index=file_index,
-                file=f,
-                dryrun=args.dryrun,
-            )
-
-    # Import processes
-    logger.info("Importing wet processes.")
-    logger.info(
-        "Wet processes IDs = %s",
-        str(import_bundle.wet_processes.get_process_ids()),
-    )
-    import_processes(
-        es_import_conn,
-        f"{args.es_index_prefix}-wet_processes",
-        import_bundle.wet_processes,
-    )
-
-    logger.info("Importing bio info processes.")
-    logger.info(
-        "Bio info processes IDs = %s",
-        str(import_bundle.bi_processes.get_process_ids()),
-    )
-    import_processes(
-        es_import_conn,
-        f"{args.es_index_prefix}-bi_processes",
-        import_bundle.bi_processes,
-    )
-
-
-if __name__ == "__main__":
-    main()