urgap 3.3.5__py2.py3-none-any.whl

urgap/__init__.py

@@ -0,0 +1,116 @@
+"""urgap."""
+
+import atexit
+import logging
+import os
+import sys
+
+from importlib.metadata import PackageNotFoundError, version
+from pathlib import Path
+from types import SimpleNamespace
+
+# Urgap imports
+import urgap.uconvert
+import urgap.ucore
+import urgap.ufile
+import urgap.uftypes
+import urgap.uinit
+import urgap.util
+import urgap.utree
+
+from .ucredentials.ucredentials import UCredentialManager
+from .ufile.ufile import UFile
+from .ufile.ufile_io_manager import UFileIOManager
+from .ufile.uuri import UUri
+from .ufile_list import UFileList
+
+# Importing into namespace a level higher
+from .umeta import io
+from .umeta.umeta import UMeta
+from .unode import UNodeBase
+from .unode_manager import UNodeManager
+from .ureport.ureport import UReport
+from .urun_dict import URunDict
+from .utelemetry import UTelemetry
+from .utrace import UTrace
+from .uwid import UWIDGenerator
+
+__all__ = [
+    "UCredentialManager",
+    "UFile",
+    "UFileIOManager",
+    "UFileList",
+    "UMeta",
+    "UNodeBase",
+    "UNodeManager",
+    "UReport",
+    "URunDict",
+    "UTelemetry",
+    "UTrace",
+    "UUri",
+    "UWIDGenerator",
+    "io",
+    "util",
+]
+
+try:
+    __version__ = version(__name__)
+except PackageNotFoundError:
+    __version__ = None
+
+__version_str__ = str(__version__)
+package_dir = Path(__file__).parent
+if not hasattr(sys, "version_info") or sys.version_info < (3, 10):
+    msg = "Urgap requires Python 3.10 or later."
+    raise RuntimeError(msg)
+
+urgap.uinit.configure_logger()
+logger = logging.getLogger(__name__)
+uwid_obj = urgap.UWIDGenerator()
+
+home = Path(os.getenv("URGAP_HOME", str(Path.home() / ".urgap")))
+info_box = [urgap.uinit.show_banner()]
+info_format_string = "  {k: <23}: {v}"
+info_box.append(info_format_string.format(k="version", v=f"{urgap.__version__}"))
+
+if urgap.home.exists() is False:
+    urgap.uinit.create_home_folder(home_dir_parent=urgap.home.parent)
+else:
+    info_box.append(info_format_string.format(k="urgap home", v=f"{home}"))
+
+project_folder = Path.cwd()
+info_box.append(info_format_string.format(k="project_folder", v=f"{project_folder}"))
+info_box.append(
+    info_format_string.format(k="urgap config", v=f"{urgap.home}/urgap.json"),
+)
+
+urgap.config = urgap.uinit.read_config()
+urgap.uinit.load_certificates()
+
+urgap.session_uwid = uwid_obj.generate_wid()
+scratch_disk_base = urgap.uinit.set_scratch_disk_path(wid=urgap.session_uwid)
+scratch_disk = scratch_disk_base  # Temporary as we will update during run with WID
+info_box.append(info_format_string.format(k="scratch disk", v=scratch_disk_base))
+logger.info("\n".join([*info_box]))
+
+urgap.uinit.copy_resources_if_needed(
+    target_dir=urgap.home,
+    force=urgap.config.get("update_resources", False),
+)
+# Instances
+urgap.instances = SimpleNamespace()
+urgap.instances.unode_manager = urgap.UNodeManager()
+urgap.instances.ufile_io_manager = urgap.UFileIOManager()
+urgap.instances.ucredential_manager = urgap.UCredentialManager()
+urgap.instances.utelemetry_manager = urgap.UTelemetry()
+urgap.utl = urgap.instances.utelemetry_manager
+urgap.instances.utree_querier = urgap.utree.UTreeQuerier(namespace=urgap.uftypes)
+
+urgap.init_node = urgap.instances.unode_manager.init_unode
+urgap.init_unode = urgap.instances.unode_manager.init_unode
+
+atexit.register(urgap.ucore.shutdown_local_upi_servers)
+if urgap.config.get("clean_scratch_at_exit", True) is True:
+    atexit.register(urgap.ucore.clean_up_scratch_space)
+
+atexit.register(urgap.ucore.shutdown_telemetry)

urgap/citation.txt

@@ -0,0 +1 @@
+<put in your name>, Fufezan, C. (2022) Journal of awesomeness

urgap/config_defaults/credentials_lookup.json

@@ -0,0 +1,86 @@
+{
+    "description": "user and password are pointing to the ENV variables that contain the actual data. host and schema are required for urgap lookup",
+    "credentials": [
+        {
+            "description": "github token https raw, here user=password will be formatted into the query",
+            "scheme": "https",
+            "host": "raw.githubusercontent.com",
+            "user": "U_GITHUB_GSK_USER",
+            "password": "U_GITHUB_GSK_PASSWORD",
+            "secure": true,
+            "secret_store": "env",
+            "base_url": "https://raw.githubusercontent.com"
+        },
+        {
+            "description": "local mongodb",
+            "scheme": "mongodb",
+            "host": "localhost:27017",
+            "user": "U_MONGODB_USER",
+            "password": "U_MONGODB_PASSWORD",
+            "secure": true,
+            "secret_store": "env",
+            "base_url": "mongodb://localhost:27017"
+        },
+        {
+            "description": "local ssh",
+            "scheme": "ssh",
+            "host": "localhost",
+            "user": "ENV_VARIABLE_WITH_SSH_USER",
+            "password": "ENV_VARIABLE_WITH_SSH_KEY",
+            "secure": true,
+            "secret_store": "env",
+            "base_url": "ssh://localhost"
+        },
+        {
+            "description": "if project_id is defined, it will be used instead of host for GCP secret extraction",
+            "scheme": "postgresql+pg8000",
+            "host": "10.138.0.5:5432",
+            "user": "U_GCS_SQL_USER",
+            "password": "U_GCS_SQL_PASSWORD",
+            "secure": true,
+            "secret_store": "gcp",
+            "cloud_host_pid": "gsk-rd-ngs-dev",
+            "base_url": "postgresql+pg8000://10.138.0.5:5432"
+        },
+        {
+            "description": "mylabdata interface",
+            "scheme": "mylabdata",
+            "host": "<mylabdata-files-api-url-without-https>",
+            "user": "U_MYLABDATA_USER",
+            "password": "U_MYLABDATA_PASSWORD",
+            "secure": true,
+            "secret_store": "env",
+            "base_url": "mylabdata://<mylabdata-files-api-url-without-https>"
+        },
+        {
+            "description": "smb test interface",
+            "scheme": "smb",
+            "host": "10.0.20.165:445",
+            "user": "U_SMB_TEST_USER",
+            "password": "U_SMB_TEST_PASSWORD",
+            "secure": true,
+            "secret_store": "env",
+            "base_url": "smb://10.0.20.165:445"
+        },
+        {
+            "description": "OpenTelemetry Azure (dsoappinstprod). We store the connection string in password and user is just a dummy.",
+            "scheme": "https",
+            "host": "eastus2-3.in.applicationinsights.azure.com",
+            "user": "U_AZ_INSIGHT_USER",
+            "password": "U_AZ_INSIGHT_PASSWORD",
+            "secure": true,
+            "secret_store": "env",
+            "base_url": "https://eastus2-3.in.applicationinsights.azure.com"
+        },
+        {
+            "description": "Azure Service Bus namespace connection string. Password env var must contain full Service Bus connection string (starts with 'Endpoint=sb://'). User not used (set to null).",
+            "scheme": "azure-servicebus",
+            "host": "<namespace>.servicebus.windows.net",
+            "user": null,
+            "password": "U_AZ_SERVICEBUS_CONNECTION_STRING",
+            "secure": true,
+            "secret_store": "env",
+            "base_url": "azure-servicebus://<namespace>.servicebus.windows.net"
+        }
+    ]
+}

urgap/config_defaults/urgap.json

@@ -0,0 +1,90 @@
+{
+    "info": "Urgap will internally reformat this config into a simple dictionary (k, v[v'value']) available under urgap.config",
+    "description": "urgap default config",
+    "umeta": {
+        "value": "sqlite3",
+        "description": "Storage interface for meta data. Might require 3rd party installations",
+        "options": ["mongodb", "sqlite3", "postgresql", "gcpsql"]
+    },
+    "logging_level": {
+        "value": "DEBUG",
+        "description": "Verbosity of urgap, can be set to any python logger level"
+    },
+    "update_resources": {
+        "value": false,
+        "description": "Define whether resources from the repo will be force copied into $URGAP_HOME on startup. This is useful if resources are developed in the urgap repo, yet code execution will use $URGAP_HOME/resource code."
+    },
+    "update_uparma": {
+        "value": false,
+        "description": "Define whether uparma json download should be forced (part of u2 dev work). Not required for release"
+    },
+    "hash_algorithm": {
+        "value": "md5",
+        "description": "Hash algorithm used in Urgap in order to create unique identifiers"
+    },
+    "max_parallel_cores": {
+        "value": null,
+        "description": "Number of CPU cores to use for parallel processing."
+    },
+    "umeta-postgresql-url": {
+        "value": "postgresql://localhost:5432",
+        "description": "url for postgresql instance, used if umeta is set to postgresql. Login and password are extracted via ENV or alternative secret store defined in credentials_lookup.json"
+    },
+    "umeta-sqlite3-url": {
+        "value": null,
+        "description": "url for sqlite3 db file, used if umeta is set to sqlite. Should start with file:///"
+    },
+    "umeta-gcpsql-url": {
+        "value": null
+    },
+    "scratch_disk": {
+        "value": null,
+        "description": "Temporary urgap processing disk"
+    },
+    "mode": {
+        "value": "dev",
+        "description": "Mode in which urgap is running. Can be set to prod for stricter rules.",
+        "options": ["dev", "prod"]
+    },
+    "clean_scratch_at_exit": {
+        "value": true,
+        "description": "Urgap uses scratch <scratch_disk>/<wid> as temporary spaces. This flag will ensure that temporary directories (<wid>s) will be deleted when interpreter exits."
+    },
+    "requests_timeout_connect": {
+        "value": null,
+        "description": "Connect timeout for python requests"
+    },
+    "requests_timeout_read": {
+        "value": null,
+        "description": "Read timeout for python requests"
+    },
+    "terminate_remote_servers_on_exit": {
+        "value": false,
+        "description": "Send a termination signal to all ports on localhost during urgap exit"
+    },
+    "opentelemetry_collector_url": {
+        "value": null,
+        "description": "Open Telementry Collector url, e.g. http://localhost:4317"
+    },
+    "opentelemetry_exporter_type": {
+        "value": null,
+        "description": "Open Telementry Exporter Type. Null will disable OTEL.",
+        "options": [null, "OTPL", "AZ-Insights", "Console"]
+    },
+    "service_bus_topic": {
+        "value": "urgap_queue",
+        "description": "Topic for pulling work from Service Bus worker."
+    },
+    "service_bus_completion_topic": {
+        "value": "urgap_completed",
+        "description": "Optional topic to publish completion events to."
+    },
+    "service_bus_exit_after_first_message": {
+        "value": true,
+        "description": "If true, worker exits successfully after processing one message."
+    },
+    "service_bus_max_autorenewal_minutes": {
+        "value": 0,
+        "description": "Maximum time after which to stop auto renewing lock on message. 0 disables this."
+    }
+}

urgap/resources/Compressor/1_0_0/__init__.py

@@ -0,0 +1 @@
+"""Init Compressor 1_0_0 resource."""

urgap/resources/Compressor/1_0_0/compressor.py

@@ -0,0 +1,109 @@
+"""Compressor resource."""
+
+import argparse
+import logging
+import os.path
+import subprocess
+import tarfile
+
+from ast import literal_eval
+from pathlib import Path
+from zipfile import ZipFile
+
+logger = logging.getLogger(__name__)
+
+
+def main(
+    compression_format: str,
+    file_list: list,
+    output_file_path: str,
+    max_tar_size: int | None = None,
+) -> None:
+    """Create tar or zip archive from input file list.
+
+    Selecting split_tar splits tar archive into parts.
+
+    Args:
+        compression_format: Currently supported are tar, zip and split_tar.
+        file_list (list): list of tuples of file Path objects and tag Path objects (or None).
+        max_tar_size: Maximum size of the split tar.
+        output_file_path (Path): path object for output.
+    """
+    if max_tar_size is not None:
+        files_to_archive = [files[0] for files in file_list]
+        common_base = Path(os.path.commonpath(files_to_archive)).parent
+        relative_files = [Path(f).relative_to(common_base) for f in files_to_archive]
+        tar_process = subprocess.Popen(
+            ["tar", "-cf", "-", "-C", str(common_base), *map(str, relative_files)],
+            stdout=subprocess.PIPE,
+        )
+        split_process = subprocess.Popen(
+            [
+                "split",
+                "-b",
+                f"{max_tar_size}",
+                "-",
+                f"{Path(output_file_path).parent / 'part.'}",
+            ],
+            stdin=tar_process.stdout,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        tar_process.stdout.close()
+        split_process.communicate()
+        tar_process.wait()
+        if tar_process.returncode != 0:
+            msg = f"Tar process failed with code {tar_process.returncode}"
+            logger.error(msg)
+        if split_process.returncode != 0:
+            msg = f"Split process failed with code {split_process.returncode}"
+            logger.error(msg)
+        else:
+            logger.info("Tar archive created and split successfully")
+    elif compression_format == "tar":
+        with tarfile.open(output_file_path, mode="w:") as tar:
+            for file, tag in file_list:
+                tar.add(file, arcname=Path(file).name)
+                if tag is not None:
+                    tar.add(tag, arcname=Path(tag).name)
+    elif compression_format == "zip":
+        with ZipFile(output_file_path, "w") as zip_file:
+            for file, tag in file_list:
+                zip_file.write(file, arcname=Path(file).name)
+                if tag is not None:
+                    zip_file.write(tag, arcname=Path(tag).name)
+    else:
+        msg = "Only zip and tar are valid compression formats."
+        raise NotImplementedError(msg)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-i",
+        "--input_files",
+        dest="input_files",
+        action="append",
+        help="input files to be compressed",
+    )
+    parser.add_argument("-o", "--output_file", dest="output_file", help="output file")
+    parser.add_argument(
+        "-cf",
+        "--compression_format",
+        dest="compression_format",
+        help="compression format: tar or zip",
+    )
+    parser.add_argument(
+        "-s",
+        "--size",
+        dest="max_tar_size",
+        help="maximum tar size",
+    )
+    args = parser.parse_args()
+
+    main(
+        file_list=[literal_eval(str_tuple) for str_tuple in args.input_files],
+        output_file_path=args.output_file,
+        compression_format=args.compression_format,
+        max_tar_size=args.max_tar_size,
+    )

urgap/resources/Compressor/__init__.py

@@ -0,0 +1 @@
+"""Init Compressor resources."""

urgap/resources/FilterTabular/1_0_0/__init__.py

@@ -0,0 +1 @@
+"""Init FilterTabular 1_0_0 resource."""

urgap/resources/FilterTabular/1_0_0/filter_tabular.py

@@ -0,0 +1,229 @@
+"""Filter / and or merge files with pandas operations."""
+
+import argparse
+import logging
+import pprint
+import zipfile
+
+from pathlib import Path
+
+import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+
+def sense_file_format(file: str) -> str:
+    """Sense the format of a file.
+
+    Args:
+        file: File to analyze.
+
+    Returns:
+        File format as string.
+    """
+    with Path(file).open("rb") as f:
+        match f.read(4):
+            case b"PAR1":
+                f.seek(-4, 2)
+                if f.read(4) == b"PAR1":
+                    return "parquet"
+            case b"PK\x03\x04":
+                with zipfile.ZipFile(file, "r") as zip_file:
+                    required_files = ["[Content_Types].xml", "xl/"]
+                    if any(file in zip_file.namelist() for file in required_files):
+                        return "xlsx"
+        return "csv"
+
+
+def get_input_file_lists(input_files: list) -> dict:
+    """Get the separate input file lists for csv, parquet and xlsx.
+
+    Args:
+        input_files: All input files.
+
+    Returns:
+        Input files split up in corresponding lists.
+    """
+    files = {"csv": [], "parquet": [], "xlsx": []}
+    for file in input_files:
+        input_file_type = sense_file_format(file)
+        files[input_file_type].append(file)
+    return files
+
+
+def read_dfs(grouped_input_files: dict, sep: str | None) -> list:
+    """Read any of the CSV, Parquet and XLSX files.
+
+    Args:
+        grouped_input_files: Dict with all files mapped to filetype.
+        sep: Seperator or delimiter for read csv function.
+
+    Returns:
+        List with Dataframe from each file.
+    """
+    pd_functions = {
+        "csv": {"function": pd.read_csv},
+        "parquet": {"function": pd.read_parquet},
+        "xlsx": {"function": pd.read_excel},
+    }
+    dfs = []
+    for file_type, file_list in grouped_input_files.items():
+        if file_type == "csv":
+            dfs.extend(
+                pd_functions[file_type]["function"](file, sep=sep, engine="python")
+                for file in file_list
+            )
+        else:
+            dfs.extend(pd_functions[file_type]["function"](file) for file in file_list)
+    return dfs
+
+
+def write_dfs(df: pd.DataFrame, mode: str, output: str) -> None:
+    """Write any of the CSV, Parquet and XLSX files.
+
+    Args:
+        df: Dataframe to write.
+        mode: Defines output file format.
+        output: Output path.
+
+    Raises:
+        ValueError: Unsupported mode is requested.
+    """
+    if mode == "csv":
+        df.to_csv(output, index=False)
+    elif mode == "parquet":
+        df.to_parquet(output, index=False)
+    elif mode == "xlsx":
+        df.to_excel(output, index=False)
+    else:
+        msg = (
+            f"Filter mode has to be defined with -m."
+            f"You defined filter mode {mode} which is currently not supported."
+            f"Supported modes are 'csv', 'parquet' and 'xlsx'."
+        )
+        raise ValueError(msg)
+
+
+def merge_parquet_files(grouped_input_files: dict, output_file_path: str) -> None:
+    """Merge all parquet files and write to the destination file path.
+
+    Args:
+        grouped_input_files: Input grouped_input_files.
+        output_file_path: Output parquet file path.
+
+    Returns:
+        True when successful else False.
+    """
+    import pyarrow as pa
+    import pyarrow.parquet as pq
+
+    parquet_files = grouped_input_files["parquet"]
+    with pq.ParquetWriter(
+        output_file_path,
+        schema=pq.read_table(parquet_files[0]).schema,
+    ) as writer:
+        for parquet in parquet_files:
+            if Path(parquet).exists() is False:
+                msg = f"Parquet file {parquet} does not exist!"
+                raise FileNotFoundError(msg)
+            pf = pq.ParquetFile(parquet)
+
+            for batch in pf.iter_batches(batch_size=6553600):
+                table = pa.Table.from_batches([batch])
+                writer.write_table(table)
+
+
+def main(
+    input_files: str | Path | list | None = None,
+    query_string: str | None = None,
+    output: str | None = None,
+    mode: str | None = None,
+    sep: str | None = None,
+) -> None:
+    r"""Filter and merge node.
+
+    Args:
+        sep: Seperator or delimiter. Defaults to ','
+        mode: Defines the output file type.
+        input_files: Path to files (can be multiple).
+
+        query_string (str): pandas df.query style strings.
+            Note: It is worth selecting each column by encapsulating it with
+            `\ in order to avoid errors, e.g.:
+                " -1 < \`Accuracy (ppm)\` < 1 and \`MS-GF:RawScore\` > 10"
+
+        output (str): Path to output file post filtering.
+    """
+    if len(input_files) == 0:
+        msg = "I need input files!..."
+        raise OSError(msg)
+
+    grouped_input_files = get_input_file_lists(input_files)
+    logger.info(pprint.pformat(grouped_input_files))
+    if (
+        mode == "parquet"
+        and len(grouped_input_files.get("csv", [])) == 0
+        and len(grouped_input_files.get("parquet", [])) > 1
+        and len(grouped_input_files.get("xlsx", [])) == 0
+        and query_string is None
+    ):
+        merge_parquet_files(grouped_input_files, output)
+
+    else:
+        dfs = read_dfs(grouped_input_files, sep)
+        concatenated_df = pd.concat(dfs)
+        if query_string is not None:
+            old_len = len(concatenated_df)
+            try:
+                concatenated_df = concatenated_df.query(query_string)
+            except (pd.errors.UndefinedVariableError, SyntaxError, ValueError) as e:
+                msg = f"Query string {query_string} is invalid"
+                f"Dataframe columns: {list(concatenated_df.columns)}"
+                logger.warning(msg)
+                raise RuntimeError(msg) from e
+
+            msg = f"Filtered {old_len - concatenated_df.shape[0]} rows"
+
+            logger.info(msg)
+        if concatenated_df.empty is True:
+            logger.warning("All rows have been filtered out!")
+        concatenated_df = concatenated_df.reset_index(drop=True)
+        write_dfs(concatenated_df, mode=mode, output=output)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-i",
+        "--input_files",
+        dest="input_files",
+        action="append",
+        help="input files to be compressed",
+    )
+    parser.add_argument("-o", "--output_file", dest="output_file", help="output file")
+    parser.add_argument(
+        "-q",
+        "--query_string",
+        dest="query_string",
+        help="pandas query string",
+    )
+    parser.add_argument(
+        "-m",
+        "--mode",
+        dest="mode",
+        help="filter mode like csv or parquet",
+    )
+    parser.add_argument(
+        "-s",
+        "--sep",
+        dest="sep",
+        help="Seperator or delimiter. Defaults to ','",
+    )
+    args = parser.parse_args()
+    main(
+        input_files=args.input_files,
+        output=args.output_file,
+        query_string=args.query_string,
+        mode=args.mode,
+        sep=args.sep,
+    )

urgap/resources/FilterTabular/2_0_0/__init__.py

@@ -0,0 +1 @@
+"""Init FilterTabular 2_0_0 resource."""