miqatools 2.0.0rc0__py3-none-any.whl

miqatools/datamanagement/datasethelpers/__init__.py

@@ -0,0 +1,180 @@
+from ...utilities.miqa_client import api_get, api_post
+
+
+def get_ds_ids_by_name(pipeline_id, **kwargs):
+    """
+    Gets a lookup of dataset names and corresponding IDs for a particular pipeline
+
+    :param pipeline_id: Miqa pipeline ID to get dataset info for
+    :return: A lookup of dataset information
+    :rtype: dict
+    """
+    info = get_ds_ids_by_name_internal(pipeline_id, **kwargs)
+    return info
+
+
+def get_ds_ids_by_name_internal(pipeline_id, **kwargs):
+    endpoint = "get_ds_ids_by_name"
+    params = {"pipeline_id": pipeline_id}
+    return api_get(endpoint, params=params, **kwargs)
+
+
+def tag_ds_ids_by_name(pipeline_id, name, tag, **kwargs):
+    """
+    Tags dataset(s) matching the name provided with the tag provided
+
+    :param pipeline_id: Miqa pipeline ID to get dataset info for
+    :param str name: Dataset name corresponding to the dataset to apply the tag to
+    :param str tag: Tag to use for the dataset
+    :return: Response from the API call
+    :rtype: object
+    """
+    info = get_ds_ids_by_name_internal(pipeline_id, **kwargs)
+    source_ids_lookup = info.get("data", {})
+
+    if source_ids_lookup and source_ids_lookup.get(name):
+        endpoint = f"/batch_tag_datasources/{tag}"
+        params = {"inline": 1, "source_ids": source_ids_lookup.get(name)}
+        return api_get(endpoint, params=params, **kwargs)
+    else:
+        raise Exception("No matching datasource IDs found")
+
+
+def tag_ds_ids_by_names(pipeline_id, names, tag, raise_if_names_not_found=False, **kwargs):
+    """
+    Tags dataset(s) matching the names provided with the tag provided
+
+    :param pipeline_id: Miqa pipeline ID to get dataset info for
+    :param list name: Dataset name corresponding to the dataset to apply the tag to
+    :param str tag: Tag to use for the dataset
+    :param bool raise_if_names_not_found: Whether to raise an exception if no datasets are found
+        matching one of the provided names
+    :return: Response from the API call
+    :rtype: object
+    """
+    info = get_ds_ids_by_name_internal(pipeline_id, **kwargs)
+    source_ids_lookup = info.get("data", {})
+    source_ids_for_names = [source_ids_lookup.get(name) for name in names]
+    source_ids_for_names = [str(i) for i in source_ids_for_names if i]
+
+    if source_ids_for_names and (
+        not raise_if_names_not_found or len(source_ids_for_names) == len(names)
+    ):
+        tag_url = (
+            f"/batch_tag_datasources/{tag}"
+            f"?inline=1"
+            f"&source_ids={','.join(source_ids_for_names)}"
+        )
+        resp = api_get(tag_url, **kwargs)
+        return resp
+    else:
+        raise Exception("Matching datasource IDs not found")
+
+
+def batch_create_datasets_and_tag_or_group(
+    org_id, pipeline_id, new_ds_names, tags, dsg_name=None, **kwargs
+):
+    """
+    Create stand-in datasets (i.e. no data upload) and tag them
+
+    :param int org_id: Miqa organization ID
+    :param int pipeline_id: Miqa pipeline ID to add the datasets to
+    :param list new_ds_names: Names for new datasets
+    :param list tags: Tags to use for the dataset
+    :param str dsg_name: Name to use to create a dataset group,
+        if wanting to create a group (Optional)
+    :return: Response from the API call
+    :rtype: Response
+    """
+    tag_url = (
+        f"/batch_create_dataset_and_tag"
+        f"?org_id={org_id}"
+        f"&pipeline_id={pipeline_id}"
+        f"&names={','.join(new_ds_names)}"
+    )
+
+    if tags:
+        tag_url += f"&tags={','.join(tags)}"
+    if dsg_name:
+        tag_url += f"&create_dsg={dsg_name}"
+
+    resp = api_get(tag_url, **kwargs)
+    return resp
+
+
+def add_truths_to_dataset(dataset_id, pipeline_id, truth_file_path, **kwargs):
+    """
+    Create a single stand-in dataset (i.e. no data upload) and optionally tag it
+
+    :param int dataset_id: Miqa dataset ID to add the truths to
+    :param int pipeline_id: Miqa pipeline ID corresponding to the dataset and truth type
+    :param str truth_file_path: Filepath pointing to the truth TSV file. Should be a tab-delimited
+        file with columns and headers corresponding to your pipeline's truth type,
+        e.g. "chr   pos ref alt"
+    """
+    upload_truths_url = (
+        f"/datasource/{dataset_id}/upload_truths" f"?pipeline_id={pipeline_id}" f"&inline=1"
+    )
+    response = api_post(
+        upload_truths_url, files={"file": open(truth_file_path, "rb")}, data={}, **kwargs
+    )
+    if response:
+        raise Exception(
+            f"Unable to add truths to dataset {dataset_id}: received an error response from the "
+            f"server at url {upload_truths_url}. Response: {response}"
+        )
+    return response
+
+
+def create_dataset_with_name(org_id, pipeline_id, dataset_name, tags=None, **kwargs):
+    """
+    Create a single stand-in dataset (i.e. no data upload) and optionally tag it
+
+    :param int org_id: Miqa organization ID
+    :param int pipeline_id: Miqa pipeline ID to add the datasets to
+    :param str dataset_name: Name for new dataset
+    :param list tags: Tags to use for the dataset (optional)
+
+    :return: New Dataset ID
+    :rtype: int
+    """
+    create_ds_response = batch_create_datasets_and_tag_or_group(
+        org_id,
+        pipeline_id,
+        new_ds_names=[dataset_name],
+        tags=tags if tags else [],
+        **kwargs,
+    )  # Add tags if desired
+    if create_ds_response:
+        create_ds_response = create_ds_response.json()
+        ds_id_lookup = create_ds_response.get("data")
+        ds_id = ds_id_lookup.get(dataset_name)
+        return ds_id
+    else:
+        raise Exception(
+            f"Unable to create dataset with name '{dataset_name}': received an error response from "
+            f"the endpoint. Response: {create_ds_response}"
+        )
+
+
+def create_dataset_and_add_truths(
+    org_id, pipeline_id, dataset_name, truth_file_path, tags=None, **kwargs
+):
+    """
+    Create a single stand-in dataset (i.e. no input file upload), add truth data,
+    and optionally tag it
+
+    :param int org_id: Miqa organization ID
+    :param int pipeline_id: Miqa pipeline ID corresponding to the dataset and truth type
+    :param str dataset_name: Name for new dataset
+    :param str truth_file_path: Filepath pointing to the truth TSV file. Should be a tab-delimited
+        file with columns and headers corresponding to your pipeline's truth type,
+        e.g. "chr   pos ref alt"
+    :param list tags: Tags to use for the dataset (optional)
+
+    :return: New Dataset ID
+    :rtype: int
+    """
+    dataset_id = create_dataset_with_name(org_id, pipeline_id, dataset_name, tags=tags, **kwargs)
+    add_truths_to_dataset(dataset_id, pipeline_id, truth_file_path, **kwargs)
+    return dataset_id

miqatools/remoteexecution/adhoc_execution/__init__.py

@@ -0,0 +1,230 @@
+from ...datamanagement.datasethelpers import get_ds_ids_by_name
+from ...utilities.miqa_client import api_get
+from ..executionhelpers import complete_exec
+from ..offlineexecution import upload_files_or_folder_for_exec
+from ..triggertest_helpers import update_execution_start_time
+
+
+def create_workflow_version(workflow_id, version_name, **kwargs):
+    """
+    Create a new manual-upload version for the specified workflow
+
+    :param int workflow_id: Miqa implemented workflow ID
+    :param str version_name: Name for new version
+    :param kwargs: Additional parameters including miqa_server and api_key
+
+    :return: New workflow version ID
+    :rtype: int
+    """
+    create_version_url = (
+        f"/api/create_uploadable_version" f"?pskel_id={workflow_id}" f"&version_name={version_name}"
+    )
+    headers = {"accept": "application/json"}
+    try:
+        create_version_response = api_get(create_version_url, headers=headers, **kwargs)
+        wfv_id = create_version_response.get("workflowversion_id")
+        return wfv_id
+    except Exception as e:
+        raise Exception(
+            f"Unable to create workflow version with name '{version_name}' for {workflow_id}: "
+            f"received an error response from the endpoint. Error: {e}"
+        )
+
+
+def create_execution(workflowversion_id, dataset_id, org_config_id, **kwargs):
+    """
+    Create a new execution for the specified dataset on the specified workflow version
+
+    :param int workflowversion_id: Miqa workflow version ID
+    :param int dataset_id: Miqa dataset ID
+    :param int org_config_id: Miqa organization configuration ID
+    :param kwargs: Additional parameters including miqa_server and api_key
+
+    :return: New execution ID
+    :rtype: int
+    """
+    create_execution_url = (
+        f"/api/workflowversion/{workflowversion_id}/execute_on_dataset/{dataset_id}"  # noqa
+        f"?org_config_id={org_config_id}"
+    )
+    headers = {"accept": "application/json"}
+    try:
+        create_execution_response = api_get(create_execution_url, headers=headers, **kwargs)
+        exec_id = create_execution_response.get("exec_id")
+        return exec_id
+    except Exception as e:
+        raise Exception(
+            f"Unable to create execution for {dataset_id} on {workflowversion_id}: "
+            f"received an error response from the endpoint. Error: {e}"
+        )
+
+
+def create_and_upload_to_new_version_for_ds(
+    version_name,
+    dataset_name,
+    pipeline_id,
+    workflow_id,
+    org_config_id,
+    directory_containing_outputs,
+    raise_on_fail=False,
+    filepattern_end=None,
+    filepattern_start=None,
+    exclude_filepattern_end=None,
+    max_filesize=None,
+    max_connections=None,
+    filepatterns=None,
+    quiet=True,
+    per_request_timeout=None,
+    parallel_file_upload=True,
+    detailed_file_logs=False,
+    **kwargs,
+):
+    """
+    Create and execute a new version for an existing dataset, retrieved by name
+
+    :param str version_name: Name for new version
+    :param str dataset_name: Name for new dataset
+    :param int pipeline_id: Miqa pipeline ID to add the datasets to
+    :param int workflow_id: Miqa implemented workflow ID for the new version
+    :param int org_config_id: Miqa organization configuration ID
+    :param str directory_containing_outputs: Folder containing files to upload (Optional)
+    :param bool raise_on_fail: Whether to raise an exception if the file upload fails.
+        Default: False, the execution will be failed but an exception won't be raised (Optional)
+    :param str filepattern_end: Pattern to match in the file suffix, e.g. '.bam' (Optional)
+    :param str filepattern_start: Pattern to match in the file prefix, e.g. 'Analysis/' (Optional)
+    :param str exclude_filepattern_end:
+        Pattern to exclude in the file suffix, e.g. '.bam' (Optional)
+    :param int max_connections: Maximum number of connections for parallel upload (Optional)
+    :param int max_filesize: Max filesize to upload, i.e. if avoiding very large files (Optional)
+    :param filepatterns:
+        List of regex strings representing file patterns to match, e.g. ['*.csv','*.vcf'] (Optional)
+    :param bool quiet: Whether to skip logging. Default True. (Optional)
+    :param int per_request_timeout: Timeout for individual requests made during uploading.
+        Default None. (Optional)
+    :param int parallel_file_upload: Whether to allow uploads of each file to proceed in parallel.
+        Default True. (Optional)
+    :param bool detailed_file_logs: If not in quiet mode, whether to show detailed logs of each file
+        uploaded. Default False. (Optional)
+    :param kwargs: Additional parameters including miqa_server and api_key
+
+    """
+    exec_id = create_and_execute_for_new_version_for_ds(
+        version_name, dataset_name, pipeline_id, workflow_id, org_config_id, **kwargs
+    )
+    upload_and_complete_execution(
+        exec_id,
+        directory_containing_outputs,
+        raise_on_fail=raise_on_fail,
+        filepattern_end=filepattern_end,
+        filepattern_start=filepattern_start,
+        exclude_filepattern_end=exclude_filepattern_end,
+        max_filesize=max_filesize,
+        max_connections=max_connections,
+        filepatterns=filepatterns,
+        quiet=quiet,
+        per_request_timeout=per_request_timeout,
+        parallel_file_upload=parallel_file_upload,
+        detailed_file_logs=detailed_file_logs,
+        **kwargs,
+    )
+    return exec_id
+
+
+def upload_and_complete_execution(
+    exec_id,
+    directory_containing_outputs,
+    raise_on_fail=False,
+    filepattern_end=None,
+    filepattern_start=None,
+    exclude_filepattern_end=None,
+    max_filesize=None,
+    max_connections=None,
+    filepatterns=None,
+    quiet=True,
+    per_request_timeout=None,
+    parallel_file_upload=True,
+    detailed_file_logs=False,
+    **kwargs,
+):
+    """
+    Uploads the test output files for each specified dataset, from the specified filepath to the
+    corresponding Miqa execution, and then marks that execution as complete.
+
+    :param int exec_id: Miqa execution ID
+    :param str directory_containing_outputs: Folder containing files to upload (Optional)
+    :param bool raise_on_fail: Whether to raise an exception if the file upload fails.
+        Default: False, the execution will be failed but an exception won't be raised (Optional)
+    :param str filepattern_end: Pattern to match in the file suffix, e.g. '.bam' (Optional)
+    :param str filepattern_start: Pattern to match in the file prefix, e.g. 'Analysis/' (Optional)
+    :param str exclude_filepattern_end:
+        Pattern to exclude in the file suffix, e.g. '.bam' (Optional)
+    :param int max_connections: Maximum number of connections for parallel upload (Optional)
+    :param int max_filesize: Max filesize to upload, i.e. if avoiding very large files (Optional)
+    :param filepatterns: List of regex strings representing file patterns to match,
+        e.g. ['*.csv','*.vcf'] (Optional)
+    :param bool quiet: Whether to skip logging. Default True. (Optional)
+    :param int per_request_timeout: Timeout for individual requests made during uploading.
+        Default None. (Optional)
+    :param int parallel_file_upload: Whether to allow uploads of each file to proceed in parallel.
+        Default True. (Optional)
+    :param bool detailed_file_logs: If not in quiet mode, whether to show detailed logs of each file
+        uploaded. Default False. (Optional)
+    :param kwargs: Additional parameters including miqa_server and api_key
+    """
+    update_execution_start_time(exec_id, quiet=quiet, **kwargs)
+    failed = False
+    try:
+        upload_files_or_folder_for_exec(
+            exec_id,
+            directory_containing_outputs,
+            args_files=None,
+            filepattern_end=filepattern_end,
+            filepattern_start=filepattern_start,
+            exclude_filepattern_end=exclude_filepattern_end,
+            max_filesize=max_filesize,
+            max_connections=max_connections,
+            filepatterns=filepatterns,
+            quiet=quiet,
+            per_request_timeout=per_request_timeout,
+            parallel_file_upload=parallel_file_upload,
+            detailed_file_logs=detailed_file_logs,
+            **kwargs,
+        )
+
+    except Exception as e2:
+        if raise_on_fail:
+            raise
+        failed = True
+        if not quiet:
+            print(
+                f"Failing execution {exec_id} due to unhandled exception (see below). Will mark "
+                "execution as failed and continue any remaining samples."
+            )
+            print(e2)
+            complete_exec(exec_id, quiet=quiet, failed=failed, **kwargs)
+    complete_exec(exec_id, quiet=quiet, failed=failed, **kwargs)
+
+
+def create_and_execute_for_new_version_for_ds(
+    version_name, dataset_name, pipeline_id, workflow_id, org_config_id, **kwargs
+):
+    """
+    Create and execute a new version for an existing dataset, retrieved by name
+
+    :param str version_name: Name for new version
+    :param str dataset_name: Name for new dataset
+    :param int pipeline_id: Miqa pipeline ID to add the datasets to
+    :param int workflow_id: Miqa implemented workflow ID for the new version
+    :param int org_config_id: Miqa organization configuration ID
+    :param kwargs: Additional parameters including miqa_server and api_key
+    """
+    ds_ids_by_name = get_ds_ids_by_name(pipeline_id, **kwargs)
+    ds_id = ds_ids_by_name.get("data", {}).get(dataset_name)
+    if not ds_id:
+        raise Exception(
+            f"Could not find a dataset by the name '{dataset_name}' for the provided pipeline "
+            f"(id={pipeline_id}). Do you have a typo or the wrong pipeline?"
+        )
+    wfv_id = create_workflow_version(workflow_id, version_name, **kwargs)
+    exec_id = create_execution(wfv_id, ds_id, org_config_id, **kwargs)
+    return exec_id