cpg-utils 5.7.0__tar.gz → 5.7.2__tar.gz

{cpg_utils-5.7.0 → cpg_utils-5.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cpg-utils
-Version: 5.7.0
+Version: 5.7.2
 Summary: Library of convenience functions specific to the CPG
 Home-page: https://github.com/populationgenomics/cpg-utils
 License: MIT

{cpg_utils-5.7.0 → cpg_utils-5.7.2}/cpg_utils/config.py

@@ -570,3 +570,18 @@ def get_gcloud_set_project(gcp_project: str | None = None) -> str:
     gcp_project = gcp_project or get_gcp_project()
     command = ['gcloud', 'config', 'set', 'project', gcp_project]
     return ' '.join([quote(c) for c in command])
+
+
+def dataset_for_access_level(dataset_name: str) -> str:
+    """
+    Checks access_level in config, modifying dataset with "-test" suffix if appropriate.
+
+    >>> dataset_for_access_level('dataset')  # with access_level != 'test'
+    'dataset'
+    >>> dataset_for_access_level('dataset')  # with access_level == 'test'
+    'dataset-test'
+    """
+    access_level = config_retrieve(['workflow', 'access_level'], 'standard')
+    if access_level == 'test' and not dataset_name.endswith('-test'):
+        return f'{dataset_name}-test'
+    return dataset_name

{cpg_utils-5.7.0 → cpg_utils-5.7.2}/cpg_utils/cromwell.py

@@ -2,12 +2,14 @@
 """
 Cromwell module contains helper code for submitting + watching
 jobs from within Hail batch.
+
+https://support.terra.bio/hc/en-us/articles/31190930435483-Cromwell-on-Google-Batch-API-released-May-19-Generally-Available-as-of-June-23
+- As of June 23, 2025, Cromwell only allows users to run workflows on Google Batch API.
 """
 
 import json
 import os
 import subprocess
-from enum import Enum
 from shlex import quote
 from typing import Any
 
@@ -36,15 +38,6 @@ from cpg_utils.git import (
 from cpg_utils.hail_batch import prepare_git_job, query_command
 
 
-class CromwellBackend(Enum):
-    # the string value here, is the key of the backend in our cromwell.conf
-    batch = 'batch'
-    pipelines_api = 'papi'
-
-
-DEFAULT_BACKEND = CromwellBackend.batch
-
-
 class CromwellOutputType:
     """Declares output type for cromwell -> hail batch glue"""
 
@@ -164,7 +157,6 @@ def run_cromwell_workflow(  # noqa: C901
     project: str | None = None,
     copy_outputs_to_gcp: bool = True,
     ar_guid_override: str | None = None,
-    backend: CromwellBackend = DEFAULT_BACKEND,
 ):
     """
     Run a cromwell workflow, and return a Batch.ResourceFile
@@ -229,26 +221,17 @@ def run_cromwell_workflow(  # noqa: C901
     )
 
     workflow_options = {
-        # michael configured these manually in config to match the enum values
-        'backend': backend.value,
+        'backend': 'batch',
         # pass the user-service-account-json to cromwell to submit jobs as this user
         'user_service_account_json': service_account_json,
         'google_compute_service_account': service_account_email,
         'google_project': _project,
-        # other config options that are useful
         'google_labels': google_labels,
         'final_call_logs_dir': logging_output_dir,
         'final_workflow_log_dir': logging_output_dir,
+        'gcp_batch_gcs_root': intermediate_dir,
     }
 
-    if backend == CromwellBackend.pipelines_api:
-        workflow_options['jes_gcs_root'] = intermediate_dir
-
-    if backend == CromwellBackend.batch:
-        # cromwell: /supportedBackends/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchWorkflowPaths.scala#L20
-        # this was undocumented at the time of writing
-        workflow_options['gcp_batch_gcs_root'] = intermediate_dir
-
     # if required, export the workflow outputs to GCS
     if copy_outputs_to_gcp:
         workflow_options['final_workflow_outputs_dir'] = workflow_output_dir
@@ -310,7 +293,6 @@ def run_cromwell_workflow_from_repo_and_get_outputs(
     min_watch_poll_interval: int = 5,
     max_watch_poll_interval: int = 60,
     time_limit_seconds: int | None = None,
-    backend: CromwellBackend = DEFAULT_BACKEND,
 ) -> tuple[Job, dict[str, Resource | list[Resource]]]:
     """
     This function needs to know the structure of the outputs you
@@ -363,7 +345,6 @@ def run_cromwell_workflow_from_repo_and_get_outputs(
         labels=labels,
         project=project,
         copy_outputs_to_gcp=copy_outputs_to_gcp,
-        backend=backend,
     )
 
     outputs_dict = watch_workflow_and_get_output(

{cpg_utils-5.7.0 → cpg_utils-5.7.2}/cpg_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cpg-utils
-Version: 5.7.0
+Version: 5.7.2
 Summary: Library of convenience functions specific to the CPG
 Home-page: https://github.com/populationgenomics/cpg-utils
 License: MIT

{cpg_utils-5.7.0 → cpg_utils-5.7.2}/setup.py

@@ -8,7 +8,7 @@ with open('README.md') as f:
 setup(
     name='cpg-utils',
     # This tag is automatically updated by bumpversion
-    version='5.7.0',
+    version='5.7.2',
     description='Library of convenience functions specific to the CPG',
     long_description=long_description,
     long_description_content_type='text/markdown',

{cpg_utils-5.7.0 → cpg_utils-5.7.2}/test/test_cromwell.py

@@ -2,93 +2,12 @@ import json
 import unittest
 from unittest.mock import MagicMock, patch
 
-from cpg_utils.cromwell import CromwellBackend, run_cromwell_workflow
+from cpg_utils.cromwell import run_cromwell_workflow
 from cpg_utils.cromwell_model import WorkflowMetadataModel
 
 
 class TestCromwellWrapper(unittest.TestCase):
 
-    @patch('cpg_utils.config.get_config')
-    @patch('cpg_utils.cromwell.read_secret')
-    def test_workflow_options_papi(
-        self,
-        mock_read_secret: MagicMock,
-        mock_get_config: MagicMock,
-    ):
-        """
-        Test that the workflow options are generated correctly for the PAPI backend
-
-        Args:
-            mock_read_secret (MagicMock): _description_
-            mock_get_config (MagicMock): _description_
-        """
-
-        dataset = 'test-dataset'
-        dataset_buckets = {
-            'default': 'test://default-bucket',
-            'analysis': 'test://analysis-bucket',
-            'tmp': 'test://tmp-bucket',
-        }
-
-        mock_get_config.return_value = {
-            'workflow': {
-                'ar-guid': '<test-ar-guid>',
-                'access_level': 'test',
-                'dataset_gcp_project': 'test-gcp-project',
-            },
-            'storage': {
-                'default': dataset_buckets,
-                dataset: dataset_buckets,
-            },
-        }
-
-        mock_read_secret.return_value = '{"client_email": "<secret-client-email>"}'
-
-        job: MagicMock = MagicMock()
-
-        _ = run_cromwell_workflow(
-            job=job,
-            dataset=dataset,
-            access_level='test',
-            workflow='test-workflow',
-            cwd=None,
-            libs=[],
-            output_prefix='output-prefix',
-            backend=CromwellBackend.pipelines_api,
-        )
-
-        mock_read_secret.assert_called_once_with(
-            'test-gcp-project',
-            f'{dataset}-cromwell-test-key',
-        )
-
-        # no inputs, hence there will only be one call to job.command
-
-        args, _ = job.command.call_args
-        command: str = args[0].strip()
-
-        # echo '{json.dumps(workflow_options)}' > workflow-options.json
-        wf_options_line = command.splitlines()[1]
-        wf_options_str = wf_options_line.split("'")[1]
-        wf_options = json.loads(wf_options_str)
-
-        self.assertEqual(
-            wf_options['jes_gcs_root'],
-            'test://tmp-bucket/cromwell',
-        )
-        self.assertEqual(
-            wf_options['final_call_logs_dir'],
-            'test://analysis-bucket/cromwell_logs/output-prefix',
-        )
-        self.assertEqual(
-            wf_options['final_workflow_log_dir'],
-            'test://analysis-bucket/cromwell_logs/output-prefix',
-        )
-        self.assertEqual(
-            wf_options['final_workflow_outputs_dir'],
-            'test://default-bucket/output-prefix',
-        )
-
     @patch('cpg_utils.config.get_config')
     @patch('cpg_utils.cromwell.read_secret')
     def test_workflow_options_backend_batch(
@@ -135,7 +54,6 @@ class TestCromwellWrapper(unittest.TestCase):
             cwd=None,
             libs=[],
             output_prefix='output-prefix',
-            backend=CromwellBackend.batch,
         )
 
         mock_read_secret.assert_called_once_with(