cpg-utils 5.2.1__tar.gz → 5.3.1__tar.gz

{cpg-utils-5.2.1 → cpg_utils-5.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: cpg-utils
-Version: 5.2.1
+Version: 5.3.1
 Summary: Library of convenience functions specific to the CPG
 Home-page: https://github.com/populationgenomics/cpg-utils
 License: MIT
@@ -17,6 +17,26 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: boto3>=1.28.56
+Requires-Dist: botocore>=1.31.56
+Requires-Dist: cloudpathlib[all]
+Requires-Dist: frozendict
+Requires-Dist: google-auth>=1.27.0
+Requires-Dist: google-cloud-artifact-registry
+Requires-Dist: google-cloud-secret-manager
+Requires-Dist: requests
+Requires-Dist: tabulate
+Requires-Dist: toml
+Requires-Dist: deprecated
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: summary
 
 # cpg-utils
 

{cpg-utils-5.2.1 → cpg_utils-5.3.1}/cpg_utils/cloud.py

@@ -5,7 +5,9 @@ import os
 import re
 import subprocess
 import traceback
-from typing import Any
+import urllib.parse
+from collections import defaultdict
+from typing import Any, NamedTuple
 
 # pylint: disable=no-name-in-module
 import google.api_core.exceptions
@@ -26,7 +28,7 @@ from google.auth._default import (
     _SERVICE_ACCOUNT_TYPE,
 )
 from google.auth.transport import requests
-from google.cloud import secretmanager
+from google.cloud import artifactregistry, secretmanager
 from google.oauth2 import credentials as oauth2_credentials
 from google.oauth2 import service_account
 
@@ -123,6 +125,56 @@ def write_secret(project_id: str, secret_name: str, secret_value: str) -> None:
             secret_manager.disable_secret_version(request={'name': version.name})
 
 
+class DockerImage(NamedTuple):
+    name: str
+    uri: str
+    tag_uri: str
+    size: str
+    build_time: str
+
+
+_repo_image_tags: dict[str, defaultdict[str, dict[str, DockerImage]]] = {}
+
+
+def _ensure_image_tags_loaded(project: str, location: str, repository: str) -> None:
+    """Populate _repo_image_tags as a map-of-map-of-maps of 'repository' -> 'imagename' -> 'tag' -> image."""
+    if repository in _repo_image_tags:
+        return
+
+    image_tags: defaultdict[str, dict[str, DockerImage]] = defaultdict(dict)
+
+    request = artifactregistry.ListDockerImagesRequest(
+        parent=f'projects/{project}/locations/{location}/repositories/{repository}',
+        page_size=500,  # Increase efficiency by making fewer requests
+    )
+    for image in artifactregistry.ArtifactRegistryClient().list_docker_images(request):
+        name_and_checksum = image.name.rpartition('/dockerImages/')[2]
+        name = urllib.parse.unquote(name_and_checksum).rpartition('@')[0]
+        base_uri = image.uri.rpartition('@')[0]
+        for tag in image.tags:
+            image_tags[name][tag] = DockerImage(
+                image.name,
+                image.uri,
+                f'{base_uri}@{tag}',
+                image.image_size_bytes,
+                image.build_time,
+            )
+
+    image_tags.default_factory = None
+    _repo_image_tags[repository] = image_tags
+
+
+def find_image(repository: str | None, name: str, version: str) -> DockerImage:
+    """Returns image details or raises ValueError if the image or tag does not exist."""
+    repository = f'images-{repository}' if repository is not None else 'images'
+    _ensure_image_tags_loaded('cpg-common', 'australia-southeast1', repository)
+    try:
+        return _repo_image_tags[repository][name][version]
+    except KeyError as e:
+        message = f'Image {name}:{version} not found in {repository} repository ({e} not found)'
+        raise ValueError(message) from None
+
+
 def get_google_identity_token(
     target_audience: str | None,
     request: google.auth.transport.Request | None = None,

{cpg-utils-5.2.1 → cpg_utils-5.3.1}/cpg_utils/config.py

@@ -9,6 +9,7 @@ import toml
 from frozendict import frozendict
 
 from cpg_utils import Path, to_path
+from cpg_utils.cloud import find_image
 
 AR_GUID_NAME = 'ar-guid'
 
@@ -474,33 +475,52 @@ def output_path(
     )
 
 
-def image_path(key: str) -> str:
+def image_path(
+    key: str,
+    version: str | list[str] | None = None,
+    repository: str | None = None,
+) -> str:
     """
-    Returns a path to a container image using key in config's "images" section.
+    Returns a path to a container image for the given key (i.e., image name)
+    and version.
 
     Examples
     --------
-    >> image_path('bcftools')
-    'australia-southeast1-docker.pkg.dev/cpg-common/images/bcftools:1.10.2'
-
-    Assuming config structure as follows:
-
-    ```toml
-    [images]
-    bcftools = 'australia-southeast1-docker.pkg.dev/cpg-common/images/bcftools:1.10.2'
-    ```
+    >> image_path('bcftools', '1.16-1')
+    'australia-southeast1-docker.pkg.dev/cpg-common/images/bcftools:1.16-1'
 
     Parameters
     ----------
     key : str
+        Specifies the image name.
+        When `version` is not specified:
         Describes the key within the `images` config section. Can list sections
         separated with '/'.
 
+    version : str or list[str], optional
+        Specifies the desired image version, e.g., '1.18-1', either directly as
+        a version number string or indirectly via a config key list which will
+        be used to retrieve a version number string via `config_retrieve`.
+
+    repository : str, optional
+        The suffix (e.g., 'dev' for images-dev) of an artifact registry repository
+        to be used instead of the default production images repository.
+
+    Using `image_path(key)` without giving `version` is deprecated. In future,
+    specifying it will be required.
+
     Returns
     -------
     str
     """
-    return config_retrieve(['images', *key.strip('/').split('/')])
+    if version is None:
+        return config_retrieve(['images', *key.strip('/').split('/')])
+
+    if isinstance(version, list):
+        version = config_retrieve(version)
+
+    assert isinstance(version, str)
+    return find_image(repository, key, version).tag_uri
 
 
 def reference_path(key: str) -> str:

{cpg-utils-5.2.1 → cpg_utils-5.3.1}/cpg_utils/cromwell.py

@@ -42,7 +42,7 @@ class CromwellBackend(Enum):
     pipelines_api = 'papi'
 
 
-DEFAULT_BACKEND = CromwellBackend.pipelines_api
+DEFAULT_BACKEND = CromwellBackend.batch
 
 
 class CromwellOutputType:
@@ -643,7 +643,8 @@ def watch_workflow_and_get_output(
         array_length = output.array_length
         if array_length is None:
             # is single
-            j = b.new_job(f'{job_prefix}_collect_{output_name}')
+            j = b.new_bash_job(f'{job_prefix}_collect_{output_name}')
+            j.image(driver_image)
             if output.resource_group:
                 # is single resource group
                 out_file_map[oname] = _copy_resource_group_into_batch(
@@ -660,13 +661,13 @@ def watch_workflow_and_get_output(
                     output_name=output_name,
                     idx=None,
                     copy_file_into_batch=output.copy_file_into_batch,
-                    driver_image=driver_image,
                 )
         else:
             # is array
             outs: list[Resource] = []
             for idx in range(array_length):
-                j = b.new_job(f'{job_prefix}_collect_{output_name}[{idx}]')
+                j = b.new_bash_job(f'{job_prefix}_collect_{output_name}[{idx}]')
+                j.image(driver_image)
                 if output.resource_group:
                     # is array output group
                     outs.append(
@@ -685,7 +686,6 @@ def watch_workflow_and_get_output(
                             output_name=output_name,
                             idx=idx,
                             copy_file_into_batch=output.copy_file_into_batch,
-                            driver_image=driver_image,
                         ),
                     )
 
@@ -701,7 +701,6 @@ def _copy_basic_file_into_batch(
     output_name: str,
     idx: int | None,
     copy_file_into_batch: bool,
-    driver_image: str,
 ) -> Resource:
     """
     1. Take the file-pointer to the dictionary `rdict`,
@@ -725,7 +724,6 @@ def _copy_basic_file_into_batch(
         jq_el = f'"{output_name}"[{idx}]'
 
     # activate to gcloud storage cp
-    j.image(driver_image)
     j.env('GOOGLE_APPLICATION_CREDENTIALS', '/gsa-key/key.json')
     j.command(GCLOUD_ACTIVATE_AUTH)
 

{cpg-utils-5.2.1 → cpg_utils-5.3.1}/cpg_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: cpg-utils
-Version: 5.2.1
+Version: 5.3.1
 Summary: Library of convenience functions specific to the CPG
 Home-page: https://github.com/populationgenomics/cpg-utils
 License: MIT
@@ -17,6 +17,26 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: boto3>=1.28.56
+Requires-Dist: botocore>=1.31.56
+Requires-Dist: cloudpathlib[all]
+Requires-Dist: frozendict
+Requires-Dist: google-auth>=1.27.0
+Requires-Dist: google-cloud-artifact-registry
+Requires-Dist: google-cloud-secret-manager
+Requires-Dist: requests
+Requires-Dist: tabulate
+Requires-Dist: toml
+Requires-Dist: deprecated
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: summary
 
 # cpg-utils
 

{cpg-utils-5.2.1 → cpg_utils-5.3.1}/cpg_utils.egg-info/requires.txt

@@ -3,6 +3,7 @@ botocore>=1.31.56
 cloudpathlib[all]
 frozendict
 google-auth>=1.27.0
+google-cloud-artifact-registry
 google-cloud-secret-manager
 requests
 tabulate

{cpg-utils-5.2.1 → cpg_utils-5.3.1}/setup.py

@@ -8,7 +8,7 @@ with open('README.md') as f:
 setup(
     name='cpg-utils',
     # This tag is automatically updated by bumpversion
-    version='5.2.1',
+    version='5.3.1',
     description='Library of convenience functions specific to the CPG',
     long_description=long_description,
     long_description_content_type='text/markdown',
@@ -24,6 +24,7 @@ setup(
         'cloudpathlib[all]',
         'frozendict',
         'google-auth>=1.27.0',
+        'google-cloud-artifact-registry',
         'google-cloud-secret-manager',
         'requests',
         'tabulate',