cpg-utils 5.2.0__tar.gz → 5.3.0__tar.gz

{cpg-utils-5.2.0 → cpg_utils-5.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: cpg-utils
-Version: 5.2.0
+Version: 5.3.0
 Summary: Library of convenience functions specific to the CPG
 Home-page: https://github.com/populationgenomics/cpg-utils
 License: MIT
@@ -17,6 +17,26 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: boto3>=1.28.56
+Requires-Dist: botocore>=1.31.56
+Requires-Dist: cloudpathlib[all]
+Requires-Dist: frozendict
+Requires-Dist: google-auth>=1.27.0
+Requires-Dist: google-cloud-artifact-registry
+Requires-Dist: google-cloud-secret-manager
+Requires-Dist: requests
+Requires-Dist: tabulate
+Requires-Dist: toml
+Requires-Dist: deprecated
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: summary
 
 # cpg-utils
 

{cpg-utils-5.2.0 → cpg_utils-5.3.0}/cpg_utils/cloud.py

@@ -5,7 +5,9 @@ import os
 import re
 import subprocess
 import traceback
-from typing import Any
+import urllib.parse
+from collections import defaultdict
+from typing import Any, NamedTuple
 
 # pylint: disable=no-name-in-module
 import google.api_core.exceptions
@@ -26,7 +28,7 @@ from google.auth._default import (
     _SERVICE_ACCOUNT_TYPE,
 )
 from google.auth.transport import requests
-from google.cloud import secretmanager
+from google.cloud import artifactregistry, secretmanager
 from google.oauth2 import credentials as oauth2_credentials
 from google.oauth2 import service_account
 
@@ -123,6 +125,56 @@ def write_secret(project_id: str, secret_name: str, secret_value: str) -> None:
             secret_manager.disable_secret_version(request={'name': version.name})
 
 
+class DockerImage(NamedTuple):
+    name: str
+    uri: str
+    tag_uri: str
+    size: str
+    build_time: str
+
+
+_repo_image_tags: dict[str, defaultdict[str, dict[str, DockerImage]]] = {}
+
+
+def _ensure_image_tags_loaded(project: str, location: str, repository: str) -> None:
+    """Populate _repo_image_tags as a map-of-map-of-maps of 'repository' -> 'imagename' -> 'tag' -> image."""
+    if repository in _repo_image_tags:
+        return
+
+    image_tags: defaultdict[str, dict[str, DockerImage]] = defaultdict(dict)
+
+    request = artifactregistry.ListDockerImagesRequest(
+        parent=f'projects/{project}/locations/{location}/repositories/{repository}',
+        page_size=500,  # Increase efficiency by making fewer requests
+    )
+    for image in artifactregistry.ArtifactRegistryClient().list_docker_images(request):
+        name_and_checksum = image.name.rpartition('/dockerImages/')[2]
+        name = urllib.parse.unquote(name_and_checksum).rpartition('@')[0]
+        base_uri = image.uri.rpartition('@')[0]
+        for tag in image.tags:
+            image_tags[name][tag] = DockerImage(
+                image.name,
+                image.uri,
+                f'{base_uri}@{tag}',
+                image.image_size_bytes,
+                image.build_time,
+            )
+
+    image_tags.default_factory = None
+    _repo_image_tags[repository] = image_tags
+
+
+def find_image(repository: str | None, name: str, version: str) -> DockerImage:
+    """Returns image details or raises ValueError if the image or tag does not exist."""
+    repository = f'images-{repository}' if repository is not None else 'images'
+    _ensure_image_tags_loaded('cpg-common', 'australia-southeast1', repository)
+    try:
+        return _repo_image_tags[repository][name][version]
+    except KeyError as e:
+        message = f'Image {name}:{version} not found in {repository} repository ({e} not found)'
+        raise ValueError(message) from None
+
+
 def get_google_identity_token(
     target_audience: str | None,
     request: google.auth.transport.Request | None = None,

{cpg-utils-5.2.0 → cpg_utils-5.3.0}/cpg_utils/config.py

@@ -9,6 +9,7 @@ import toml
 from frozendict import frozendict
 
 from cpg_utils import Path, to_path
+from cpg_utils.cloud import find_image
 
 AR_GUID_NAME = 'ar-guid'
 
@@ -474,33 +475,52 @@ def output_path(
     )
 
 
-def image_path(key: str) -> str:
+def image_path(
+    key: str,
+    version: str | list[str] | None = None,
+    repository: str | None = None,
+) -> str:
     """
-    Returns a path to a container image using key in config's "images" section.
+    Returns a path to a container image for the given key (i.e., image name)
+    and version.
 
     Examples
     --------
-    >> image_path('bcftools')
-    'australia-southeast1-docker.pkg.dev/cpg-common/images/bcftools:1.10.2'
-
-    Assuming config structure as follows:
-
-    ```toml
-    [images]
-    bcftools = 'australia-southeast1-docker.pkg.dev/cpg-common/images/bcftools:1.10.2'
-    ```
+    >> image_path('bcftools', '1.16-1')
+    'australia-southeast1-docker.pkg.dev/cpg-common/images/bcftools:1.16-1'
 
     Parameters
     ----------
     key : str
+        Specifies the image name.
+        When `version` is not specified:
         Describes the key within the `images` config section. Can list sections
         separated with '/'.
 
+    version : str or list[str], optional
+        Specifies the desired image version, e.g., '1.18-1', either directly as
+        a version number string or indirectly via a config key list which will
+        be used to retrieve a version number string via `config_retrieve`.
+
+    repository : str, optional
+        The suffix (e.g., 'dev' for images-dev) of an artifact registry repository
+        to be used instead of the default production images repository.
+
+    Using `image_path(key)` without giving `version` is deprecated. In future,
+    specifying it will be required.
+
     Returns
     -------
     str
     """
-    return config_retrieve(['images', *key.strip('/').split('/')])
+    if version is None:
+        return config_retrieve(['images', *key.strip('/').split('/')])
+
+    if isinstance(version, list):
+        version = config_retrieve(version)
+
+    assert isinstance(version, str)
+    return find_image(repository, key, version).tag_uri
 
 
 def reference_path(key: str) -> str:

{cpg-utils-5.2.0 → cpg_utils-5.3.0}/cpg_utils/cromwell.py

@@ -708,7 +708,7 @@ def _copy_basic_file_into_batch(
     2. the output name `output`,
     3. check that the value we select is a string,
     4. either:
-        (a) gsutil cp it into `output_filename`
+        (a) gcloud storage cp it into `output_filename`
         (b) write the value into `output_filename`
     """
     output_filename = j.out
@@ -724,14 +724,14 @@ def _copy_basic_file_into_batch(
         # wrap this in quotes, because output often contains a '.', which has to be escaped in jq
         jq_el = f'"{output_name}"[{idx}]'
 
-    # activate to gsutil cp
+    # activate to gcloud storage cp
     j.image(driver_image)
     j.env('GOOGLE_APPLICATION_CREDENTIALS', '/gsa-key/key.json')
     j.command(GCLOUD_ACTIVATE_AUTH)
 
     # this has to be in bash unfortunately :(
     # we want to check that the output we get is a string
-    # if it starts with gs://, then we'll `gsutil cp` it into output_filename
+    # if it starts with gs://, then we'll `gcloud storage cp` it into output_filename
     # otherwise write the value into output_filename.
 
     # in future, add s3://* or AWS handling here
@@ -751,7 +751,7 @@ fi
 OUTPUT_VALUE=$(cat {rdict} | jq -r '.{jq_el}')
 if [[ "$OUTPUT_VALUE" == gs://* ]]; then
     echo "Copying file from $OUTPUT_VALUE";
-    gsutil cp $OUTPUT_VALUE {output_filename};
+    gcloud storage cp $OUTPUT_VALUE {output_filename};
 else
     # cleaner to directly pipe into file
     cat {rdict} | jq -r '.{jq_el}' > {output_filename}
@@ -798,13 +798,13 @@ def _copy_resource_group_into_batch(
         # wrap this in quotes, because output often contains a '.', which has to be escaped in jq
         jq_els = [f'"{output_source}"[{idx}]' for output_source in rg.values()]
 
-    # activate to gsutil cp
+    # activate to use a gcloud cp
     j.env('GOOGLE_APPLICATION_CREDENTIALS', '/gsa-key/key.json')
     j.command(GCLOUD_ACTIVATE_AUTH)
 
     # this has to be in bash unfortunately :(
     # we want to check that the output we get is a string
-    # if it starts with gs://, then we'll `gsutil cp` it into output_filename
+    # if it starts with gs://, then we'll `gcloud storage cp` it into output_filename
     # otherwise write the value into output_filename.
 
     # in future, add s3://* or AWS handling here
@@ -825,7 +825,7 @@ def _copy_resource_group_into_batch(
         OUTPUT_VALUE=$(cat {rdict} | jq -r '.{jq_el}')
         if [[ "$OUTPUT_VALUE" == gs://* ]]; then
             echo "Copying file from $OUTPUT_VALUE";
-            gsutil cp $OUTPUT_VALUE {output_filename}.{output_name};
+            gcloud storage cp $OUTPUT_VALUE {output_filename}.{output_name};
         else
             # cleaner to directly pipe into file
             cat {rdict} | jq -r '.{jq_el}' > {output_filename}.{output_name};

{cpg-utils-5.2.0 → cpg_utils-5.3.0}/cpg_utils/hail_batch.py

@@ -555,7 +555,7 @@ function retry_gs_cp {
     dst=/io/batch/${basename $src}
   fi
 
-  retry gsutil -o GSUtil:check_hashes=never cp $src $dst
+  retry gcloud storage cp $src $dst
 }
 """
 

{cpg-utils-5.2.0 → cpg_utils-5.3.0}/cpg_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: cpg-utils
-Version: 5.2.0
+Version: 5.3.0
 Summary: Library of convenience functions specific to the CPG
 Home-page: https://github.com/populationgenomics/cpg-utils
 License: MIT
@@ -17,6 +17,26 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: boto3>=1.28.56
+Requires-Dist: botocore>=1.31.56
+Requires-Dist: cloudpathlib[all]
+Requires-Dist: frozendict
+Requires-Dist: google-auth>=1.27.0
+Requires-Dist: google-cloud-artifact-registry
+Requires-Dist: google-cloud-secret-manager
+Requires-Dist: requests
+Requires-Dist: tabulate
+Requires-Dist: toml
+Requires-Dist: deprecated
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: summary
 
 # cpg-utils
 

{cpg-utils-5.2.0 → cpg_utils-5.3.0}/cpg_utils.egg-info/requires.txt

@@ -3,6 +3,7 @@ botocore>=1.31.56
 cloudpathlib[all]
 frozendict
 google-auth>=1.27.0
+google-cloud-artifact-registry
 google-cloud-secret-manager
 requests
 tabulate

{cpg-utils-5.2.0 → cpg_utils-5.3.0}/setup.py

@@ -8,7 +8,7 @@ with open('README.md') as f:
 setup(
     name='cpg-utils',
     # This tag is automatically updated by bumpversion
-    version='5.2.0',
+    version='5.3.0',
     description='Library of convenience functions specific to the CPG',
     long_description=long_description,
     long_description_content_type='text/markdown',
@@ -24,6 +24,7 @@ setup(
         'cloudpathlib[all]',
         'frozendict',
         'google-auth>=1.27.0',
+        'google-cloud-artifact-registry',
         'google-cloud-secret-manager',
         'requests',
         'tabulate',