konduktor-nightly 0.1.0.dev20250327104656__py3-none-any.whl → 0.1.0.dev20250328104606__py3-none-any.whl

konduktor/__init__.py

@@ -14,7 +14,8 @@ __all__ = [
 ]
 
 # Replaced with the current commit when building the wheels.
-_KONDUKTOR_COMMIT_SHA = 'ac5a9936d3c1b6a76bffd3f660d68ac80634bda5'
+_KONDUKTOR_COMMIT_SHA = '139a1d21db826f4beb908a929f2a02c514e1adef'
+os.makedirs(os.path.expanduser('~/.konduktor'), exist_ok=True)
 
 
 def _get_git_commit():
@@ -47,5 +48,5 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev0.1.0.dev20250327104656'
+__version__ = '1.0.0.dev0.1.0.dev20250328104606'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))

konduktor/adaptors/gcp.py

@@ -12,13 +12,16 @@
 
 """GCP cloud adaptors"""
 
-# pylint: disable=import-outside-toplevel
 import json
+import os
+from functools import wraps
+
+from filelock import FileLock
 
 from konduktor.adaptors import common
 
 _IMPORT_ERROR_MESSAGE = (
-    'Failed to import dependencies for GCP. ' 'Try pip install "skypilot[gcp]"'
+    'Failed to import dependencies for GCP. ' 'Try pip install "konduktor[gcp]"'
 )
 googleapiclient = common.LazyImport(
     'googleapiclient', import_error_message=_IMPORT_ERROR_MESSAGE
@@ -27,6 +30,34 @@ google = common.LazyImport('google', import_error_message=_IMPORT_ERROR_MESSAGE)
 _LAZY_MODULES = (google, googleapiclient)
 
 
+class LockedClientProxy:
+    """Proxy for GCP client that locks access to the client."""
+
+    def __init__(
+        self,
+        client,
+        lock_path=os.path.expanduser('~/.konduktor/gcs_storage.lock'),
+        timeout=10,
+    ):
+        self._client = client
+        self._lock = FileLock(lock_path, timeout=timeout)
+
+    def __getattr__(self, attr):
+        target = getattr(self._client, attr)
+
+        if callable(target):
+
+            @wraps(target)
+            def locked_method(*args, **kwargs):
+                with self._lock:
+                    return target(*args, **kwargs)
+
+            return locked_method
+        else:
+            # Attribute (not method) access just passes through
+            return target
+
+
 @common.load_lazy_modules(_LAZY_MODULES)
 def build(service_name: str, version: str, *args, **kwargs):
     """Build a GCP service.
@@ -44,7 +75,7 @@ def storage_client():
     """Helper that connects to GCS Storage Client for GCS Bucket"""
     from google.cloud import storage
 
-    return storage.Client()
+    return LockedClientProxy(storage.Client())
 
 
 @common.load_lazy_modules(_LAZY_MODULES)
@@ -52,7 +83,7 @@ def anonymous_storage_client():
     """Helper that connects to GCS Storage Client for Public GCS Buckets"""
     from google.cloud import storage
 
-    return storage.Client.create_anonymous_client()
+    return LockedClientProxy(storage.Client.create_anonymous_client())
 
 
 @common.load_lazy_modules(_LAZY_MODULES)
@@ -71,6 +102,14 @@ def forbidden_exception():
     return gcs_exceptions.Forbidden
 
 
+@common.load_lazy_modules(_LAZY_MODULES)
+def conflict_exception():
+    """Conflict exception."""
+    from google.api_core import exceptions as gcs_exceptions
+
+    return gcs_exceptions.Conflict
+
+
 @common.load_lazy_modules(_LAZY_MODULES)
 def http_error_exception():
     """HttpError exception."""

konduktor/backends/jobset_utils.py

@@ -12,7 +12,8 @@ from urllib.parse import urlparse
 import colorama
 
 import konduktor
-from konduktor import cloud_stores, constants, kube_client, logging
+from konduktor import constants, kube_client, logging
+from konduktor.data import registry
 from konduktor.utils import common_utils, kubernetes_utils, log_utils
 
 if typing.TYPE_CHECKING:
@@ -102,7 +103,7 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
         if '/tmp/konduktor-job-filemounts-files' in dst:
             continue
         # should impelement a method here instead of raw dog dict access
-        cloud_store = cloud_stores._REGISTRY[store_scheme]
+        cloud_store = registry._REGISTRY[store_scheme]
         storage_secrets[store_scheme] = cloud_store._STORE.get_k8s_credential_name()
         mkdir_commands.append(
             f'cd {constants.KONDUKTOR_REMOTE_WORKDIR};' f'mkdir -p {dst}'
@@ -118,7 +119,7 @@ def create_pod_spec(task: 'konduktor.Task') -> Dict[str, Any]:
     assert task.file_mounts is not None
     for dst, src in task.file_mounts.items():
         store_scheme = str(urlparse(store.source).scheme)
-        cloud_store = cloud_stores._REGISTRY[store_scheme]
+        cloud_store = registry._REGISTRY[store_scheme]
         mkdir_commands.append(
             f'cd {constants.KONDUKTOR_REMOTE_WORKDIR};'
             f'mkdir -p {os.path.dirname(dst)}'

konduktor/check.py

@@ -34,12 +34,13 @@ from typing import Iterable, List, Optional, Tuple
 import click
 import colorama
 
-from konduktor import cloud_stores, logging
 from konduktor import config as konduktor_config
+from konduktor import logging
+from konduktor.data import registry
 from konduktor.utils import rich_utils
 
 if typing.TYPE_CHECKING:
-    pass
+    from konduktor.data import storage_utils
 
 logger = logging.get_logger(__name__)
 
@@ -58,7 +59,7 @@ def check(
     disabled_clouds = []
 
     def check_one_cloud(
-        cloud_tuple: Tuple[str, 'cloud_stores.CloudStorage'],
+        cloud_tuple: Tuple[str, 'storage_utils.CloudStorage'],
     ) -> None:
         cloud_repr, cloud = cloud_tuple
         with rich_utils.safe_status(f'Checking {cloud_repr}...'):
@@ -80,15 +81,15 @@ def check(
             disabled_clouds.append(cloud_repr)
             echo(f'    Reason: {reason}')
 
-    def get_cloud_tuple(cloud_name: str) -> Tuple[str, 'cloud_stores.CloudStorage']:
+    def get_cloud_tuple(cloud_name: str) -> Tuple[str, 'storage_utils.CloudStorage']:
         # Validates cloud_name and returns a tuple of the cloud's name and
         # the cloud object. Includes special handling for Cloudflare.
-        cloud_obj = cloud_stores._REGISTRY.get(cloud_name, None)
+        cloud_obj = registry._REGISTRY.get(cloud_name, None)
         assert cloud_obj is not None, f'Cloud {cloud_name!r} not found'
         return cloud_name, cloud_obj
 
     def get_all_clouds():
-        return tuple([c for c in cloud_stores._REGISTRY.keys()])
+        return tuple([c for c in registry._REGISTRY.keys()])
 
     if clouds is not None:
         cloud_list = clouds
@@ -158,7 +159,7 @@ def check(
             + '\n'
             + click.style(
                 'If any problems remain, refer to detailed docs at: '
-                'https://konduktor.readthedocs.io/en/latest/admin/installation.html',  # pylint: disable=line-too-long
+                'https://trainy.mintlify.app',  # pylint: disable=line-too-long
                 dim=True,
             )
         )
@@ -179,14 +180,5 @@ def check(
     return enabled_clouds
 
 
-# === Helper functions ===
-def storage_in_iterable(
-    cloud: 'cloud_stores.GcsCloudStorage',
-    cloud_list: Iterable['cloud_stores.GcsCloudStorage'],
-) -> bool:
-    """Returns whether the cloud is in the given cloud list."""
-    return any(cloud == c for c in cloud_list)
-
-
 def _format_enabled_storage(cloud_name: str) -> str:
     return f'{colorama.Fore.GREEN}{cloud_name}{colorama.Style.RESET_ALL}'

konduktor/data/constants.py

@@ -4,9 +4,4 @@ Path = str
 SourceType = Union[Path]
 StorageHandle = Any
 
-# TODO(asaiacai) This should match the cloud store
-# classes in cloud_stores.py,
-# should honestly just use one or the other instead of both
-STORE_ENABLED_CLOUDS = ['gs']
-
 _STORAGE_LOG_FILE_NAME = 'storage.log'

konduktor/data/gcp/__init__.py

@@ -1,18 +1,18 @@
 """Data sync between workstation <--> blob (s3, gcs, etc.) <--> worker pods"""
 
 from konduktor.data.gcp.constants import (
-    DEFAULT_SERVICE_ACCOUNT_ROLES,
     STORAGE_MINIMAL_PERMISSIONS,
 )
 from konduktor.data.gcp.gcs import (
     DEFAULT_GCP_APPLICATION_CREDENTIAL_PATH,
     GOOGLE_SDK_INSTALLATION_COMMAND,
+    GcsCloudStorage,
     GcsStore,
 )
 
 __all__ = [
     'GcsStore',
-    'DEFAULT_SERVICE_ACCOUNT_ROLES',
+    'GcsCloudStorage',
     'STORAGE_MINIMAL_PERMISSIONS',
     'GOOGLE_SDK_INSTALLATION_COMMAND',
     'DEFAULT_GCP_APPLICATION_CREDENTIAL_PATH',

konduktor/data/gcp/gcs.py

@@ -578,6 +578,10 @@ class GcsStore(storage_utils.AbstractStore):
             bucket = self.client.bucket(bucket_name)
             bucket.storage_class = 'STANDARD'
             new_bucket = self.client.create_bucket(bucket, location=region)
+        except gcp.conflict_exception():
+            # it's fine to pass this exception since
+            # this means the bucket already exists
+            pass
         except Exception as e:  # pylint: disable=broad-except
             with ux_utils.print_exception_no_traceback():
                 raise exceptions.StorageBucketCreateError(
@@ -904,3 +908,79 @@ class GcsStore(storage_utils.AbstractStore):
     @classmethod
     def get_k8s_credential_name(cls) -> str:
         return cls._GCP_SECRET_NAME
+
+
+class GcsCloudStorage(storage_utils.CloudStorage):
+    """Google Cloud Storage."""
+
+    # We use gsutil as a basic implementation.  One pro is that its -m
+    # multi-threaded download is nice, which frees us from implementing
+    # parellel workers on our end.
+    # The gsutil command is part of the Google Cloud SDK, and we reuse
+    # the installation logic here.
+    _INSTALL_GSUTIL = GOOGLE_SDK_INSTALLATION_COMMAND
+    _STORE: typing.Type[storage_utils.AbstractStore] = GcsStore
+
+    @property
+    def _gsutil_command(self):
+        gsutil_alias, alias_gen = data_utils.get_gsutil_command()
+        return (
+            f'{alias_gen}; GOOGLE_APPLICATION_CREDENTIALS='
+            f'{DEFAULT_GCP_APPLICATION_CREDENTIAL_PATH}; '
+            # Explicitly activate service account. Unlike the gcp packages
+            # and other GCP commands, gsutil does not automatically pick up
+            # the default credential keys when it is a service account.
+            'gcloud auth activate-service-account '
+            '--key-file=$GOOGLE_APPLICATION_CREDENTIALS '
+            '2> /dev/null || true; '
+            f'{gsutil_alias}'
+        )
+
+    def is_directory(self, url: str) -> bool:
+        """Returns whether 'url' is a directory.
+        In cloud object stores, a "directory" refers to a regular object whose
+        name is a prefix of other objects.
+        """
+        commands = [self._INSTALL_GSUTIL]
+        commands.append(f'{self._gsutil_command} ls -d {url}')
+        command = ' && '.join(commands)
+        p = subprocess.run(
+            command,
+            stdout=subprocess.PIPE,
+            shell=True,
+            check=True,
+            executable='/bin/bash',
+        )
+        out = p.stdout.decode().strip()
+        # Edge Case: Gcloud command is run for first time #437
+        out = out.split('\n')[-1]
+        # If <url> is a bucket root, then we only need `gsutil` to succeed
+        # to make sure the bucket exists. It is already a directory.
+        _, key = data_utils.split_gcs_path(url)
+        if not key:
+            return True
+        # Otherwise, gsutil ls -d url will return:
+        #   --> url.rstrip('/')          if url is not a directory
+        #   --> url with an ending '/'   if url is a directory
+        if not out.endswith('/'):
+            assert out == url.rstrip('/'), (out, url)
+            return False
+        url = url if url.endswith('/') else (url + '/')
+        assert out == url, (out, url)
+        return True
+
+    def make_sync_dir_command(self, source: str, destination: str) -> str:
+        """Downloads a directory using gsutil."""
+        download_via_gsutil = (
+            f'{self._gsutil_command} ' f'rsync -e -r {source} {destination}'
+        )
+        all_commands = [self._INSTALL_GSUTIL]
+        all_commands.append(download_via_gsutil)
+        return ' && '.join(all_commands)
+
+    def make_sync_file_command(self, source: str, destination: str) -> str:
+        """Downloads a file using gsutil."""
+        download_via_gsutil = f'{self._gsutil_command} ' f'cp {source} {destination}'
+        all_commands = [self._INSTALL_GSUTIL]
+        all_commands.append(download_via_gsutil)
+        return ' && '.join(all_commands)

konduktor/data/registry.py

@@ -0,0 +1,18 @@
+from konduktor.data.gcp import GcsCloudStorage
+
+# Maps bucket's URIs prefix(scheme) to its corresponding storage class
+
+_REGISTRY = {
+    'gs': GcsCloudStorage(),
+    # TODO(asaiacai): Add other cloud stores here
+    # 's3': S3CloudStorage(),
+    # 'r2': R2CloudStorage(),
+    # 'cos': IBMCosCloudStorage(),
+    # 'oci': OciCloudStorage(),
+    # # TODO: This is a hack, as Azure URL starts with https://, we should
+    # # refactor the registry to be able to take regex, so that Azure blob can
+    # # be identified with `https://(.*?)\.blob\.core\.windows\.net`
+    # 'https': AzureBlobCloudStorage()
+}
+
+_STORE_ENABLED_CLOUDS = list(_REGISTRY.keys())

konduktor/data/storage.py

@@ -28,7 +28,7 @@ import urllib.parse
 from typing import Any, Dict, List, Literal, Optional, Tuple, Type, Union
 
 from konduktor import check, config, logging
-from konduktor.data import constants, data_utils, gcp, storage_utils
+from konduktor.data import constants, data_utils, gcp, registry, storage_utils
 from konduktor.utils import annotations, common_utils, exceptions, schemas, ux_utils
 
 logger = logging.get_logger(__file__)
@@ -40,13 +40,13 @@ def get_cached_enabled_storage_clouds_or_refresh(
 ) -> List[str]:
     # This is a temporary solution until https://github.com/skypilot-org/skypilot/issues/1943 # noqa: E501
     # (asaiacai): This function does not do any actual checking right now.
-    # this is temporary.In the future, we can cache to disk.
+    # this is temporary. In the future, we can cache to disk.
     # For now, we just print a warning to the user saying what
     # clouds are enabled and if the task fails to run `konduktor check`
     # to update the credentials.
     enabled_clouds = config.get_nested(('allowed_clouds',), [])
     if len(enabled_clouds) == 0:
-        enabled_clouds = constants.STORE_ENABLED_CLOUDS
+        enabled_clouds = registry._STORE_ENABLED_CLOUDS
     else:
         enabled_clouds = [str(cloud) for cloud in enabled_clouds]
     logger.warning(

konduktor/data/storage_utils.py

@@ -498,3 +498,38 @@ class AbstractStore:
     def get_k8s_credential_name(cls) -> str:
         """Returns the name of the k8s secret storing the credentials for the store."""
         raise NotImplementedError
+
+
+class CloudStorage:
+    """Interface for a cloud object store."""
+
+    # this needs to be overridden by the subclass
+    _STORE: typing.Type[AbstractStore]
+
+    def is_directory(self, url: str) -> bool:
+        """Returns whether 'url' is a directory.
+
+        In cloud object stores, a "directory" refers to a regular object whose
+        name is a prefix of other objects.
+        """
+        raise NotImplementedError
+
+    def make_sync_dir_command(self, source: str, destination: str) -> str:
+        """Makes a runnable bash command to sync a 'directory'."""
+        raise NotImplementedError
+
+    def make_sync_file_command(self, source: str, destination: str) -> str:
+        """Makes a runnable bash command to sync a file."""
+        raise NotImplementedError
+
+    def check_credentials(self):
+        """Checks if the user has access credentials to this cloud."""
+        return self._STORE.check_credentials()
+
+    def check_credentials_from_secret(self):
+        """Checks if the user has access credentials to this cloud."""
+        return self._STORE.check_credentials_from_secret()
+
+    def set_secret_credentials(self):
+        """Set the credentials from the secret"""
+        return self._STORE.set_secret_credentials()

konduktor/execution.py

@@ -13,8 +13,8 @@ if typing.TYPE_CHECKING:
 from konduktor import config, constants
 from konduktor import logging as konduktor_logging
 from konduktor.backends import JobsetBackend
-from konduktor.data import constants as storage_constants
 from konduktor.data import data_utils
+from konduktor.data import registry as storage_registry
 from konduktor.data import storage as storage_lib
 from konduktor.utils import common_utils, exceptions, rich_utils, ux_utils
 
@@ -345,7 +345,7 @@ def maybe_translate_local_file_mounts_and_sync_up(
                         data_src += ' and workdir'
                     else:
                         data_src = 'workdir'
-                store_enabled_clouds = ', '.join(storage_constants.STORE_ENABLED_CLOUDS)
+                store_enabled_clouds = ', '.join(storage_registry._STORE_ENABLED_CLOUDS)
                 with ux_utils.print_exception_no_traceback():
                     raise exceptions.NotSupportedError(
                         f'Unable to use {data_src} - no cloud with object '

konduktor/kube_client.py

@@ -18,7 +18,7 @@ DEFAULT_IN_CLUSTER_REGION = 'in-cluster'
 # for Kubernetes clusters. This is used to associate a name with the current
 # context when running with in-cluster auth. If not set, the context name is
 # set to DEFAULT_IN_CLUSTER_REGION.
-IN_CLUSTER_CONTEXT_NAME_ENV_VAR = 'SKYPILOT_IN_CLUSTER_CONTEXT_NAME'
+IN_CLUSTER_CONTEXT_NAME_ENV_VAR = 'KONDUKTOR_IN_CLUSTER_CONTEXT_NAME'
 
 
 def _decorate_methods(obj: Any, decorator: Callable, decoration_type: str):

konduktor/templates/pod.yaml.j2

@@ -85,7 +85,7 @@ kubernetes:
               PACKAGES="$PACKAGES git";
               {% endif %}
 
-              if [ -z "${PACKAGES}" ]; then
+              if [ ! -z "${PACKAGES}" ]; then
                 # Run apt update, install missing packages
                 DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get update > ~/.konduktor/tmp/apt-update.log 2>&1 || \
                 $(prefix_cmd) echo "Warning: apt-get update failed. Continuing anyway..." >> ~/.konduktor/tmp/apt-update.log

konduktor/utils/schemas.py

@@ -454,7 +454,8 @@ _REMOTE_IDENTITY_SCHEMA_KUBERNETES = {
 
 def get_storage_schema():
     # pylint: disable=import-outside-toplevel
-    from konduktor import cloud_stores
+    from knoduktor.registry import registry
+
     from konduktor.data import storage
 
     return {
@@ -476,7 +477,7 @@ def get_storage_schema():
             },
             'store': {
                 'type': 'string',
-                'case_insensitive_enum': [type for type in cloud_stores._REGISTRY],
+                'case_insensitive_enum': [type for type in registry._REGISTRY],
             },
             'persistent': {
                 'type': 'boolean',
@@ -497,7 +498,7 @@ def get_storage_schema():
 
 def get_config_schema():
     # pylint: disable=import-outside-toplevel
-    from konduktor import cloud_stores
+    from konduktor.data import registry
     from konduktor.utils import kubernetes_enums
 
     cloud_configs = {
@@ -547,7 +548,7 @@ def get_config_schema():
         'required': ['items'],
         'items': {
             'type': 'string',
-            'case_insensitive_enum': (list(cloud_stores._REGISTRY.keys())),
+            'case_insensitive_enum': (list(registry._REGISTRY.keys())),
         },
     }
 

{konduktor_nightly-0.1.0.dev20250327104656.dist-info → konduktor_nightly-0.1.0.dev20250328104606.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: konduktor-nightly
-Version: 0.1.0.dev20250327104656
+Version: 0.1.0.dev20250328104606
 Summary: GPU Cluster Health Management
 Author: Andrew Aikawa
 Author-email: asai@berkeley.edu
@@ -12,6 +12,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: click (>=8.1.7,<9.0.0)
 Requires-Dist: colorama (>=0.4.6,<0.5.0)
+Requires-Dist: filelock (>=3.18.0,<4.0.0)
 Requires-Dist: google-api-python-client[gcp] (>=2.161.0,<3.0.0)
 Requires-Dist: google-cloud-storage[gcp] (>=3.0.0,<4.0.0)
 Requires-Dist: jinja2 (>=3.1.5,<4.0.0)

{konduktor_nightly-0.1.0.dev20250327104656.dist-info → konduktor_nightly-0.1.0.dev20250328104606.dist-info}/RECORD

@@ -1,14 +1,13 @@
-konduktor/__init__.py,sha256=CWG3yvw6TSNGuQHtxiuStEVcTZ-0X7So3cCPi0XyJks,1477
+konduktor/__init__.py,sha256=2U2pj_Uu1x76859aRUOI6UapnthXIO_rYCUUhV4K8oQ,1540
 konduktor/adaptors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/adaptors/common.py,sha256=mYb_6c3u5MghtiFfiW5OO-EH6t7cIR5npbkgUmz6FYE,3517
-konduktor/adaptors/gcp.py,sha256=liCm4_D_qSci0DZA2t5bckLIoGDkJ8qx31EO_hSBzo0,3751
+konduktor/adaptors/gcp.py,sha256=pOQA2q8fFyr97Htn8EqvNM0XT-Ao8UwvExviiLaDats,4746
 konduktor/backends/__init__.py,sha256=1Q6sqqdeMYarpTX_U-QVywJYf7idiUTRsyP-E4BQSOw,129
 konduktor/backends/backend.py,sha256=qh0bp94lzoTYZkzyQv2-CVrB5l91FkG2vclXg24UFC0,2910
 konduktor/backends/jobset.py,sha256=lh_PihQgM0tmVryCpjSsZjWug8hBnJr7ua9lqk0qEAM,8251
-konduktor/backends/jobset_utils.py,sha256=1LOiP-UVtezbLP0jbtQR7pwBQH3B5b5sqptVVzri-48,17222
-konduktor/check.py,sha256=hIrxDMKaGX2eZP-Pj9TCymGUHQAp93m48Gj3XMiqadA,7833
+konduktor/backends/jobset_utils.py,sha256=5xeFPNN724SX6M4p4Koump5yCcdRJdiSUs03TexSwMQ,17236
+konduktor/check.py,sha256=JennyWoaqSKhdyfUldd266KwVXTPJpcYQa4EED4a_BA,7569
 konduktor/cli.py,sha256=90bnh3nIobfBkzqS_SXgw9Z8Zqh4ouwpLDj0kx_6kL8,23562
-konduktor/cloud_stores.py,sha256=KX3u5YlXGslMCe_q8zYtFy62_KGCmmLTrYuK7Y9jFIM,6277
 konduktor/config.py,sha256=J50JxC6MsXMnlrJPXdDUMr38C89xvOO7mR8KJ6fyils,15520
 konduktor/constants.py,sha256=T3AeXXxuQHINW_bAWyztvDeS8r4g8kXBGIwIq13cys0,1814
 konduktor/controller/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -49,16 +48,17 @@ konduktor/dashboard/frontend/postcss.config.mjs,sha256=rDHiqV72T-J860Ek4QFnUnMQe
 konduktor/dashboard/frontend/server.js,sha256=jcp6_Ww9YJD3uKY07jR3KMlAM6n1QZdxZnVY6Kh-J6A,1789
 konduktor/dashboard/frontend/tailwind.config.js,sha256=fCnc48wvioIDOe5ldQ_6RE7F76cP7aU7pDrxBPJx-Fk,366
 konduktor/data/__init__.py,sha256=KMR2i3E9YcIpiIuCxtRdS7BQ1w2vUAbbve7agziJrLo,213
-konduktor/data/constants.py,sha256=tAYHrmzPCI2lwK_Fy7AfwE1RTcb38AYW99Zkw8fEtDM,299
+konduktor/data/constants.py,sha256=yXVEoTI2we1xOjVSU-bjRCQCLpVvpEvJ0GedXvSwEfw,127
 konduktor/data/data_utils.py,sha256=aIv3q2H1GSiN2w8WNjZgVaglm-hoiHSb4KR-MAiKKXs,8383
-konduktor/data/gcp/__init__.py,sha256=m40OpIx2UDGnREokqfQ52OsAfo9WXC748hF98YWyG-A,517
+konduktor/data/gcp/__init__.py,sha256=rlQxACBC_Vu36mdgPyJgUy4mGc_6Nt_a96JAuaPz2pQ,489
 konduktor/data/gcp/constants.py,sha256=dMfOiFccM8O6rUi9kClJcbvw1K1VnS1JzzQk3apq8ho,1483
-konduktor/data/gcp/gcs.py,sha256=wJri7wG7FJBQvEkAqWQsNHJMLWqMH2n10d8vyy0M_5o,38650
+konduktor/data/gcp/gcs.py,sha256=kDbUzf8ALYzsw_G3sBRn_enQ8fjI-UKV0jeWuFZiULA,42018
 konduktor/data/gcp/utils.py,sha256=FJQcMXZqtMIzjZ98b3lTTc0UbdPUKTDLsOsfJaaH5-s,214
-konduktor/data/storage.py,sha256=KppuO1bYSyMA7RSkmBbJnw8xwBPjKYQ8IbtxYv3abz4,34731
-konduktor/data/storage_utils.py,sha256=v_OZyWEb9DmNeLxn_OEJLCzKa0Y3MxcKI5CmPL3jQ6I,19566
-konduktor/execution.py,sha256=wwJUQJO3Rc0u0TiF78ilwsEw3a7gGa6uEwBULfeIzAQ,18403
-konduktor/kube_client.py,sha256=Dhza1605wmS4TaFCrw63Y7nh3oSc2P3ekUE2-RI-Qlw,6155
+konduktor/data/registry.py,sha256=eLs8Wr5ugwOfXGPtg1utTvGIqdbVLsCf-a3PFS1NELc,652
+konduktor/data/storage.py,sha256=sm0ZfGZUZRiChza_jMRQY1xDIWtZpFQqwPuVOF8PM_Y,34742
+konduktor/data/storage_utils.py,sha256=n4GivkN0KMqmyOTDznF0Z-hzsJvm7KCEh5i5HgFAT-4,20806
+konduktor/execution.py,sha256=UaHUdBmDaIYgiAXkRKJQOHniYPVIR4sr4yUbIqpgMrQ,18401
+konduktor/kube_client.py,sha256=aqwjDfNSneB5NOxV6CtqhkBeNl0UQNUt730R3ujG9Ow,6156
 konduktor/logging.py,sha256=mBCoCTNhDEkUxd4tsse4mw-aVzSGohhXYf16ViR0ch4,2722
 konduktor/manifests/controller_deployment.yaml,sha256=6p3oSLkEVONZsvKZGqVop0Dhn4bo3lrigRmhf8NXBHE,1730
 konduktor/manifests/dashboard_deployment.yaml,sha256=xJLd4FbPMAosI0fIv5_8y7dV9bw0Vsf81l-w4MB_aU8,2837
@@ -67,7 +67,7 @@ konduktor/manifests/pod_cleanup_controller.yaml,sha256=hziL1Ka1kCAEL9R7Tjvpb80iw
 konduktor/resource.py,sha256=68z8gC8Ivqktwv0R6ylMn9ZNocgkcRT0yIRGGKOdwcM,18491
 konduktor/task.py,sha256=edHgMLYECGux6WLCilqsNZNYr3dEcw_miWvu4FYpu5U,34713
 konduktor/templates/jobset.yaml.j2,sha256=onYiHtXAgk-XBtji994hPu_g0hxnLzvmfxwjbdKdeZc,960
-konduktor/templates/pod.yaml.j2,sha256=XLQ2dD7jq9yeF-eKtweXaMrHWj5cFI-DIZwcJ3qkANQ,8433
+konduktor/templates/pod.yaml.j2,sha256=rO5rDfM2XYCRbc5tD0bXzIK_ulZGlVISZdQdJzr-Gfk,8435
 konduktor/usage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 konduktor/usage/constants.py,sha256=gCL8afIHZhO0dcxbJGpESE9sCC1cBSbeRnQ8GwNOY4M,612
 konduktor/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -83,12 +83,12 @@ konduktor/utils/kubernetes_utils.py,sha256=NGBredKPWpZC8VNlwTfWLhHnc-p68d5xlxT-0
 konduktor/utils/log_utils.py,sha256=lgHCq4OdtJNfbpso-uYGONUCVNsUrUkUWjROarsHt6s,9897
 konduktor/utils/loki_utils.py,sha256=ND1pbbbFhLhLKw3870j44LpR_9MB0EkDJSs5K7nWdY4,3473
 konduktor/utils/rich_utils.py,sha256=kdjNe6S2LlpOxyzhFHqMzCz7g4ROC4e7TPWgcbRsrQE,3577
-konduktor/utils/schemas.py,sha256=4Goihc-NpFQpiJ7RSiKirAIPNWqw_DV_TRqVwejqTDY,17479
+konduktor/utils/schemas.py,sha256=Gv7SEhFpv-eO5izqRz8d-eQ9z-lVmY05akm6HEXIIdc,17478
 konduktor/utils/subprocess_utils.py,sha256=WoFkoFhGecPR8-rF8WJxbIe-YtV94LXz9UG64SDhCY4,9448
 konduktor/utils/ux_utils.py,sha256=NPNu3Igu2Z9Oq77ghJhy_fIxQZTXWr9BtKyxN3Wslzo,7164
 konduktor/utils/validator.py,sha256=tgBghVyedyzGx84-U2Qfoh_cJBE3oUk9gclMW90ORks,691
-konduktor_nightly-0.1.0.dev20250327104656.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
-konduktor_nightly-0.1.0.dev20250327104656.dist-info/METADATA,sha256=EX_G_loycMXJbLC5OQc05ikBf0RbbGbA2zMGia3xNyc,4070
-konduktor_nightly-0.1.0.dev20250327104656.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-konduktor_nightly-0.1.0.dev20250327104656.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
-konduktor_nightly-0.1.0.dev20250327104656.dist-info/RECORD,,
+konduktor_nightly-0.1.0.dev20250328104606.dist-info/LICENSE,sha256=MuuqTZbHvmqXR_aNKAXzggdV45ANd3wQ5YI7tnpZhm0,6586
+konduktor_nightly-0.1.0.dev20250328104606.dist-info/METADATA,sha256=AEritN7-smRecfPlFGJTxTtmqRImheYS4fW-KsGCBBo,4112
+konduktor_nightly-0.1.0.dev20250328104606.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+konduktor_nightly-0.1.0.dev20250328104606.dist-info/entry_points.txt,sha256=k3nG5wDFIJhNqsZWrHk4d0irIB2Ns9s47cjRWYsTCT8,48
+konduktor_nightly-0.1.0.dev20250328104606.dist-info/RECORD,,

konduktor/cloud_stores.py

@@ -1,158 +0,0 @@
-# Proprietary Changes made for Trainy under the Trainy Software License
-# Original source: skypilot: https://github.com/skypilot-org/skypilot
-# which is Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#     http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Cloud object stores.
-
-Currently, used for transferring data in bulk.  Thus, this module does not
-offer file-level calls (e.g., open, reading, writing).
-
-TODO:
-* Better interface.
-* Better implementation (e.g., fsspec, smart_open, using each cloud's SDK).
-"""
-
-import subprocess
-import typing
-
-from konduktor import logging
-from konduktor.data import data_utils, gcp, storage_utils
-
-logger = logging.get_logger(__name__)
-
-# TODO(asaiacai): this internal API is shit and should just be unified with
-# the storage_utils.AbstractStore class. Shit Berkeley EECS as usual.
-
-
-class CloudStorage:
-    """Interface for a cloud object store."""
-
-    # this needs to be overridden by the subclass
-    _STORE: typing.Type[storage_utils.AbstractStore]
-
-    def is_directory(self, url: str) -> bool:
-        """Returns whether 'url' is a directory.
-
-        In cloud object stores, a "directory" refers to a regular object whose
-        name is a prefix of other objects.
-        """
-        raise NotImplementedError
-
-    def make_sync_dir_command(self, source: str, destination: str) -> str:
-        """Makes a runnable bash command to sync a 'directory'."""
-        raise NotImplementedError
-
-    def make_sync_file_command(self, source: str, destination: str) -> str:
-        """Makes a runnable bash command to sync a file."""
-        raise NotImplementedError
-
-    def check_credentials(self):
-        """Checks if the user has access credentials to this cloud."""
-        return self._STORE.check_credentials()
-
-    def check_credentials_from_secret(self):
-        """Checks if the user has access credentials to this cloud."""
-        return self._STORE.check_credentials_from_secret()
-
-    def set_secret_credentials(self):
-        """Set the credentials from the secret"""
-        return self._STORE.set_secret_credentials()
-
-
-class GcsCloudStorage(CloudStorage):
-    """Google Cloud Storage."""
-
-    # We use gsutil as a basic implementation.  One pro is that its -m
-    # multi-threaded download is nice, which frees us from implementing
-    # parellel workers on our end.
-    # The gsutil command is part of the Google Cloud SDK, and we reuse
-    # the installation logic here.
-    _INSTALL_GSUTIL = gcp.GOOGLE_SDK_INSTALLATION_COMMAND
-    _STORE: typing.Type[storage_utils.AbstractStore] = gcp.GcsStore
-
-    @property
-    def _gsutil_command(self):
-        gsutil_alias, alias_gen = data_utils.get_gsutil_command()
-        return (
-            f'{alias_gen}; GOOGLE_APPLICATION_CREDENTIALS='
-            f'{gcp.DEFAULT_GCP_APPLICATION_CREDENTIAL_PATH}; '
-            # Explicitly activate service account. Unlike the gcp packages
-            # and other GCP commands, gsutil does not automatically pick up
-            # the default credential keys when it is a service account.
-            'gcloud auth activate-service-account '
-            '--key-file=$GOOGLE_APPLICATION_CREDENTIALS '
-            '2> /dev/null || true; '
-            f'{gsutil_alias}'
-        )
-
-    def is_directory(self, url: str) -> bool:
-        """Returns whether 'url' is a directory.
-        In cloud object stores, a "directory" refers to a regular object whose
-        name is a prefix of other objects.
-        """
-        commands = [self._INSTALL_GSUTIL]
-        commands.append(f'{self._gsutil_command} ls -d {url}')
-        command = ' && '.join(commands)
-        p = subprocess.run(
-            command,
-            stdout=subprocess.PIPE,
-            shell=True,
-            check=True,
-            executable='/bin/bash',
-        )
-        out = p.stdout.decode().strip()
-        # Edge Case: Gcloud command is run for first time #437
-        out = out.split('\n')[-1]
-        # If <url> is a bucket root, then we only need `gsutil` to succeed
-        # to make sure the bucket exists. It is already a directory.
-        _, key = data_utils.split_gcs_path(url)
-        if not key:
-            return True
-        # Otherwise, gsutil ls -d url will return:
-        #   --> url.rstrip('/')          if url is not a directory
-        #   --> url with an ending '/'   if url is a directory
-        if not out.endswith('/'):
-            assert out == url.rstrip('/'), (out, url)
-            return False
-        url = url if url.endswith('/') else (url + '/')
-        assert out == url, (out, url)
-        return True
-
-    def make_sync_dir_command(self, source: str, destination: str) -> str:
-        """Downloads a directory using gsutil."""
-        download_via_gsutil = (
-            f'{self._gsutil_command} ' f'rsync -e -r {source} {destination}'
-        )
-        all_commands = [self._INSTALL_GSUTIL]
-        all_commands.append(download_via_gsutil)
-        return ' && '.join(all_commands)
-
-    def make_sync_file_command(self, source: str, destination: str) -> str:
-        """Downloads a file using gsutil."""
-        download_via_gsutil = f'{self._gsutil_command} ' f'cp {source} {destination}'
-        all_commands = [self._INSTALL_GSUTIL]
-        all_commands.append(download_via_gsutil)
-        return ' && '.join(all_commands)
-
-
-# Maps bucket's URIs prefix(scheme) to its corresponding storage class
-_REGISTRY = {
-    'gs': GcsCloudStorage(),
-    # TODO(asaiacai): Add other cloud stores here
-    # 's3': S3CloudStorage(),
-    # 'r2': R2CloudStorage(),
-    # 'cos': IBMCosCloudStorage(),
-    # 'oci': OciCloudStorage(),
-    # # TODO: This is a hack, as Azure URL starts with https://, we should
-    # # refactor the registry to be able to take regex, so that Azure blob can
-    # # be identified with `https://(.*?)\.blob\.core\.windows\.net`
-    # 'https': AzureBlobCloudStorage()
-}