proximl 0.5.8__tar.gz → 0.5.10__tar.gz

{proximl-0.5.8/proximl.egg-info → proximl-0.5.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: proximl
-Version: 0.5.8
+Version: 0.5.10
 Summary: proxiML client SDK and command line utilities
 Home-page: https://github.com/proxiML/python-sdk
 Author: proxiML

{proximl-0.5.8 → proximl-0.5.10}/proximl/__init__.py

@@ -13,5 +13,5 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 
 
-__version__ = "0.5.8"
+__version__ = "0.5.10"
 __all__ = "ProxiML"

{proximl-0.5.8 → proximl-0.5.10}/proximl/checkpoints.py

@@ -31,13 +31,24 @@ class Checkpoints(object):
         datasets = [Checkpoint(self.proximl, **dataset) for dataset in resp]
         return datasets
 
-    async def create(self, name, source_type, source_uri, **kwargs):
+    async def create(
+        self,
+        name,
+        source_type,
+        source_uri,
+        type="evefs",
+        project_uuid=None,
+        **kwargs,
+    ):
+        if not project_uuid:
+            project_uuid = self.proximl.active_project
         data = dict(
             name=name,
             source_type=source_type,
             source_uri=source_uri,
-            source_options=kwargs.get("source_options"),
-            project_uuid=kwargs.get("project_uuid") or self.proximl.active_project,
+            project_uuid=project_uuid,
+            type=type,
+            **kwargs,
         )
         payload = {k: v for k, v in data.items() if v is not None}
         logging.info(f"Creating Checkpoint {name}")

{proximl-0.5.8 → proximl-0.5.10}/proximl/cli/cloudbender/datastore.py

@@ -29,13 +29,11 @@ def datastore(config):
 def list(config, provider, region):
     """List datastores."""
     data = [
-        ["ID", "NAME", "TYPE", "URI", "ROOT"],
+        ["ID", "NAME", "TYPE"],
         [
             "-" * 80,
             "-" * 80,
             "-" * 80,
-            "-" * 80,
-            "-" * 80,
         ],
     ]
 
@@ -51,15 +49,12 @@ def list(config, provider, region):
                 datastore.id,
                 datastore.name,
                 datastore.type,
-                datastore.uri,
-                datastore.root,
             ]
         )
 
     for row in data:
         click.echo(
-            "{: >37.36} {: >29.28} {: >9.8} {: >12.11} {: >12.11}"
-            "".format(*row),
+            "{: >37.36} {: >29.28} {: >9.8} " "".format(*row),
             file=config.stdout,
         )
 

proximl-0.5.8/proximl/cli/project.py → proximl-0.5.10/proximl/cli/project/__init__.py

@@ -147,3 +147,7 @@ def list_services(config):
             "{: >38.36} {: >30.28} {: >30.28} {: >38.36}" "".format(*row),
             file=config.stdout,
         )
+
+
+from proximl.cli.project.secret import secret
+from proximl.cli.project.key import key

proximl-0.5.10/proximl/cli/project/key.py

@@ -0,0 +1,124 @@
+import click
+import os
+import json
+import base64
+from pathlib import Path
+from proximl.cli import pass_config
+from proximl.cli.project import project
+
+
+@project.group()
+@pass_config
+def key(config):
+    """proxiML project key commands."""
+    pass
+
+
+@key.command()
+@pass_config
+def list(config):
+    """List keys."""
+    data = [
+        ["TYPE", "KEY ID", "UPDATED AT"],
+        [
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+        ],
+    ]
+    project = config.proximl.run(config.proximl.client.projects.get_current())
+    keys = config.proximl.run(project.keys.list())
+
+    for key in keys:
+        data.append(
+            [
+                key.type,
+                key.key_id,
+                key.updated_at.isoformat(timespec="seconds"),
+            ]
+        )
+
+    for row in data:
+        click.echo(
+            "{: >13.11} {: >37.35} {: >28.26}" "".format(*row),
+            file=config.stdout,
+        )
+
+
+@key.command()
+@click.argument(
+    "type",
+    type=click.Choice(
+        [
+            "aws",
+            "azure",
+            "docker",
+            "gcp",
+            "huggingface",
+            "kaggle",
+            "ngc",
+            "wasabi",
+        ],
+        case_sensitive=False,
+    ),
+)
+@pass_config
+def put(config, type):
+    """
+    Set a key.
+
+    A key is uploaded.
+    """
+    project = config.proximl.run(config.proximl.client.projects.get_current())
+
+    tenant = None
+
+    if type in ["aws", "wasabi"]:
+        key_id = click.prompt("Enter the key ID", type=str, hide_input=False)
+        secret = click.prompt("Enter the secret key", type=str, hide_input=True)
+    elif type == "azure":
+        key_id = click.prompt(
+            "Enter the Application (client) ID", type=str, hide_input=False
+        )
+        tenant = click.prompt(
+            "Enter the Directory (tenant) ley", type=str, hide_input=False
+        )
+        secret = click.prompt("Enter the client secret", type=str, hide_input=True)
+    elif type in ["docker", "huggingface"]:
+        key_id = click.prompt("Enter the username", type=str, hide_input=False)
+        secret = click.prompt("Enter the access token", type=str, hide_input=True)
+    elif type in ["gcp", "kaggle"]:
+        file_name = click.prompt(
+            "Enter the path of the credentials file",
+            type=click.Path(
+                exists=True, file_okay=True, dir_okay=False, resolve_path=True
+            ),
+            hide_input=False,
+        )
+        key_id = os.path.basename(file_name)
+        with open(file_name) as f:
+            secret = json.load(f)
+        secret = json.dumps(secret)
+    elif type == "ngc":
+        key_id = "$oauthtoken"
+        secret = click.prompt("Enter the access token", type=str, hide_input=True)
+    else:
+        raise click.UsageError("Unsupported key type")
+
+    return config.proximl.run(
+        project.keys.put(type=type, key_id=key_id, secret=secret, tenant=tenant)
+    )
+
+
+@key.command()
+@click.argument("name", type=click.STRING)
+@pass_config
+def remove(config, name):
+    """
+    Remove a key.
+
+
+    """
+    project = config.proximl.run(config.proximl.client.projects.get_current())
+
+    return config.proximl.run(project.key.remove(name))

proximl-0.5.10/proximl/cli/project/secret.py

@@ -0,0 +1,71 @@
+import click
+from proximl.cli import pass_config
+from proximl.cli.project import project
+
+
+@project.group()
+@pass_config
+def secret(config):
+    """proxiML project secret commands."""
+    pass
+
+
+@secret.command()
+@pass_config
+def list(config):
+    """List secrets."""
+    data = [
+        ["NAME", "CREATED BY", "UPDATED AT"],
+        [
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+        ],
+    ]
+    project = config.proximl.run(config.proximl.client.projects.get_current())
+    secrets = config.proximl.run(project.secrets.list())
+
+    for secret in secrets:
+        data.append(
+            [
+                secret.name,
+                secret.created_by,
+                secret.updated_at.isoformat(timespec="seconds"),
+            ]
+        )
+
+    for row in data:
+        click.echo(
+            "{: >38.36} {: >30.28} {: >28.26}" "".format(*row),
+            file=config.stdout,
+        )
+
+
+@secret.command()
+@click.argument("name", type=click.STRING)
+@pass_config
+def put(config, name):
+    """
+    Set a secret value.
+
+    Secret is created with the specified NAME.
+    """
+    project = config.proximl.run(config.proximl.client.projects.get_current())
+
+    value = click.prompt("Enter the secret value", type=str, hide_input=True)
+
+    return config.proximl.run(project.secrets.put(name=name, value=value))
+
+
+@secret.command()
+@click.argument("name", type=click.STRING)
+@pass_config
+def remove(config, name):
+    """
+    Remove a secret.
+
+
+    """
+    project = config.proximl.run(config.proximl.client.projects.get_current())
+
+    return config.proximl.run(project.secret.remove(name))

{proximl-0.5.8 → proximl-0.5.10}/proximl/cloudbender/data_connectors.py

@@ -1,5 +1,13 @@
 import json
 import logging
+import asyncio
+import math
+
+from proximl.exceptions import (
+    ApiError,
+    SpecificationError,
+    ProxiMLException,
+)
 
 
 class DataConnectors(object):

{proximl-0.5.8 → proximl-0.5.10}/proximl/cloudbender/datastores.py

@@ -1,5 +1,13 @@
 import json
 import logging
+import asyncio
+import math
+
+from proximl.exceptions import (
+    ApiError,
+    SpecificationError,
+    ProxiMLException,
+)
 
 
 class Datastores(object):
@@ -20,9 +28,7 @@ class Datastores(object):
             "GET",
             kwargs,
         )
-        datastores = [
-            Datastore(self.proximl, **datastore) for datastore in resp
-        ]
+        datastores = [Datastore(self.proximl, **datastore) for datastore in resp]
         return datastores
 
     async def create(
@@ -31,18 +37,12 @@ class Datastores(object):
         region_uuid,
         name,
         type,
-        uri,
-        root,
-        options=None,
         **kwargs,
     ):
         logging.info(f"Creating Datastore {name}")
         data = dict(
             name=name,
             type=type,
-            uri=uri,
-            root=root,
-            options=options,
             **kwargs,
         )
         payload = {k: v for k, v in data.items() if v is not None}
@@ -73,8 +73,6 @@ class Datastore:
         self._region_uuid = self._datastore.get("region_uuid")
         self._type = self._datastore.get("type")
         self._name = self._datastore.get("name")
-        self._uri = self._datastore.get("uri")
-        self._root = self._datastore.get("root")
 
     @property
     def id(self) -> str:
@@ -96,14 +94,6 @@ class Datastore:
     def name(self) -> str:
         return self._name
 
-    @property
-    def uri(self) -> str:
-        return self._uri
-
-    @property
-    def root(self) -> str:
-        return self._root
-
     def __str__(self):
         return json.dumps({k: v for k, v in self._datastore.items()})
 

{proximl-0.5.8 → proximl-0.5.10}/proximl/cloudbender/nodes.py

@@ -1,5 +1,13 @@
 import json
 import logging
+import asyncio
+import math
+
+from proximl.exceptions import (
+    ApiError,
+    SpecificationError,
+    ProxiMLException,
+)
 
 
 class Nodes(object):
@@ -153,3 +161,40 @@ class Node:
             None,
             dict(command=command),
         )
+
+    async def wait_for(self, status, timeout=300):
+        if self.status == status:
+            return
+        valid_statuses = ["active", "maintenance", "offline", "stopped", "archived"]
+        if not status in valid_statuses:
+            raise SpecificationError(
+                "status",
+                f"Invalid wait_for status {status}.  Valid statuses are: {valid_statuses}",
+            )
+        MAX_TIMEOUT = 24 * 60 * 60
+        if timeout > MAX_TIMEOUT:
+            raise SpecificationError(
+                "timeout",
+                f"timeout must be less than {MAX_TIMEOUT} seconds.",
+            )
+
+        POLL_INTERVAL_MIN = 5
+        POLL_INTERVAL_MAX = 60
+        POLL_INTERVAL = max(min(timeout / 60, POLL_INTERVAL_MAX), POLL_INTERVAL_MIN)
+        retry_count = math.ceil(timeout / POLL_INTERVAL)
+        count = 0
+        while count < retry_count:
+            await asyncio.sleep(POLL_INTERVAL)
+            try:
+                await self.refresh()
+            except ApiError as e:
+                if status == "archived" and e.status == 404:
+                    return
+                raise e
+            if self.status == status:
+                return self
+            else:
+                count += 1
+                logging.debug(f"self: {self}, retry count {count}")
+
+        raise ProxiMLException(f"Timeout waiting for {status}")

{proximl-0.5.8 → proximl-0.5.10}/proximl/datasets.py

@@ -31,13 +31,24 @@ class Datasets(object):
         datasets = [Dataset(self.proximl, **dataset) for dataset in resp]
         return datasets
 
-    async def create(self, name, source_type, source_uri, **kwargs):
+    async def create(
+        self,
+        name,
+        source_type,
+        source_uri,
+        type="evefs",
+        project_uuid=None,
+        **kwargs,
+    ):
+        if not project_uuid:
+            project_uuid = self.proximl.active_project
         data = dict(
             name=name,
             source_type=source_type,
             source_uri=source_uri,
-            source_options=kwargs.get("source_options"),
-            project_uuid=kwargs.get("project_uuid") or self.proximl.active_project,
+            project_uuid=project_uuid,
+            type=type,
+            **kwargs,
         )
         payload = {k: v for k, v in data.items() if v is not None}
         logging.info(f"Creating Dataset {name}")

{proximl-0.5.8 → proximl-0.5.10}/proximl/jobs.py

@@ -476,7 +476,6 @@ class Job:
             return
         valid_statuses = [
             "waiting for data/model download",
-            "waiting for GPUs",
             "waiting for resources",
             "running",
             "stopped",
@@ -488,11 +487,6 @@ class Job:
                 "status",
                 f"Invalid wait_for status {status}.  Valid statuses are: {valid_statuses}",
             )
-        if status == "waiting for GPUs":
-            warnings.warn(
-                "'waiting for GPUs' status is deprecated, use 'waiting for resources' instead.",
-                DeprecationWarning,
-            )
         if (self.type == "training") and status == "stopped":
             warnings.warn(
                 "'stopped' status is deprecated for training jobs, use 'finished' instead.",
@@ -523,12 +517,8 @@ class Job:
                 self.status == status
                 or (
                     status
-                    in [
-                        "waiting for GPUs",
-                        "waiting for resources",
-                    ]  ## this status could be very short and the polling could miss it
-                    and self.status
-                    not in ["new", "waiting for GPUs", "waiting for resources"]
+                    == "waiting for resources"  ## this status could be very short and the polling could miss it
+                    and self.status not in ["new", "waiting for resources"]
                 )
                 or (
                     status
@@ -536,7 +526,6 @@ class Job:
                     and self.status
                     not in [
                         "new",
-                        "waiting for GPUs",
                         "waiting for resources",
                         "waiting for data/model download",
                     ]

{proximl-0.5.8 → proximl-0.5.10}/proximl/models.py

@@ -26,13 +26,24 @@ class Models(object):
         models = [Model(self.proximl, **model) for model in resp]
         return models
 
-    async def create(self, name, source_type, source_uri, **kwargs):
+    async def create(
+        self,
+        name,
+        source_type,
+        source_uri,
+        type="evefs",
+        project_uuid=None,
+        **kwargs,
+    ):
+        if not project_uuid:
+            project_uuid = self.proximl.active_project
         data = dict(
             name=name,
             source_type=source_type,
             source_uri=source_uri,
-            source_options=kwargs.get("source_options"),
-            project_uuid=kwargs.get("project_uuid") or self.proximl.active_project,
+            project_uuid=project_uuid,
+            type=type,
+            **kwargs,
         )
         payload = {k: v for k, v in data.items() if v is not None}
         logging.info(f"Creating Model {name}")

proximl-0.5.10/proximl/projects/__init__.py

@@ -0,0 +1,3 @@
+from .projects import Projects, Project
+
+__all__ = ["Projects", "Project"]

proximl-0.5.10/proximl/projects/data_connectors.py

@@ -0,0 +1,63 @@
+import json
+import logging
+
+
+class ProjectDataConnectors(object):
+    def __init__(self, proximl, project_id):
+        self.proximl = proximl
+        self.project_id = project_id
+
+    async def list(self, **kwargs):
+        resp = await self.proximl._query(
+            f"/project/{self.project_id}/data_connectors", "GET", kwargs
+        )
+        data_connectors = [
+            ProjectDataConnector(self.proximl, **data_connector)
+            for data_connector in resp
+        ]
+        return data_connectors
+
+    async def refresh(self):
+        await self.proximl._query(
+            f"/project/{self.project_id}/data_connectors", "PATCH"
+        )
+
+
+class ProjectDataConnector:
+    def __init__(self, proximl, **kwargs):
+        self.proximl = proximl
+        self._entity = kwargs
+        self._id = self._entity.get("id")
+        self._project_uuid = self._entity.get("project_uuid")
+        self._name = self._entity.get("name")
+        self._type = self._entity.get("type")
+        self._region_uuid = self._entity.get("region_uuid")
+
+    @property
+    def id(self) -> str:
+        return self._id
+
+    @property
+    def project_uuid(self) -> str:
+        return self._project_uuid
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def type(self) -> str:
+        return self._type
+
+    @property
+    def region_uuid(self) -> str:
+        return self._region_uuid
+
+    def __str__(self):
+        return json.dumps({k: v for k, v in self._entity.items()})
+
+    def __repr__(self):
+        return f"ProjectDataConnector( proximl , **{self._entity.__repr__()})"
+
+    def __bool__(self):
+        return bool(self._id)

proximl-0.5.10/proximl/projects/datastores.py

@@ -0,0 +1,58 @@
+import json
+import logging
+
+
+class ProjectDatastores(object):
+    def __init__(self, proximl, project_id):
+        self.proximl = proximl
+        self.project_id = project_id
+
+    async def list(self, **kwargs):
+        resp = await self.proximl._query(
+            f"/project/{self.project_id}/datastores", "GET", kwargs
+        )
+        datastores = [ProjectDatastore(self.proximl, **datastore) for datastore in resp]
+        return datastores
+
+    async def refresh(self):
+        await self.proximl._query(f"/project/{self.project_id}/datastores", "PATCH")
+
+
+class ProjectDatastore:
+    def __init__(self, proximl, **kwargs):
+        self.proximl = proximl
+        self._entity = kwargs
+        self._id = self._entity.get("id")
+        self._project_uuid = self._entity.get("project_uuid")
+        self._name = self._entity.get("name")
+        self._type = self._entity.get("type")
+        self._region_uuid = self._entity.get("region_uuid")
+
+    @property
+    def id(self) -> str:
+        return self._id
+
+    @property
+    def project_uuid(self) -> str:
+        return self._project_uuid
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def type(self) -> str:
+        return self._type
+
+    @property
+    def region_uuid(self) -> str:
+        return self._region_uuid
+
+    def __str__(self):
+        return json.dumps({k: v for k, v in self._entity.items()})
+
+    def __repr__(self):
+        return f"ProjectDatastore( proximl , **{self._entity.__repr__()})"
+
+    def __bool__(self):
+        return bool(self._id)