proximl 0.5.10__py3-none-any.whl → 0.5.12__py3-none-any.whl

proximl/__init__.py

@@ -13,5 +13,5 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 
 
-__version__ = "0.5.5"
+__version__ = "0.5.11"
 __all__ = "ProxiML"

proximl/checkpoints.py

@@ -23,9 +23,7 @@ class Checkpoints(object):
 
     async def list(self, **kwargs):
         resp = await self.proximl._query(f"/checkpoint", "GET", kwargs)
-        checkpoints = [
-            Checkpoint(self.proximl, **checkpoint) for checkpoint in resp
-        ]
+        checkpoints = [Checkpoint(self.proximl, **checkpoint) for checkpoint in resp]
         return checkpoints
 
     async def list_public(self, **kwargs):
@@ -33,14 +31,24 @@ class Checkpoints(object):
         datasets = [Checkpoint(self.proximl, **dataset) for dataset in resp]
         return datasets
 
-    async def create(self, name, source_type, source_uri, **kwargs):
+    async def create(
+        self,
+        name,
+        source_type,
+        source_uri,
+        type="evefs",
+        project_uuid=None,
+        **kwargs,
+    ):
+        if not project_uuid:
+            project_uuid = self.proximl.active_project
         data = dict(
             name=name,
             source_type=source_type,
             source_uri=source_uri,
-            source_options=kwargs.get("source_options"),
-            project_uuid=kwargs.get("project_uuid")
-            or self.proximl.active_project,
+            project_uuid=project_uuid,
+            type=type,
+            **kwargs,
         )
         payload = {k: v for k, v in data.items() if v is not None}
         logging.info(f"Creating Checkpoint {name}")
@@ -60,12 +68,13 @@ class Checkpoint:
     def __init__(self, proximl, **kwargs):
         self.proximl = proximl
         self._checkpoint = kwargs
-        self._id = self._checkpoint.get(
-            "id", self._checkpoint.get("checkpoint_uuid")
-        )
+        self._id = self._checkpoint.get("id", self._checkpoint.get("checkpoint_uuid"))
         self._status = self._checkpoint.get("status")
         self._name = self._checkpoint.get("name")
-        self._size = self._checkpoint.get("size")
+        self._size = self._checkpoint.get("size") or self._checkpoint.get("used_size")
+        self._billed_size = self._checkpoint.get("billed_size") or self._checkpoint.get(
+            "size"
+        )
         self._project_uuid = self._checkpoint.get("project_uuid")
 
     @property
@@ -84,6 +93,10 @@ class Checkpoint:
     def size(self) -> int:
         return self._size
 
+    @property
+    def billed_size(self) -> int:
+        return self._billed_size
+
     def __str__(self):
         return json.dumps({k: v for k, v in self._checkpoint.items()})
 
@@ -123,15 +136,17 @@ class Checkpoint:
                 entity_type="checkpoint",
                 project_uuid=self._checkpoint.get("project_uuid"),
                 cidr=self._checkpoint.get("vpn").get("cidr"),
-                ssh_port=self._checkpoint.get("vpn")
-                .get("client")
-                .get("ssh_port"),
-                input_path=self._checkpoint.get("source_uri")
-                if self.status in ["new", "downloading"]
-                else None,
-                output_path=self._checkpoint.get("output_uri")
-                if self.status == "exporting"
-                else None,
+                ssh_port=self._checkpoint.get("vpn").get("client").get("ssh_port"),
+                input_path=(
+                    self._checkpoint.get("source_uri")
+                    if self.status in ["new", "downloading"]
+                    else None
+                ),
+                output_path=(
+                    self._checkpoint.get("output_uri")
+                    if self.status == "exporting"
+                    else None
+                ),
             )
         else:
             details = dict()
@@ -195,9 +210,7 @@ class Checkpoint:
                 if msg_handler:
                     msg_handler(data)
                 else:
-                    timestamp = datetime.fromtimestamp(
-                        int(data.get("time")) / 1000
-                    )
+                    timestamp = datetime.fromtimestamp(int(data.get("time")) / 1000)
                     print(
                         f"{timestamp.strftime('%m/%d/%Y, %H:%M:%S')}: {data.get('msg').rstrip()}"
                     )
@@ -224,19 +237,24 @@ class Checkpoint:
         return self
 
     async def wait_for(self, status, timeout=300):
+        if self.status == status:
+            return
         valid_statuses = ["downloading", "ready", "archived"]
         if not status in valid_statuses:
             raise SpecificationError(
                 "status",
                 f"Invalid wait_for status {status}.  Valid statuses are: {valid_statuses}",
             )
-        if self.status == status:
-            return
+
+        MAX_TIMEOUT = 24 * 60 * 60
+        if timeout > MAX_TIMEOUT:
+            raise SpecificationError(
+                "timeout",
+                f"timeout must be less than {MAX_TIMEOUT} seconds.",
+            )
         POLL_INTERVAL_MIN = 5
         POLL_INTERVAL_MAX = 60
-        POLL_INTERVAL = max(
-            min(timeout / 60, POLL_INTERVAL_MAX), POLL_INTERVAL_MIN
-        )
+        POLL_INTERVAL = max(min(timeout / 60, POLL_INTERVAL_MAX), POLL_INTERVAL_MIN)
         retry_count = math.ceil(timeout / POLL_INTERVAL)
         count = 0
         while count < retry_count:

proximl/cli/cloudbender/__init__.py

@@ -15,4 +15,5 @@ from proximl.cli.cloudbender.region import region
 from proximl.cli.cloudbender.node import node
 from proximl.cli.cloudbender.device import device
 from proximl.cli.cloudbender.datastore import datastore
-from proximl.cli.cloudbender.reservation import reservation
+from proximl.cli.cloudbender.data_connector import data_connector
+from proximl.cli.cloudbender.service import service

proximl/cli/cloudbender/datastore.py

@@ -29,13 +29,11 @@ def datastore(config):
 def list(config, provider, region):
     """List datastores."""
     data = [
-        ["ID", "NAME", "TYPE", "URI", "ROOT"],
+        ["ID", "NAME", "TYPE"],
         [
             "-" * 80,
             "-" * 80,
             "-" * 80,
-            "-" * 80,
-            "-" * 80,
         ],
     ]
 
@@ -51,15 +49,12 @@ def list(config, provider, region):
                 datastore.id,
                 datastore.name,
                 datastore.type,
-                datastore.uri,
-                datastore.root,
             ]
         )
 
     for row in data:
         click.echo(
-            "{: >37.36} {: >29.28} {: >9.8} {: >12.11} {: >12.11}"
-            "".format(*row),
+            "{: >37.36} {: >29.28} {: >9.8} " "".format(*row),
             file=config.stdout,
         )
 

proximl/cli/cloudbender/service.py

@@ -74,6 +74,19 @@ def list(config, provider, region):
     required=True,
     help="The region ID to create the service in.",
 )
+@click.option(
+    "--type",
+    "-t",
+    type=click.Choice(
+        [
+            "https",
+            "tcp",
+            "udp",
+        ],
+    ),
+    required=True,
+    help="The type of regional service.",
+)
 @click.option(
     "--public/--no-public",
     default=True,
@@ -82,13 +95,17 @@ def list(config, provider, region):
 )
 @click.argument("name", type=click.STRING, required=True)
 @pass_config
-def create(config, provider, region, public, name):
+def create(config, provider, region, type, public, name):
     """
     Creates a service.
     """
     return config.proximl.run(
         config.proximl.client.cloudbender.services.create(
-            provider_uuid=provider, region_uuid=region, name=name, public=public
+            provider_uuid=provider,
+            region_uuid=region,
+            name=name,
+            type=type,
+            public=public,
         )
     )
 

proximl/cli/project/__init__.py

@@ -77,77 +77,8 @@ def remove(config, project):
     return config.proximl.run(found.remove())
 
 
-@project.command()
-@pass_config
-def list_datastores(config):
-    """List project datastores."""
-    data = [
-        ["ID", "NAME", "TYPE", "REGION_UUID"],
-        [
-            "-" * 80,
-            "-" * 80,
-            "-" * 80,
-            "-" * 80,
-        ],
-    ]
-    project = config.proximl.run(
-        config.proximl.client.projects.get(config.proximl.client.project)
-    )
-
-    datastores = config.proximl.run(project.list_datastores())
-
-    for datastore in datastores:
-        data.append(
-            [
-                datastore.id,
-                datastore.name,
-                datastore.type,
-                datastore.region_uuid,
-            ]
-        )
-
-    for row in data:
-        click.echo(
-            "{: >38.36} {: >30.28} {: >15.13} {: >38.36}" "".format(*row),
-            file=config.stdout,
-        )
-
-
-@project.command()
-@pass_config
-def list_services(config):
-    """List project services."""
-    data = [
-        ["ID", "NAME", "HOSTNAME", "REGION_UUID"],
-        [
-            "-" * 80,
-            "-" * 80,
-            "-" * 80,
-            "-" * 80,
-        ],
-    ]
-    project = config.proximl.run(
-        config.proximl.client.projects.get(config.proximl.client.project)
-    )
-
-    services = config.proximl.run(project.list_services())
-
-    for service in services:
-        data.append(
-            [
-                service.id,
-                service.name,
-                service.hostname,
-                service.region_uuid,
-            ]
-        )
-
-    for row in data:
-        click.echo(
-            "{: >38.36} {: >30.28} {: >30.28} {: >38.36}" "".format(*row),
-            file=config.stdout,
-        )
-
-
 from proximl.cli.project.secret import secret
 from proximl.cli.project.key import key
+from proximl.cli.project.data_connector import data_connector
+from proximl.cli.project.datastore import datastore
+from proximl.cli.project.service import service

proximl/cli/project/credential.py

@@ -0,0 +1,128 @@
+import click
+import os
+import json
+import base64
+from pathlib import Path
+from proximl.cli import pass_config
+from proximl.cli.project import project
+
+
+@project.group()
+@pass_config
+def credential(config):
+    """proxiML project credential commands."""
+    pass
+
+
+@credential.command()
+@pass_config
+def list(config):
+    """List credentials."""
+    data = [
+        ["TYPE", "KEY ID", "UPDATED AT"],
+        [
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+        ],
+    ]
+    project = config.proximl.run(config.proximl.client.projects.get_current())
+    credentials = config.proximl.run(project.credentials.list())
+
+    for credential in credentials:
+        data.append(
+            [
+                credential.type,
+                credential.key_id,
+                credential.updated_at.isoformat(timespec="seconds"),
+            ]
+        )
+
+    for row in data:
+        click.echo(
+            "{: >13.11} {: >37.35} {: >28.26}" "".format(*row),
+            file=config.stdout,
+        )
+
+
+@credential.command()
+@click.argument(
+    "type",
+    type=click.Choice(
+        [
+            "aws",
+            "azure",
+            "docker",
+            "gcp",
+            "huggingface",
+            "kaggle",
+            "ngc",
+            "wasabi",
+        ],
+        case_sensitive=False,
+    ),
+)
+@pass_config
+def put(config, type):
+    """
+    Set a credential.
+
+    A credential is uploaded.
+    """
+    project = config.proximl.run(config.proximl.client.projects.get_current())
+
+    tenant = None
+
+    if type in ["aws", "wasabi"]:
+        credential_id = click.prompt(
+            "Enter the credential ID", type=str, hide_input=False
+        )
+        secret = click.prompt("Enter the secret credential", type=str, hide_input=True)
+    elif type == "azure":
+        credential_id = click.prompt(
+            "Enter the Application (client) ID", type=str, hide_input=False
+        )
+        tenant = click.prompt(
+            "Enter the Directory (tenant) ley", type=str, hide_input=False
+        )
+        secret = click.prompt("Enter the client secret", type=str, hide_input=True)
+    elif type in ["docker", "huggingface"]:
+        credential_id = click.prompt("Enter the username", type=str, hide_input=False)
+        secret = click.prompt("Enter the access token", type=str, hide_input=True)
+    elif type in ["gcp", "kaggle"]:
+        file_name = click.prompt(
+            "Enter the path of the credentials file",
+            type=click.Path(
+                exists=True, file_okay=True, dir_okay=False, resolve_path=True
+            ),
+            hide_input=False,
+        )
+        credential_id = os.path.basename(file_name)
+        with open(file_name) as f:
+            secret = json.load(f)
+        secret = json.dumps(secret)
+    elif type == "ngc":
+        credential_id = "$oauthtoken"
+        secret = click.prompt("Enter the access token", type=str, hide_input=True)
+    else:
+        raise click.UsageError("Unsupported credential type")
+
+    return config.proximl.run(
+        project.credentials.put(
+            type=type, credential_id=credential_id, secret=secret, tenant=tenant
+        )
+    )
+
+
+@credential.command()
+@click.argument("name", type=click.STRING)
+@pass_config
+def remove(config, name):
+    """
+    Remove a credential.
+
+
+    """
+    project = config.proximl.run(config.proximl.client.projects.get_current())
+
+    return config.proximl.run(project.credential.remove(name))

proximl/cli/project/data_connector.py

@@ -0,0 +1,61 @@
+import click
+import os
+import json
+import base64
+from pathlib import Path
+from proximl.cli import pass_config
+from proximl.cli.project import project
+
+
+@project.group()
+@pass_config
+def data_connector(config):
+    """proxiML project data_connector commands."""
+    pass
+
+
+@data_connector.command()
+@pass_config
+def list(config):
+    """List project data_connectors."""
+    data = [
+        ["ID", "NAME", "TYPE", "REGION_UUID"],
+        [
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+        ],
+    ]
+    project = config.proximl.run(
+        config.proximl.client.projects.get(config.proximl.client.project)
+    )
+
+    data_connectors = config.proximl.run(project.data_connectors.list())
+
+    for data_connector in data_connectors:
+        data.append(
+            [
+                data_connector.id,
+                data_connector.name,
+                data_connector.type,
+                data_connector.region_uuid,
+            ]
+        )
+
+    for row in data:
+        click.echo(
+            "{: >38.36} {: >30.28} {: >15.13} {: >38.36}" "".format(*row),
+            file=config.stdout,
+        )
+
+
+@data_connector.command()
+@pass_config
+def refresh(config):
+    """
+    Refresh project data_connector list.
+    """
+    project = config.proximl.run(config.proximl.client.projects.get_current())
+
+    return config.proximl.run(project.data_connectors.refresh())

proximl/cli/project/datastore.py

@@ -0,0 +1,61 @@
+import click
+import os
+import json
+import base64
+from pathlib import Path
+from proximl.cli import pass_config
+from proximl.cli.project import project
+
+
+@project.group()
+@pass_config
+def datastore(config):
+    """proxiML project datastore commands."""
+    pass
+
+
+@datastore.command()
+@pass_config
+def list(config):
+    """List project datastores."""
+    data = [
+        ["ID", "NAME", "TYPE", "REGION_UUID"],
+        [
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+        ],
+    ]
+    project = config.proximl.run(
+        config.proximl.client.projects.get(config.proximl.client.project)
+    )
+
+    datastores = config.proximl.run(project.datastores.list())
+
+    for datastore in datastores:
+        data.append(
+            [
+                datastore.id,
+                datastore.name,
+                datastore.type,
+                datastore.region_uuid,
+            ]
+        )
+
+    for row in data:
+        click.echo(
+            "{: >38.36} {: >30.28} {: >15.13} {: >38.36}" "".format(*row),
+            file=config.stdout,
+        )
+
+
+@datastore.command()
+@pass_config
+def refresh(config):
+    """
+    Refresh project datastore list.
+    """
+    project = config.proximl.run(config.proximl.client.projects.get_current())
+
+    return config.proximl.run(project.datastores.refresh())

proximl/cli/project/service.py

@@ -0,0 +1,61 @@
+import click
+import os
+import json
+import base64
+from pathlib import Path
+from proximl.cli import pass_config
+from proximl.cli.project import project
+
+
+@project.group()
+@pass_config
+def service(config):
+    """proxiML project service commands."""
+    pass
+
+
+@service.command()
+@pass_config
+def list(config):
+    """List project services."""
+    data = [
+        ["ID", "NAME", "TYPE", "REGION_UUID"],
+        [
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+        ],
+    ]
+    project = config.proximl.run(
+        config.proximl.client.projects.get(config.proximl.client.project)
+    )
+
+    services = config.proximl.run(project.services.list())
+
+    for service in services:
+        data.append(
+            [
+                service.id,
+                service.name,
+                service.hostname,
+                service.region_uuid,
+            ]
+        )
+
+    for row in data:
+        click.echo(
+            "{: >38.36} {: >30.28} {: >15.13} {: >38.36}" "".format(*row),
+            file=config.stdout,
+        )
+
+
+@service.command()
+@pass_config
+def refresh(config):
+    """
+    Refresh project service list.
+    """
+    project = config.proximl.run(config.proximl.client.projects.get_current())
+
+    return config.proximl.run(project.services.refresh())

proximl/cloudbender/cloudbender.py

@@ -3,7 +3,8 @@ from .regions import Regions
 from .nodes import Nodes
 from .devices import Devices
 from .datastores import Datastores
-from .reservations import Reservations
+from .data_connectors import DataConnectors
+from .services import Services
 from .device_configs import DeviceConfigs
 
 
@@ -15,5 +16,6 @@ class Cloudbender(object):
         self.nodes = Nodes(proximl)
         self.devices = Devices(proximl)
         self.datastores = Datastores(proximl)
-        self.reservations = Reservations(proximl)
+        self.data_connectors = DataConnectors(proximl)
+        self.services = Services(proximl)
         self.device_configs = DeviceConfigs(proximl)

proximl/cloudbender/data_connectors.py

@@ -1,5 +1,13 @@
 import json
 import logging
+import asyncio
+import math
+
+from proximl.exceptions import (
+    ApiError,
+    SpecificationError,
+    ProxiMLException,
+)
 
 
 class DataConnectors(object):