proximl 0.5.6__tar.gz → 0.5.8__tar.gz

{proximl-0.5.6/proximl.egg-info → proximl-0.5.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: proximl
-Version: 0.5.6
+Version: 0.5.8
 Summary: proxiML client SDK and command line utilities
 Home-page: https://github.com/proxiML/python-sdk
 Author: proxiML

{proximl-0.5.6 → proximl-0.5.8}/proximl/__init__.py

@@ -13,5 +13,5 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 
 
-__version__ = "0.5.6"
+__version__ = "0.5.8"
 __all__ = "ProxiML"

{proximl-0.5.6 → proximl-0.5.8}/proximl/checkpoints.py

@@ -23,9 +23,7 @@ class Checkpoints(object):
 
     async def list(self, **kwargs):
         resp = await self.proximl._query(f"/checkpoint", "GET", kwargs)
-        checkpoints = [
-            Checkpoint(self.proximl, **checkpoint) for checkpoint in resp
-        ]
+        checkpoints = [Checkpoint(self.proximl, **checkpoint) for checkpoint in resp]
         return checkpoints
 
     async def list_public(self, **kwargs):
@@ -39,8 +37,7 @@ class Checkpoints(object):
             source_type=source_type,
             source_uri=source_uri,
             source_options=kwargs.get("source_options"),
-            project_uuid=kwargs.get("project_uuid")
-            or self.proximl.active_project,
+            project_uuid=kwargs.get("project_uuid") or self.proximl.active_project,
         )
         payload = {k: v for k, v in data.items() if v is not None}
         logging.info(f"Creating Checkpoint {name}")
@@ -60,12 +57,13 @@ class Checkpoint:
     def __init__(self, proximl, **kwargs):
         self.proximl = proximl
         self._checkpoint = kwargs
-        self._id = self._checkpoint.get(
-            "id", self._checkpoint.get("checkpoint_uuid")
-        )
+        self._id = self._checkpoint.get("id", self._checkpoint.get("checkpoint_uuid"))
         self._status = self._checkpoint.get("status")
         self._name = self._checkpoint.get("name")
-        self._size = self._checkpoint.get("size")
+        self._size = self._checkpoint.get("size") or self._checkpoint.get("used_size")
+        self._billed_size = self._checkpoint.get("billed_size") or self._checkpoint.get(
+            "size"
+        )
         self._project_uuid = self._checkpoint.get("project_uuid")
 
     @property
@@ -84,6 +82,10 @@ class Checkpoint:
     def size(self) -> int:
         return self._size
 
+    @property
+    def billed_size(self) -> int:
+        return self._billed_size
+
     def __str__(self):
         return json.dumps({k: v for k, v in self._checkpoint.items()})
 
@@ -123,15 +125,17 @@ class Checkpoint:
                 entity_type="checkpoint",
                 project_uuid=self._checkpoint.get("project_uuid"),
                 cidr=self._checkpoint.get("vpn").get("cidr"),
-                ssh_port=self._checkpoint.get("vpn")
-                .get("client")
-                .get("ssh_port"),
-                input_path=self._checkpoint.get("source_uri")
-                if self.status in ["new", "downloading"]
-                else None,
-                output_path=self._checkpoint.get("output_uri")
-                if self.status == "exporting"
-                else None,
+                ssh_port=self._checkpoint.get("vpn").get("client").get("ssh_port"),
+                input_path=(
+                    self._checkpoint.get("source_uri")
+                    if self.status in ["new", "downloading"]
+                    else None
+                ),
+                output_path=(
+                    self._checkpoint.get("output_uri")
+                    if self.status == "exporting"
+                    else None
+                ),
             )
         else:
             details = dict()
@@ -195,9 +199,7 @@ class Checkpoint:
                 if msg_handler:
                     msg_handler(data)
                 else:
-                    timestamp = datetime.fromtimestamp(
-                        int(data.get("time")) / 1000
-                    )
+                    timestamp = datetime.fromtimestamp(int(data.get("time")) / 1000)
                     print(
                         f"{timestamp.strftime('%m/%d/%Y, %H:%M:%S')}: {data.get('msg').rstrip()}"
                     )
@@ -224,19 +226,24 @@ class Checkpoint:
         return self
 
     async def wait_for(self, status, timeout=300):
+        if self.status == status:
+            return
         valid_statuses = ["downloading", "ready", "archived"]
         if not status in valid_statuses:
             raise SpecificationError(
                 "status",
                 f"Invalid wait_for status {status}.  Valid statuses are: {valid_statuses}",
             )
-        if self.status == status:
-            return
+
+        MAX_TIMEOUT = 24 * 60 * 60
+        if timeout > MAX_TIMEOUT:
+            raise SpecificationError(
+                "timeout",
+                f"timeout must be less than {MAX_TIMEOUT} seconds.",
+            )
         POLL_INTERVAL_MIN = 5
         POLL_INTERVAL_MAX = 60
-        POLL_INTERVAL = max(
-            min(timeout / 60, POLL_INTERVAL_MAX), POLL_INTERVAL_MIN
-        )
+        POLL_INTERVAL = max(min(timeout / 60, POLL_INTERVAL_MAX), POLL_INTERVAL_MIN)
         retry_count = math.ceil(timeout / POLL_INTERVAL)
         count = 0
         while count < retry_count:

{proximl-0.5.6 → proximl-0.5.8}/proximl/cli/cloudbender/__init__.py

@@ -15,4 +15,5 @@ from proximl.cli.cloudbender.region import region
 from proximl.cli.cloudbender.node import node
 from proximl.cli.cloudbender.device import device
 from proximl.cli.cloudbender.datastore import datastore
+from proximl.cli.cloudbender.data_connector import data_connector
 from proximl.cli.cloudbender.service import service

proximl-0.5.8/proximl/cli/cloudbender/data_connector.py

@@ -0,0 +1,159 @@
+import click
+from proximl.cli import cli, pass_config, search_by_id_name
+from proximl.cli.cloudbender import cloudbender
+
+
+@cloudbender.group()
+@pass_config
+def data_connector(config):
+    """proxiML CloudBender data connector commands."""
+    pass
+
+
+@data_connector.command()
+@click.option(
+    "--provider",
+    "-p",
+    type=click.STRING,
+    required=True,
+    help="The provider ID of the region.",
+)
+@click.option(
+    "--region",
+    "-r",
+    type=click.STRING,
+    required=True,
+    help="The region ID to list data connectors for.",
+)
+@pass_config
+def list(config, provider, region):
+    """List data connectors."""
+    data = [
+        ["ID", "NAME", "TYPE"],
+        [
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+        ],
+    ]
+
+    data_connectors = config.proximl.run(
+        config.proximl.client.cloudbender.data_connectors.list(
+            provider_uuid=provider, region_uuid=region
+        )
+    )
+
+    for data_connector in data_connectors:
+        data.append(
+            [
+                data_connector.id,
+                data_connector.name,
+                data_connector.type,
+            ]
+        )
+
+    for row in data:
+        click.echo(
+            "{: >37.36} {: >29.28} {: >9.8}" "".format(*row),
+            file=config.stdout,
+        )
+
+
+@data_connector.command()
+@click.option(
+    "--provider",
+    "-p",
+    type=click.STRING,
+    required=True,
+    help="The provider ID of the region.",
+)
+@click.option(
+    "--region",
+    "-r",
+    type=click.STRING,
+    required=True,
+    help="The region ID to create the data_connector in.",
+)
+@click.option(
+    "--type",
+    "-t",
+    type=click.Choice(
+        [
+            "custom",
+        ],
+        case_sensitive=False,
+    ),
+    required=True,
+    help="The type of data connector to create.",
+)
+@click.option(
+    "--protocol",
+    "-r",
+    type=click.STRING,
+    help="The transport protocol of the data connector",
+)
+@click.option(
+    "--port-range",
+    "-p",
+    type=click.STRING,
+    help="The port range of the data connector",
+)
+@click.option(
+    "--cidr",
+    "-i",
+    type=click.STRING,
+    help="The IP range to allow in CIDR notation",
+)
+@click.argument("name", type=click.STRING, required=True)
+@pass_config
+def create(config, provider, region, type, protocol, port_range, cidr, name):
+    """
+    Creates a data_connector.
+    """
+    return config.proximl.run(
+        config.proximl.client.cloudbender.data_connectors.create(
+            provider_uuid=provider,
+            region_uuid=region,
+            name=name,
+            type=type,
+            protocol=protocol,
+            port_range=port_range,
+            cidr=cidr,
+        )
+    )
+
+
+@data_connector.command()
+@click.option(
+    "--provider",
+    "-p",
+    type=click.STRING,
+    required=True,
+    help="The provider ID of the region.",
+)
+@click.option(
+    "--region",
+    "-r",
+    type=click.STRING,
+    required=True,
+    help="The region ID to remove the data_connector from.",
+)
+@click.argument("data_connector", type=click.STRING)
+@pass_config
+def remove(config, provider, region, data_connector):
+    """
+    Remove a data_connector.
+
+    DATASTORE may be specified by name or ID, but ID is preferred.
+    """
+    data_connectors = config.proximl.run(
+        config.proximl.client.cloudbender.data_connectors.list(
+            provider_uuid=provider, region_uuid=region
+        )
+    )
+
+    found = search_by_id_name(data_connector, data_connectors)
+    if None is found:
+        raise click.UsageError("Cannot find specified data_connector.")
+
+    return config.proximl.run(found.remove())

{proximl-0.5.6 → proximl-0.5.8}/proximl/cli/cloudbender/service.py

@@ -74,6 +74,19 @@ def list(config, provider, region):
     required=True,
     help="The region ID to create the service in.",
 )
+@click.option(
+    "--type",
+    "-t",
+    type=click.Choice(
+        [
+            "https",
+            "tcp",
+            "udp",
+        ],
+    ),
+    required=True,
+    help="The type of regional service.",
+)
 @click.option(
     "--public/--no-public",
     default=True,
@@ -82,13 +95,17 @@ def list(config, provider, region):
 )
 @click.argument("name", type=click.STRING, required=True)
 @pass_config
-def create(config, provider, region, public, name):
+def create(config, provider, region, type, public, name):
     """
     Creates a service.
     """
     return config.proximl.run(
         config.proximl.client.cloudbender.services.create(
-            provider_uuid=provider, region_uuid=region, name=name, public=public
+            provider_uuid=provider,
+            region_uuid=region,
+            name=name,
+            type=type,
+            public=public,
         )
     )
 

{proximl-0.5.6 → proximl-0.5.8}/proximl/cloudbender/cloudbender.py

@@ -3,6 +3,7 @@ from .regions import Regions
 from .nodes import Nodes
 from .devices import Devices
 from .datastores import Datastores
+from .data_connectors import DataConnectors
 from .services import Services
 from .device_configs import DeviceConfigs
 
@@ -15,5 +16,6 @@ class Cloudbender(object):
         self.nodes = Nodes(proximl)
         self.devices = Devices(proximl)
         self.datastores = Datastores(proximl)
+        self.data_connectors = DataConnectors(proximl)
         self.services = Services(proximl)
         self.device_configs = DeviceConfigs(proximl)

proximl-0.5.8/proximl/cloudbender/data_connectors.py

@@ -0,0 +1,112 @@
+import json
+import logging
+
+
+class DataConnectors(object):
+    def __init__(self, proximl):
+        self.proximl = proximl
+
+    async def get(self, provider_uuid, region_uuid, id, **kwargs):
+        resp = await self.proximl._query(
+            f"/provider/{provider_uuid}/region/{region_uuid}/data_connector/{id}",
+            "GET",
+            kwargs,
+        )
+        return DataConnector(self.proximl, **resp)
+
+    async def list(self, provider_uuid, region_uuid, **kwargs):
+        resp = await self.proximl._query(
+            f"/provider/{provider_uuid}/region/{region_uuid}/data_connector",
+            "GET",
+            kwargs,
+        )
+        data_connectors = [
+            DataConnector(self.proximl, **data_connector) for data_connector in resp
+        ]
+        return data_connectors
+
+    async def create(
+        self,
+        provider_uuid,
+        region_uuid,
+        name,
+        type,
+        **kwargs,
+    ):
+        logging.info(f"Creating Data Connector {name}")
+        data = dict(
+            name=name,
+            type=type,
+            **kwargs,
+        )
+        payload = {k: v for k, v in data.items() if v is not None}
+        resp = await self.proximl._query(
+            f"/provider/{provider_uuid}/region/{region_uuid}/data_connector",
+            "POST",
+            None,
+            payload,
+        )
+        data_connector = DataConnector(self.proximl, **resp)
+        logging.info(f"Created Data Connector {name} with id {data_connector.id}")
+        return data_connector
+
+    async def remove(self, provider_uuid, region_uuid, id, **kwargs):
+        await self.proximl._query(
+            f"/provider/{provider_uuid}/region/{region_uuid}/data_connector/{id}",
+            "DELETE",
+            kwargs,
+        )
+
+
+class DataConnector:
+    def __init__(self, proximl, **kwargs):
+        self.proximl = proximl
+        self._data_connector = kwargs
+        self._id = self._data_connector.get("connector_id")
+        self._provider_uuid = self._data_connector.get("provider_uuid")
+        self._region_uuid = self._data_connector.get("region_uuid")
+        self._type = self._data_connector.get("type")
+        self._name = self._data_connector.get("name")
+
+    @property
+    def id(self) -> str:
+        return self._id
+
+    @property
+    def provider_uuid(self) -> str:
+        return self._provider_uuid
+
+    @property
+    def region_uuid(self) -> str:
+        return self._region_uuid
+
+    @property
+    def type(self) -> str:
+        return self._type
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    def __str__(self):
+        return json.dumps({k: v for k, v in self._data_connector.items()})
+
+    def __repr__(self):
+        return f"DataConnector( proximl , **{self._data_connector.__repr__()})"
+
+    def __bool__(self):
+        return bool(self._id)
+
+    async def remove(self):
+        await self.proximl._query(
+            f"/provider/{self._provider_uuid}/region/{self._region_uuid}/data_connector/{self._id}",
+            "DELETE",
+        )
+
+    async def refresh(self):
+        resp = await self.proximl._query(
+            f"/provider/{self._provider_uuid}/region/{self._region_uuid}/data_connector/{self._id}",
+            "GET",
+        )
+        self.__init__(self.proximl, **resp)
+        return self

{proximl-0.5.6 → proximl-0.5.8}/proximl/cloudbender/services.py

@@ -1,5 +1,13 @@
 import json
 import logging
+import asyncio
+import math
+
+from proximl.exceptions import (
+    ApiError,
+    SpecificationError,
+    ProxiMLException,
+)
 
 
 class Services(object):
@@ -28,12 +36,14 @@ class Services(object):
         provider_uuid,
         region_uuid,
         name,
+        type,
         public,
         **kwargs,
     ):
         logging.info(f"Creating Service {name}")
         data = dict(
             name=name,
+            type=type,
             public=public,
             **kwargs,
         )
@@ -65,7 +75,12 @@ class Service:
         self._region_uuid = self._service.get("region_uuid")
         self._public = self._service.get("public")
         self._name = self._service.get("name")
-        self._hostname = self._service.get("hostname")
+        self._type = self._service.get("type")
+        self._hostname = self._service.get("custom_hostname") or self._service.get(
+            "hostname"
+        )
+        self._status = self._service.get("status")
+        self._port = self._service.get("port")
 
     @property
     def id(self) -> str:
@@ -91,6 +106,18 @@ class Service:
     def hostname(self) -> str:
         return self._hostname
 
+    @property
+    def status(self) -> str:
+        return self._status
+
+    @property
+    def type(self) -> str:
+        return self._type
+
+    @property
+    def port(self) -> str:
+        return self._port
+
     def __str__(self):
         return json.dumps({k: v for k, v in self._service.items()})
 
@@ -113,3 +140,40 @@ class Service:
         )
         self.__init__(self.proximl, **resp)
         return self
+
+    async def wait_for(self, status, timeout=300):
+        if self.status == status:
+            return
+        valid_statuses = ["active", "archived"]
+        if not status in valid_statuses:
+            raise SpecificationError(
+                "status",
+                f"Invalid wait_for status {status}.  Valid statuses are: {valid_statuses}",
+            )
+        MAX_TIMEOUT = 24 * 60 * 60
+        if timeout > MAX_TIMEOUT:
+            raise SpecificationError(
+                "timeout",
+                f"timeout must be less than {MAX_TIMEOUT} seconds.",
+            )
+
+        POLL_INTERVAL_MIN = 5
+        POLL_INTERVAL_MAX = 60
+        POLL_INTERVAL = max(min(timeout / 60, POLL_INTERVAL_MAX), POLL_INTERVAL_MIN)
+        retry_count = math.ceil(timeout / POLL_INTERVAL)
+        count = 0
+        while count < retry_count:
+            await asyncio.sleep(POLL_INTERVAL)
+            try:
+                await self.refresh()
+            except ApiError as e:
+                if status == "archived" and e.status == 404:
+                    return
+                raise e
+            if self.status == status:
+                return self
+            else:
+                count += 1
+                logging.debug(f"self: {self}, retry count {count}")
+
+        raise ProxiMLException(f"Timeout waiting for {status}")

{proximl-0.5.6 → proximl-0.5.8}/proximl/datasets.py

@@ -60,7 +60,10 @@ class Dataset:
         self._id = self._dataset.get("id", self._dataset.get("dataset_uuid"))
         self._status = self._dataset.get("status")
         self._name = self._dataset.get("name")
-        self._size = self._dataset.get("size")
+        self._size = self._dataset.get("size") or self._dataset.get("used_size")
+        self._billed_size = self._dataset.get("billed_size") or self._dataset.get(
+            "size"
+        )
         self._project_uuid = self._dataset.get("project_uuid")
 
     @property
@@ -79,6 +82,10 @@ class Dataset:
     def size(self) -> int:
         return self._size or 0
 
+    @property
+    def billed_size(self) -> int:
+        return self._billed_size
+
     def __str__(self):
         return json.dumps({k: v for k, v in self._dataset.items()})
 
@@ -119,12 +126,16 @@ class Dataset:
                 project_uuid=self._dataset.get("project_uuid"),
                 cidr=self._dataset.get("vpn").get("cidr"),
                 ssh_port=self._dataset.get("vpn").get("client").get("ssh_port"),
-                input_path=self._dataset.get("source_uri")
-                if self.status in ["new", "downloading"]
-                else None,
-                output_path=self._dataset.get("output_uri")
-                if self.status == "exporting"
-                else None,
+                input_path=(
+                    self._dataset.get("source_uri")
+                    if self.status in ["new", "downloading"]
+                    else None
+                ),
+                output_path=(
+                    self._dataset.get("output_uri")
+                    if self.status == "exporting"
+                    else None
+                ),
             )
         else:
             details = dict()
@@ -215,14 +226,21 @@ class Dataset:
         return self
 
     async def wait_for(self, status, timeout=300):
+        if self.status == status:
+            return
         valid_statuses = ["downloading", "ready", "archived"]
         if not status in valid_statuses:
             raise SpecificationError(
                 "status",
                 f"Invalid wait_for status {status}.  Valid statuses are: {valid_statuses}",
             )
-        if self.status == status:
-            return
+        MAX_TIMEOUT = 24 * 60 * 60
+        if timeout > MAX_TIMEOUT:
+            raise SpecificationError(
+                "timeout",
+                f"timeout must be less than {MAX_TIMEOUT} seconds.",
+            )
+
         POLL_INTERVAL_MIN = 5
         POLL_INTERVAL_MAX = 60
         POLL_INTERVAL = max(min(timeout / 60, POLL_INTERVAL_MAX), POLL_INTERVAL_MIN)

{proximl-0.5.6 → proximl-0.5.8}/proximl/jobs.py

@@ -468,6 +468,12 @@ class Job:
         return job
 
     async def wait_for(self, status, timeout=300):
+        if self.status == status or (
+            self.type == "training"
+            and status == "finished"
+            and self.status == "stopped"
+        ):
+            return
         valid_statuses = [
             "waiting for data/model download",
             "waiting for GPUs",
@@ -492,12 +498,13 @@ class Job:
                 "'stopped' status is deprecated for training jobs, use 'finished' instead.",
                 DeprecationWarning,
             )
-        if self.status == status or (
-            self.type == "training"
-            and status == "finished"
-            and self.status == "stopped"
-        ):
-            return
+
+        MAX_TIMEOUT = 24 * 60 * 60
+        if timeout > MAX_TIMEOUT:
+            raise SpecificationError(
+                "timeout",
+                f"timeout must be less than {MAX_TIMEOUT} seconds.",
+            )
 
         POLL_INTERVAL_MIN = 5
         POLL_INTERVAL_MAX = 60