proximl 0.5.10__py3-none-any.whl → 0.5.12__py3-none-any.whl

proximl/cloudbender/datastores.py

@@ -1,5 +1,13 @@
 import json
 import logging
+import asyncio
+import math
+
+from proximl.exceptions import (
+    ApiError,
+    SpecificationError,
+    ProxiMLException,
+)
 
 
 class Datastores(object):
@@ -20,9 +28,7 @@ class Datastores(object):
             "GET",
             kwargs,
         )
-        datastores = [
-            Datastore(self.proximl, **datastore) for datastore in resp
-        ]
+        datastores = [Datastore(self.proximl, **datastore) for datastore in resp]
         return datastores
 
     async def create(
@@ -31,18 +37,12 @@ class Datastores(object):
         region_uuid,
         name,
         type,
-        uri,
-        root,
-        options=None,
         **kwargs,
     ):
         logging.info(f"Creating Datastore {name}")
         data = dict(
             name=name,
             type=type,
-            uri=uri,
-            root=root,
-            options=options,
             **kwargs,
         )
         payload = {k: v for k, v in data.items() if v is not None}
@@ -73,8 +73,6 @@ class Datastore:
         self._region_uuid = self._datastore.get("region_uuid")
         self._type = self._datastore.get("type")
         self._name = self._datastore.get("name")
-        self._uri = self._datastore.get("uri")
-        self._root = self._datastore.get("root")
 
     @property
     def id(self) -> str:
@@ -96,14 +94,6 @@ class Datastore:
     def name(self) -> str:
         return self._name
 
-    @property
-    def uri(self) -> str:
-        return self._uri
-
-    @property
-    def root(self) -> str:
-        return self._root
-
     def __str__(self):
         return json.dumps({k: v for k, v in self._datastore.items()})
 

proximl/cloudbender/nodes.py

@@ -1,5 +1,9 @@
 import json
 import logging
+import asyncio
+import math
+
+from proximl.exceptions import ApiError, SpecificationError, ProxiMLException, NodeError
 
 
 class Nodes(object):
@@ -29,7 +33,7 @@ class Nodes(object):
         region_uuid,
         friendly_name,
         hostname,
-        minion_id,
+        minion_id=None,
         type="permanent",
         service="compute",
         **kwargs,
@@ -153,3 +157,42 @@ class Node:
             None,
             dict(command=command),
         )
+
+    async def wait_for(self, status, timeout=300):
+        if self.status == status:
+            return
+        valid_statuses = ["active", "maintenance", "offline", "stopped", "archived"]
+        if not status in valid_statuses:
+            raise SpecificationError(
+                "status",
+                f"Invalid wait_for status {status}.  Valid statuses are: {valid_statuses}",
+            )
+        MAX_TIMEOUT = 24 * 60 * 60
+        if timeout > MAX_TIMEOUT:
+            raise SpecificationError(
+                "timeout",
+                f"timeout must be less than {MAX_TIMEOUT} seconds.",
+            )
+
+        POLL_INTERVAL_MIN = 5
+        POLL_INTERVAL_MAX = 60
+        POLL_INTERVAL = max(min(timeout / 60, POLL_INTERVAL_MAX), POLL_INTERVAL_MIN)
+        retry_count = math.ceil(timeout / POLL_INTERVAL)
+        count = 0
+        while count < retry_count:
+            await asyncio.sleep(POLL_INTERVAL)
+            try:
+                await self.refresh()
+            except ApiError as e:
+                if status == "archived" and e.status == 404:
+                    return
+                raise e
+            if self.status in ["errored", "failed"]:
+                raise NodeError(self.status, self)
+            if self.status == status:
+                return self
+            else:
+                count += 1
+                logging.debug(f"self: {self}, retry count {count}")
+
+        raise ProxiMLException(f"Timeout waiting for {status}")

proximl/cloudbender/providers.py

@@ -1,7 +1,16 @@
 import json
 import logging
+import asyncio
+import math
 from datetime import datetime
 
+from proximl.exceptions import (
+    ApiError,
+    SpecificationError,
+    ProxiMLException,
+    ProviderError,
+)
+
 
 class Providers(object):
     def __init__(self, proximl):
@@ -36,6 +45,7 @@ class Provider:
         self._provider = kwargs
         self._id = self._provider.get("provider_uuid")
         self._type = self._provider.get("type")
+        self._status = self._provider.get("status")
         self._credits = self._provider.get("credits")
 
     @property
@@ -46,6 +56,10 @@ class Provider:
     def type(self) -> str:
         return self._type
 
+    @property
+    def status(self) -> str:
+        return self._status
+
     @property
     def credits(self) -> float:
         return self._credits
@@ -69,3 +83,42 @@ class Provider:
         )
         self.__init__(self.proximl, **resp)
         return self
+
+    async def wait_for(self, status, timeout=300):
+        if self.status == status:
+            return
+        valid_statuses = ["ready", "archived"]
+        if not status in valid_statuses:
+            raise SpecificationError(
+                "status",
+                f"Invalid wait_for status {status}.  Valid statuses are: {valid_statuses}",
+            )
+        MAX_TIMEOUT = 24 * 60 * 60
+        if timeout > MAX_TIMEOUT:
+            raise SpecificationError(
+                "timeout",
+                f"timeout must be less than {MAX_TIMEOUT} seconds.",
+            )
+
+        POLL_INTERVAL_MIN = 5
+        POLL_INTERVAL_MAX = 60
+        POLL_INTERVAL = max(min(timeout / 60, POLL_INTERVAL_MAX), POLL_INTERVAL_MIN)
+        retry_count = math.ceil(timeout / POLL_INTERVAL)
+        count = 0
+        while count < retry_count:
+            await asyncio.sleep(POLL_INTERVAL)
+            try:
+                await self.refresh()
+            except ApiError as e:
+                if status == "archived" and e.status == 404:
+                    return
+                raise e
+            if self.status in ["errored", "failed"]:
+                raise ProviderError(self.status, self)
+            if self.status == status:
+                return self
+            else:
+                count += 1
+                logging.debug(f"self: {self}, retry count {count}")
+
+        raise ProxiMLException(f"Timeout waiting for {status}")

proximl/cloudbender/regions.py

@@ -1,5 +1,14 @@
 import json
 import logging
+import asyncio
+import math
+
+from proximl.exceptions import (
+    ApiError,
+    SpecificationError,
+    ProxiMLException,
+    RegionError,
+)
 
 
 class Regions(object):
@@ -111,3 +120,42 @@ class Region:
             None,
             dict(project_uuid=project_uuid, checkpoint_uuid=checkpoint_uuid),
         )
+
+    async def wait_for(self, status, timeout=300):
+        if self.status == status:
+            return
+        valid_statuses = ["healthy", "offline", "archived"]
+        if not status in valid_statuses:
+            raise SpecificationError(
+                "status",
+                f"Invalid wait_for status {status}.  Valid statuses are: {valid_statuses}",
+            )
+        MAX_TIMEOUT = 24 * 60 * 60
+        if timeout > MAX_TIMEOUT:
+            raise SpecificationError(
+                "timeout",
+                f"timeout must be less than {MAX_TIMEOUT} seconds.",
+            )
+
+        POLL_INTERVAL_MIN = 5
+        POLL_INTERVAL_MAX = 60
+        POLL_INTERVAL = max(min(timeout / 60, POLL_INTERVAL_MAX), POLL_INTERVAL_MIN)
+        retry_count = math.ceil(timeout / POLL_INTERVAL)
+        count = 0
+        while count < retry_count:
+            await asyncio.sleep(POLL_INTERVAL)
+            try:
+                await self.refresh()
+            except ApiError as e:
+                if status == "archived" and e.status == 404:
+                    return
+                raise e
+            if self.status in ["errored", "failed"]:
+                raise RegionError(self.status, self)
+            if self.status == status:
+                return self
+            else:
+                count += 1
+                logging.debug(f"self: {self}, retry count {count}")
+
+        raise ProxiMLException(f"Timeout waiting for {status}")

proximl/cloudbender/services.py

@@ -1,5 +1,13 @@
 import json
 import logging
+import asyncio
+import math
+
+from proximl.exceptions import (
+    ApiError,
+    SpecificationError,
+    ProxiMLException,
+)
 
 
 class Services(object):
@@ -28,12 +36,14 @@ class Services(object):
         provider_uuid,
         region_uuid,
         name,
+        type,
         public,
         **kwargs,
     ):
         logging.info(f"Creating Service {name}")
         data = dict(
             name=name,
+            type=type,
             public=public,
             **kwargs,
         )
@@ -65,7 +75,12 @@ class Service:
         self._region_uuid = self._service.get("region_uuid")
         self._public = self._service.get("public")
         self._name = self._service.get("name")
-        self._hostname = self._service.get("hostname")
+        self._type = self._service.get("type")
+        self._hostname = self._service.get("custom_hostname") or self._service.get(
+            "hostname"
+        )
+        self._status = self._service.get("status")
+        self._port = self._service.get("port")
 
     @property
     def id(self) -> str:
@@ -91,6 +106,18 @@ class Service:
     def hostname(self) -> str:
         return self._hostname
 
+    @property
+    def status(self) -> str:
+        return self._status
+
+    @property
+    def type(self) -> str:
+        return self._type
+
+    @property
+    def port(self) -> str:
+        return self._port
+
     def __str__(self):
         return json.dumps({k: v for k, v in self._service.items()})
 
@@ -113,3 +140,40 @@ class Service:
         )
         self.__init__(self.proximl, **resp)
         return self
+
+    async def wait_for(self, status, timeout=300):
+        if self.status == status:
+            return
+        valid_statuses = ["active", "archived"]
+        if not status in valid_statuses:
+            raise SpecificationError(
+                "status",
+                f"Invalid wait_for status {status}.  Valid statuses are: {valid_statuses}",
+            )
+        MAX_TIMEOUT = 24 * 60 * 60
+        if timeout > MAX_TIMEOUT:
+            raise SpecificationError(
+                "timeout",
+                f"timeout must be less than {MAX_TIMEOUT} seconds.",
+            )
+
+        POLL_INTERVAL_MIN = 5
+        POLL_INTERVAL_MAX = 60
+        POLL_INTERVAL = max(min(timeout / 60, POLL_INTERVAL_MAX), POLL_INTERVAL_MIN)
+        retry_count = math.ceil(timeout / POLL_INTERVAL)
+        count = 0
+        while count < retry_count:
+            await asyncio.sleep(POLL_INTERVAL)
+            try:
+                await self.refresh()
+            except ApiError as e:
+                if status == "archived" and e.status == 404:
+                    return
+                raise e
+            if self.status == status:
+                return self
+            else:
+                count += 1
+                logging.debug(f"self: {self}, retry count {count}")
+
+        raise ProxiMLException(f"Timeout waiting for {status}")

proximl/datasets.py

@@ -31,13 +31,24 @@ class Datasets(object):
         datasets = [Dataset(self.proximl, **dataset) for dataset in resp]
         return datasets
 
-    async def create(self, name, source_type, source_uri, **kwargs):
+    async def create(
+        self,
+        name,
+        source_type,
+        source_uri,
+        type="evefs",
+        project_uuid=None,
+        **kwargs,
+    ):
+        if not project_uuid:
+            project_uuid = self.proximl.active_project
         data = dict(
             name=name,
             source_type=source_type,
             source_uri=source_uri,
-            source_options=kwargs.get("source_options"),
-            project_uuid=kwargs.get("project_uuid") or self.proximl.active_project,
+            project_uuid=project_uuid,
+            type=type,
+            **kwargs,
         )
         payload = {k: v for k, v in data.items() if v is not None}
         logging.info(f"Creating Dataset {name}")
@@ -60,7 +71,10 @@ class Dataset:
         self._id = self._dataset.get("id", self._dataset.get("dataset_uuid"))
         self._status = self._dataset.get("status")
         self._name = self._dataset.get("name")
-        self._size = self._dataset.get("size")
+        self._size = self._dataset.get("size") or self._dataset.get("used_size")
+        self._billed_size = self._dataset.get("billed_size") or self._dataset.get(
+            "size"
+        )
         self._project_uuid = self._dataset.get("project_uuid")
 
     @property
@@ -79,6 +93,10 @@ class Dataset:
     def size(self) -> int:
         return self._size or 0
 
+    @property
+    def billed_size(self) -> int:
+        return self._billed_size
+
     def __str__(self):
         return json.dumps({k: v for k, v in self._dataset.items()})
 
@@ -119,12 +137,16 @@ class Dataset:
                 project_uuid=self._dataset.get("project_uuid"),
                 cidr=self._dataset.get("vpn").get("cidr"),
                 ssh_port=self._dataset.get("vpn").get("client").get("ssh_port"),
-                input_path=self._dataset.get("source_uri")
-                if self.status in ["new", "downloading"]
-                else None,
-                output_path=self._dataset.get("output_uri")
-                if self.status == "exporting"
-                else None,
+                input_path=(
+                    self._dataset.get("source_uri")
+                    if self.status in ["new", "downloading"]
+                    else None
+                ),
+                output_path=(
+                    self._dataset.get("output_uri")
+                    if self.status == "exporting"
+                    else None
+                ),
             )
         else:
             details = dict()
@@ -215,14 +237,21 @@ class Dataset:
         return self
 
     async def wait_for(self, status, timeout=300):
+        if self.status == status:
+            return
         valid_statuses = ["downloading", "ready", "archived"]
         if not status in valid_statuses:
             raise SpecificationError(
                 "status",
                 f"Invalid wait_for status {status}.  Valid statuses are: {valid_statuses}",
             )
-        if self.status == status:
-            return
+        MAX_TIMEOUT = 24 * 60 * 60
+        if timeout > MAX_TIMEOUT:
+            raise SpecificationError(
+                "timeout",
+                f"timeout must be less than {MAX_TIMEOUT} seconds.",
+            )
+
         POLL_INTERVAL_MIN = 5
         POLL_INTERVAL_MAX = 60
         POLL_INTERVAL = max(min(timeout / 60, POLL_INTERVAL_MAX), POLL_INTERVAL_MIN)

proximl/exceptions.py

@@ -147,3 +147,54 @@ class SpecificationError(ProxiMLException):
 
     def __str__(self):
         return "SpecificationError({self.attribute}, {self.message})".format(self=self)
+
+
+class NodeError(ProxiMLException):
+    def __init__(self, status, data, *args):
+        super().__init__(data, *args)
+        self._status = status
+        self._message = data
+
+    @property
+    def status(self) -> str:
+        return self._status
+
+    def __repr__(self):
+        return "NodeError({self.status}, {self.message})".format(self=self)
+
+    def __str__(self):
+        return "NodeError({self.status}, {self.message})".format(self=self)
+
+
+class ProviderError(ProxiMLException):
+    def __init__(self, status, data, *args):
+        super().__init__(data, *args)
+        self._status = status
+        self._message = data
+
+    @property
+    def status(self) -> str:
+        return self._status
+
+    def __repr__(self):
+        return "ProviderError({self.status}, {self.message})".format(self=self)
+
+    def __str__(self):
+        return "ProviderError({self.status}, {self.message})".format(self=self)
+
+
+class RegionError(ProxiMLException):
+    def __init__(self, status, data, *args):
+        super().__init__(data, *args)
+        self._status = status
+        self._message = data
+
+    @property
+    def status(self) -> str:
+        return self._status
+
+    def __repr__(self):
+        return "RegionError({self.status}, {self.message})".format(self=self)
+
+    def __str__(self):
+        return "RegionError({self.status}, {self.message})".format(self=self)

proximl/jobs.py

@@ -468,9 +468,14 @@ class Job:
         return job
 
     async def wait_for(self, status, timeout=300):
+        if self.status == status or (
+            self.type == "training"
+            and status == "finished"
+            and self.status == "stopped"
+        ):
+            return
         valid_statuses = [
             "waiting for data/model download",
-            "waiting for GPUs",
             "waiting for resources",
             "running",
             "stopped",
@@ -482,22 +487,18 @@ class Job:
                 "status",
                 f"Invalid wait_for status {status}.  Valid statuses are: {valid_statuses}",
             )
-        if status == "waiting for GPUs":
-            warnings.warn(
-                "'waiting for GPUs' status is deprecated, use 'waiting for resources' instead.",
-                DeprecationWarning,
-            )
         if (self.type == "training") and status == "stopped":
             warnings.warn(
                 "'stopped' status is deprecated for training jobs, use 'finished' instead.",
                 DeprecationWarning,
             )
-        if self.status == status or (
-            self.type == "training"
-            and status == "finished"
-            and self.status == "stopped"
-        ):
-            return
+
+        MAX_TIMEOUT = 24 * 60 * 60
+        if timeout > MAX_TIMEOUT:
+            raise SpecificationError(
+                "timeout",
+                f"timeout must be less than {MAX_TIMEOUT} seconds.",
+            )
 
         POLL_INTERVAL_MIN = 5
         POLL_INTERVAL_MAX = 60
@@ -516,12 +517,8 @@ class Job:
                 self.status == status
                 or (
                     status
-                    in [
-                        "waiting for GPUs",
-                        "waiting for resources",
-                    ]  ## this status could be very short and the polling could miss it
-                    and self.status
-                    not in ["new", "waiting for GPUs", "waiting for resources"]
+                    == "waiting for resources"  ## this status could be very short and the polling could miss it
+                    and self.status not in ["new", "waiting for resources"]
                 )
                 or (
                     status
@@ -529,7 +526,6 @@ class Job:
                     and self.status
                     not in [
                         "new",
-                        "waiting for GPUs",
                         "waiting for resources",
                         "waiting for data/model download",
                     ]