proximl 0.5.0__tar.gz → 0.5.2__tar.gz

{proximl-0.5.0/proximl.egg-info → proximl-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: proximl
-Version: 0.5.0
+Version: 0.5.2
 Summary: proxiML client SDK and command line utilities
 Home-page: https://github.com/proxiML/python-sdk
 Author: proxiML

{proximl-0.5.0 → proximl-0.5.2}/proximl/__init__.py

@@ -13,5 +13,5 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 
 
-__version__ = "0.5.0"
+__version__ = "0.5.2"
 __all__ = "ProxiML"

{proximl-0.5.0 → proximl-0.5.2}/proximl/auth.py

@@ -222,10 +222,6 @@ n_hex = (
 g_hex = "2"
 info_bits = bytearray("Caldera Derived Key", "utf-8")
 
-CONFIG_DIR = os.path.expanduser(
-    os.environ.get("PROXIML_CONFIG_DIR") or "~/.proximl"
-)
-
 
 def hash_sha256(buf):
     """AuthenticationHelper.hash"""
@@ -512,9 +508,9 @@ class AWSSRP(object):
 
 
 class Auth(object):
-    def __init__(self, domain_suffix="proximl.ai", **kwargs):
+    def __init__(self, config_dir, domain_suffix="proximl.ai", **kwargs):
         try:
-            with open(f"{CONFIG_DIR}/environment.json", "r") as file:
+            with open(f"{config_dir}/environment.json", "r") as file:
                 env_str = file.read().replace("\n", "")
             env = json.loads(env_str)
         except:
@@ -544,7 +540,7 @@ class Auth(object):
         )
 
         try:
-            with open(f"{CONFIG_DIR}/credentials.json", "r") as file:
+            with open(f"{config_dir}/credentials.json", "r") as file:
                 key_str = file.read().replace("\n", "")
             keys = json.loads(key_str)
         except:

{proximl-0.5.0 → proximl-0.5.2}/proximl/cli/cloudbender/__init__.py

@@ -13,5 +13,6 @@ def cloudbender(config):
 from proximl.cli.cloudbender.provider import provider
 from proximl.cli.cloudbender.region import region
 from proximl.cli.cloudbender.node import node
+from proximl.cli.cloudbender.device import device
 from proximl.cli.cloudbender.datastore import datastore
 from proximl.cli.cloudbender.reservation import reservation

proximl-0.5.2/proximl/cli/cloudbender/device.py

@@ -0,0 +1,157 @@
+import click
+from proximl.cli import cli, pass_config, search_by_id_name
+from proximl.cli.cloudbender import cloudbender
+
+
+@cloudbender.group()
+@pass_config
+def device(config):
+    """proxiML CloudBender device commands."""
+    pass
+
+
+@device.command()
+@click.option(
+    "--provider",
+    "-p",
+    type=click.STRING,
+    required=True,
+    help="The provider ID of the region.",
+)
+@click.option(
+    "--region",
+    "-r",
+    type=click.STRING,
+    required=True,
+    help="The region ID to list devices for.",
+)
+@pass_config
+def list(config, provider, region):
+    """List devices."""
+    data = [
+        [
+            "ID",
+            "NAME",
+            "STATUS",
+            "JOB STATUS",
+            "ONLINE",
+            "MAINTENANCE",
+        ],
+        [
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+            "-" * 80,
+        ],
+    ]
+
+    devices = config.proximl.run(
+        config.proximl.client.cloudbender.devices.list(
+            provider_uuid=provider, region_uuid=region
+        )
+    )
+
+    for device in devices:
+        data.append(
+            [
+                device.id,
+                device.name,
+                device.status,
+                device.job_status,
+                "X" if device.online else "",
+                "X" if device.maintenance_mode else "",
+            ]
+        )
+
+    for row in data:
+        click.echo(
+            "{: >37.36} {: >29.28} {: >9.8} {: >11.10} {: >7.6} {: >12.11}"
+            "".format(*row),
+            file=config.stdout,
+        )
+
+
+@device.command()
+@click.option(
+    "--provider",
+    "-p",
+    type=click.STRING,
+    required=True,
+    help="The provider ID of the region.",
+)
+@click.option(
+    "--region",
+    "-r",
+    type=click.STRING,
+    required=True,
+    help="The region ID to create the region in.",
+)
+@click.option(
+    "--minion-id",
+    "-m",
+    type=click.STRING,
+    required=True,
+    help="The minion_id of the new node.",
+)
+@click.option(
+    "--hostname",
+    "-h",
+    type=click.STRING,
+    help="The hostname (if different from name)",
+)
+@click.argument("name", type=click.STRING, required=True)
+@pass_config
+def create(config, provider, region, minion_id, hostname, name):
+    """
+    Creates a node.
+    """
+    if not hostname:
+        hostname = name
+    return config.proximl.run(
+        config.proximl.client.cloudbender.devices.create(
+            provider_uuid=provider,
+            region_uuid=region,
+            friendly_name=name,
+            hostname=hostname,
+            minion_id=minion_id,
+        )
+    )
+
+
+@device.command()
+@click.option(
+    "--provider",
+    "-p",
+    type=click.STRING,
+    required=True,
+    help="The provider ID of the region.",
+)
+@click.option(
+    "--region",
+    "-r",
+    type=click.STRING,
+    required=True,
+    help="The region ID to delete the node from.",
+)
+@click.argument("device", type=click.STRING)
+@pass_config
+def remove(config, provider, region, node):
+    """
+    Remove a device.
+
+    DEVICE may be specified by name or ID, but ID is preferred.
+    """
+    devices = config.proximl.run(
+        config.proximl.client.cloudbender.devices.list(
+            provider_uuid=provider, region_uuid=region
+        )
+    )
+
+    found = search_by_id_name(device, devices)
+    if None is found:
+        raise click.UsageError("Cannot find specified device.")
+
+    return config.proximl.run(found.remove())

{proximl-0.5.0 → proximl-0.5.2}/proximl/cli/dataset.py

@@ -252,9 +252,7 @@ def rename(config, dataset, name):
     DATASET may be specified by name or ID, but ID is preferred.
     """
     try:
-        dataset = config.proximl.run(
-            config.proximl.client.datasets.get(dataset)
-        )
+        dataset = config.proximl.run(config.proximl.client.datasets.get(dataset))
         if dataset is None:
             raise click.UsageError("Cannot find specified dataset.")
     except:

{proximl-0.5.0 → proximl-0.5.2}/proximl/cli/job/create.py

@@ -683,7 +683,7 @@ def training(
         config.proximl.client.jobs.create(
             name=name,
             type="training",
-            gpu_type=gpu_type,
+            gpu_types=gpu_type,
             cpu_count=cpu_count,
             disk_size=disk_size,
             workers=[command for command in commands],
@@ -1021,7 +1021,7 @@ def inference(
         config.proximl.client.jobs.create(
             name=name,
             type="inference",
-            gpu_type=gpu_type,
+            gpu_types=gpu_type,
             cpu_count=cpu_count,
             disk_size=disk_size,
             workers=[command],
@@ -1330,7 +1330,7 @@ def endpoint(
         config.proximl.client.jobs.create(
             name=name,
             type="endpoint",
-            gpu_type=gpu_type,
+            gpu_types=gpu_type,
             cpu_count=cpu_count,
             disk_size=disk_size,
             endpoint=dict(routes=routes),

{proximl-0.5.0 → proximl-0.5.2}/proximl/cli/model.py

@@ -1,4 +1,5 @@
 import click
+import logging
 from proximl.cli import cli, pass_config, search_by_id_name
 
 
@@ -64,6 +65,7 @@ def connect(config, model, attach):
     models = config.proximl.run(config.proximl.client.models.list())
 
     found = search_by_id_name(model, models)
+    logging.debug(found)
     if None is found:
         raise click.UsageError("Cannot find specified model.")
 

{proximl-0.5.0 → proximl-0.5.2}/proximl/cloudbender/cloudbender.py

@@ -1,6 +1,7 @@
 from .providers import Providers
 from .regions import Regions
 from .nodes import Nodes
+from .devices import Devices
 from .datastores import Datastores
 from .reservations import Reservations
 from .device_configs import DeviceConfigs
@@ -12,6 +13,7 @@ class Cloudbender(object):
         self.providers = Providers(proximl)
         self.regions = Regions(proximl)
         self.nodes = Nodes(proximl)
+        self.devices = Devices(proximl)
         self.datastores = Datastores(proximl)
         self.reservations = Reservations(proximl)
         self.device_configs = DeviceConfigs(proximl)

{proximl-0.5.0 → proximl-0.5.2}/proximl/cloudbender/datastores.py

@@ -118,3 +118,11 @@ class Datastore:
             f"/provider/{self._provider_uuid}/region/{self._region_uuid}/datastore/{self._id}",
             "DELETE",
         )
+
+    async def refresh(self):
+        resp = await self.proximl._query(
+            f"/provider/{self._provider_uuid}/region/{self._region_uuid}/datastore/{self._id}",
+            "GET",
+        )
+        self.__init__(self.proximl, **resp)
+        return self

{proximl-0.5.0 → proximl-0.5.2}/proximl/cloudbender/device_configs.py

@@ -107,3 +107,11 @@ class DeviceConfig:
             f"/provider/{self._provider_uuid}/region/{self._region_uuid}/device/config/{self._id}",
             "DELETE",
         )
+
+    async def refresh(self):
+        resp = await self.proximl._query(
+            f"/provider/{self._provider_uuid}/region/{self._region_uuid}/device/config/{self._id}",
+            "GET",
+        )
+        self.__init__(self.proximl, **resp)
+        return self

proximl-0.5.2/proximl/cloudbender/devices.py

@@ -0,0 +1,190 @@
+import json
+import logging
+
+
+class Devices(object):
+    def __init__(self, proximl):
+        self.proximl = proximl
+
+    async def get(self, provider_uuid, region_uuid, id, **kwargs):
+        resp = await self.proximl._query(
+            f"/provider/{provider_uuid}/region/{region_uuid}/device/{id}",
+            "GET",
+            kwargs,
+        )
+        return Device(self.proximl, **resp)
+
+    async def list(self, provider_uuid, region_uuid, **kwargs):
+        resp = await self.proximl._query(
+            f"/provider/{provider_uuid}/region/{region_uuid}/device",
+            "GET",
+            kwargs,
+        )
+        devices = [Device(self.proximl, **device) for device in resp]
+        return devices
+
+    async def create(
+        self,
+        provider_uuid,
+        region_uuid,
+        friendly_name,
+        hostname,
+        minion_id,
+        **kwargs,
+    ):
+        logging.info(f"Creating Device {friendly_name}")
+        data = dict(
+            friendly_name=friendly_name,
+            hostname=hostname,
+            minion_id=minion_id,
+            type="device",
+            service="compute",
+            **kwargs,
+        )
+        payload = {k: v for k, v in data.items() if v is not None}
+        resp = await self.proximl._query(
+            f"/provider/{provider_uuid}/region/{region_uuid}/device",
+            "POST",
+            None,
+            payload,
+        )
+        device = Device(self.proximl, **resp)
+        logging.info(f"Created Device {friendly_name} with id {device.id}")
+        return device
+
+    async def remove(self, provider_uuid, region_uuid, id, **kwargs):
+        await self.proximl._query(
+            f"/provider/{provider_uuid}/region/{region_uuid}/device/{id}",
+            "DELETE",
+            kwargs,
+        )
+
+
+class Device:
+    def __init__(self, proximl, **kwargs):
+        self.proximl = proximl
+        self._device = kwargs
+        self._id = self._device.get("device_id")
+        self._provider_uuid = self._device.get("provider_uuid")
+        self._region_uuid = self._device.get("region_uuid")
+        self._name = self._device.get("friendly_name")
+        self._hostname = self._device.get("hostname")
+        self._status = self._device.get("status")
+        self._online = self._device.get("online")
+        self._maintenance_mode = self._device.get("maintenance_mode")
+        self._device_config_id = self._device.get("device_config_id")
+        self._job_status = self._device.get("job_status")
+        self._job_last_deployed = self._device.get("job_last_deployed")
+        self._job_config_id = self._device.get("job_config_id")
+        self._job_config_revision = self._device.get("job_config_revision")
+
+    @property
+    def id(self) -> str:
+        return self._id
+
+    @property
+    def provider_uuid(self) -> str:
+        return self._provider_uuid
+
+    @property
+    def region_uuid(self) -> str:
+        return self._region_uuid
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def hostname(self) -> str:
+        return self._hostname
+
+    @property
+    def status(self) -> str:
+        return self._status
+
+    @property
+    def online(self) -> bool:
+        return self._online
+
+    @property
+    def maintenance_mode(self) -> bool:
+        return self._maintenance_mode
+
+    @property
+    def device_config_id(self) -> str:
+        return self._device_config_id
+
+    @property
+    def job_status(self) -> str:
+        return self._job_status
+
+    @property
+    def job_last_deployed(self) -> str:
+        return self._job_last_deployed
+
+    @property
+    def job_config_id(self) -> str:
+        return self._job_config_id
+
+    @property
+    def job_config_revision(self) -> str:
+        return self._job_config_revision
+
+    def __str__(self):
+        return json.dumps({k: v for k, v in self._device.items()})
+
+    def __repr__(self):
+        return f"Device( proximl , **{self._device.__repr__()})"
+
+    def __bool__(self):
+        return bool(self._id)
+
+    async def remove(self):
+        await self.proximl._query(
+            f"/provider/{self._provider_uuid}/region/{self._region_uuid}/device/{self._id}",
+            "DELETE",
+        )
+
+    async def refresh(self):
+        resp = await self.proximl._query(
+            f"/provider/{self._provider_uuid}/region/{self._region_uuid}/device/{self._id}",
+            "GET",
+        )
+        self.__init__(self.proximl, **resp)
+        return self
+
+    async def toggle_maintenance(self):
+        await self.proximl._query(
+            f"/provider/{self._provider_uuid}/region/{self._region_uuid}/device/{self._id}/maintenance",
+            "PATCH",
+        )
+
+    async def run_action(self, command):
+        await self.proximl._query(
+            f"/provider/{self._provider_uuid}/region/{self._region_uuid}/device/{self._id}/action",
+            "POST",
+            None,
+            dict(command=command),
+        )
+
+    async def set_config(self, device_config_id):
+        resp = await self.proximl._query(
+            f"/provider/{self._provider_uuid}/region/{self._region_uuid}/device/{self._id}",
+            "PATCH",
+            None,
+            dict(device_config_id=device_config_id),
+        )
+        self.__init__(self.proximl, **resp)
+        return self
+
+    async def deploy_endpoint(self):
+        await self.proximl._query(
+            f"/provider/{self._provider_uuid}/region/{self._region_uuid}/device/{self._id}/deploy",
+            "PUT",
+        )
+
+    async def stop_endpoint(self):
+        await self.proximl._query(
+            f"/provider/{self._provider_uuid}/region/{self._region_uuid}/device/{self._id}/stop",
+            "PUT",
+        )

{proximl-0.5.0 → proximl-0.5.2}/proximl/cloudbender/nodes.py

@@ -131,3 +131,25 @@ class Node:
             f"/provider/{self._provider_uuid}/region/{self._region_uuid}/node/{self._id}",
             "DELETE",
         )
+
+    async def refresh(self):
+        resp = await self.proximl._query(
+            f"/provider/{self._provider_uuid}/region/{self._region_uuid}/node/{self._id}",
+            "GET",
+        )
+        self.__init__(self.proximl, **resp)
+        return self
+
+    async def toggle_maintenance(self):
+        await self.proximl._query(
+            f"/provider/{self._provider_uuid}/region/{self._region_uuid}/node/{self._id}/maintenance",
+            "PATCH",
+        )
+
+    async def run_action(self, command):
+        await self.proximl._query(
+            f"/provider/{self._provider_uuid}/region/{self._region_uuid}/node/{self._id}/action",
+            "POST",
+            None,
+            dict(command=command),
+        )

{proximl-0.5.0 → proximl-0.5.2}/proximl/cloudbender/providers.py

@@ -61,3 +61,11 @@ class Provider:
 
     async def remove(self):
         await self.proximl._query(f"/provider/{self._id}", "DELETE")
+
+    async def refresh(self):
+        resp = await self.proximl._query(
+            f"/provider/{self._id}",
+            "GET",
+        )
+        self.__init__(self.proximl, **resp)
+        return self

{proximl-0.5.0 → proximl-0.5.2}/proximl/cloudbender/regions.py

@@ -80,6 +80,14 @@ class Region:
             f"/provider/{self._provider_uuid}/region/{self._id}", "DELETE"
         )
 
+    async def refresh(self):
+        resp = await self.proximl._query(
+            f"/provider/{self._provider_uuid}/region/{self._id}",
+            "GET",
+        )
+        self.__init__(self.proximl, **resp)
+        return self
+
     async def add_dataset(self, project_uuid, dataset_uuid, **kwargs):
         await self.proximl._query(
             f"/provider/{self._provider_uuid}/region/{self._id}/dataset",

{proximl-0.5.0 → proximl-0.5.2}/proximl/cloudbender/reservations.py

@@ -116,3 +116,11 @@ class Reservation:
             f"/provider/{self._provider_uuid}/region/{self._region_uuid}/reservation/{self._id}",
             "DELETE",
         )
+
+    async def refresh(self):
+        resp = await self.proximl._query(
+            f"/provider/{self._provider_uuid}/region/{self._region_uuid}/reservation/{self._id}",
+            "GET",
+        )
+        self.__init__(self.proximl, **resp)
+        return self

{proximl-0.5.0 → proximl-0.5.2}/proximl/datasets.py

@@ -37,8 +37,7 @@ class Datasets(object):
             source_type=source_type,
             source_uri=source_uri,
             source_options=kwargs.get("source_options"),
-            project_uuid=kwargs.get("project_uuid")
-            or self.proximl.active_project,
+            project_uuid=kwargs.get("project_uuid") or self.proximl.active_project,
         )
         payload = {k: v for k, v in data.items() if v is not None}
         logging.info(f"Creating Dataset {name}")
@@ -119,11 +118,13 @@ class Dataset:
                 entity_type="dataset",
                 project_uuid=self._dataset.get("project_uuid"),
                 cidr=self._dataset.get("vpn").get("cidr"),
-                ssh_port=self._dataset.get("vpn")
-                .get("client")
-                .get("ssh_port"),
-                input_path=self._dataset.get("source_uri"),
-                output_path=None,
+                ssh_port=self._dataset.get("vpn").get("client").get("ssh_port"),
+                input_path=self._dataset.get("source_uri")
+                if self.status in ["new", "downloading"]
+                else None,
+                output_path=self._dataset.get("output_uri")
+                if self.status == "exporting"
+                else None,
             )
         else:
             details = dict()
@@ -133,7 +134,7 @@ class Dataset:
         if self.status in ["ready", "failed"]:
             raise SpecificationError(
                 "status",
-                f"You can only connect to new or downloading datasets.",
+                f"You can only connect to downloading or exporting datasets.",
             )
         if self.status == "new":
             await self.wait_for("downloading")
@@ -167,15 +168,27 @@ class Dataset:
         self.__init__(self.proximl, **resp)
         return self
 
+    async def export(self, output_type, output_uri, output_options=dict()):
+        resp = await self.proximl._query(
+            f"/dataset/{self._id}/export",
+            "POST",
+            dict(project_uuid=self._project_uuid),
+            dict(
+                output_type=output_type,
+                output_uri=output_uri,
+                output_options=output_options,
+            ),
+        )
+        self.__init__(self.proximl, **resp)
+        return self
+
     def _get_msg_handler(self, msg_handler):
         def handler(data):
             if data.get("type") == "subscription":
                 if msg_handler:
                     msg_handler(data)
                 else:
-                    timestamp = datetime.fromtimestamp(
-                        int(data.get("time")) / 1000
-                    )
+                    timestamp = datetime.fromtimestamp(int(data.get("time")) / 1000)
                     print(
                         f"{timestamp.strftime('%m/%d/%Y, %H:%M:%S')}: {data.get('msg').rstrip()}"
                     )
@@ -212,9 +225,7 @@ class Dataset:
             return
         POLL_INTERVAL_MIN = 5
         POLL_INTERVAL_MAX = 60
-        POLL_INTERVAL = max(
-            min(timeout / 60, POLL_INTERVAL_MAX), POLL_INTERVAL_MIN
-        )
+        POLL_INTERVAL = max(min(timeout / 60, POLL_INTERVAL_MAX), POLL_INTERVAL_MIN)
         retry_count = math.ceil(timeout / POLL_INTERVAL)
         count = 0
         while count < retry_count: