proximl 0.5.4__py3-none-any.whl → 0.5.6__py3-none-any.whl

tests/integration/test_jobs_integration.py

@@ -269,7 +269,7 @@ class JobAPIDataValidationTests:
                 ),
             )
         assert (
-            "Invalid Request - Only regional datastore are allowed for Notebook and Endpoint output locations"
+            "Invalid Request - output_type invalid for Notebook and Endpoint jobs"
             in error.value.message
         )
 
@@ -298,7 +298,45 @@ class JobAPIDataValidationTests:
                 ),
             )
         assert (
-            "Invalid Request - Only regional datastore are allowed for Notebook and Endpoint output locations"
+            "Invalid Request - output_type invalid for Notebook and Endpoint jobs"
+            in error.value.message
+        )
+
+    async def test_invalid_volumes_for_training(self, proximl):
+        with raises(ApiError) as error:
+            await proximl.jobs.create(
+                name="Invalid Volumes for Training",
+                type="training",
+                gpu_types=["rtx3090"],
+                disk_size=10,
+                data=dict(
+                    output_uri="s3://proximl-examples/output/resnet_cifar10",
+                    output_type="aws",
+                    volumes=["volume-id"],
+                ),
+                workers=["python train.py"],
+            )
+        assert (
+            "Invalid Request - Only Notebook and Endpoint job types can use writable volumes"
+            in error.value.message
+        )
+
+    async def test_invalid_volumes_for_inference(self, proximl):
+        with raises(ApiError) as error:
+            await proximl.jobs.create(
+                name="Invalid Volumes for Inference",
+                type="inference",
+                gpu_types=["rtx3090"],
+                disk_size=10,
+                data=dict(
+                    output_uri="s3://proximl-examples/output/resnet_cifar10",
+                    output_type="aws",
+                    volumes=["volume-id"],
+                ),
+                workers=["python predict.py"],
+            )
+        assert (
+            "Invalid Request - Only Notebook and Endpoint job types can use writable volumes"
             in error.value.message
         )
 

tests/integration/test_models_integration.py

@@ -43,29 +43,27 @@ class GetModelTests:
 
     async def test_model_repr(self, model):
         string = repr(model)
-        regex = (
-            r"^Model\( proximl , \*\*{.*'model_uuid': '"
-            + model.id
-            + r"'.*}\)$"
-        )
+        regex = r"^Model\( proximl , \*\*{.*'model_uuid': '" + model.id + r"'.*}\)$"
         assert isinstance(string, str)
         assert re.match(regex, string)
 
 
 @mark.create
 @mark.asyncio
-async def test_model_aws(proximl, capsys):
+async def test_model_wasabi(proximl, capsys):
     model = await proximl.models.create(
-        name="CLI Automated AWS",
-        source_type="aws",
-        source_uri="s3://proximl-examples/models/mxnet-model.zip",
+        name="CLI Automated Wasabi",
+        source_type="wasabi",
+        source_uri="s3://proximl-example/models/proximl-examples",
+        capacity="10G",
+        source_options=dict(endpoint_url="https://s3.wasabisys.com"),
     )
     model = await model.wait_for("ready", 300)
     status = model.status
     size = model.size
     await model.remove()
     assert status == "ready"
-    assert size >= 1000000
+    assert size >= 500000
 
 
 @mark.create

tests/integration/test_projects_integration.py

@@ -11,9 +11,7 @@ pytestmark = [mark.sdk, mark.integration, mark.projects]
 class GetProjectsTests:
     @fixture(scope="class")
     async def project(self, proximl):
-        project = await proximl.projects.create(
-            name="New Project", copy_keys=False
-        )
+        project = await proximl.projects.create(name="New Project", copy_keys=False)
         yield project
         await project.remove()
 
@@ -41,8 +39,6 @@ class GetProjectsTests:
 
     async def test_project_repr(self, project):
         string = repr(project)
-        regex = (
-            r"^Project\( proximl , \*\*{.*'id': '" + project.id + r"'.*}\)$"
-        )
+        regex = r"^Project\( proximl , \*\*{.*'id': '" + project.id + r"'.*}\)$"
         assert isinstance(string, str)
         assert re.match(regex, string)

tests/integration/test_volumes_integration.py

@@ -0,0 +1,100 @@
+import re
+import sys
+import asyncio
+from pytest import mark, fixture
+
+pytestmark = [mark.sdk, mark.integration, mark.volumes]
+
+
+@mark.create
+@mark.asyncio
+class GetVolumeTests:
+    @fixture(scope="class")
+    async def volume(self, proximl):
+        volume = await proximl.volumes.create(
+            name="CLI Automated",
+            source_type="git",
+            source_uri="git@github.com:proxiML/environment-tests.git",
+            capacity="10G",
+        )
+        volume = await volume.wait_for("ready", 120)
+        yield volume
+        await volume.remove()
+        volume = await volume.wait_for("archived", 60)
+
+    async def test_get_volumes(self, proximl, volume):
+        volumes = await proximl.volumes.list()
+        assert len(volumes) > 0
+
+    async def test_get_volume(self, proximl, volume):
+        response = await proximl.volumes.get(volume.id)
+        assert response.id == volume.id
+
+    async def test_volume_properties(self, volume):
+        assert isinstance(volume.id, str)
+        assert isinstance(volume.status, str)
+        assert isinstance(volume.name, str)
+        assert isinstance(volume.capacity, str)
+        assert isinstance(volume.used_size, int)
+        assert isinstance(volume.billed_size, int)
+
+    async def test_volume_str(self, volume):
+        string = str(volume)
+        regex = r"^{.*\"id\": \"" + volume.id + r"\".*}$"
+        assert isinstance(string, str)
+        assert re.match(regex, string)
+
+    async def test_volume_repr(self, volume):
+        string = repr(volume)
+        regex = r"^Volume\( proximl , \*\*{.*'id': '" + volume.id + r"'.*}\)$"
+        assert isinstance(string, str)
+        assert re.match(regex, string)
+
+
+@mark.create
+@mark.asyncio
+async def test_volume_wasabi(proximl, capsys):
+    volume = await proximl.volumes.create(
+        name="CLI Automated Wasabi",
+        source_type="wasabi",
+        source_uri="s3://proximl-example/models/proximl-examples",
+        capacity="10G",
+        source_options=dict(endpoint_url="https://s3.wasabisys.com"),
+    )
+    volume = await volume.wait_for("ready", 300)
+    status = volume.status
+    billed_size = volume.billed_size
+    used_size = volume.used_size
+    await volume.remove()
+    assert status == "ready"
+    assert billed_size >= 10000000
+    assert used_size >= 500000
+
+
+@mark.create
+@mark.asyncio
+async def test_volume_local(proximl, capsys):
+    volume = await proximl.volumes.create(
+        name="CLI Automated Local",
+        source_type="local",
+        source_uri="~/tensorflow-model",
+        capacity="10G",
+    )
+    attach_task = asyncio.create_task(volume.attach())
+    connect_task = asyncio.create_task(volume.connect())
+    await asyncio.gather(attach_task, connect_task)
+    await volume.disconnect()
+    await volume.refresh()
+    status = volume.status
+    billed_size = volume.billed_size
+    used_size = volume.used_size
+    await volume.remove()
+    assert status == "ready"
+    assert billed_size >= 10000000
+    assert used_size >= 1000000
+    captured = capsys.readouterr()
+    sys.stdout.write(captured.out)
+    sys.stderr.write(captured.err)
+    assert "Starting data upload from local" in captured.out
+    assert "official/LICENSE  11456 bytes" in captured.out
+    assert "Upload complete" in captured.out

tests/unit/cli/test_cli_volume_unit.py

@@ -0,0 +1,20 @@
+import re
+import json
+import click
+from unittest.mock import AsyncMock, patch
+from pytest import mark, fixture, raises
+
+pytestmark = [mark.cli, mark.unit, mark.volumes]
+
+from proximl.cli import volume as specimen
+from proximl.volumes import Volume
+
+
+def test_list(runner, mock_my_volumes):
+    with patch("proximl.cli.ProxiML", new=AsyncMock) as mock_proximl:
+        mock_proximl.volumes = AsyncMock()
+        mock_proximl.volumes.list = AsyncMock(return_value=mock_my_volumes)
+        result = runner.invoke(specimen, ["list"])
+        print(result)
+        assert result.exit_code == 0
+        mock_proximl.volumes.list.assert_called_once()

tests/unit/cloudbender/test_services_unit.py

@@ -0,0 +1,161 @@
+import re
+import json
+import logging
+from unittest.mock import AsyncMock, patch
+from pytest import mark, fixture, raises
+from aiohttp import WSMessage, WSMsgType
+
+import proximl.cloudbender.services as specimen
+from proximl.exceptions import (
+    ApiError,
+    SpecificationError,
+    ProxiMLException,
+)
+
+pytestmark = [mark.sdk, mark.unit, mark.cloudbender, mark.services]
+
+
+@fixture
+def services(mock_proximl):
+    yield specimen.Services(mock_proximl)
+
+
+@fixture
+def service(mock_proximl):
+    yield specimen.Service(
+        mock_proximl,
+        provider_uuid="1",
+        region_uuid="a",
+        service_id="x",
+        name="On-Prem Service",
+        public=False,
+        hostname="app1.proximl.cloud",
+    )
+
+
+class RegionsTests:
+    @mark.asyncio
+    async def test_get_service(
+        self,
+        services,
+        mock_proximl,
+    ):
+        api_response = dict()
+        mock_proximl._query = AsyncMock(return_value=api_response)
+        await services.get("1234", "5687", "91011")
+        mock_proximl._query.assert_called_once_with(
+            "/provider/1234/region/5687/service/91011", "GET", {}
+        )
+
+    @mark.asyncio
+    async def test_list_services(
+        self,
+        services,
+        mock_proximl,
+    ):
+        api_response = dict()
+        mock_proximl._query = AsyncMock(return_value=api_response)
+        await services.list("1234", "5687")
+        mock_proximl._query.assert_called_once_with(
+            "/provider/1234/region/5687/service", "GET", {}
+        )
+
+    @mark.asyncio
+    async def test_remove_service(
+        self,
+        services,
+        mock_proximl,
+    ):
+        api_response = dict()
+        mock_proximl._query = AsyncMock(return_value=api_response)
+        await services.remove("1234", "4567", "8910")
+        mock_proximl._query.assert_called_once_with(
+            "/provider/1234/region/4567/service/8910", "DELETE", {}
+        )
+
+    @mark.asyncio
+    async def test_create_service(self, services, mock_proximl):
+        requested_config = dict(
+            provider_uuid="provider-id-1",
+            region_uuid="region-id-1",
+            name="On-Prem Service",
+            public=False,
+        )
+        expected_payload = dict(
+            name="On-Prem Service",
+            public=False,
+        )
+        api_response = {
+            "provider_uuid": "provider-id-1",
+            "region_uuid": "region-id-1",
+            "service_id": "service-id-1",
+            "name": "On-Prem Service",
+            "public": False,
+            "hostname": "app1.proximl.cloud",
+            "createdAt": "2020-12-31T23:59:59.000Z",
+        }
+
+        mock_proximl._query = AsyncMock(return_value=api_response)
+        response = await services.create(**requested_config)
+        mock_proximl._query.assert_called_once_with(
+            "/provider/provider-id-1/region/region-id-1/service",
+            "POST",
+            None,
+            expected_payload,
+        )
+        assert response.id == "service-id-1"
+
+
+class serviceTests:
+    def test_service_properties(self, service):
+        assert isinstance(service.id, str)
+        assert isinstance(service.provider_uuid, str)
+        assert isinstance(service.region_uuid, str)
+        assert isinstance(service.public, bool)
+        assert isinstance(service.name, str)
+        assert isinstance(service.hostname, str)
+
+    def test_service_str(self, service):
+        string = str(service)
+        regex = r"^{.*\"service_id\": \"" + service.id + r"\".*}$"
+        assert isinstance(string, str)
+        assert re.match(regex, string)
+
+    def test_service_repr(self, service):
+        string = repr(service)
+        regex = r"^Service\( proximl , \*\*{.*'service_id': '" + service.id + r"'.*}\)$"
+        assert isinstance(string, str)
+        assert re.match(regex, string)
+
+    def test_service_bool(self, service, mock_proximl):
+        empty_service = specimen.Service(mock_proximl)
+        assert bool(service)
+        assert not bool(empty_service)
+
+    @mark.asyncio
+    async def test_service_remove(self, service, mock_proximl):
+        api_response = dict()
+        mock_proximl._query = AsyncMock(return_value=api_response)
+        await service.remove()
+        mock_proximl._query.assert_called_once_with(
+            "/provider/1/region/a/service/x", "DELETE"
+        )
+
+    @mark.asyncio
+    async def test_service_refresh(self, service, mock_proximl):
+        api_response = {
+            "provider_uuid": "provider-id-1",
+            "region_uuid": "region-id-1",
+            "service_id": "service-id-1",
+            "name": "On-Prem Service",
+            "public": False,
+            "hostname": "app1.proximl.cloud",
+            "createdAt": "2020-12-31T23:59:59.000Z",
+        }
+        mock_proximl._query = AsyncMock(return_value=api_response)
+        response = await service.refresh()
+        mock_proximl._query.assert_called_once_with(
+            f"/provider/1/region/a/service/x", "GET"
+        )
+        assert service.id == "service-id-1"
+        assert response.id == "service-id-1"

tests/unit/conftest.py

@@ -7,6 +7,7 @@ from proximl.proximl import ProxiML
 from proximl.auth import Auth
 from proximl.datasets import Dataset, Datasets
 from proximl.checkpoints import Checkpoint, Checkpoints
+from proximl.volumes import Volume, Volumes
 from proximl.models import Model, Models
 from proximl.gpu_types import GpuType, GpuTypes
 from proximl.environments import Environment, Environments
@@ -258,6 +259,79 @@ def mock_models():
     ]
 
 
+@fixture(scope="session")
+def mock_my_volumes():
+    proximl = Mock()
+    yield [
+        Volume(
+            proximl,
+            id="1",
+            project_uuid="proj-id-1",
+            name="first one",
+            status="ready",
+            capacity="10G",
+            used_size=100000000,
+            billed_size=100000000,
+            createdAt="2020-12-31T23:59:59.000Z",
+        ),
+        Volume(
+            proximl,
+            id="2",
+            project_uuid="proj-id-1",
+            name="second one",
+            status="ready",
+            capacity="10G",
+            used_size=100000000,
+            billed_size=100000000,
+            createdAt="2021-01-01T00:00:01.000Z",
+        ),
+        Volume(
+            proximl,
+            id="3",
+            project_uuid="proj-id-1",
+            name="first one",
+            status="ready",
+            capacity="10G",
+            used_size=100000000,
+            billed_size=100000000,
+            createdAt="2021-01-01T00:00:01.000Z",
+        ),
+        Volume(
+            proximl,
+            id="4",
+            project_uuid="proj-id-1",
+            name="other one",
+            status="ready",
+            capacity="10G",
+            used_size=100000000,
+            billed_size=100000000,
+            createdAt="2020-12-31T23:59:59.000Z",
+        ),
+        Volume(
+            proximl,
+            id="5",
+            project_uuid="proj-id-1",
+            name="not ready",
+            status="new",
+            capacity="10G",
+            used_size=100000000,
+            billed_size=100000000,
+            createdAt="2021-01-01T00:00:01.000Z",
+        ),
+        Volume(
+            proximl,
+            id="6",
+            project_uuid="proj-id-1",
+            name="failed",
+            status="failed",
+            capacity="10G",
+            used_size=100000000,
+            billed_size=100000000,
+            createdAt="2021-01-01T00:00:01.000Z",
+        ),
+    ]
+
+
 @fixture(scope="session")
 def mock_gpu_types():
     proximl = Mock()
@@ -903,6 +977,9 @@ def mock_device_configs():
 def mock_proximl(
     mock_my_datasets,
     mock_public_datasets,
+    mock_my_checkpoints,
+    mock_public_checkpoints,
+    mock_my_volumes,
     mock_models,
     mock_gpu_types,
     mock_environments,
@@ -921,6 +998,7 @@ def mock_proximl(
     proximl.project = "proj-id-1"
     proximl.datasets = create_autospec(Datasets)
     proximl.checkpoints = create_autospec(Checkpoints)
+    proximl.volumes = create_autospec(Volumes)
     proximl.models = create_autospec(Models)
     proximl.gpu_types = create_autospec(GpuTypes)
     proximl.environments = create_autospec(Environments)
@@ -930,10 +1008,9 @@ def mock_proximl(
     proximl.datasets.list = AsyncMock(return_value=mock_my_datasets)
     proximl.datasets.list_public = AsyncMock(return_value=mock_public_datasets)
     proximl.checkpoints.list = AsyncMock(return_value=mock_my_checkpoints)
-    proximl.checkpoints.list_public = AsyncMock(
-        return_value=mock_public_checkpoints
-    )
+    proximl.checkpoints.list_public = AsyncMock(return_value=mock_public_checkpoints)
     proximl.models.list = AsyncMock(return_value=mock_models)
+    proximl.volumes.list = AsyncMock(return_value=mock_my_volumes)
     proximl.gpu_types.list = AsyncMock(return_value=mock_gpu_types)
     proximl.environments.list = AsyncMock(return_value=mock_environments)
     proximl.jobs.list = AsyncMock(return_value=mock_jobs)
@@ -950,13 +1027,9 @@ def mock_proximl(
     proximl.cloudbender.devices = create_autospec(Nodes)
     proximl.cloudbender.devices.list = AsyncMock(return_value=mock_devices)
     proximl.cloudbender.datastores = create_autospec(Datastores)
-    proximl.cloudbender.datastores.list = AsyncMock(
-        return_value=mock_datastores
-    )
+    proximl.cloudbender.datastores.list = AsyncMock(return_value=mock_datastores)
     proximl.cloudbender.reservations = create_autospec(Reservations)
-    proximl.cloudbender.reservations.list = AsyncMock(
-        return_value=mock_reservations
-    )
+    proximl.cloudbender.reservations.list = AsyncMock(return_value=mock_reservations)
     proximl.cloudbender.device_configs = create_autospec(DeviceConfigs)
     proximl.cloudbender.device_configs.list = AsyncMock(
         return_value=mock_device_configs