proximl 0.5.13.post1__tar.gz → 0.5.15__tar.gz

{proximl-0.5.13.post1/proximl.egg-info → proximl-0.5.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: proximl
-Version: 0.5.13.post1
+Version: 0.5.15
 Summary: proxiML client SDK and command line utilities
 Home-page: https://github.com/proxiML/python-sdk
 Author: proxiML

{proximl-0.5.13.post1 → proximl-0.5.15}/proximl/__init__.py

@@ -13,5 +13,5 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 
 
-__version__ = "0.5.13.post1"
+__version__ = "0.5.15"
 __all__ = "ProxiML"

{proximl-0.5.13.post1 → proximl-0.5.15}/proximl/cli/job/create.py

@@ -180,6 +180,12 @@ def create(config):
     help="Third Party Credentials to add to the job environment",
     multiple=True,
 )
+@click.option(
+    "--secret",
+    type=click.STRING,
+    help="Project secrets to add to the job environment",
+    multiple=True,
+)
 @click.option(
     "--git-uri",
     type=click.STRING,
@@ -222,6 +228,7 @@ def notebook(
     custom_image,
     env,
     credential,
+    secret,
     apt_packages,
     pip_packages,
     conda_packages,
@@ -254,7 +261,7 @@ def notebook(
         data=dict(datasets=datasets),
         model=dict(checkpoints=checkpoints),
         environment=dict(
-            credentials=[k for k in credential],
+            credentials=[k for k in credential], secrets=[s for s in secret]
         ),
     )
 
@@ -508,6 +515,12 @@ def notebook(
     help="Third Party Credentials to add to the job environment",
     multiple=True,
 )
+@click.option(
+    "--secret",
+    type=click.STRING,
+    help="Project secrets to add to the job environment",
+    multiple=True,
+)
 @click.option(
     "--apt-packages",
     type=click.STRING,
@@ -564,6 +577,7 @@ def training(
     custom_image,
     env,
     credential,
+    secret,
     apt_packages,
     pip_packages,
     conda_packages,
@@ -596,7 +610,7 @@ def training(
         data=dict(datasets=datasets),
         model=dict(checkpoints=checkpoints),
         environment=dict(
-            credentials=[k for k in credential],
+            credentials=[k for k in credential], secrets=[s for s in secret]
         ),
     )
 
@@ -852,6 +866,12 @@ def training(
     help="Third Party Credentials to add to the job environment",
     multiple=True,
 )
+@click.option(
+    "--secret",
+    type=click.STRING,
+    help="Project secrets to add to the job environment",
+    multiple=True,
+)
 @click.option(
     "--apt-packages",
     type=click.STRING,
@@ -908,6 +928,7 @@ def inference(
     custom_image,
     env,
     credential,
+    secret,
     apt_packages,
     pip_packages,
     conda_packages,
@@ -934,6 +955,7 @@ def inference(
         model=dict(checkpoints=checkpoints),
         environment=dict(
             credentials=[k for k in credential],
+            secrets=[s for s in secret],
         ),
     )
 
@@ -1171,6 +1193,12 @@ def from_json(config, attach, connect, file):
     help="Third Party Credentials to add to the job environment",
     multiple=True,
 )
+@click.option(
+    "--secret",
+    type=click.STRING,
+    help="Project secrets to add to the job environment",
+    multiple=True,
+)
 @click.option(
     "--apt-packages",
     type=click.STRING,
@@ -1231,6 +1259,7 @@ def endpoint(
     custom_image,
     env,
     credential,
+    secret,
     apt_packages,
     pip_packages,
     conda_packages,
@@ -1258,6 +1287,7 @@ def endpoint(
         model=dict(checkpoints=checkpoints),
         environment=dict(
             credentials=[k for k in credential],
+            secrets=[s for s in secret],
         ),
     )
 

{proximl-0.5.13.post1 → proximl-0.5.15}/proximl/cloudbender/services.py

@@ -67,7 +67,7 @@ class Services(object):
 
 
 class Service:
-    def __init__(self, proximl, **kwargs):
+    def __init__(self, proximl,  **kwargs):
         self.proximl = proximl
         self._service = kwargs
         self._id = self._service.get("service_id")
@@ -177,3 +177,21 @@ class Service:
                 logging.debug(f"self: {self}, retry count {count}")
 
         raise ProxiMLException(f"Timeout waiting for {status}")
+    
+    async def generate_certificate(self, **kwargs):
+        resp = await self.proximl._query(
+            f"/provider/{self._provider_uuid}/region/{self._region_uuid}/service/{self._id}/certificate",
+            "POST",
+            kwargs
+        )
+        self.__init__(self.proximl, **resp)
+        return self
+    
+    async def sign_client_certificate(self, csr, **kwargs):
+        certificate = await self.proximl._query(
+            f"/provider/{self._provider_uuid}/region/{self._region_uuid}/service/{self._id}/certificate/sign",
+            "POST",
+            kwargs,
+            dict(csr=csr)
+        )
+        return certificate

{proximl-0.5.13.post1 → proximl-0.5.15}/proximl/jobs.py

@@ -530,6 +530,15 @@ class Job:
                         "waiting for data/model download",
                     ]
                 )
+                or (
+                    status
+                    == "running"  ## this status could be too short for polling could miss it
+                    and self.status
+                    in [
+                        "uploading",
+                        "finished"
+                    ]
+                )
             ):
                 return self
             elif self.status == "failed":

proximl-0.5.15/proximl/projects/members.py

@@ -0,0 +1,98 @@
+import json
+import logging
+from typing import Literal
+
+class ProjectMembers(object):
+    def __init__(self, proximl, project_id):
+        self.proximl = proximl
+        self.project_id = project_id
+
+    async def list(self, **kwargs):
+        resp = await self.proximl._query(
+            f"/project/{self.project_id}/access", "GET", kwargs
+        )
+        members = [ProjectMember(self.proximl, **member) for member in resp]
+        return members
+    
+    async def add(self, email: str, job: Literal["all", "read"], dataset: Literal["all", "read"], model: Literal["all", "read"], checkpoint: Literal["all", "read"], volume: Literal["all", "read"],  **kwargs):
+        data = dict(
+            email=email,
+            job=job,
+            dataset=dataset,
+            model=model,
+            checkpoint=checkpoint,
+            volume=volume,
+        )
+        payload = {k: v for k, v in data.items() if v is not None}
+        resp = await self.proximl._query(
+            f"/project/{self.project_id}/access", "POST",kwargs, payload)
+        member = ProjectMember(self.proximl, **resp)
+        logging.info(f"Added Project Member {email} to project {self.project_id}")
+        return member
+
+    
+    async def remove(self, email, **kwargs):
+        await self.proximl._query(
+            f"/project/{self.project_id}/access", "DELETE", dict(**kwargs, email=email)
+        )
+
+
+class ProjectMember:
+    def __init__(self, proximl, **kwargs):
+        self.proximl = proximl
+        self._entity = kwargs
+        self._id = self._entity.get("email")
+        self._project_uuid = self._entity.get("project_uuid")
+        self._owner = self._entity.get("owner")
+        self._job = self._entity.get("job")
+        self._dataset = self._entity.get("dataset")
+        self._model = self._entity.get("model")
+        self._checkpoint = self._entity.get("checkpoint")
+        self._volume = self._entity.get("volume")
+
+    @property
+    def id(self) -> str:
+        return self._id
+
+    @property
+    def project_uuid(self) -> str:
+        return self._project_uuid
+    
+    @property
+    def email(self) -> str:
+        return self._id
+
+    @property
+    def owner(self) -> bool:
+        return self._owner
+
+    @property
+    def job(self) -> str:
+        return self._job
+
+    @property
+    def dataset(self) -> str:
+        return self._dataset
+    
+    @property
+    def model(self) -> str:
+        return self._model
+    
+    @property
+    def checkpoint(self) -> str:
+        return self._checkpoint
+    
+    @property
+    def volume(self) -> str:
+        return self._volume
+
+    def __str__(self):
+        return json.dumps({k: v for k, v in self._entity.items()})
+
+    def __repr__(self):
+        return f"ProjectMember( proximl , **{self._entity.__repr__()})"
+
+    def __bool__(self):
+        return bool(self._id)
+
+

{proximl-0.5.13.post1 → proximl-0.5.15}/proximl/projects/projects.py

@@ -5,6 +5,7 @@ from .data_connectors import ProjectDataConnectors
 from .services import ProjectServices
 from .credentials import ProjectCredentials
 from .secrets import ProjectSecrets
+from .members import ProjectMembers
 
 
 class Projects(object):
@@ -56,6 +57,7 @@ class Project:
         self.services = ProjectServices(self.proximl, self._id)
         self.credentials = ProjectCredentials(self.proximl, self._id)
         self.secrets = ProjectSecrets(self.proximl, self._id)
+        self.members = ProjectMembers(self.proximl, self._id)
 
     @property
     def id(self) -> str:

{proximl-0.5.13.post1 → proximl-0.5.15}/proximl/projects/services.py

@@ -77,3 +77,20 @@ class ProjectService:
         await self.proximl._query(
             f"/project/{self._project_uuid}/services/{self._id}/disable", "PATCH"
         )
+
+    async def get_service_ca_certificate(self, **kwargs):
+        certificate = await self.proximl._query(
+            f"/project/{self._project_uuid}/services/{self._id}/certificate/ca",
+            "GET",
+            kwargs,
+        )
+        return certificate
+    
+    async def sign_client_certificate(self, csr, **kwargs):
+        certificate = await self.proximl._query(
+            f"/project/{self._project_uuid}/services/{self._id}/certificate/sign",
+            "POST",
+            kwargs,
+            dict(csr=csr)
+        )
+        return certificate

{proximl-0.5.13.post1 → proximl-0.5.15}/proximl/proximl.py

@@ -4,6 +4,7 @@ import asyncio
 import aiohttp
 import logging
 import traceback
+import random
 from importlib.metadata import version
 
 from proximl.auth import Auth
@@ -91,7 +92,7 @@ class ProxiML(object):
     def project(self) -> str:
         return self.active_project
 
-    async def _query(self, path, method, params=None, data=None, headers=None):
+    async def _query(self, path, method, params=None, data=None, headers=None,max_retries=3, backoff_factor=0.5):
         try:
             tokens = self.auth.get_tokens()
         except ProxiMLException as e:
@@ -142,24 +143,40 @@ class ProxiML(object):
         logging.debug(
             f"Request - Url: {url}, Method: {method}, Params: {params}, Body: {data}, Headers: {headers}"
         )
-        async with aiohttp.ClientSession() as session:
-            async with session.request(
-                method,
-                url,
-                data=json.dumps(data),
-                headers=headers,
-                params=params,
-            ) as resp:
-                if (resp.status // 100) in [4, 5]:
-                    what = await resp.read()
-                    content_type = resp.headers.get("content-type", "")
-                    resp.close()
-                    if content_type == "application/json":
-                        raise ApiError(resp.status, json.loads(what.decode("utf8")))
-                    else:
-                        raise ApiError(resp.status, {"message": what.decode("utf8")})
-                results = await resp.json()
-                return results
+        for attempt in range(max_retries):
+            try:
+                async with aiohttp.ClientSession() as session:
+                    async with session.request(
+                        method,
+                        url,
+                        data=json.dumps(data),
+                        headers=headers,
+                        params=params,
+                    ) as resp:
+                        if (resp.status // 100) in [4, 5]:
+                            if resp.status == 502 and attempt < max_retries - 1:
+                                wait_time = (2 ** attempt) * backoff_factor * (random.random() + 0.5)
+                                await asyncio.sleep(wait_time)
+                                continue
+                            else:
+                                what = await resp.read()
+                                content_type = resp.headers.get("content-type", "")
+                                resp.close()
+                                if content_type == "application/json":
+                                    raise ApiError(resp.status, json.loads(what.decode("utf8")))
+                                else:
+                                    raise ApiError(resp.status, {"message": what.decode("utf8")})
+                        results = await resp.json()
+                        return results
+            except aiohttp.ClientResponseError as e:
+                if e.status == 502 and attempt < max_retries - 1:
+                    wait_time = (2 ** attempt) * backoff_factor * (random.random() + 0.5)
+                    await asyncio.sleep(wait_time)
+                    continue
+                else:
+                    raise  ApiError(e.status, f"Error {e.message}")
+
+        raise ProxiMLException("Unexpected API failure")
 
     async def _ws_subscribe(self, entity, project_uuid, id, msg_handler):
         headers = {

{proximl-0.5.13.post1 → proximl-0.5.15/proximl.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: proximl
-Version: 0.5.13.post1
+Version: 0.5.15
 Summary: proxiML client SDK and command line utilities
 Home-page: https://github.com/proxiML/python-sdk
 Author: proxiML

{proximl-0.5.13.post1 → proximl-0.5.15}/proximl.egg-info/SOURCES.txt

@@ -63,6 +63,7 @@ proximl/projects/__init__.py
 proximl/projects/credentials.py
 proximl/projects/data_connectors.py
 proximl/projects/datastores.py
+proximl/projects/members.py
 proximl/projects/projects.py
 proximl/projects/secrets.py
 proximl/projects/services.py
@@ -76,13 +77,17 @@ tests/integration/test_jobs_integration.py
 tests/integration/test_models_integration.py
 tests/integration/test_volumes_integration.py
 tests/integration/cloudbender/__init__.py
+tests/integration/cloudbender/conftest.py
 tests/integration/cloudbender/test_providers_integration.py
+tests/integration/cloudbender/test_regions_integration.py
+tests/integration/cloudbender/test_services_integration.py
 tests/integration/projects/__init__.py
 tests/integration/projects/conftest.py
 tests/integration/projects/test_projects_credentials_integration.py
 tests/integration/projects/test_projects_data_connectors_integration.py
 tests/integration/projects/test_projects_datastores_integration.py
 tests/integration/projects/test_projects_integration.py
+tests/integration/projects/test_projects_members_integration.py
 tests/integration/projects/test_projects_secrets_integration.py
 tests/integration/projects/test_projects_services_integration.py
 tests/unit/__init__.py
@@ -134,6 +139,7 @@ tests/unit/projects/__init__.py
 tests/unit/projects/test_project_credentials_unit.py
 tests/unit/projects/test_project_data_connectors_unit.py
 tests/unit/projects/test_project_datastores_unit.py
+tests/unit/projects/test_project_members_unit.py
 tests/unit/projects/test_project_secrets_unit.py
 tests/unit/projects/test_project_services_unit.py
 tests/unit/projects/test_projects_unit.py

{proximl-0.5.13.post1 → proximl-0.5.15}/pyproject.toml

@@ -1,4 +1,5 @@
 [tool.pytest.ini_options]
+addopts = "--cov-report term-missing --cov=proximl -n 4 --dist=loadgroup"
 python_files = "test_*"
 python_classes = "*Tests"
 python_functions = "test_*"

proximl-0.5.15/tests/integration/cloudbender/conftest.py

@@ -0,0 +1,28 @@
+
+from pytest import fixture, mark
+
+
+pytestmark = [mark.integration,mark.cloudbender]
+
+@fixture(scope="session")
+@mark.create
+@mark.asyncio
+@mark.xdist_group("cloudbender_resources")
+async def provider( proximl):
+    provider = await proximl.cloudbender.providers.enable(type="test")
+    await provider.wait_for("ready")
+    yield provider
+    await provider.remove()
+
+@fixture(scope="session")
+@mark.create
+@mark.asyncio
+@mark.xdist_group("cloudbender_resources")
+async def region(proximl, provider):
+    region = await proximl.cloudbender.regions.create(provider_uuid=provider.id,name="test-region",
+        public=False,
+        storage=dict(mode="local"),)
+    await region.wait_for("healthy")
+    yield region
+    await region.remove()
+    await region.wait_for("archived")

{proximl-0.5.13.post1 → proximl-0.5.15}/tests/integration/cloudbender/test_providers_integration.py

@@ -8,14 +8,9 @@ pytestmark = [mark.sdk, mark.integration, mark.cloudbender, mark.providers]
 
 @mark.create
 @mark.asyncio
-class GetProvidersTests:
-    @fixture(scope="class")
-    async def provider(self, proximl):
-        provider = await proximl.cloudbender.providers.enable(type="test")
-        yield provider
-        await provider.remove()
-
-    async def test_get_providers(self, proximl):
+@mark.xdist_group("cloudbender_resources")
+class GetProviderTests:
+    async def test_get_providers(self, proximl, provider):
         providers = await proximl.cloudbender.providers.list()
         assert len(providers) > 0
 

proximl-0.5.15/tests/integration/cloudbender/test_regions_integration.py

@@ -0,0 +1,42 @@
+import re
+import sys
+import asyncio
+from pytest import mark, fixture
+
+pytestmark = [mark.sdk, mark.integration, mark.cloudbender, mark.regions]
+
+
+@mark.create
+@mark.asyncio
+@mark.xdist_group("cloudbender_resources")
+class GetRegionTests:
+    async def test_get_regions(self, proximl, provider, region):
+        regions = await proximl.cloudbender.regions.list(provider_uuid=provider.id)
+        assert len(regions) > 0
+
+    async def test_get_region(self, proximl, provider, region):
+        response = await proximl.cloudbender.regions.get(provider.id, region.id)
+        assert response.id == region.id
+
+    async def test_region_properties(self, provider, region):
+        assert isinstance(region.id, str)
+        assert isinstance(region.provider_uuid, str)
+        assert isinstance(region.type, str)
+        assert region.type == "test"
+        assert region.provider_uuid == provider.id
+
+    async def test_region_str(self, region):
+        string = str(region)
+        regex = r"^{.*\"region_uuid\": \"" + region.id + r"\".*}$"
+        assert isinstance(string, str)
+        assert re.match(regex, string)
+
+    async def test_region_repr(self, region):
+        string = repr(region)
+        regex = (
+            r"^Region\( proximl , \*\*{.*'region_uuid': '"
+            + region.id
+            + r"'.*}\)$"
+        )
+        assert isinstance(string, str)
+        assert re.match(regex, string)

proximl-0.5.15/tests/integration/cloudbender/test_services_integration.py

@@ -0,0 +1,87 @@
+import re
+import sys
+import asyncio
+from pytest import mark, fixture
+from cryptography import x509
+from cryptography.x509.oid import NameOID, ExtendedKeyUsageOID
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.hazmat.primitives import serialization, hashes
+
+pytestmark = [mark.sdk, mark.integration, mark.cloudbender, mark.regions]
+
+def get_csr(service_id):
+    private_key = rsa.generate_private_key(
+        public_exponent=65537,
+        key_size=4096
+    )
+    csr = x509.CertificateSigningRequestBuilder().subject_name(x509.Name([
+        x509.NameAttribute(NameOID.COUNTRY_NAME, "US"),
+        x509.NameAttribute(NameOID.ORGANIZATION_NAME, "proxiML"),
+        x509.NameAttribute(NameOID.ORGANIZATIONAL_UNIT_NAME, service_id),
+        x509.NameAttribute(NameOID.COMMON_NAME, "test-client"),  # Client identity
+    ])).add_extension(
+        x509.ExtendedKeyUsage([
+            ExtendedKeyUsageOID.CLIENT_AUTH  # Client authentication usage
+        ]),
+        critical=True
+    ).sign(private_key, hashes.SHA256())
+    return csr.public_bytes(serialization.Encoding.PEM).decode("utf-8")
+
+@mark.create
+@mark.asyncio
+@mark.xdist_group("cloudbender_resources")
+class GetServiceTests:
+    @fixture(scope="class")
+    async def service(self, proximl, region):
+        service = await proximl.cloudbender.services.create(
+            provider_uuid=region.provider_uuid,
+            region_uuid=region.id,
+            name="CLI Automated Service",
+            type="tcp",
+            port="8989",
+            public=False,
+        )
+        await service.wait_for("active")
+        yield service
+        await service.remove()
+        await service.wait_for("archived")
+
+    async def test_get_services(self, proximl, region,service):
+        services = await proximl.cloudbender.services.list(provider_uuid=region.provider_uuid, region_uuid=region.id)
+        assert len(services) > 0
+
+    async def test_get_service(self, proximl, provider, region, service):
+        response = await proximl.cloudbender.services.get(provider.id, region.id, service.id)
+        assert response.id == service.id
+
+    async def test_service_properties(self, region, service):
+        assert isinstance(service.id, str)
+        assert isinstance(service.provider_uuid, str)
+        assert isinstance(service.region_uuid, str)
+        assert isinstance(service.public, bool)
+        assert service.port == "8989"
+        assert service.provider_uuid == region.provider_uuid
+        assert service.region_uuid == region.id
+
+    async def test_service_str(self, service):
+        string = str(service)
+        regex = r"^{.*\"service_id\": \"" + service.id + r"\".*}$"
+        assert isinstance(string, str)
+        assert re.match(regex, string)
+
+    async def test_service_repr(self, service):
+        string = repr(service)
+        regex = (
+            r"^Service\( proximl , \*\*{.*'service_id': '"
+            + service.id
+            + r"'.*}\)$"
+        )
+        assert isinstance(string, str)
+        assert re.match(regex, string)
+
+    async def test_service_certificate(self, service):
+        service = await service.generate_certificate()
+        assert isinstance(service._service.get("auth_cert"), str)
+        csr = get_csr(service.id)
+        certificate = await service.sign_client_certificate(csr)
+        assert isinstance(certificate, str)

{proximl-0.5.13.post1 → proximl-0.5.15}/tests/integration/conftest.py

@@ -40,7 +40,7 @@ def env(request):
     yield ENVS[env]
 
 
-@fixture(scope="module")
+@fixture(scope="session")
 def proximl(env):
     proximl = ProxiML(**env)
     yield proximl

{proximl-0.5.13.post1 → proximl-0.5.15}/tests/integration/projects/conftest.py

@@ -1,7 +1,8 @@
-from pytest import fixture
+from pytest import fixture, mark
 
 
 @fixture(scope="module")
+@mark.xdist_group("project_resources")
 async def project(proximl):
     project = await proximl.projects.create(
         name="New Project", copy_credentials=False, copy_secrets=False

{proximl-0.5.13.post1 → proximl-0.5.15}/tests/integration/projects/test_projects_credentials_integration.py

@@ -8,6 +8,7 @@ pytestmark = [mark.sdk, mark.integration, mark.projects]
 
 @mark.create
 @mark.asyncio
+@mark.xdist_group("project_resources")
 class ProjectCredentialsTests:
     @fixture(scope="class")
     async def project_credential(self, project):

{proximl-0.5.13.post1 → proximl-0.5.15}/tests/integration/projects/test_projects_data_connectors_integration.py

@@ -8,6 +8,7 @@ pytestmark = [mark.sdk, mark.integration, mark.projects]
 
 @mark.create
 @mark.asyncio
+@mark.xdist_group("project_resources")
 class ProjectDataConnectorsTests:
     @fixture(scope="class")
     async def project_data_connector(self, project):

{proximl-0.5.13.post1 → proximl-0.5.15}/tests/integration/projects/test_projects_datastores_integration.py

@@ -8,6 +8,7 @@ pytestmark = [mark.sdk, mark.integration, mark.projects]
 
 @mark.create
 @mark.asyncio
+@mark.xdist_group("project_resources")
 class ProjectDatastoresTests:
     @fixture(scope="class")
     async def project_datastore(self, project):