proximl 0.5.13__py3-none-any.whl → 0.5.14__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- proximl/__init__.py +1 -1
- proximl/cli/job/create.py +46 -16
- proximl/cli/project/__init__.py +1 -1
- proximl/cli/project/secret.py +12 -3
- proximl/jobs.py +9 -0
- proximl/projects/data_connectors.py +16 -0
- proximl/projects/datastores.py +16 -0
- proximl/projects/members.py +98 -0
- proximl/projects/projects.py +9 -4
- proximl/projects/secrets.py +1 -1
- proximl/projects/services.py +16 -0
- proximl/proximl.py +36 -19
- {proximl-0.5.13.dist-info → proximl-0.5.14.dist-info}/METADATA +1 -1
- {proximl-0.5.13.dist-info → proximl-0.5.14.dist-info}/RECORD +25 -27
- tests/integration/projects/conftest.py +1 -1
- tests/integration/projects/test_projects_members_integration.py +49 -0
- tests/integration/projects/test_projects_secrets_integration.py +1 -1
- tests/integration/test_jobs_integration.py +9 -9
- tests/unit/conftest.py +9 -9
- tests/unit/projects/test_project_members_unit.py +107 -0
- tests/unit/projects/test_projects_unit.py +1 -1
- proximl/cli/project/key.py +0 -124
- proximl/projects/keys.py +0 -71
- tests/integration/projects/test_projects_keys_integration.py +0 -43
- tests/unit/cli/projects/test_cli_project_key_unit.py +0 -26
- tests/unit/projects/test_project_keys_unit.py +0 -96
- {proximl-0.5.13.dist-info → proximl-0.5.14.dist-info}/LICENSE +0 -0
- {proximl-0.5.13.dist-info → proximl-0.5.14.dist-info}/WHEEL +0 -0
- {proximl-0.5.13.dist-info → proximl-0.5.14.dist-info}/entry_points.txt +0 -0
- {proximl-0.5.13.dist-info → proximl-0.5.14.dist-info}/top_level.txt +0 -0
proximl/__init__.py
CHANGED
proximl/cli/job/create.py
CHANGED
|
@@ -172,12 +172,18 @@ def create(config):
|
|
|
172
172
|
help="Conda packages to install as a comma separated list 'p1,\"p2=v2\",p3'",
|
|
173
173
|
)
|
|
174
174
|
@click.option(
|
|
175
|
-
"--
|
|
175
|
+
"--credential",
|
|
176
176
|
type=click.Choice(
|
|
177
177
|
["aws", "gcp", "kaggle", "azure", "wasabi"],
|
|
178
178
|
case_sensitive=False,
|
|
179
179
|
),
|
|
180
|
-
help="Third Party
|
|
180
|
+
help="Third Party Credentials to add to the job environment",
|
|
181
|
+
multiple=True,
|
|
182
|
+
)
|
|
183
|
+
@click.option(
|
|
184
|
+
"--secret",
|
|
185
|
+
type=click.STRING,
|
|
186
|
+
help="Project secrets to add to the job environment",
|
|
181
187
|
multiple=True,
|
|
182
188
|
)
|
|
183
189
|
@click.option(
|
|
@@ -221,7 +227,8 @@ def notebook(
|
|
|
221
227
|
environment,
|
|
222
228
|
custom_image,
|
|
223
229
|
env,
|
|
224
|
-
|
|
230
|
+
credential,
|
|
231
|
+
secret,
|
|
225
232
|
apt_packages,
|
|
226
233
|
pip_packages,
|
|
227
234
|
conda_packages,
|
|
@@ -254,7 +261,7 @@ def notebook(
|
|
|
254
261
|
data=dict(datasets=datasets),
|
|
255
262
|
model=dict(checkpoints=checkpoints),
|
|
256
263
|
environment=dict(
|
|
257
|
-
|
|
264
|
+
credentials=[k for k in credential], secrets=[s for s in secret]
|
|
258
265
|
),
|
|
259
266
|
)
|
|
260
267
|
|
|
@@ -500,12 +507,18 @@ def notebook(
|
|
|
500
507
|
multiple=True,
|
|
501
508
|
)
|
|
502
509
|
@click.option(
|
|
503
|
-
"--
|
|
510
|
+
"--credential",
|
|
504
511
|
type=click.Choice(
|
|
505
512
|
["aws", "gcp", "kaggle", "azure", "wasabi"],
|
|
506
513
|
case_sensitive=False,
|
|
507
514
|
),
|
|
508
|
-
help="Third Party
|
|
515
|
+
help="Third Party Credentials to add to the job environment",
|
|
516
|
+
multiple=True,
|
|
517
|
+
)
|
|
518
|
+
@click.option(
|
|
519
|
+
"--secret",
|
|
520
|
+
type=click.STRING,
|
|
521
|
+
help="Project secrets to add to the job environment",
|
|
509
522
|
multiple=True,
|
|
510
523
|
)
|
|
511
524
|
@click.option(
|
|
@@ -563,7 +576,8 @@ def training(
|
|
|
563
576
|
environment,
|
|
564
577
|
custom_image,
|
|
565
578
|
env,
|
|
566
|
-
|
|
579
|
+
credential,
|
|
580
|
+
secret,
|
|
567
581
|
apt_packages,
|
|
568
582
|
pip_packages,
|
|
569
583
|
conda_packages,
|
|
@@ -596,7 +610,7 @@ def training(
|
|
|
596
610
|
data=dict(datasets=datasets),
|
|
597
611
|
model=dict(checkpoints=checkpoints),
|
|
598
612
|
environment=dict(
|
|
599
|
-
|
|
613
|
+
credentials=[k for k in credential], secrets=[s for s in secret]
|
|
600
614
|
),
|
|
601
615
|
)
|
|
602
616
|
|
|
@@ -844,12 +858,18 @@ def training(
|
|
|
844
858
|
multiple=True,
|
|
845
859
|
)
|
|
846
860
|
@click.option(
|
|
847
|
-
"--
|
|
861
|
+
"--credential",
|
|
848
862
|
type=click.Choice(
|
|
849
863
|
["aws", "gcp", "kaggle", "azure", "wasabi"],
|
|
850
864
|
case_sensitive=False,
|
|
851
865
|
),
|
|
852
|
-
help="Third Party
|
|
866
|
+
help="Third Party Credentials to add to the job environment",
|
|
867
|
+
multiple=True,
|
|
868
|
+
)
|
|
869
|
+
@click.option(
|
|
870
|
+
"--secret",
|
|
871
|
+
type=click.STRING,
|
|
872
|
+
help="Project secrets to add to the job environment",
|
|
853
873
|
multiple=True,
|
|
854
874
|
)
|
|
855
875
|
@click.option(
|
|
@@ -907,7 +927,8 @@ def inference(
|
|
|
907
927
|
environment,
|
|
908
928
|
custom_image,
|
|
909
929
|
env,
|
|
910
|
-
|
|
930
|
+
credential,
|
|
931
|
+
secret,
|
|
911
932
|
apt_packages,
|
|
912
933
|
pip_packages,
|
|
913
934
|
conda_packages,
|
|
@@ -933,7 +954,8 @@ def inference(
|
|
|
933
954
|
data=dict(datasets=[]),
|
|
934
955
|
model=dict(checkpoints=checkpoints),
|
|
935
956
|
environment=dict(
|
|
936
|
-
|
|
957
|
+
credentials=[k for k in credential],
|
|
958
|
+
secrets=[s for s in secret],
|
|
937
959
|
),
|
|
938
960
|
)
|
|
939
961
|
|
|
@@ -1163,12 +1185,18 @@ def from_json(config, attach, connect, file):
|
|
|
1163
1185
|
multiple=True,
|
|
1164
1186
|
)
|
|
1165
1187
|
@click.option(
|
|
1166
|
-
"--
|
|
1188
|
+
"--credential",
|
|
1167
1189
|
type=click.Choice(
|
|
1168
1190
|
["aws", "gcp", "kaggle", "azure", "wasabi"],
|
|
1169
1191
|
case_sensitive=False,
|
|
1170
1192
|
),
|
|
1171
|
-
help="Third Party
|
|
1193
|
+
help="Third Party Credentials to add to the job environment",
|
|
1194
|
+
multiple=True,
|
|
1195
|
+
)
|
|
1196
|
+
@click.option(
|
|
1197
|
+
"--secret",
|
|
1198
|
+
type=click.STRING,
|
|
1199
|
+
help="Project secrets to add to the job environment",
|
|
1172
1200
|
multiple=True,
|
|
1173
1201
|
)
|
|
1174
1202
|
@click.option(
|
|
@@ -1230,7 +1258,8 @@ def endpoint(
|
|
|
1230
1258
|
environment,
|
|
1231
1259
|
custom_image,
|
|
1232
1260
|
env,
|
|
1233
|
-
|
|
1261
|
+
credential,
|
|
1262
|
+
secret,
|
|
1234
1263
|
apt_packages,
|
|
1235
1264
|
pip_packages,
|
|
1236
1265
|
conda_packages,
|
|
@@ -1257,7 +1286,8 @@ def endpoint(
|
|
|
1257
1286
|
max_price=max_price,
|
|
1258
1287
|
model=dict(checkpoints=checkpoints),
|
|
1259
1288
|
environment=dict(
|
|
1260
|
-
|
|
1289
|
+
credentials=[k for k in credential],
|
|
1290
|
+
secrets=[s for s in secret],
|
|
1261
1291
|
),
|
|
1262
1292
|
)
|
|
1263
1293
|
|
proximl/cli/project/__init__.py
CHANGED
|
@@ -78,7 +78,7 @@ def remove(config, project):
|
|
|
78
78
|
|
|
79
79
|
|
|
80
80
|
from proximl.cli.project.secret import secret
|
|
81
|
-
from proximl.cli.project.
|
|
81
|
+
from proximl.cli.project.credential import credential
|
|
82
82
|
from proximl.cli.project.data_connector import data_connector
|
|
83
83
|
from proximl.cli.project.datastore import datastore
|
|
84
84
|
from proximl.cli.project.service import service
|
proximl/cli/project/secret.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import click
|
|
2
|
+
import os
|
|
2
3
|
from proximl.cli import pass_config
|
|
3
4
|
from proximl.cli.project import project
|
|
4
5
|
|
|
@@ -42,17 +43,25 @@ def list(config):
|
|
|
42
43
|
|
|
43
44
|
|
|
44
45
|
@secret.command()
|
|
46
|
+
@click.option(
|
|
47
|
+
"--file",
|
|
48
|
+
type=click.Path(exists=True, file_okay=True, dir_okay=False, resolve_path=True),
|
|
49
|
+
help="Load the secret value from the file at the provided path",
|
|
50
|
+
)
|
|
45
51
|
@click.argument("name", type=click.STRING)
|
|
46
52
|
@pass_config
|
|
47
|
-
def put(config, name):
|
|
53
|
+
def put(config, file, name):
|
|
48
54
|
"""
|
|
49
55
|
Set a secret value.
|
|
50
56
|
|
|
51
57
|
Secret is created with the specified NAME.
|
|
52
58
|
"""
|
|
53
59
|
project = config.proximl.run(config.proximl.client.projects.get_current())
|
|
54
|
-
|
|
55
|
-
|
|
60
|
+
if file:
|
|
61
|
+
with open(os.path.expanduser(file)) as f:
|
|
62
|
+
value = f.read()
|
|
63
|
+
else:
|
|
64
|
+
value = click.prompt("Enter the secret value", type=str, hide_input=True)
|
|
56
65
|
|
|
57
66
|
return config.proximl.run(project.secrets.put(name=name, value=value))
|
|
58
67
|
|
proximl/jobs.py
CHANGED
|
@@ -530,6 +530,15 @@ class Job:
|
|
|
530
530
|
"waiting for data/model download",
|
|
531
531
|
]
|
|
532
532
|
)
|
|
533
|
+
or (
|
|
534
|
+
status
|
|
535
|
+
== "running" ## this status could be too short for polling could miss it
|
|
536
|
+
and self.status
|
|
537
|
+
in [
|
|
538
|
+
"uploading",
|
|
539
|
+
"finished"
|
|
540
|
+
]
|
|
541
|
+
)
|
|
533
542
|
):
|
|
534
543
|
return self
|
|
535
544
|
elif self.status == "failed":
|
|
@@ -7,6 +7,12 @@ class ProjectDataConnectors(object):
|
|
|
7
7
|
self.proximl = proximl
|
|
8
8
|
self.project_id = project_id
|
|
9
9
|
|
|
10
|
+
async def get(self, id, **kwargs):
|
|
11
|
+
resp = await self.proximl._query(
|
|
12
|
+
f"/project/{self.project_id}/data_connectors/{id}", "GET", kwargs
|
|
13
|
+
)
|
|
14
|
+
return ProjectDataConnector(self.proximl, **resp)
|
|
15
|
+
|
|
10
16
|
async def list(self, **kwargs):
|
|
11
17
|
resp = await self.proximl._query(
|
|
12
18
|
f"/project/{self.project_id}/data_connectors", "GET", kwargs
|
|
@@ -61,3 +67,13 @@ class ProjectDataConnector:
|
|
|
61
67
|
|
|
62
68
|
def __bool__(self):
|
|
63
69
|
return bool(self._id)
|
|
70
|
+
|
|
71
|
+
async def enable(self):
|
|
72
|
+
await self.proximl._query(
|
|
73
|
+
f"/project/{self._project_uuid}/data_connectors/{self._id}/enable", "PATCH"
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
async def disable(self):
|
|
77
|
+
await self.proximl._query(
|
|
78
|
+
f"/project/{self._project_uuid}/data_connectors/{self._id}/disable", "PATCH"
|
|
79
|
+
)
|
proximl/projects/datastores.py
CHANGED
|
@@ -7,6 +7,12 @@ class ProjectDatastores(object):
|
|
|
7
7
|
self.proximl = proximl
|
|
8
8
|
self.project_id = project_id
|
|
9
9
|
|
|
10
|
+
async def get(self, id, **kwargs):
|
|
11
|
+
resp = await self.proximl._query(
|
|
12
|
+
f"/project/{self.project_id}/datastores/{id}", "GET", kwargs
|
|
13
|
+
)
|
|
14
|
+
return ProjectDatastore(self.proximl, **resp)
|
|
15
|
+
|
|
10
16
|
async def list(self, **kwargs):
|
|
11
17
|
resp = await self.proximl._query(
|
|
12
18
|
f"/project/{self.project_id}/datastores", "GET", kwargs
|
|
@@ -56,3 +62,13 @@ class ProjectDatastore:
|
|
|
56
62
|
|
|
57
63
|
def __bool__(self):
|
|
58
64
|
return bool(self._id)
|
|
65
|
+
|
|
66
|
+
async def enable(self):
|
|
67
|
+
await self.proximl._query(
|
|
68
|
+
f"/project/{self._project_uuid}/datastores/{self._id}/enable", "PATCH"
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
async def disable(self):
|
|
72
|
+
await self.proximl._query(
|
|
73
|
+
f"/project/{self._project_uuid}/datastores/{self._id}/disable", "PATCH"
|
|
74
|
+
)
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
from typing import Literal
|
|
4
|
+
|
|
5
|
+
class ProjectMembers(object):
|
|
6
|
+
def __init__(self, proximl, project_id):
|
|
7
|
+
self.proximl = proximl
|
|
8
|
+
self.project_id = project_id
|
|
9
|
+
|
|
10
|
+
async def list(self, **kwargs):
|
|
11
|
+
resp = await self.proximl._query(
|
|
12
|
+
f"/project/{self.project_id}/access", "GET", kwargs
|
|
13
|
+
)
|
|
14
|
+
members = [ProjectMember(self.proximl, **member) for member in resp]
|
|
15
|
+
return members
|
|
16
|
+
|
|
17
|
+
async def add(self, email: str, job: Literal["all", "read"], dataset: Literal["all", "read"], model: Literal["all", "read"], checkpoint: Literal["all", "read"], volume: Literal["all", "read"], **kwargs):
|
|
18
|
+
data = dict(
|
|
19
|
+
email=email,
|
|
20
|
+
job=job,
|
|
21
|
+
dataset=dataset,
|
|
22
|
+
model=model,
|
|
23
|
+
checkpoint=checkpoint,
|
|
24
|
+
volume=volume,
|
|
25
|
+
)
|
|
26
|
+
payload = {k: v for k, v in data.items() if v is not None}
|
|
27
|
+
resp = await self.proximl._query(
|
|
28
|
+
f"/project/{self.project_id}/access", "POST",kwargs, payload)
|
|
29
|
+
member = ProjectMember(self.proximl, **resp)
|
|
30
|
+
logging.info(f"Added Project Member {email} to project {self.project_id}")
|
|
31
|
+
return member
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
async def remove(self, email, **kwargs):
|
|
35
|
+
await self.proximl._query(
|
|
36
|
+
f"/project/{self.project_id}/access", "DELETE", dict(**kwargs, email=email)
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class ProjectMember:
|
|
41
|
+
def __init__(self, proximl, **kwargs):
|
|
42
|
+
self.proximl = proximl
|
|
43
|
+
self._entity = kwargs
|
|
44
|
+
self._id = self._entity.get("email")
|
|
45
|
+
self._project_uuid = self._entity.get("project_uuid")
|
|
46
|
+
self._owner = self._entity.get("owner")
|
|
47
|
+
self._job = self._entity.get("job")
|
|
48
|
+
self._dataset = self._entity.get("dataset")
|
|
49
|
+
self._model = self._entity.get("model")
|
|
50
|
+
self._checkpoint = self._entity.get("checkpoint")
|
|
51
|
+
self._volume = self._entity.get("volume")
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def id(self) -> str:
|
|
55
|
+
return self._id
|
|
56
|
+
|
|
57
|
+
@property
|
|
58
|
+
def project_uuid(self) -> str:
|
|
59
|
+
return self._project_uuid
|
|
60
|
+
|
|
61
|
+
@property
|
|
62
|
+
def email(self) -> str:
|
|
63
|
+
return self._id
|
|
64
|
+
|
|
65
|
+
@property
|
|
66
|
+
def owner(self) -> bool:
|
|
67
|
+
return self._owner
|
|
68
|
+
|
|
69
|
+
@property
|
|
70
|
+
def job(self) -> str:
|
|
71
|
+
return self._job
|
|
72
|
+
|
|
73
|
+
@property
|
|
74
|
+
def dataset(self) -> str:
|
|
75
|
+
return self._dataset
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def model(self) -> str:
|
|
79
|
+
return self._model
|
|
80
|
+
|
|
81
|
+
@property
|
|
82
|
+
def checkpoint(self) -> str:
|
|
83
|
+
return self._checkpoint
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def volume(self) -> str:
|
|
87
|
+
return self._volume
|
|
88
|
+
|
|
89
|
+
def __str__(self):
|
|
90
|
+
return json.dumps({k: v for k, v in self._entity.items()})
|
|
91
|
+
|
|
92
|
+
def __repr__(self):
|
|
93
|
+
return f"ProjectMember( proximl , **{self._entity.__repr__()})"
|
|
94
|
+
|
|
95
|
+
def __bool__(self):
|
|
96
|
+
return bool(self._id)
|
|
97
|
+
|
|
98
|
+
|
proximl/projects/projects.py
CHANGED
|
@@ -3,8 +3,9 @@ import logging
|
|
|
3
3
|
from .datastores import ProjectDatastores
|
|
4
4
|
from .data_connectors import ProjectDataConnectors
|
|
5
5
|
from .services import ProjectServices
|
|
6
|
-
from .
|
|
6
|
+
from .credentials import ProjectCredentials
|
|
7
7
|
from .secrets import ProjectSecrets
|
|
8
|
+
from .members import ProjectMembers
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
class Projects(object):
|
|
@@ -26,8 +27,11 @@ class Projects(object):
|
|
|
26
27
|
projects = [Project(self.proximl, **project) for project in resp]
|
|
27
28
|
return projects
|
|
28
29
|
|
|
29
|
-
async def create(self, name,
|
|
30
|
-
data = dict(
|
|
30
|
+
async def create(self, name, copy_credentials=False, **kwargs):
|
|
31
|
+
data = dict(
|
|
32
|
+
name=name,
|
|
33
|
+
copy_credentials=copy_credentials,
|
|
34
|
+
)
|
|
31
35
|
payload = {k: v for k, v in data.items() if v is not None}
|
|
32
36
|
logging.info(f"Creating Project {name}")
|
|
33
37
|
resp = await self.proximl._query("/project", "POST", None, payload)
|
|
@@ -51,8 +55,9 @@ class Project:
|
|
|
51
55
|
self.datastores = ProjectDatastores(self.proximl, self._id)
|
|
52
56
|
self.data_connectors = ProjectDataConnectors(self.proximl, self._id)
|
|
53
57
|
self.services = ProjectServices(self.proximl, self._id)
|
|
54
|
-
self.
|
|
58
|
+
self.credentials = ProjectCredentials(self.proximl, self._id)
|
|
55
59
|
self.secrets = ProjectSecrets(self.proximl, self._id)
|
|
60
|
+
self.members = ProjectMembers(self.proximl, self._id)
|
|
56
61
|
|
|
57
62
|
@property
|
|
58
63
|
def id(self) -> str:
|
proximl/projects/secrets.py
CHANGED
|
@@ -24,7 +24,7 @@ class ProjectSecrets(object):
|
|
|
24
24
|
f"/project/{self.project_id}/secret/{name}", "PUT", None, payload
|
|
25
25
|
)
|
|
26
26
|
secret = ProjectSecret(self.proximl, **resp)
|
|
27
|
-
logging.info(f"Created Project
|
|
27
|
+
logging.info(f"Created Project Secret {name} in project {self.project_id}")
|
|
28
28
|
return secret
|
|
29
29
|
|
|
30
30
|
async def remove(self, name, **kwargs):
|
proximl/projects/services.py
CHANGED
|
@@ -7,6 +7,12 @@ class ProjectServices(object):
|
|
|
7
7
|
self.proximl = proximl
|
|
8
8
|
self.project_id = project_id
|
|
9
9
|
|
|
10
|
+
async def get(self, id, **kwargs):
|
|
11
|
+
resp = await self.proximl._query(
|
|
12
|
+
f"/project/{self.project_id}/services/{id}", "GET", kwargs
|
|
13
|
+
)
|
|
14
|
+
return ProjectService(self.proximl, **resp)
|
|
15
|
+
|
|
10
16
|
async def list(self, **kwargs):
|
|
11
17
|
resp = await self.proximl._query(
|
|
12
18
|
f"/project/{self.project_id}/services", "GET", kwargs
|
|
@@ -61,3 +67,13 @@ class ProjectService:
|
|
|
61
67
|
|
|
62
68
|
def __bool__(self):
|
|
63
69
|
return bool(self._id)
|
|
70
|
+
|
|
71
|
+
async def enable(self):
|
|
72
|
+
await self.proximl._query(
|
|
73
|
+
f"/project/{self._project_uuid}/services/{self._id}/enable", "PATCH"
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
async def disable(self):
|
|
77
|
+
await self.proximl._query(
|
|
78
|
+
f"/project/{self._project_uuid}/services/{self._id}/disable", "PATCH"
|
|
79
|
+
)
|
proximl/proximl.py
CHANGED
|
@@ -4,6 +4,7 @@ import asyncio
|
|
|
4
4
|
import aiohttp
|
|
5
5
|
import logging
|
|
6
6
|
import traceback
|
|
7
|
+
import random
|
|
7
8
|
from importlib.metadata import version
|
|
8
9
|
|
|
9
10
|
from proximl.auth import Auth
|
|
@@ -91,7 +92,7 @@ class ProxiML(object):
|
|
|
91
92
|
def project(self) -> str:
|
|
92
93
|
return self.active_project
|
|
93
94
|
|
|
94
|
-
async def _query(self, path, method, params=None, data=None, headers=None):
|
|
95
|
+
async def _query(self, path, method, params=None, data=None, headers=None,max_retries=3, backoff_factor=0.5):
|
|
95
96
|
try:
|
|
96
97
|
tokens = self.auth.get_tokens()
|
|
97
98
|
except ProxiMLException as e:
|
|
@@ -142,24 +143,40 @@ class ProxiML(object):
|
|
|
142
143
|
logging.debug(
|
|
143
144
|
f"Request - Url: {url}, Method: {method}, Params: {params}, Body: {data}, Headers: {headers}"
|
|
144
145
|
)
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
146
|
+
for attempt in range(max_retries):
|
|
147
|
+
try:
|
|
148
|
+
async with aiohttp.ClientSession() as session:
|
|
149
|
+
async with session.request(
|
|
150
|
+
method,
|
|
151
|
+
url,
|
|
152
|
+
data=json.dumps(data),
|
|
153
|
+
headers=headers,
|
|
154
|
+
params=params,
|
|
155
|
+
) as resp:
|
|
156
|
+
if (resp.status // 100) in [4, 5]:
|
|
157
|
+
if resp.status == 502 and attempt < max_retries - 1:
|
|
158
|
+
wait_time = (2 ** attempt) * backoff_factor * (random.random() + 0.5)
|
|
159
|
+
await asyncio.sleep(wait_time)
|
|
160
|
+
continue
|
|
161
|
+
else:
|
|
162
|
+
what = await resp.read()
|
|
163
|
+
content_type = resp.headers.get("content-type", "")
|
|
164
|
+
resp.close()
|
|
165
|
+
if content_type == "application/json":
|
|
166
|
+
raise ApiError(resp.status, json.loads(what.decode("utf8")))
|
|
167
|
+
else:
|
|
168
|
+
raise ApiError(resp.status, {"message": what.decode("utf8")})
|
|
169
|
+
results = await resp.json()
|
|
170
|
+
return results
|
|
171
|
+
except aiohttp.ClientResponseError as e:
|
|
172
|
+
if e.status == 502 and attempt < max_retries - 1:
|
|
173
|
+
wait_time = (2 ** attempt) * backoff_factor * (random.random() + 0.5)
|
|
174
|
+
await asyncio.sleep(wait_time)
|
|
175
|
+
continue
|
|
176
|
+
else:
|
|
177
|
+
raise ApiError(e.status, f"Error {e.message}")
|
|
178
|
+
|
|
179
|
+
raise ProxiMLException("Unexpected API failure")
|
|
163
180
|
|
|
164
181
|
async def _ws_subscribe(self, entity, project_uuid, id, msg_handler):
|
|
165
182
|
headers = {
|
|
@@ -2,7 +2,7 @@ examples/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
|
2
2
|
examples/create_dataset_and_training_job.py,sha256=Fqueoz2KD1MTxnU960iqHUdxvo68Xe64HT5XS55lI9w,1178
|
|
3
3
|
examples/local_storage.py,sha256=6K6LMO7ZPI7N2KdBcgqXSvdsqJfjISzN4yRO9YrJqbA,1745
|
|
4
4
|
examples/training_inference_pipeline.py,sha256=pxux0QUUtRXxKj2rX-6fPEKBIi43mhRd1zJ-Lf1ZJGI,2334
|
|
5
|
-
proximl/__init__.py,sha256=
|
|
5
|
+
proximl/__init__.py,sha256=iO6-pU6BIwDgKVSlSYqbBe88EBQnbAkS6-sVe7xBBJA,433
|
|
6
6
|
proximl/__main__.py,sha256=JgErYkiskih8Y6oRwowALtR-rwQhAAdqOYWjQraRIPI,59
|
|
7
7
|
proximl/auth.py,sha256=LacGBDAVel5HcJx7JXp1wVn3s0gZc7nf--vDfSFOXYU,26565
|
|
8
8
|
proximl/checkpoints.py,sha256=Ezpiab9Wfcmd2h5Slm9U5lekZGXiPzvhDKxXXgla1yo,8790
|
|
@@ -11,9 +11,9 @@ proximl/datasets.py,sha256=dBDzf7DZ1rS5LmJPy5oV-JC0PXGTf-NZuBDG1KPyoA4,8600
|
|
|
11
11
|
proximl/environments.py,sha256=L_cRmau1wJxpGvnJAqgms-GiNdDhiuOntrlBqsdoE3A,1507
|
|
12
12
|
proximl/exceptions.py,sha256=q1YfsqbLw3y1DJHP1FcMM8xG5CEhQsTIvRfq4v5PyVw,5468
|
|
13
13
|
proximl/gpu_types.py,sha256=V-EZzE-hDLi5eVQ2_9yGLTm8-Qk1AnnzctfSVC44yLY,1901
|
|
14
|
-
proximl/jobs.py,sha256=
|
|
14
|
+
proximl/jobs.py,sha256=HiGs2gRKPFZReHLg9DLPsDWEecxZpafWv6XFBSGXYj0,17947
|
|
15
15
|
proximl/models.py,sha256=gJQ1C3HGEHScW41PAQS3Q8IKSQSauIUIJxOZvkDzizQ,8284
|
|
16
|
-
proximl/proximl.py,sha256=
|
|
16
|
+
proximl/proximl.py,sha256=7UyqYbGO9edFJec54xqk_wwahKldXQlY5112Izg3saA,11902
|
|
17
17
|
proximl/volumes.py,sha256=qRIgzvJ4NTHuLi7QN_6v61Urns8l4Y9KjWladCIQ2gk,8443
|
|
18
18
|
proximl/cli/__init__.py,sha256=R_8ExAKQp67N2xtwGM00gtK3zSoWPGruHAP_XFLddSI,4346
|
|
19
19
|
proximl/cli/checkpoint.py,sha256=Iv1i1EAt2LJey2wy2ioQ6-ZysqwRG4kFj0lnE6INZCM,7170
|
|
@@ -32,13 +32,12 @@ proximl/cli/cloudbender/provider.py,sha256=qhWbDK1tWi00wQWEYqGw7yGoZx0nEjV40GLHR
|
|
|
32
32
|
proximl/cli/cloudbender/region.py,sha256=WnSkY4dXKRJ-FNaoxMfmoh6iuUx5dXCNJmEFT34Xtao,2892
|
|
33
33
|
proximl/cli/cloudbender/service.py,sha256=6NuwlFULJLtOmMy9OFA8GkW4MLvNemAcd_KitQyDXV8,3147
|
|
34
34
|
proximl/cli/job/__init__.py,sha256=s8mU2PvCWDcv4gGT3EmjHn8MIZlXBAoayoZKmnKpXnY,6545
|
|
35
|
-
proximl/cli/job/create.py,sha256=
|
|
36
|
-
proximl/cli/project/__init__.py,sha256=
|
|
35
|
+
proximl/cli/job/create.py,sha256=plRiX9lRqt6lrjWvTIVbLC-EoDgSL0-oYqwIfuWb_ek,35098
|
|
36
|
+
proximl/cli/project/__init__.py,sha256=ZSiXhsq7O1JwGecsBKBOD1W0VuiwErs-FFsfrXMOiFs,1918
|
|
37
37
|
proximl/cli/project/credential.py,sha256=5vPI6VBIo_YyvAzfUa9ARmz-LUnkdIVW0zbko9t0Q38,3443
|
|
38
38
|
proximl/cli/project/data_connector.py,sha256=CKF4snfrzDmeDXb4au8nV2W9jTweRMDZ89U2GAlBedQ,1411
|
|
39
39
|
proximl/cli/project/datastore.py,sha256=YusrUTKY_qIwEOBN2gXOUoKCUp2qbWTKzquKc0DcUsE,1336
|
|
40
|
-
proximl/cli/project/
|
|
41
|
-
proximl/cli/project/secret.py,sha256=uys2FsXuO_8_EoolQNuOZ9btX4I0LXBeKG8dGx6StMM,1621
|
|
40
|
+
proximl/cli/project/secret.py,sha256=jWj9SNcx4jNKX1E84Fjkj0hRzF1XU0CrdYqNtxYuTpI,1927
|
|
42
41
|
proximl/cli/project/service.py,sha256=06UoGbAWbPPh3kBdH3QXBTzj_k7AfKcD_bAyWGpYeDE,1310
|
|
43
42
|
proximl/cloudbender/__init__.py,sha256=iE29obtC0_9f0IhRvHQcG5aY58fVhVYipTakpjAhdss,64
|
|
44
43
|
proximl/cloudbender/cloudbender.py,sha256=k9Naj_le32YZotM7PRrhml3DxjqRetsHcgg5J4RtJ_4,717
|
|
@@ -52,34 +51,34 @@ proximl/cloudbender/regions.py,sha256=6doBdfMXIQI-uexzvwmNg2ATDzruatw-ixviZSMjon
|
|
|
52
51
|
proximl/cloudbender/services.py,sha256=km3KcIgaRRVb3iuZEIn6ONIekuyXhBgk_brAFJcMMl4,5126
|
|
53
52
|
proximl/projects/__init__.py,sha256=6NKCcHtQMeGB1IyU-djANphfnDX6MEkrXUM5Fyq9fWg,75
|
|
54
53
|
proximl/projects/credentials.py,sha256=hWz6EUEAgltkAQMDjQVsFlVvfWUbZfY3KoWpldnCrXY,2255
|
|
55
|
-
proximl/projects/data_connectors.py,sha256=
|
|
56
|
-
proximl/projects/datastores.py,sha256=
|
|
57
|
-
proximl/projects/
|
|
58
|
-
proximl/projects/projects.py,sha256=
|
|
59
|
-
proximl/projects/secrets.py,sha256=
|
|
60
|
-
proximl/projects/services.py,sha256=
|
|
54
|
+
proximl/projects/data_connectors.py,sha256=o2-K40BdF85TBXJgj1UMsznBmNHmnYh1qbZtHmmOa2Y,2216
|
|
55
|
+
proximl/projects/datastores.py,sha256=8nHpMEHn3UzsIhmPMoXympbRRJ10XkHdyRhBXUVXyeE,2095
|
|
56
|
+
proximl/projects/members.py,sha256=siREAa-Oac1pLGRHMXmj3ShoB0DVl8uY34eFdTlSezA,2826
|
|
57
|
+
proximl/projects/projects.py,sha256=PJYUaXbYMINcvlDQePKUSAz6uO11jg9tj5gJW-cNjGQ,2890
|
|
58
|
+
proximl/projects/secrets.py,sha256=xlfzRONjT2ySJDcq8FYRQmSU2ZJMdPLS8NbePWYNCeM,2160
|
|
59
|
+
proximl/projects/services.py,sha256=C1YUxixkio8n4AlgmGqZJ93hP3VKOoTXVXTNCkRVd9Y,2206
|
|
61
60
|
tests/integration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
62
61
|
tests/integration/conftest.py,sha256=zRWpherX-yfbpk7xqZk9bIZCyJ-dwVeszY_7kekn2M4,1134
|
|
63
62
|
tests/integration/test_checkpoints_integration.py,sha256=m-m6KM5Iy99bbCyA6VQVBFWc9MI7VItuoAPb_Q6Q0Fk,3206
|
|
64
63
|
tests/integration/test_datasets_integration.py,sha256=Ndp8itnncDSjVH0t5BM5R_-_yL4qt6JrkQAVOTMi1R8,3499
|
|
65
64
|
tests/integration/test_environments_integration.py,sha256=7P6pKSyxA7rTwyNCD9HEaM2ablMG8WcBesOzkG-BgsQ,1403
|
|
66
65
|
tests/integration/test_gpu_types_integration.py,sha256=Zv-yrHcAgKau9BJQvzi92bdrRHhLPl_hbhhzNLEWJ9w,1256
|
|
67
|
-
tests/integration/test_jobs_integration.py,sha256=
|
|
66
|
+
tests/integration/test_jobs_integration.py,sha256=TAezhtUhe-Prigo6mwnNOivycbnPGBhAXZslvujMRI4,25100
|
|
68
67
|
tests/integration/test_models_integration.py,sha256=cnsao7Zo4PwVCbdAdcBs0xombAslI_X_JOw6ipxdzOA,2878
|
|
69
68
|
tests/integration/test_volumes_integration.py,sha256=Xo2Whw2U-7jyvESIkyex3f0SMXKlExe3kLmsbpXTHhQ,3270
|
|
70
69
|
tests/integration/cloudbender/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
71
70
|
tests/integration/cloudbender/test_providers_integration.py,sha256=gFqPQom-Cn1iZC50_ChQ2us2_f4tIPATQSAUcWdf7ss,1473
|
|
72
71
|
tests/integration/projects/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
73
|
-
tests/integration/projects/conftest.py,sha256=
|
|
72
|
+
tests/integration/projects/conftest.py,sha256=4WbknSwdJgzPAr5qumCJD3Cl1rQFWgGZ5-Wro1ka0xA,249
|
|
74
73
|
tests/integration/projects/test_projects_credentials_integration.py,sha256=_JqE2sGN09gZpvhtMRFjBPXumJ-PqKl8iXOgcYHlvSY,1636
|
|
75
74
|
tests/integration/projects/test_projects_data_connectors_integration.py,sha256=ag6w2FFQYNlAdV83rOqat0FVZnP4JFhDunCvRwMJdzo,1630
|
|
76
75
|
tests/integration/projects/test_projects_datastores_integration.py,sha256=DCHFnCYguhtrZvfTeWgKpPxlkeAXxVfE3IWkM4KjgAc,1469
|
|
77
76
|
tests/integration/projects/test_projects_integration.py,sha256=wlRLxZkMnJZHlNPaeRFjAeyUlMfbgdrrFEI2eMBcoUI,1241
|
|
78
|
-
tests/integration/projects/
|
|
79
|
-
tests/integration/projects/test_projects_secrets_integration.py,sha256=
|
|
77
|
+
tests/integration/projects/test_projects_members_integration.py,sha256=3URbefe0DD0rMjwcWpBV521Yt9k9tLVL3nn6mNRPp4g,1843
|
|
78
|
+
tests/integration/projects/test_projects_secrets_integration.py,sha256=wRkcVkaf-zLWvCdJx9ikJYAoiJXU1esv0diRxwwRFdc,1481
|
|
80
79
|
tests/integration/projects/test_projects_services_integration.py,sha256=MV9tZFTgrUtzImNrc-ZAoHYLbf_ZxAAAeVpxxr5AGdg,1530
|
|
81
80
|
tests/unit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
82
|
-
tests/unit/conftest.py,sha256=
|
|
81
|
+
tests/unit/conftest.py,sha256=11Z49O4mKXtm6xl18kWPomZvbZJQpqdaDYeJHFQbO1A,35588
|
|
83
82
|
tests/unit/test_auth_unit.py,sha256=IJZHT5CZLNfu3MybTdEWIsKlvxNfFYpZ6oWYzbS856g,858
|
|
84
83
|
tests/unit/test_checkpoints_unit.py,sha256=FbBM8Ffqy8JGTpWq2zWAWle2CI8A0MB_zABL_9Mezbk,15965
|
|
85
84
|
tests/unit/test_connections_unit.py,sha256=LFAZzlrvL9oM8rZJTiC1oA9quw1KA2vMUCc3LV6SjXs,5507
|
|
@@ -111,7 +110,6 @@ tests/unit/cli/projects/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG
|
|
|
111
110
|
tests/unit/cli/projects/test_cli_project_credential_unit.py,sha256=wehwquMGwUzwc8iH0bWletS3-kwIXKshoOuXHhJxRUc,943
|
|
112
111
|
tests/unit/cli/projects/test_cli_project_data_connector_unit.py,sha256=RlqgHOUkFv3bmc19pUtGNVD0W1UWBJSDfaI_lzN0ZEA,993
|
|
113
112
|
tests/unit/cli/projects/test_cli_project_datastore_unit.py,sha256=6UMgFwqxBtH65VeWTzXzaXhVfCRoG3Tef6aflqcDlV4,936
|
|
114
|
-
tests/unit/cli/projects/test_cli_project_key_unit.py,sha256=Xb-O4cltVMNvHpSoPtIdfY4hmXUiMv-z5ixY49FKKyM,894
|
|
115
113
|
tests/unit/cli/projects/test_cli_project_secret_unit.py,sha256=tygmgNOOf7C1J5Flo-YRo1-QLI0He5Yc_4Wx50ssC4o,915
|
|
116
114
|
tests/unit/cli/projects/test_cli_project_service_unit.py,sha256=P-u_2-Gpbu0P_MjH6niAMDzj5M8oJYJ5adWcfU4LNpA,922
|
|
117
115
|
tests/unit/cli/projects/test_cli_project_unit.py,sha256=64tR6ae-dQHVEE86N8tDrxOQypvyk8gANXGg6SBEUUI,627
|
|
@@ -128,13 +126,13 @@ tests/unit/projects/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSu
|
|
|
128
126
|
tests/unit/projects/test_project_credentials_unit.py,sha256=y528DyvIm7uc5mFuwiUPjOMO1dkVcMEeYQqzMmKLOuk,3480
|
|
129
127
|
tests/unit/projects/test_project_data_connectors_unit.py,sha256=Yx2yCUgcuN_KfTKynZxse0Nx8jRrSlMygI7R8a-NjE4,3385
|
|
130
128
|
tests/unit/projects/test_project_datastores_unit.py,sha256=WZMKkhpEg2dcevJqT7k6QRcYJF6FJfq177BDk2GWOBk,3129
|
|
131
|
-
tests/unit/projects/
|
|
129
|
+
tests/unit/projects/test_project_members_unit.py,sha256=dtbRfP454mviQYn6Dc0Uzyb3BBIuR3Ive6g3mnUAR6M,3466
|
|
132
130
|
tests/unit/projects/test_project_secrets_unit.py,sha256=VE9L91FJodcwVGizfF65WYMiHZaF0s2AdW1aiJ3z7xA,3276
|
|
133
131
|
tests/unit/projects/test_project_services_unit.py,sha256=PzeNuJRuAG7RkrPWX0FfgFTt6-63FviecrDY06rLQ6A,3331
|
|
134
|
-
tests/unit/projects/test_projects_unit.py,sha256=
|
|
135
|
-
proximl-0.5.
|
|
136
|
-
proximl-0.5.
|
|
137
|
-
proximl-0.5.
|
|
138
|
-
proximl-0.5.
|
|
139
|
-
proximl-0.5.
|
|
140
|
-
proximl-0.5.
|
|
132
|
+
tests/unit/projects/test_projects_unit.py,sha256=4vMo7TF-SjduoT2ejpuyM3dULslpdqE8-c25M9X-iYc,3810
|
|
133
|
+
proximl-0.5.14.dist-info/LICENSE,sha256=ADFxLEZDxKY0j4MdyUd5GNuhQ18rnWH5rOz1ZG7yiOA,1069
|
|
134
|
+
proximl-0.5.14.dist-info/METADATA,sha256=gGFu62XfxbC3WtzrYBSfHygX-5l2n2Gg3jz39NCy9ck,7345
|
|
135
|
+
proximl-0.5.14.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
|
|
136
|
+
proximl-0.5.14.dist-info/entry_points.txt,sha256=HmI311IIabkZReMCXu-nGbvIEW-KfaduAOyfiSqt5SY,63
|
|
137
|
+
proximl-0.5.14.dist-info/top_level.txt,sha256=-TWqc9tAaxmWmW4c7uYsmzPEYUIoh6z02xxqPbv7Kys,23
|
|
138
|
+
proximl-0.5.14.dist-info/RECORD,,
|