proximl 0.5.17__py3-none-any.whl → 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- examples/local_storage.py +0 -2
- proximl/__init__.py +1 -1
- proximl/checkpoints.py +56 -57
- proximl/cli/__init__.py +6 -3
- proximl/cli/checkpoint.py +18 -57
- proximl/cli/dataset.py +17 -57
- proximl/cli/job/__init__.py +11 -53
- proximl/cli/job/create.py +51 -24
- proximl/cli/model.py +14 -56
- proximl/cli/volume.py +18 -57
- proximl/datasets.py +50 -55
- proximl/jobs.py +239 -68
- proximl/models.py +51 -55
- proximl/proximl.py +50 -16
- proximl/utils/__init__.py +1 -0
- proximl/{auth.py → utils/auth.py} +4 -3
- proximl/utils/transfer.py +587 -0
- proximl/volumes.py +48 -53
- {proximl-0.5.17.dist-info → proximl-1.0.0.dist-info}/METADATA +3 -3
- {proximl-0.5.17.dist-info → proximl-1.0.0.dist-info}/RECORD +52 -50
- tests/integration/test_checkpoints_integration.py +4 -3
- tests/integration/test_datasets_integration.py +5 -3
- tests/integration/test_jobs_integration.py +33 -27
- tests/integration/test_models_integration.py +7 -3
- tests/integration/test_volumes_integration.py +2 -2
- tests/unit/cli/test_cli_checkpoint_unit.py +312 -1
- tests/unit/cloudbender/test_nodes_unit.py +112 -0
- tests/unit/cloudbender/test_providers_unit.py +96 -0
- tests/unit/cloudbender/test_regions_unit.py +106 -0
- tests/unit/cloudbender/test_services_unit.py +141 -0
- tests/unit/conftest.py +23 -10
- tests/unit/projects/test_project_data_connectors_unit.py +39 -0
- tests/unit/projects/test_project_datastores_unit.py +37 -0
- tests/unit/projects/test_project_members_unit.py +46 -0
- tests/unit/projects/test_project_services_unit.py +65 -0
- tests/unit/projects/test_projects_unit.py +16 -0
- tests/unit/test_auth_unit.py +17 -2
- tests/unit/test_checkpoints_unit.py +256 -71
- tests/unit/test_datasets_unit.py +218 -68
- tests/unit/test_exceptions.py +133 -0
- tests/unit/test_gpu_types_unit.py +11 -1
- tests/unit/test_jobs_unit.py +1014 -95
- tests/unit/test_main_unit.py +20 -0
- tests/unit/test_models_unit.py +218 -70
- tests/unit/test_proximl_unit.py +627 -3
- tests/unit/test_volumes_unit.py +211 -70
- tests/unit/utils/__init__.py +1 -0
- tests/unit/utils/test_transfer_unit.py +4260 -0
- proximl/cli/connection.py +0 -61
- proximl/connections.py +0 -621
- tests/unit/test_connections_unit.py +0 -182
- {proximl-0.5.17.dist-info → proximl-1.0.0.dist-info}/LICENSE +0 -0
- {proximl-0.5.17.dist-info → proximl-1.0.0.dist-info}/WHEEL +0 -0
- {proximl-0.5.17.dist-info → proximl-1.0.0.dist-info}/entry_points.txt +0 -0
- {proximl-0.5.17.dist-info → proximl-1.0.0.dist-info}/top_level.txt +0 -0
tests/unit/test_volumes_unit.py
CHANGED
|
@@ -47,7 +47,9 @@ class VolumesTests:
|
|
|
47
47
|
api_response = dict()
|
|
48
48
|
mock_proximl._query = AsyncMock(return_value=api_response)
|
|
49
49
|
await volumes.get("1234")
|
|
50
|
-
mock_proximl._query.assert_called_once_with(
|
|
50
|
+
mock_proximl._query.assert_called_once_with(
|
|
51
|
+
"/volume/1234", "GET", dict()
|
|
52
|
+
)
|
|
51
53
|
|
|
52
54
|
@mark.asyncio
|
|
53
55
|
async def test_list_volumes(
|
|
@@ -137,9 +139,7 @@ class VolumeTests:
|
|
|
137
139
|
|
|
138
140
|
@mark.asyncio
|
|
139
141
|
async def test_volume_get_log_url(self, volume, mock_proximl):
|
|
140
|
-
api_response =
|
|
141
|
-
"https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
|
|
142
|
-
)
|
|
142
|
+
api_response = "https://trainml-jobs-dev.s3.us-east-2.amazonaws.com/1/logs/first_one.zip"
|
|
143
143
|
mock_proximl._query = AsyncMock(return_value=api_response)
|
|
144
144
|
response = await volume.get_log_url()
|
|
145
145
|
mock_proximl._query.assert_called_once_with(
|
|
@@ -164,79 +164,73 @@ class VolumeTests:
|
|
|
164
164
|
assert response == api_response
|
|
165
165
|
|
|
166
166
|
@mark.asyncio
|
|
167
|
-
async def
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
167
|
+
async def test_volume_connect_downloading_status(self, mock_proximl):
|
|
168
|
+
volume = specimen.Volume(
|
|
169
|
+
mock_proximl,
|
|
170
|
+
id="1",
|
|
171
|
+
project_uuid="proj-id-1",
|
|
172
|
+
name="test volume",
|
|
173
|
+
status="downloading",
|
|
174
|
+
auth_token="test-token",
|
|
175
|
+
hostname="example.com",
|
|
176
|
+
source_uri="/path/to/source",
|
|
175
177
|
)
|
|
176
|
-
assert response == api_response
|
|
177
178
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
179
|
+
with patch(
|
|
180
|
+
"proximl.volumes.Volume.refresh", new_callable=AsyncMock
|
|
181
|
+
) as mock_refresh:
|
|
182
|
+
with patch(
|
|
183
|
+
"proximl.volumes.upload", new_callable=AsyncMock
|
|
184
|
+
) as mock_upload:
|
|
185
|
+
await volume.connect()
|
|
186
|
+
mock_refresh.assert_called_once()
|
|
187
|
+
mock_upload.assert_called_once_with(
|
|
188
|
+
"example.com", "test-token", "/path/to/source"
|
|
189
|
+
)
|
|
182
190
|
|
|
183
|
-
|
|
191
|
+
@mark.asyncio
|
|
192
|
+
async def test_volume_connect_exporting_status(
|
|
193
|
+
self, mock_proximl, tmp_path
|
|
194
|
+
):
|
|
195
|
+
output_dir = str(tmp_path / "output")
|
|
184
196
|
volume = specimen.Volume(
|
|
185
197
|
mock_proximl,
|
|
186
198
|
id="1",
|
|
187
|
-
project_uuid="
|
|
188
|
-
name="
|
|
189
|
-
status="
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
vpn={
|
|
195
|
-
"status": "new",
|
|
196
|
-
"cidr": "10.106.171.0/24",
|
|
197
|
-
"client": {
|
|
198
|
-
"port": "36017",
|
|
199
|
-
"id": "cus-id-1",
|
|
200
|
-
"address": "10.106.171.253",
|
|
201
|
-
"ssh_port": 46600,
|
|
202
|
-
},
|
|
203
|
-
"net_prefix_type_id": 1,
|
|
204
|
-
},
|
|
205
|
-
)
|
|
206
|
-
details = volume.get_connection_details()
|
|
207
|
-
expected_details = dict(
|
|
208
|
-
project_uuid="a",
|
|
209
|
-
entity_type="volume",
|
|
210
|
-
cidr="10.106.171.0/24",
|
|
211
|
-
ssh_port=46600,
|
|
212
|
-
input_path="~/tensorflow-example",
|
|
213
|
-
output_path=None,
|
|
214
|
-
)
|
|
215
|
-
assert details == expected_details
|
|
199
|
+
project_uuid="proj-id-1",
|
|
200
|
+
name="test volume",
|
|
201
|
+
status="exporting",
|
|
202
|
+
auth_token="test-token",
|
|
203
|
+
hostname="example.com",
|
|
204
|
+
output_uri=output_dir,
|
|
205
|
+
)
|
|
216
206
|
|
|
217
|
-
@mark.asyncio
|
|
218
|
-
async def test_volume_connect(self, volume, mock_proximl):
|
|
219
207
|
with patch(
|
|
220
|
-
"proximl.volumes.
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
208
|
+
"proximl.volumes.Volume.refresh", new_callable=AsyncMock
|
|
209
|
+
) as mock_refresh:
|
|
210
|
+
with patch(
|
|
211
|
+
"proximl.volumes.download", new_callable=AsyncMock
|
|
212
|
+
) as mock_download:
|
|
213
|
+
await volume.connect()
|
|
214
|
+
mock_refresh.assert_called_once()
|
|
215
|
+
mock_download.assert_called_once_with(
|
|
216
|
+
"example.com", "test-token", output_dir
|
|
217
|
+
)
|
|
228
218
|
|
|
229
219
|
@mark.asyncio
|
|
230
|
-
async def
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
220
|
+
async def test_volume_connect_invalid_status(self, mock_proximl):
|
|
221
|
+
volume = specimen.Volume(
|
|
222
|
+
mock_proximl,
|
|
223
|
+
id="1",
|
|
224
|
+
project_uuid="proj-id-1",
|
|
225
|
+
name="test volume",
|
|
226
|
+
status="ready",
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
with raises(
|
|
230
|
+
SpecificationError,
|
|
231
|
+
match="You can only connect to downloading or exporting volumes",
|
|
232
|
+
):
|
|
233
|
+
await volume.connect()
|
|
240
234
|
|
|
241
235
|
@mark.asyncio
|
|
242
236
|
async def test_volume_remove(self, volume, mock_proximl):
|
|
@@ -375,7 +369,9 @@ class VolumeTests:
|
|
|
375
369
|
mock_proximl._query.assert_not_called()
|
|
376
370
|
|
|
377
371
|
@mark.asyncio
|
|
378
|
-
async def test_volume_wait_for_incorrect_status(
|
|
372
|
+
async def test_volume_wait_for_incorrect_status(
|
|
373
|
+
self, volume, mock_proximl
|
|
374
|
+
):
|
|
379
375
|
api_response = None
|
|
380
376
|
mock_proximl._query = AsyncMock(return_value=api_response)
|
|
381
377
|
with raises(SpecificationError):
|
|
@@ -433,7 +429,9 @@ class VolumeTests:
|
|
|
433
429
|
mock_proximl._query.assert_called()
|
|
434
430
|
|
|
435
431
|
@mark.asyncio
|
|
436
|
-
async def test_volume_wait_for_archived_succeeded(
|
|
432
|
+
async def test_volume_wait_for_archived_succeeded(
|
|
433
|
+
self, volume, mock_proximl
|
|
434
|
+
):
|
|
437
435
|
mock_proximl._query = AsyncMock(
|
|
438
436
|
side_effect=ApiError(404, dict(errorMessage="Volume Not Found"))
|
|
439
437
|
)
|
|
@@ -441,10 +439,153 @@ class VolumeTests:
|
|
|
441
439
|
mock_proximl._query.assert_called()
|
|
442
440
|
|
|
443
441
|
@mark.asyncio
|
|
444
|
-
async def test_volume_wait_for_unexpected_api_error(
|
|
442
|
+
async def test_volume_wait_for_unexpected_api_error(
|
|
443
|
+
self, volume, mock_proximl
|
|
444
|
+
):
|
|
445
445
|
mock_proximl._query = AsyncMock(
|
|
446
446
|
side_effect=ApiError(404, dict(errorMessage="Volume Not Found"))
|
|
447
447
|
)
|
|
448
448
|
with raises(ApiError):
|
|
449
449
|
await volume.wait_for("ready")
|
|
450
450
|
mock_proximl._query.assert_called()
|
|
451
|
+
|
|
452
|
+
@mark.asyncio
|
|
453
|
+
async def test_volume_rename(self, volume, mock_proximl):
|
|
454
|
+
api_response = dict(
|
|
455
|
+
id="1",
|
|
456
|
+
name="renamed volume",
|
|
457
|
+
project_uuid="proj-id-1",
|
|
458
|
+
status="ready",
|
|
459
|
+
)
|
|
460
|
+
mock_proximl._query = AsyncMock(return_value=api_response)
|
|
461
|
+
result = await volume.rename("renamed volume")
|
|
462
|
+
mock_proximl._query.assert_called_once_with(
|
|
463
|
+
"/volume/1",
|
|
464
|
+
"PATCH",
|
|
465
|
+
dict(project_uuid="proj-id-1"),
|
|
466
|
+
dict(name="renamed volume"),
|
|
467
|
+
)
|
|
468
|
+
assert result == volume
|
|
469
|
+
assert volume.name == "renamed volume"
|
|
470
|
+
|
|
471
|
+
@mark.asyncio
|
|
472
|
+
async def test_volume_export(self, volume, mock_proximl):
|
|
473
|
+
api_response = dict(
|
|
474
|
+
id="1",
|
|
475
|
+
name="first one",
|
|
476
|
+
project_uuid="proj-id-1",
|
|
477
|
+
status="exporting",
|
|
478
|
+
)
|
|
479
|
+
mock_proximl._query = AsyncMock(return_value=api_response)
|
|
480
|
+
result = await volume.export("aws", "s3://bucket/path", dict(key="value"))
|
|
481
|
+
mock_proximl._query.assert_called_once_with(
|
|
482
|
+
"/volume/1/export",
|
|
483
|
+
"POST",
|
|
484
|
+
dict(project_uuid="proj-id-1"),
|
|
485
|
+
dict(
|
|
486
|
+
output_type="aws",
|
|
487
|
+
output_uri="s3://bucket/path",
|
|
488
|
+
output_options=dict(key="value"),
|
|
489
|
+
),
|
|
490
|
+
)
|
|
491
|
+
assert result == volume
|
|
492
|
+
assert volume.status == "exporting"
|
|
493
|
+
|
|
494
|
+
@mark.asyncio
|
|
495
|
+
async def test_volume_export_default_options(self, volume, mock_proximl):
|
|
496
|
+
api_response = dict(
|
|
497
|
+
id="1",
|
|
498
|
+
name="first one",
|
|
499
|
+
project_uuid="proj-id-1",
|
|
500
|
+
status="exporting",
|
|
501
|
+
)
|
|
502
|
+
mock_proximl._query = AsyncMock(return_value=api_response)
|
|
503
|
+
result = await volume.export("aws", "s3://bucket/path")
|
|
504
|
+
mock_proximl._query.assert_called_once_with(
|
|
505
|
+
"/volume/1/export",
|
|
506
|
+
"POST",
|
|
507
|
+
dict(project_uuid="proj-id-1"),
|
|
508
|
+
dict(
|
|
509
|
+
output_type="aws",
|
|
510
|
+
output_uri="s3://bucket/path",
|
|
511
|
+
output_options=dict(),
|
|
512
|
+
),
|
|
513
|
+
)
|
|
514
|
+
assert result == volume
|
|
515
|
+
|
|
516
|
+
@mark.asyncio
|
|
517
|
+
async def test_volume_wait_for_timeout_validation(
|
|
518
|
+
self, volume, mock_proximl
|
|
519
|
+
):
|
|
520
|
+
with raises(SpecificationError) as exc_info:
|
|
521
|
+
await volume.wait_for("ready", timeout=25 * 60 * 60) # > 24 hours
|
|
522
|
+
assert "timeout" in str(exc_info.value.attribute).lower()
|
|
523
|
+
assert "less than" in str(exc_info.value.message).lower()
|
|
524
|
+
|
|
525
|
+
@mark.asyncio
|
|
526
|
+
async def test_volume_connect_new_status_waits_for_downloading(
|
|
527
|
+
self, volume, mock_proximl
|
|
528
|
+
):
|
|
529
|
+
"""Test that connect() waits for downloading status when status is 'new'."""
|
|
530
|
+
volume._volume["status"] = "new"
|
|
531
|
+
volume._status = "new"
|
|
532
|
+
api_response_new = dict(
|
|
533
|
+
id="1",
|
|
534
|
+
name="first one",
|
|
535
|
+
status="new",
|
|
536
|
+
)
|
|
537
|
+
api_response_downloading = dict(
|
|
538
|
+
id="1",
|
|
539
|
+
name="first one",
|
|
540
|
+
status="downloading",
|
|
541
|
+
auth_token="token",
|
|
542
|
+
hostname="host",
|
|
543
|
+
source_uri="s3://bucket/path",
|
|
544
|
+
)
|
|
545
|
+
# wait_for calls refresh multiple times, then connect calls refresh again
|
|
546
|
+
mock_proximl._query = AsyncMock(
|
|
547
|
+
side_effect=[
|
|
548
|
+
api_response_new, # wait_for refresh 1
|
|
549
|
+
api_response_downloading, # wait_for refresh 2 (status matches, wait_for returns)
|
|
550
|
+
api_response_downloading, # connect refresh
|
|
551
|
+
]
|
|
552
|
+
)
|
|
553
|
+
with patch("proximl.volumes.upload", new_callable=AsyncMock) as mock_upload:
|
|
554
|
+
await volume.connect()
|
|
555
|
+
# After refresh, status should be downloading
|
|
556
|
+
assert volume.status == "downloading"
|
|
557
|
+
mock_upload.assert_called_once()
|
|
558
|
+
|
|
559
|
+
@mark.asyncio
|
|
560
|
+
async def test_volume_connect_downloading_missing_properties(
|
|
561
|
+
self, volume, mock_proximl
|
|
562
|
+
):
|
|
563
|
+
"""Test connect() raises error when downloading status missing properties."""
|
|
564
|
+
volume._volume["status"] = "downloading"
|
|
565
|
+
api_response = dict(
|
|
566
|
+
id="1",
|
|
567
|
+
name="first one",
|
|
568
|
+
status="downloading",
|
|
569
|
+
# Missing auth_token, hostname, or source_uri
|
|
570
|
+
)
|
|
571
|
+
mock_proximl._query = AsyncMock(return_value=api_response)
|
|
572
|
+
with raises(SpecificationError) as exc_info:
|
|
573
|
+
await volume.connect()
|
|
574
|
+
assert "missing required connection properties" in str(exc_info.value.message).lower()
|
|
575
|
+
|
|
576
|
+
@mark.asyncio
|
|
577
|
+
async def test_volume_connect_exporting_missing_properties(
|
|
578
|
+
self, volume, mock_proximl
|
|
579
|
+
):
|
|
580
|
+
"""Test connect() raises error when exporting status missing properties."""
|
|
581
|
+
volume._volume["status"] = "exporting"
|
|
582
|
+
api_response = dict(
|
|
583
|
+
id="1",
|
|
584
|
+
name="first one",
|
|
585
|
+
status="exporting",
|
|
586
|
+
# Missing auth_token, hostname, or output_uri
|
|
587
|
+
)
|
|
588
|
+
mock_proximl._query = AsyncMock(return_value=api_response)
|
|
589
|
+
with raises(SpecificationError) as exc_info:
|
|
590
|
+
await volume.connect()
|
|
591
|
+
assert "missing required connection properties" in str(exc_info.value.message).lower()
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Unit tests for utility modules."""
|