digitalhub 0.9.0b3__py3-none-any.whl → 0.9.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of digitalhub might be problematic. Click here for more details.
- digitalhub/client/_base/client.py +56 -20
- digitalhub/client/_base/key_builder.py +52 -0
- digitalhub/client/dhcore/client.py +4 -0
- digitalhub/client/dhcore/key_builder.py +58 -0
- digitalhub/client/local/client.py +2 -0
- digitalhub/client/local/key_builder.py +58 -0
- digitalhub/entities/_base/material/entity.py +1 -1
- digitalhub/entities/_base/project/entity.py +1 -1
- digitalhub/entities/_base/unversioned/entity.py +2 -1
- digitalhub/entities/_base/versioned/entity.py +2 -1
- digitalhub/entities/_operations/processor.py +123 -0
- digitalhub/stores/_base/store.py +6 -2
- digitalhub/stores/local/store.py +6 -2
- digitalhub/stores/remote/store.py +6 -2
- digitalhub/stores/s3/store.py +100 -46
- digitalhub/stores/sql/store.py +10 -2
- digitalhub/utils/git_utils.py +3 -3
- digitalhub/utils/s3_utils.py +17 -0
- digitalhub/utils/uri_utils.py +68 -32
- {digitalhub-0.9.0b3.dist-info → digitalhub-0.9.1.dist-info}/LICENSE.txt +1 -1
- {digitalhub-0.9.0b3.dist-info → digitalhub-0.9.1.dist-info}/METADATA +3 -3
- {digitalhub-0.9.0b3.dist-info → digitalhub-0.9.1.dist-info}/RECORD +24 -21
- {digitalhub-0.9.0b3.dist-info → digitalhub-0.9.1.dist-info}/WHEEL +1 -1
- {digitalhub-0.9.0b3.dist-info → digitalhub-0.9.1.dist-info}/top_level.txt +0 -0
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import typing
|
|
3
4
|
from abc import abstractmethod
|
|
4
5
|
|
|
5
|
-
|
|
6
|
+
if typing.TYPE_CHECKING:
|
|
7
|
+
from digitalhub.client._base.api_builder import ClientApiBuilder
|
|
8
|
+
from digitalhub.client._base.key_builder import ClientKeyBuilder
|
|
6
9
|
|
|
7
10
|
|
|
8
11
|
class Client:
|
|
@@ -16,26 +19,11 @@ class Client:
|
|
|
16
19
|
|
|
17
20
|
def __init__(self) -> None:
|
|
18
21
|
self._api_builder: ClientApiBuilder = None
|
|
22
|
+
self._key_builder: ClientKeyBuilder = None
|
|
19
23
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
Parameters
|
|
25
|
-
----------
|
|
26
|
-
category : str
|
|
27
|
-
API category.
|
|
28
|
-
operation : str
|
|
29
|
-
API operation.
|
|
30
|
-
**kwargs : dict
|
|
31
|
-
Additional parameters.
|
|
32
|
-
|
|
33
|
-
Returns
|
|
34
|
-
-------
|
|
35
|
-
str
|
|
36
|
-
API formatted.
|
|
37
|
-
"""
|
|
38
|
-
return self._api_builder.build_api(category, operation, **kwargs)
|
|
24
|
+
##############################
|
|
25
|
+
# CRUD methods
|
|
26
|
+
##############################
|
|
39
27
|
|
|
40
28
|
@abstractmethod
|
|
41
29
|
def create_object(self, api: str, obj: dict, **kwargs) -> dict:
|
|
@@ -79,6 +67,54 @@ class Client:
|
|
|
79
67
|
Search objects method.
|
|
80
68
|
"""
|
|
81
69
|
|
|
70
|
+
##############################
|
|
71
|
+
# Build methods
|
|
72
|
+
##############################
|
|
73
|
+
|
|
74
|
+
def build_api(self, category: str, operation: str, **kwargs) -> str:
|
|
75
|
+
"""
|
|
76
|
+
Build the API for the client.
|
|
77
|
+
|
|
78
|
+
Parameters
|
|
79
|
+
----------
|
|
80
|
+
category : str
|
|
81
|
+
API category.
|
|
82
|
+
operation : str
|
|
83
|
+
API operation.
|
|
84
|
+
**kwargs : dict
|
|
85
|
+
Additional parameters.
|
|
86
|
+
|
|
87
|
+
Returns
|
|
88
|
+
-------
|
|
89
|
+
str
|
|
90
|
+
API formatted.
|
|
91
|
+
"""
|
|
92
|
+
return self._api_builder.build_api(category, operation, **kwargs)
|
|
93
|
+
|
|
94
|
+
def build_key(self, category: str, *args, **kwargs) -> str:
|
|
95
|
+
"""
|
|
96
|
+
Build the key for the client.
|
|
97
|
+
|
|
98
|
+
Parameters
|
|
99
|
+
----------
|
|
100
|
+
category : str
|
|
101
|
+
Key category.
|
|
102
|
+
*args : tuple
|
|
103
|
+
Additional arguments.
|
|
104
|
+
**kwargs : dict
|
|
105
|
+
Additional parameters.
|
|
106
|
+
|
|
107
|
+
Returns
|
|
108
|
+
-------
|
|
109
|
+
str
|
|
110
|
+
Key formatted.
|
|
111
|
+
"""
|
|
112
|
+
return self._key_builder.build_key(category, *args, **kwargs)
|
|
113
|
+
|
|
114
|
+
##############################
|
|
115
|
+
# Interface methods
|
|
116
|
+
##############################
|
|
117
|
+
|
|
82
118
|
@staticmethod
|
|
83
119
|
@abstractmethod
|
|
84
120
|
def is_local() -> bool:
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from abc import abstractmethod
|
|
4
|
+
|
|
5
|
+
from digitalhub.entities._commons.enums import ApiCategories
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ClientKeyBuilder:
|
|
9
|
+
"""
|
|
10
|
+
Class that build the key of entities.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
def build_key(self, category: str, *args, **kwargs) -> str:
|
|
14
|
+
"""
|
|
15
|
+
Build key.
|
|
16
|
+
|
|
17
|
+
Parameters
|
|
18
|
+
----------
|
|
19
|
+
category : str
|
|
20
|
+
Key category.
|
|
21
|
+
*args : tuple
|
|
22
|
+
Positional arguments.
|
|
23
|
+
**kwargs : dict
|
|
24
|
+
Keyword arguments.
|
|
25
|
+
|
|
26
|
+
Returns
|
|
27
|
+
-------
|
|
28
|
+
str
|
|
29
|
+
Key.
|
|
30
|
+
"""
|
|
31
|
+
if category == ApiCategories.BASE.value:
|
|
32
|
+
return self.base_entity_key(*args, **kwargs)
|
|
33
|
+
return self.context_entity_key(*args, **kwargs)
|
|
34
|
+
|
|
35
|
+
@abstractmethod
|
|
36
|
+
def base_entity_key(self, entity_id: str) -> str:
|
|
37
|
+
"""
|
|
38
|
+
Build for base entity key.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
@abstractmethod
|
|
42
|
+
def context_entity_key(
|
|
43
|
+
self,
|
|
44
|
+
project: str,
|
|
45
|
+
entity_type: str,
|
|
46
|
+
entity_kind: str,
|
|
47
|
+
entity_name: str,
|
|
48
|
+
entity_id: str | None = None,
|
|
49
|
+
) -> str:
|
|
50
|
+
"""
|
|
51
|
+
Build for context entity key.
|
|
52
|
+
"""
|
|
@@ -14,6 +14,7 @@ from digitalhub.client._base.client import Client
|
|
|
14
14
|
from digitalhub.client.dhcore.api_builder import ClientDHCoreApiBuilder
|
|
15
15
|
from digitalhub.client.dhcore.enums import AuthType, DhcoreEnvVar
|
|
16
16
|
from digitalhub.client.dhcore.env import ENV_FILE, FALLBACK_USER, LIB_VERSION, MAX_API_LEVEL, MIN_API_LEVEL
|
|
17
|
+
from digitalhub.client.dhcore.key_builder import ClientDHCoreKeyBuilder
|
|
17
18
|
from digitalhub.client.dhcore.models import BasicAuth, OAuth2TokenAuth
|
|
18
19
|
from digitalhub.utils.exceptions import (
|
|
19
20
|
BackendError,
|
|
@@ -52,6 +53,9 @@ class ClientDHCore(Client):
|
|
|
52
53
|
# API builder
|
|
53
54
|
self._api_builder = ClientDHCoreApiBuilder()
|
|
54
55
|
|
|
56
|
+
# Key builder
|
|
57
|
+
self._key_builder = ClientDHCoreKeyBuilder()
|
|
58
|
+
|
|
55
59
|
# Endpoints
|
|
56
60
|
self._endpoint_core: str | None = None
|
|
57
61
|
self._endpoint_issuer: str | None = None
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from digitalhub.client._base.key_builder import ClientKeyBuilder
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class ClientDHCoreKeyBuilder(ClientKeyBuilder):
|
|
7
|
+
"""
|
|
8
|
+
Class that build the key of entities.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
def base_entity_key(self, entity_id: str) -> str:
|
|
12
|
+
"""
|
|
13
|
+
Build for base entity key.
|
|
14
|
+
|
|
15
|
+
Parameters
|
|
16
|
+
----------
|
|
17
|
+
entity_id : str
|
|
18
|
+
Entity id.
|
|
19
|
+
|
|
20
|
+
Returns
|
|
21
|
+
-------
|
|
22
|
+
str
|
|
23
|
+
Key.
|
|
24
|
+
"""
|
|
25
|
+
return f"store://{entity_id}"
|
|
26
|
+
|
|
27
|
+
def context_entity_key(
|
|
28
|
+
self,
|
|
29
|
+
project: str,
|
|
30
|
+
entity_type: str,
|
|
31
|
+
entity_kind: str,
|
|
32
|
+
entity_name: str,
|
|
33
|
+
entity_id: str | None = None,
|
|
34
|
+
) -> str:
|
|
35
|
+
"""
|
|
36
|
+
Build for context entity key.
|
|
37
|
+
|
|
38
|
+
Parameters
|
|
39
|
+
----------
|
|
40
|
+
project : str
|
|
41
|
+
Project name.
|
|
42
|
+
entity_type : str
|
|
43
|
+
Entity type.
|
|
44
|
+
entity_kind : str
|
|
45
|
+
Entity kind.
|
|
46
|
+
entity_name : str
|
|
47
|
+
Entity name.
|
|
48
|
+
entity_id : str
|
|
49
|
+
Entity ID.
|
|
50
|
+
|
|
51
|
+
Returns
|
|
52
|
+
-------
|
|
53
|
+
str
|
|
54
|
+
Key.
|
|
55
|
+
"""
|
|
56
|
+
if entity_id is None:
|
|
57
|
+
return f"store://{project}/{entity_type}/{entity_kind}/{entity_name}"
|
|
58
|
+
return f"store://{project}/{entity_type}/{entity_kind}/{entity_name}:{entity_id}"
|
|
@@ -5,6 +5,7 @@ from datetime import datetime, timezone
|
|
|
5
5
|
|
|
6
6
|
from digitalhub.client._base.client import Client
|
|
7
7
|
from digitalhub.client.local.api_builder import ClientLocalApiBuilder
|
|
8
|
+
from digitalhub.client.local.key_builder import ClientLocalKeyBuilder
|
|
8
9
|
from digitalhub.utils.exceptions import BackendError
|
|
9
10
|
|
|
10
11
|
|
|
@@ -24,6 +25,7 @@ class ClientLocal(Client):
|
|
|
24
25
|
def __init__(self) -> None:
|
|
25
26
|
super().__init__()
|
|
26
27
|
self._api_builder = ClientLocalApiBuilder()
|
|
28
|
+
self._key_builder = ClientLocalKeyBuilder()
|
|
27
29
|
self._db: dict[str, dict[str, dict]] = {}
|
|
28
30
|
|
|
29
31
|
##############################
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from digitalhub.client._base.key_builder import ClientKeyBuilder
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class ClientLocalKeyBuilder(ClientKeyBuilder):
|
|
7
|
+
"""
|
|
8
|
+
Class that build the key of entities.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
def base_entity_key(self, entity_id: str) -> str:
|
|
12
|
+
"""
|
|
13
|
+
Build for base entity key.
|
|
14
|
+
|
|
15
|
+
Parameters
|
|
16
|
+
----------
|
|
17
|
+
entity_id : str
|
|
18
|
+
Entity id.
|
|
19
|
+
|
|
20
|
+
Returns
|
|
21
|
+
-------
|
|
22
|
+
str
|
|
23
|
+
Key.
|
|
24
|
+
"""
|
|
25
|
+
return f"store://{entity_id}"
|
|
26
|
+
|
|
27
|
+
def context_entity_key(
|
|
28
|
+
self,
|
|
29
|
+
project: str,
|
|
30
|
+
entity_type: str,
|
|
31
|
+
entity_kind: str,
|
|
32
|
+
entity_name: str,
|
|
33
|
+
entity_id: str | None = None,
|
|
34
|
+
) -> str:
|
|
35
|
+
"""
|
|
36
|
+
Build for context entity key.
|
|
37
|
+
|
|
38
|
+
Parameters
|
|
39
|
+
----------
|
|
40
|
+
project : str
|
|
41
|
+
Project name.
|
|
42
|
+
entity_type : str
|
|
43
|
+
Entity type.
|
|
44
|
+
entity_kind : str
|
|
45
|
+
Entity kind.
|
|
46
|
+
entity_name : str
|
|
47
|
+
Entity name.
|
|
48
|
+
entity_id : str
|
|
49
|
+
Entity ID.
|
|
50
|
+
|
|
51
|
+
Returns
|
|
52
|
+
-------
|
|
53
|
+
str
|
|
54
|
+
Key.
|
|
55
|
+
"""
|
|
56
|
+
if entity_id is None:
|
|
57
|
+
return f"store://{project}/{entity_type}/{entity_kind}/{entity_name}"
|
|
58
|
+
return f"store://{project}/{entity_type}/{entity_kind}/{entity_name}:{entity_id}"
|
|
@@ -168,7 +168,7 @@ class MaterialEntity(VersionedEntity):
|
|
|
168
168
|
paths = store.upload(source, self.spec.path)
|
|
169
169
|
|
|
170
170
|
# Update files info
|
|
171
|
-
files_info = store.get_file_info(paths)
|
|
171
|
+
files_info = store.get_file_info(self.spec.path, paths)
|
|
172
172
|
self._update_files_info(files_info)
|
|
173
173
|
|
|
174
174
|
##############################
|
|
@@ -40,7 +40,7 @@ class ProjectEntity(Entity):
|
|
|
40
40
|
super().__init__(kind, metadata, spec, status, user)
|
|
41
41
|
self.id = name
|
|
42
42
|
self.name = name
|
|
43
|
-
self.key =
|
|
43
|
+
self.key = processor.build_project_key(self.name, local=local)
|
|
44
44
|
|
|
45
45
|
self._obj_attr.extend(["id", "name"])
|
|
46
46
|
|
|
@@ -3,6 +3,7 @@ from __future__ import annotations
|
|
|
3
3
|
import typing
|
|
4
4
|
|
|
5
5
|
from digitalhub.entities._base.context.entity import ContextEntity
|
|
6
|
+
from digitalhub.entities._operations.processor import processor
|
|
6
7
|
|
|
7
8
|
if typing.TYPE_CHECKING:
|
|
8
9
|
from digitalhub.entities._base.entity.metadata import Metadata
|
|
@@ -24,4 +25,4 @@ class UnversionedEntity(ContextEntity):
|
|
|
24
25
|
super().__init__(project, kind, metadata, spec, status, user)
|
|
25
26
|
self.id = uuid
|
|
26
27
|
self.name = uuid
|
|
27
|
-
self.key =
|
|
28
|
+
self.key = processor.build_context_entity_key(project, self.ENTITY_TYPE, kind, uuid)
|
|
@@ -3,6 +3,7 @@ from __future__ import annotations
|
|
|
3
3
|
import typing
|
|
4
4
|
|
|
5
5
|
from digitalhub.entities._base.context.entity import ContextEntity
|
|
6
|
+
from digitalhub.entities._operations.processor import processor
|
|
6
7
|
|
|
7
8
|
if typing.TYPE_CHECKING:
|
|
8
9
|
from digitalhub.entities._base.entity.metadata import Metadata
|
|
@@ -25,4 +26,4 @@ class VersionedEntity(ContextEntity):
|
|
|
25
26
|
super().__init__(project, kind, metadata, spec, status, user)
|
|
26
27
|
self.name = name
|
|
27
28
|
self.id = uuid
|
|
28
|
-
self.key =
|
|
29
|
+
self.key = processor.build_context_entity_key(project, self.ENTITY_TYPE, kind, name, uuid)
|
|
@@ -427,6 +427,55 @@ class OperationsProcessor:
|
|
|
427
427
|
**kwargs,
|
|
428
428
|
)
|
|
429
429
|
|
|
430
|
+
##############################
|
|
431
|
+
# Base entity operations
|
|
432
|
+
##############################
|
|
433
|
+
|
|
434
|
+
def _build_base_entity_key(
|
|
435
|
+
self,
|
|
436
|
+
client: Client,
|
|
437
|
+
entity_id: str,
|
|
438
|
+
) -> str:
|
|
439
|
+
"""
|
|
440
|
+
Build object key.
|
|
441
|
+
|
|
442
|
+
Parameters
|
|
443
|
+
----------
|
|
444
|
+
client : Client
|
|
445
|
+
Client instance.
|
|
446
|
+
entity_id : str
|
|
447
|
+
Entity ID.
|
|
448
|
+
|
|
449
|
+
Returns
|
|
450
|
+
-------
|
|
451
|
+
str
|
|
452
|
+
Object key.
|
|
453
|
+
"""
|
|
454
|
+
return client.build_key(ApiCategories.BASE.value, entity_id)
|
|
455
|
+
|
|
456
|
+
def build_project_key(
|
|
457
|
+
self,
|
|
458
|
+
entity_id: str,
|
|
459
|
+
**kwargs,
|
|
460
|
+
) -> str:
|
|
461
|
+
"""
|
|
462
|
+
Build object key.
|
|
463
|
+
|
|
464
|
+
Parameters
|
|
465
|
+
----------
|
|
466
|
+
entity_id : str
|
|
467
|
+
Entity ID.
|
|
468
|
+
**kwargs : dict
|
|
469
|
+
Parameters to pass to entity builder.
|
|
470
|
+
|
|
471
|
+
Returns
|
|
472
|
+
-------
|
|
473
|
+
str
|
|
474
|
+
Object key.
|
|
475
|
+
"""
|
|
476
|
+
client = get_client(kwargs.pop("local", False))
|
|
477
|
+
return self._build_base_entity_key(client, entity_id)
|
|
478
|
+
|
|
430
479
|
def share_project_entity(
|
|
431
480
|
self,
|
|
432
481
|
entity_type: str,
|
|
@@ -736,6 +785,10 @@ class OperationsProcessor:
|
|
|
736
785
|
"""
|
|
737
786
|
if not identifier.startswith("store://"):
|
|
738
787
|
entity_id = identifier
|
|
788
|
+
else:
|
|
789
|
+
splt = identifier.split(":")
|
|
790
|
+
if len(splt) == 3:
|
|
791
|
+
identifier = f"{splt[0]}:{splt[1]}"
|
|
739
792
|
return self.read_context_entity(
|
|
740
793
|
identifier,
|
|
741
794
|
entity_type=entity_type,
|
|
@@ -1275,6 +1328,76 @@ class OperationsProcessor:
|
|
|
1275
1328
|
# Context entity operations
|
|
1276
1329
|
##############################
|
|
1277
1330
|
|
|
1331
|
+
def _build_context_entity_key(
|
|
1332
|
+
self,
|
|
1333
|
+
context: Context,
|
|
1334
|
+
entity_type: str,
|
|
1335
|
+
entity_kind: str,
|
|
1336
|
+
entity_name: str,
|
|
1337
|
+
entity_id: str | None = None,
|
|
1338
|
+
) -> str:
|
|
1339
|
+
"""
|
|
1340
|
+
Build object key.
|
|
1341
|
+
|
|
1342
|
+
Parameters
|
|
1343
|
+
----------
|
|
1344
|
+
context : Context
|
|
1345
|
+
Context instance.
|
|
1346
|
+
entity_type : str
|
|
1347
|
+
Entity type.
|
|
1348
|
+
entity_kind : str
|
|
1349
|
+
Entity kind.
|
|
1350
|
+
entity_name : str
|
|
1351
|
+
Entity name.
|
|
1352
|
+
entity_id : str
|
|
1353
|
+
Entity ID.
|
|
1354
|
+
|
|
1355
|
+
Returns
|
|
1356
|
+
-------
|
|
1357
|
+
str
|
|
1358
|
+
Object key.
|
|
1359
|
+
"""
|
|
1360
|
+
return context.client.build_key(
|
|
1361
|
+
ApiCategories.CONTEXT.value,
|
|
1362
|
+
project=context.name,
|
|
1363
|
+
entity_type=entity_type,
|
|
1364
|
+
entity_kind=entity_kind,
|
|
1365
|
+
entity_name=entity_name,
|
|
1366
|
+
entity_id=entity_id,
|
|
1367
|
+
)
|
|
1368
|
+
|
|
1369
|
+
def build_context_entity_key(
|
|
1370
|
+
self,
|
|
1371
|
+
project: str,
|
|
1372
|
+
entity_type: str,
|
|
1373
|
+
entity_kind: str,
|
|
1374
|
+
entity_name: str,
|
|
1375
|
+
entity_id: str | None = None,
|
|
1376
|
+
) -> str:
|
|
1377
|
+
"""
|
|
1378
|
+
Build object key.
|
|
1379
|
+
|
|
1380
|
+
Parameters
|
|
1381
|
+
----------
|
|
1382
|
+
project : str
|
|
1383
|
+
Project name.
|
|
1384
|
+
entity_type : str
|
|
1385
|
+
Entity type.
|
|
1386
|
+
entity_kind : str
|
|
1387
|
+
Entity kind.
|
|
1388
|
+
entity_name : str
|
|
1389
|
+
Entity name.
|
|
1390
|
+
entity_id : str
|
|
1391
|
+
Entity ID.
|
|
1392
|
+
|
|
1393
|
+
Returns
|
|
1394
|
+
-------
|
|
1395
|
+
str
|
|
1396
|
+
Object key.
|
|
1397
|
+
"""
|
|
1398
|
+
context = self._get_context(project)
|
|
1399
|
+
return self._build_context_entity_key(context, entity_type, entity_kind, entity_name, entity_id)
|
|
1400
|
+
|
|
1278
1401
|
def read_secret_data(
|
|
1279
1402
|
self,
|
|
1280
1403
|
project: str,
|
digitalhub/stores/_base/store.py
CHANGED
|
@@ -52,13 +52,17 @@ class Store:
|
|
|
52
52
|
"""
|
|
53
53
|
|
|
54
54
|
@abstractmethod
|
|
55
|
-
def upload(self, src: str | list[str], dst: str
|
|
55
|
+
def upload(self, src: str | list[str], dst: str) -> list[tuple[str, str]]:
|
|
56
56
|
"""
|
|
57
57
|
Method to upload artifact to storage.
|
|
58
58
|
"""
|
|
59
59
|
|
|
60
60
|
@abstractmethod
|
|
61
|
-
def get_file_info(
|
|
61
|
+
def get_file_info(
|
|
62
|
+
self,
|
|
63
|
+
root: str,
|
|
64
|
+
paths: list[tuple[str, str]],
|
|
65
|
+
) -> list[dict]:
|
|
62
66
|
"""
|
|
63
67
|
Method to get file metadata.
|
|
64
68
|
"""
|
digitalhub/stores/local/store.py
CHANGED
|
@@ -61,7 +61,7 @@ class LocalStore(Store):
|
|
|
61
61
|
"""
|
|
62
62
|
raise StoreError("Local store does not support download.")
|
|
63
63
|
|
|
64
|
-
def upload(self, src: str | list[str], dst: str
|
|
64
|
+
def upload(self, src: str | list[str], dst: str) -> list[tuple[str, str]]:
|
|
65
65
|
"""
|
|
66
66
|
Upload an artifact to storage.
|
|
67
67
|
|
|
@@ -72,7 +72,11 @@ class LocalStore(Store):
|
|
|
72
72
|
"""
|
|
73
73
|
raise StoreError("Local store does not support upload.")
|
|
74
74
|
|
|
75
|
-
def get_file_info(
|
|
75
|
+
def get_file_info(
|
|
76
|
+
self,
|
|
77
|
+
root: str,
|
|
78
|
+
paths: list[tuple[str, str]],
|
|
79
|
+
) -> list[dict]:
|
|
76
80
|
"""
|
|
77
81
|
Method to get file metadata.
|
|
78
82
|
|
|
@@ -69,7 +69,7 @@ class RemoteStore(Store):
|
|
|
69
69
|
|
|
70
70
|
return self._download_file(root, dst, overwrite)
|
|
71
71
|
|
|
72
|
-
def upload(self, src: str | list[str], dst: str
|
|
72
|
+
def upload(self, src: str | list[str], dst: str) -> list[tuple[str, str]]:
|
|
73
73
|
"""
|
|
74
74
|
Upload an artifact to storage.
|
|
75
75
|
|
|
@@ -80,7 +80,11 @@ class RemoteStore(Store):
|
|
|
80
80
|
"""
|
|
81
81
|
raise StoreError("Remote HTTP store does not support upload.")
|
|
82
82
|
|
|
83
|
-
def get_file_info(
|
|
83
|
+
def get_file_info(
|
|
84
|
+
self,
|
|
85
|
+
root: str,
|
|
86
|
+
paths: list[tuple[str, str]],
|
|
87
|
+
) -> list[dict]:
|
|
84
88
|
"""
|
|
85
89
|
Get file information from HTTP(s) storage.
|
|
86
90
|
|
digitalhub/stores/s3/store.py
CHANGED
|
@@ -13,6 +13,7 @@ from digitalhub.readers.api import get_reader_by_object
|
|
|
13
13
|
from digitalhub.stores._base.store import Store, StoreConfig
|
|
14
14
|
from digitalhub.utils.exceptions import StoreError
|
|
15
15
|
from digitalhub.utils.file_utils import get_file_info_from_s3, get_file_mime_type
|
|
16
|
+
from digitalhub.utils.s3_utils import get_bucket_name
|
|
16
17
|
|
|
17
18
|
# Type aliases
|
|
18
19
|
S3Client = Type["botocore.client.S3"]
|
|
@@ -76,7 +77,7 @@ class S3Store(Store):
|
|
|
76
77
|
str
|
|
77
78
|
Destination path of the downloaded artifact.
|
|
78
79
|
"""
|
|
79
|
-
client, bucket = self._check_factory()
|
|
80
|
+
client, bucket = self._check_factory(root)
|
|
80
81
|
|
|
81
82
|
# Build destination directory
|
|
82
83
|
if dst.suffix == "":
|
|
@@ -126,14 +127,18 @@ class S3Store(Store):
|
|
|
126
127
|
return str(Path(dst, trees[0]))
|
|
127
128
|
return str(dst)
|
|
128
129
|
|
|
129
|
-
def upload(
|
|
130
|
+
def upload(
|
|
131
|
+
self,
|
|
132
|
+
src: str | list[str],
|
|
133
|
+
dst: str,
|
|
134
|
+
) -> list[tuple[str, str]]:
|
|
130
135
|
"""
|
|
131
136
|
Upload an artifact to storage.
|
|
132
137
|
|
|
133
138
|
Parameters
|
|
134
139
|
----------
|
|
135
|
-
src : str
|
|
136
|
-
|
|
140
|
+
src : str | list[str]
|
|
141
|
+
Source(s).
|
|
137
142
|
dst : str
|
|
138
143
|
The destination of the artifact on storage.
|
|
139
144
|
|
|
@@ -143,18 +148,11 @@ class S3Store(Store):
|
|
|
143
148
|
Returns the list of destination and source paths of the uploaded artifacts.
|
|
144
149
|
"""
|
|
145
150
|
# Destination handling
|
|
151
|
+
key = self._get_key(dst)
|
|
146
152
|
|
|
147
|
-
#
|
|
148
|
-
|
|
149
|
-
if
|
|
150
|
-
raise StoreError(
|
|
151
|
-
"Destination must be provided. If source is a list of files or a directory, destination must be a partition, e.g. 's3://bucket/partition/' otherwise a destination key, e.g. 's3://bucket/key'"
|
|
152
|
-
)
|
|
153
|
-
else:
|
|
154
|
-
dst = self._get_key(dst)
|
|
155
|
-
|
|
156
|
-
# Source handling
|
|
157
|
-
if not isinstance(src, list):
|
|
153
|
+
# Source handling (files list, dir or single file)
|
|
154
|
+
src_is_list = isinstance(src, list)
|
|
155
|
+
if not src_is_list:
|
|
158
156
|
self._check_local_src(src)
|
|
159
157
|
src_is_dir = Path(src).is_dir()
|
|
160
158
|
else:
|
|
@@ -165,21 +163,31 @@ class S3Store(Store):
|
|
|
165
163
|
src = src[0]
|
|
166
164
|
|
|
167
165
|
# If source is a directory, destination must be a partition
|
|
168
|
-
if (src_is_dir or
|
|
169
|
-
raise StoreError(
|
|
166
|
+
if (src_is_dir or src_is_list) and not dst.endswith("/"):
|
|
167
|
+
raise StoreError(
|
|
168
|
+
"If source is a list of files or a directory, "
|
|
169
|
+
"destination must be a partition, e.g. 's3://bucket/partition/'"
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
# S3 client
|
|
173
|
+
client, bucket = self._check_factory(dst)
|
|
170
174
|
|
|
171
175
|
# Directory
|
|
172
176
|
if src_is_dir:
|
|
173
|
-
return self._upload_dir(src,
|
|
177
|
+
return self._upload_dir(src, key, client, bucket)
|
|
174
178
|
|
|
175
179
|
# List of files
|
|
176
|
-
elif
|
|
177
|
-
return self._upload_file_list(src,
|
|
180
|
+
elif src_is_list:
|
|
181
|
+
return self._upload_file_list(src, key, client, bucket)
|
|
178
182
|
|
|
179
183
|
# Single file
|
|
180
|
-
return self._upload_single_file(src,
|
|
184
|
+
return self._upload_single_file(src, key, client, bucket)
|
|
181
185
|
|
|
182
|
-
def upload_fileobject(
|
|
186
|
+
def upload_fileobject(
|
|
187
|
+
self,
|
|
188
|
+
src: BytesIO,
|
|
189
|
+
dst: str,
|
|
190
|
+
) -> str:
|
|
183
191
|
"""
|
|
184
192
|
Upload an BytesIO to S3 based storage.
|
|
185
193
|
|
|
@@ -188,18 +196,23 @@ class S3Store(Store):
|
|
|
188
196
|
src : BytesIO
|
|
189
197
|
The source object to be persisted.
|
|
190
198
|
dst : str
|
|
191
|
-
The destination
|
|
199
|
+
The destination path of the artifact.
|
|
192
200
|
|
|
193
201
|
Returns
|
|
194
202
|
-------
|
|
195
203
|
str
|
|
196
204
|
S3 key of the uploaded artifact.
|
|
197
205
|
"""
|
|
198
|
-
client, bucket = self._check_factory()
|
|
199
|
-
self.
|
|
200
|
-
|
|
206
|
+
client, bucket = self._check_factory(dst)
|
|
207
|
+
key = self._get_key(dst)
|
|
208
|
+
self._upload_fileobject(src, key, client, bucket)
|
|
209
|
+
return f"s3://{bucket}/{key}"
|
|
201
210
|
|
|
202
|
-
def get_file_info(
|
|
211
|
+
def get_file_info(
|
|
212
|
+
self,
|
|
213
|
+
root: str,
|
|
214
|
+
paths: list[tuple[str, str]],
|
|
215
|
+
) -> list[dict]:
|
|
203
216
|
"""
|
|
204
217
|
Method to get file metadata.
|
|
205
218
|
|
|
@@ -213,7 +226,7 @@ class S3Store(Store):
|
|
|
213
226
|
list[dict]
|
|
214
227
|
Returns files metadata.
|
|
215
228
|
"""
|
|
216
|
-
client, bucket = self._check_factory()
|
|
229
|
+
client, bucket = self._check_factory(root)
|
|
217
230
|
|
|
218
231
|
infos = []
|
|
219
232
|
for i in paths:
|
|
@@ -264,7 +277,13 @@ class S3Store(Store):
|
|
|
264
277
|
# Download file
|
|
265
278
|
client.download_file(bucket, key, dst_pth)
|
|
266
279
|
|
|
267
|
-
def _upload_dir(
|
|
280
|
+
def _upload_dir(
|
|
281
|
+
self,
|
|
282
|
+
src: str,
|
|
283
|
+
dst: str,
|
|
284
|
+
client: S3Client,
|
|
285
|
+
bucket: str,
|
|
286
|
+
) -> list[tuple[str, str]]:
|
|
268
287
|
"""
|
|
269
288
|
Upload directory to storage.
|
|
270
289
|
|
|
@@ -274,14 +293,16 @@ class S3Store(Store):
|
|
|
274
293
|
List of sources.
|
|
275
294
|
dst : str
|
|
276
295
|
The destination of the artifact on storage.
|
|
296
|
+
client : S3Client
|
|
297
|
+
The S3 client object.
|
|
298
|
+
bucket : str
|
|
299
|
+
The name of the S3 bucket.
|
|
277
300
|
|
|
278
301
|
Returns
|
|
279
302
|
-------
|
|
280
303
|
list[tuple[str, str]]
|
|
281
304
|
Returns the list of destination and source paths of the uploaded artifacts.
|
|
282
305
|
"""
|
|
283
|
-
client, bucket = self._check_factory()
|
|
284
|
-
|
|
285
306
|
# Get list of files
|
|
286
307
|
src_pth = Path(src)
|
|
287
308
|
files = [i for i in src_pth.rglob("*") if i.is_file()]
|
|
@@ -299,7 +320,13 @@ class S3Store(Store):
|
|
|
299
320
|
paths.append((k, str(f.relative_to(src_pth))))
|
|
300
321
|
return paths
|
|
301
322
|
|
|
302
|
-
def _upload_file_list(
|
|
323
|
+
def _upload_file_list(
|
|
324
|
+
self,
|
|
325
|
+
src: list[str],
|
|
326
|
+
dst: str,
|
|
327
|
+
client: S3Client,
|
|
328
|
+
bucket: str,
|
|
329
|
+
) -> list[tuple[str, str]]:
|
|
303
330
|
"""
|
|
304
331
|
Upload list of files to storage.
|
|
305
332
|
|
|
@@ -309,13 +336,16 @@ class S3Store(Store):
|
|
|
309
336
|
List of sources.
|
|
310
337
|
dst : str
|
|
311
338
|
The destination of the artifact on storage.
|
|
339
|
+
client : S3Client
|
|
340
|
+
The S3 client object.
|
|
341
|
+
bucket : str
|
|
342
|
+
The name of the S3 bucket.
|
|
312
343
|
|
|
313
344
|
Returns
|
|
314
345
|
-------
|
|
315
346
|
list[tuple[str, str]]
|
|
316
347
|
Returns the list of destination and source paths of the uploaded artifacts.
|
|
317
348
|
"""
|
|
318
|
-
client, bucket = self._check_factory()
|
|
319
349
|
files = src
|
|
320
350
|
keys = []
|
|
321
351
|
for i in files:
|
|
@@ -330,7 +360,13 @@ class S3Store(Store):
|
|
|
330
360
|
paths.append((k, Path(f).name))
|
|
331
361
|
return paths
|
|
332
362
|
|
|
333
|
-
def _upload_single_file(
|
|
363
|
+
def _upload_single_file(
|
|
364
|
+
self,
|
|
365
|
+
src: str,
|
|
366
|
+
dst: str,
|
|
367
|
+
client: S3Client,
|
|
368
|
+
bucket: str,
|
|
369
|
+
) -> str:
|
|
334
370
|
"""
|
|
335
371
|
Upload a single file to storage.
|
|
336
372
|
|
|
@@ -340,14 +376,16 @@ class S3Store(Store):
|
|
|
340
376
|
List of sources.
|
|
341
377
|
dst : str
|
|
342
378
|
The destination of the artifact on storage.
|
|
379
|
+
client : S3Client
|
|
380
|
+
The S3 client object.
|
|
381
|
+
bucket : str
|
|
382
|
+
The name of the S3 bucket.
|
|
343
383
|
|
|
344
384
|
Returns
|
|
345
385
|
-------
|
|
346
386
|
str
|
|
347
387
|
Returns the list of destination and source paths of the uploaded artifacts.
|
|
348
388
|
"""
|
|
349
|
-
client, bucket = self._check_factory()
|
|
350
|
-
|
|
351
389
|
if dst.endswith("/"):
|
|
352
390
|
dst = f"{dst.removeprefix('/')}{Path(src).name}"
|
|
353
391
|
|
|
@@ -357,7 +395,12 @@ class S3Store(Store):
|
|
|
357
395
|
return [(dst, name)]
|
|
358
396
|
|
|
359
397
|
@staticmethod
|
|
360
|
-
def _upload_file(
|
|
398
|
+
def _upload_file(
|
|
399
|
+
src: str,
|
|
400
|
+
key: str,
|
|
401
|
+
client: S3Client,
|
|
402
|
+
bucket: str,
|
|
403
|
+
) -> None:
|
|
361
404
|
"""
|
|
362
405
|
Upload a file to S3 based storage. The function checks if the
|
|
363
406
|
bucket is accessible.
|
|
@@ -384,7 +427,12 @@ class S3Store(Store):
|
|
|
384
427
|
client.upload_file(Filename=src, Bucket=bucket, Key=key, ExtraArgs=extra_args)
|
|
385
428
|
|
|
386
429
|
@staticmethod
|
|
387
|
-
def _upload_fileobject(
|
|
430
|
+
def _upload_fileobject(
|
|
431
|
+
fileobj: BytesIO,
|
|
432
|
+
key: str,
|
|
433
|
+
client: S3Client,
|
|
434
|
+
bucket: str,
|
|
435
|
+
) -> None:
|
|
388
436
|
"""
|
|
389
437
|
Upload a fileobject to S3 based storage. The function checks if the bucket is accessible.
|
|
390
438
|
|
|
@@ -409,7 +457,13 @@ class S3Store(Store):
|
|
|
409
457
|
# Datastore methods
|
|
410
458
|
##############################
|
|
411
459
|
|
|
412
|
-
def write_df(
|
|
460
|
+
def write_df(
|
|
461
|
+
self,
|
|
462
|
+
df: Any,
|
|
463
|
+
dst: str,
|
|
464
|
+
extension: str | None = None,
|
|
465
|
+
**kwargs,
|
|
466
|
+
) -> str:
|
|
413
467
|
"""
|
|
414
468
|
Write a dataframe to S3 based storage. Kwargs are passed to df.to_parquet().
|
|
415
469
|
|
|
@@ -419,6 +473,8 @@ class S3Store(Store):
|
|
|
419
473
|
The dataframe.
|
|
420
474
|
dst : str
|
|
421
475
|
The destination path on S3 based storage.
|
|
476
|
+
extension : str
|
|
477
|
+
The extension of the file.
|
|
422
478
|
**kwargs : dict
|
|
423
479
|
Keyword arguments.
|
|
424
480
|
|
|
@@ -430,15 +486,13 @@ class S3Store(Store):
|
|
|
430
486
|
fileobj = BytesIO()
|
|
431
487
|
reader = get_reader_by_object(df)
|
|
432
488
|
reader.write_df(df, fileobj, extension=extension, **kwargs)
|
|
433
|
-
|
|
434
|
-
key = self._get_key(dst)
|
|
435
|
-
return self.upload_fileobject(fileobj, key)
|
|
489
|
+
return self.upload_fileobject(fileobj, dst)
|
|
436
490
|
|
|
437
491
|
##############################
|
|
438
492
|
# Helper methods
|
|
439
493
|
##############################
|
|
440
494
|
|
|
441
|
-
def _get_bucket(self) -> str:
|
|
495
|
+
def _get_bucket(self, root: str) -> str:
|
|
442
496
|
"""
|
|
443
497
|
Get the name of the S3 bucket from the URI.
|
|
444
498
|
|
|
@@ -447,7 +501,7 @@ class S3Store(Store):
|
|
|
447
501
|
str
|
|
448
502
|
The name of the S3 bucket.
|
|
449
503
|
"""
|
|
450
|
-
return
|
|
504
|
+
return get_bucket_name(root)
|
|
451
505
|
|
|
452
506
|
def _get_client(self) -> S3Client:
|
|
453
507
|
"""
|
|
@@ -465,7 +519,7 @@ class S3Store(Store):
|
|
|
465
519
|
}
|
|
466
520
|
return boto3.client("s3", **cfg)
|
|
467
521
|
|
|
468
|
-
def _check_factory(self) -> tuple[S3Client, str]:
|
|
522
|
+
def _check_factory(self, root: str) -> tuple[S3Client, str]:
|
|
469
523
|
"""
|
|
470
524
|
Check if the S3 bucket is accessible by sending a head_bucket request.
|
|
471
525
|
|
|
@@ -475,7 +529,7 @@ class S3Store(Store):
|
|
|
475
529
|
A tuple containing the S3 client object and the name of the S3 bucket.
|
|
476
530
|
"""
|
|
477
531
|
client = self._get_client()
|
|
478
|
-
bucket = self._get_bucket()
|
|
532
|
+
bucket = self._get_bucket(root)
|
|
479
533
|
self._check_access_to_storage(client, bucket)
|
|
480
534
|
return client, bucket
|
|
481
535
|
|
digitalhub/stores/sql/store.py
CHANGED
|
@@ -99,7 +99,11 @@ class SqlStore(Store):
|
|
|
99
99
|
table = self._get_table_name(root)
|
|
100
100
|
return self._download_table(schema, table, str(dst))
|
|
101
101
|
|
|
102
|
-
def upload(
|
|
102
|
+
def upload(
|
|
103
|
+
self,
|
|
104
|
+
src: str | list[str],
|
|
105
|
+
dst: str,
|
|
106
|
+
) -> list[tuple[str, str]]:
|
|
103
107
|
"""
|
|
104
108
|
Upload an artifact to storage.
|
|
105
109
|
|
|
@@ -110,7 +114,11 @@ class SqlStore(Store):
|
|
|
110
114
|
"""
|
|
111
115
|
raise StoreError("SQL store does not support upload.")
|
|
112
116
|
|
|
113
|
-
def get_file_info(
|
|
117
|
+
def get_file_info(
|
|
118
|
+
self,
|
|
119
|
+
root: str,
|
|
120
|
+
paths: list[tuple[str, str]],
|
|
121
|
+
) -> list[dict]:
|
|
114
122
|
"""
|
|
115
123
|
Get file information from SQL based storage.
|
|
116
124
|
|
digitalhub/utils/git_utils.py
CHANGED
|
@@ -13,16 +13,16 @@ except ImportError as e:
|
|
|
13
13
|
warnings.warn("git is not installed. Please install git and try again.", RuntimeWarning)
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
def clone_repository(
|
|
16
|
+
def clone_repository(path: Path, url: str) -> None:
|
|
17
17
|
"""
|
|
18
18
|
Clone git repository.
|
|
19
19
|
|
|
20
20
|
Parameters
|
|
21
21
|
----------
|
|
22
|
-
url : str
|
|
23
|
-
URL of the repository.
|
|
24
22
|
path : Path
|
|
25
23
|
Path where to save the repository.
|
|
24
|
+
url : str
|
|
25
|
+
URL of the repository.
|
|
26
26
|
|
|
27
27
|
Returns
|
|
28
28
|
-------
|
digitalhub/utils/s3_utils.py
CHANGED
|
@@ -7,6 +7,23 @@ from urllib.parse import urlparse
|
|
|
7
7
|
from boto3 import client as boto3_client
|
|
8
8
|
|
|
9
9
|
|
|
10
|
+
def get_bucket_name(path: str) -> str:
|
|
11
|
+
"""
|
|
12
|
+
Get bucket name from path.
|
|
13
|
+
|
|
14
|
+
Parameters
|
|
15
|
+
----------
|
|
16
|
+
path : str
|
|
17
|
+
The source path to get the key from.
|
|
18
|
+
|
|
19
|
+
Returns
|
|
20
|
+
-------
|
|
21
|
+
str
|
|
22
|
+
The bucket name.
|
|
23
|
+
"""
|
|
24
|
+
return urlparse(path).netloc
|
|
25
|
+
|
|
26
|
+
|
|
10
27
|
def get_bucket_and_key(path: str) -> tuple[str, str]:
|
|
11
28
|
"""
|
|
12
29
|
Get bucket and key from path.
|
digitalhub/utils/uri_utils.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
from enum import Enum
|
|
4
|
-
from urllib.parse import urlparse
|
|
4
|
+
from urllib.parse import unquote, urlparse
|
|
5
5
|
|
|
6
6
|
from digitalhub.utils.generic_utils import list_enum
|
|
7
7
|
|
|
@@ -41,6 +41,8 @@ class RemoteSchemes(Enum):
|
|
|
41
41
|
|
|
42
42
|
HTTP = "http"
|
|
43
43
|
HTTPS = "https"
|
|
44
|
+
ZIP_HTTP = "zip+http"
|
|
45
|
+
ZIP_HTTPS = "zip+https"
|
|
44
46
|
|
|
45
47
|
|
|
46
48
|
class SqlSchemes(Enum):
|
|
@@ -97,7 +99,7 @@ def map_uri_scheme(uri: str) -> str:
|
|
|
97
99
|
if scheme in list_enum(LocalSchemes):
|
|
98
100
|
return SchemeCategory.LOCAL.value
|
|
99
101
|
if scheme in list_enum(InvalidLocalSchemes):
|
|
100
|
-
raise ValueError("For local
|
|
102
|
+
raise ValueError("For local uri, do not use any scheme.")
|
|
101
103
|
if scheme in list_enum(RemoteSchemes):
|
|
102
104
|
return SchemeCategory.REMOTE.value
|
|
103
105
|
if scheme in list_enum(S3Schemes):
|
|
@@ -109,86 +111,120 @@ def map_uri_scheme(uri: str) -> str:
|
|
|
109
111
|
raise ValueError(f"Unknown scheme '{scheme}'!")
|
|
110
112
|
|
|
111
113
|
|
|
112
|
-
def has_local_scheme(
|
|
114
|
+
def has_local_scheme(uri: str) -> bool:
|
|
113
115
|
"""
|
|
114
|
-
Check if
|
|
116
|
+
Check if uri is local.
|
|
115
117
|
|
|
116
118
|
Parameters
|
|
117
119
|
----------
|
|
118
|
-
|
|
119
|
-
|
|
120
|
+
uri : str
|
|
121
|
+
Uri of some source.
|
|
120
122
|
|
|
121
123
|
Returns
|
|
122
124
|
-------
|
|
123
125
|
bool
|
|
124
|
-
True if
|
|
126
|
+
True if uri is local.
|
|
125
127
|
"""
|
|
126
|
-
return map_uri_scheme(
|
|
128
|
+
return map_uri_scheme(uri) == SchemeCategory.LOCAL.value
|
|
127
129
|
|
|
128
130
|
|
|
129
|
-
def has_remote_scheme(
|
|
131
|
+
def has_remote_scheme(uri: str) -> bool:
|
|
130
132
|
"""
|
|
131
|
-
Check if
|
|
133
|
+
Check if uri is remote.
|
|
132
134
|
|
|
133
135
|
Parameters
|
|
134
136
|
----------
|
|
135
|
-
|
|
136
|
-
|
|
137
|
+
uri : str
|
|
138
|
+
Uri of some source.
|
|
137
139
|
|
|
138
140
|
Returns
|
|
139
141
|
-------
|
|
140
142
|
bool
|
|
141
|
-
True if
|
|
143
|
+
True if uri is remote.
|
|
142
144
|
"""
|
|
143
|
-
return map_uri_scheme(
|
|
145
|
+
return map_uri_scheme(uri) == SchemeCategory.REMOTE.value
|
|
144
146
|
|
|
145
147
|
|
|
146
|
-
def has_s3_scheme(
|
|
148
|
+
def has_s3_scheme(uri: str) -> bool:
|
|
147
149
|
"""
|
|
148
|
-
Check if
|
|
150
|
+
Check if uri is s3.
|
|
149
151
|
|
|
150
152
|
Parameters
|
|
151
153
|
----------
|
|
152
|
-
|
|
153
|
-
|
|
154
|
+
uri : str
|
|
155
|
+
Uri of some source.
|
|
154
156
|
|
|
155
157
|
Returns
|
|
156
158
|
-------
|
|
157
159
|
bool
|
|
158
|
-
True if
|
|
160
|
+
True if uri is s3.
|
|
159
161
|
"""
|
|
160
|
-
return map_uri_scheme(
|
|
162
|
+
return map_uri_scheme(uri) == SchemeCategory.S3.value
|
|
161
163
|
|
|
162
164
|
|
|
163
|
-
def has_sql_scheme(
|
|
165
|
+
def has_sql_scheme(uri: str) -> bool:
|
|
164
166
|
"""
|
|
165
|
-
Check if
|
|
167
|
+
Check if uri is sql.
|
|
166
168
|
|
|
167
169
|
Parameters
|
|
168
170
|
----------
|
|
169
|
-
|
|
170
|
-
|
|
171
|
+
uri : str
|
|
172
|
+
Uri of some source.
|
|
171
173
|
|
|
172
174
|
Returns
|
|
173
175
|
-------
|
|
174
176
|
bool
|
|
175
|
-
True if
|
|
177
|
+
True if uri is sql.
|
|
176
178
|
"""
|
|
177
|
-
return map_uri_scheme(
|
|
179
|
+
return map_uri_scheme(uri) == SchemeCategory.SQL.value
|
|
178
180
|
|
|
179
181
|
|
|
180
|
-
def has_git_scheme(
|
|
182
|
+
def has_git_scheme(uri: str) -> bool:
|
|
181
183
|
"""
|
|
182
|
-
Check if
|
|
184
|
+
Check if uri is git.
|
|
183
185
|
|
|
184
186
|
Parameters
|
|
185
187
|
----------
|
|
186
|
-
|
|
187
|
-
|
|
188
|
+
uri : str
|
|
189
|
+
Uri of some source.
|
|
188
190
|
|
|
189
191
|
Returns
|
|
190
192
|
-------
|
|
191
193
|
bool
|
|
192
|
-
True if
|
|
194
|
+
True if uri is git.
|
|
195
|
+
"""
|
|
196
|
+
return map_uri_scheme(uri) == SchemeCategory.GIT.value
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def has_zip_scheme(uri: str) -> bool:
|
|
200
|
+
"""
|
|
201
|
+
Check if uri is zip.
|
|
202
|
+
|
|
203
|
+
Parameters
|
|
204
|
+
----------
|
|
205
|
+
uri : str
|
|
206
|
+
Uri of some source.
|
|
207
|
+
|
|
208
|
+
Returns
|
|
209
|
+
-------
|
|
210
|
+
bool
|
|
211
|
+
True if uri is zip.
|
|
212
|
+
"""
|
|
213
|
+
return uri.startswith("zip+")
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def get_filename_from_uri(uri: str) -> str:
|
|
217
|
+
"""
|
|
218
|
+
Get filename from uri.
|
|
219
|
+
|
|
220
|
+
Parameters
|
|
221
|
+
----------
|
|
222
|
+
uri : str
|
|
223
|
+
Uri of some source.
|
|
224
|
+
|
|
225
|
+
Returns
|
|
226
|
+
-------
|
|
227
|
+
str
|
|
228
|
+
Filename.
|
|
193
229
|
"""
|
|
194
|
-
return
|
|
230
|
+
return unquote(urlparse(uri).path).split("/")[-1]
|
|
@@ -186,7 +186,7 @@ APPENDIX: How to apply the Apache License to your work.
|
|
|
186
186
|
same "printed page" as the copyright notice for easier
|
|
187
187
|
identification within third-party archives.
|
|
188
188
|
|
|
189
|
-
Copyright
|
|
189
|
+
Copyright 2024 DSLab, Fondazione Bruno Kessler
|
|
190
190
|
|
|
191
191
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
192
|
you may not use this file except in compliance with the License.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
2
|
Name: digitalhub
|
|
3
|
-
Version: 0.9.
|
|
3
|
+
Version: 0.9.1
|
|
4
4
|
Summary: Python SDK for Digitalhub
|
|
5
5
|
Author-email: Fondazione Bruno Kessler <dslab@fbk.eu>, Matteo Martini <mmartini@fbk.eu>
|
|
6
6
|
License: Apache License
|
|
@@ -191,7 +191,7 @@ License: Apache License
|
|
|
191
191
|
same "printed page" as the copyright notice for easier
|
|
192
192
|
identification within third-party archives.
|
|
193
193
|
|
|
194
|
-
Copyright
|
|
194
|
+
Copyright 2024 DSLab, Fondazione Bruno Kessler
|
|
195
195
|
|
|
196
196
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
197
197
|
you may not use this file except in compliance with the License.
|
|
@@ -4,17 +4,20 @@ digitalhub/client/api.py,sha256=JwYh4rFUaTeZ2MvUdhQiWEG1AZbrZjRYMhJHoDRnk30,586
|
|
|
4
4
|
digitalhub/client/builder.py,sha256=83PoMCus4s4nbkoWmvcjW2hIpXbNx74sUW93wgQgbuo,1195
|
|
5
5
|
digitalhub/client/_base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
6
|
digitalhub/client/_base/api_builder.py,sha256=kB0phBqaPTLayyXyA1GbHgnbo4bBkadBtimznVB3-jc,339
|
|
7
|
-
digitalhub/client/_base/client.py,sha256=
|
|
7
|
+
digitalhub/client/_base/client.py,sha256=Z2iVRCyEzR1-B2uUh9uGQVuCpLFOE0UL64kEXhyf2aI,2974
|
|
8
|
+
digitalhub/client/_base/key_builder.py,sha256=xl3jM7z2eLP6HfPXusmZyLqsvfKHQfxfZ3SWPS7MHuk,1155
|
|
8
9
|
digitalhub/client/dhcore/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
10
|
digitalhub/client/dhcore/api_builder.py,sha256=IkRnANGG3dQmu4t1KmSnpDxGqkZDnJpBP9CcUhWZliQ,3652
|
|
10
|
-
digitalhub/client/dhcore/client.py,sha256=
|
|
11
|
+
digitalhub/client/dhcore/client.py,sha256=rYNSv-UpxoksaZki19A4pCch0tHlGIId9eDuFCfdxDk,21297
|
|
11
12
|
digitalhub/client/dhcore/enums.py,sha256=kaVXZTTa2WmsFbcc1CKWNLOM0JtUtcjL-KpspnTOhEE,523
|
|
12
13
|
digitalhub/client/dhcore/env.py,sha256=zBUNbK8G8PyMGAw_65BnX2Z_WUkrmTyQmYhLE6Jqgvk,618
|
|
14
|
+
digitalhub/client/dhcore/key_builder.py,sha256=VyyHk18rs1z8FKoRXGD2glb_WCipCWoYsS50_jYQpC4,1317
|
|
13
15
|
digitalhub/client/dhcore/models.py,sha256=KiTg5xR8EzI7Xa1pmYmzixabLdnqlnn5kn-IILZDGIw,900
|
|
14
16
|
digitalhub/client/dhcore/utils.py,sha256=I6m26WiGMCEyPa6Yv5hk3nmqWV4jDFjYyI590f86Ebs,3136
|
|
15
17
|
digitalhub/client/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
16
18
|
digitalhub/client/local/api_builder.py,sha256=WR24zuWguJqzUkD7xReO12LtvSkn020UG6LKFcKq77g,3592
|
|
17
|
-
digitalhub/client/local/client.py,sha256=
|
|
19
|
+
digitalhub/client/local/client.py,sha256=3DZI4UOrIJ0C03uhbvmKFpeoe2wY7IhBUHO92yzCikQ,17460
|
|
20
|
+
digitalhub/client/local/key_builder.py,sha256=jO5RHMRe5bxygX3rbD7TOqBafvO6FJcEOymsnJbjyuY,1316
|
|
18
21
|
digitalhub/context/api.py,sha256=un-_HU7JC3t-eQKleVJ9cCDS8BtQFRHSrP3VDHs3gPU,1099
|
|
19
22
|
digitalhub/context/builder.py,sha256=ibsjwCbll4RbNc_NplAn-8cX0J8GIXyY41KjtcvlsI4,2037
|
|
20
23
|
digitalhub/context/context.py,sha256=-xn3ZHNqoifUdRF4vwmLzOkfQrEheFPDQNk2J-q47EI,1456
|
|
@@ -40,25 +43,25 @@ digitalhub/entities/_base/entity/_constructors/uuid.py,sha256=QzG0nsVpGU2UDD5Cjp
|
|
|
40
43
|
digitalhub/entities/_base/executable/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
41
44
|
digitalhub/entities/_base/executable/entity.py,sha256=tNQmmABuY90eKWcxePPGu4iaZhDLyL9TdugKVcnnKSY,10079
|
|
42
45
|
digitalhub/entities/_base/material/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
43
|
-
digitalhub/entities/_base/material/entity.py,sha256=
|
|
46
|
+
digitalhub/entities/_base/material/entity.py,sha256=Gm87nZCKUIredCEucqwZ5T_QGG6cjSuWXKzrw15w-l4,6930
|
|
44
47
|
digitalhub/entities/_base/material/spec.py,sha256=jL1OohnYhUSrdTaFF3Rkw817jIjNY0ewm1wB3tgQ_9c,436
|
|
45
48
|
digitalhub/entities/_base/material/status.py,sha256=I9yxNLHvsHSRDkKcKGPxlPJrJ7grsrxHxId06wtqVXk,383
|
|
46
49
|
digitalhub/entities/_base/material/utils.py,sha256=EL-p6XEYlPxQ33II2ish2sVx3hIdPcrgbSh5om7Xfls,2427
|
|
47
50
|
digitalhub/entities/_base/project/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
48
|
-
digitalhub/entities/_base/project/entity.py,sha256=
|
|
51
|
+
digitalhub/entities/_base/project/entity.py,sha256=MwK80OCNKN_vxnPajJhErdLS4VQfIWydmfQrscf5OCc,10621
|
|
49
52
|
digitalhub/entities/_base/runtime_entity/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
50
53
|
digitalhub/entities/_base/runtime_entity/builder.py,sha256=vZBftdOrAUICoEJXbqOyozrbMxNRqqoVVxDVjKs2GnI,2414
|
|
51
54
|
digitalhub/entities/_base/unversioned/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
52
55
|
digitalhub/entities/_base/unversioned/builder.py,sha256=ChEhRXlu1BGNmMHOMu9i9kVMu3uFtwxmgiTO4Il_5oU,1834
|
|
53
|
-
digitalhub/entities/_base/unversioned/entity.py,sha256=
|
|
56
|
+
digitalhub/entities/_base/unversioned/entity.py,sha256=eQ9BKtqr7-UPgS4fkNdFX2Uh-YddfYrfFmk4Y-6uj7w,862
|
|
54
57
|
digitalhub/entities/_base/versioned/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
55
58
|
digitalhub/entities/_base/versioned/builder.py,sha256=0niRbqpCnHqBK0hTQxdrtUl43BbJuGXMLVbruhpUUuU,1896
|
|
56
|
-
digitalhub/entities/_base/versioned/entity.py,sha256=
|
|
59
|
+
digitalhub/entities/_base/versioned/entity.py,sha256=xtvVJfmJTfCAA96va1-r7B2j7wbXlkutGMil--PWP4I,885
|
|
57
60
|
digitalhub/entities/_commons/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
58
61
|
digitalhub/entities/_commons/enums.py,sha256=pjhbOMzAABRPbV6XNt-2Lyn2kLWIFBAwpeboCtvnz1w,1861
|
|
59
62
|
digitalhub/entities/_commons/utils.py,sha256=_HL6zFSCL_2ug4LpXcxK1MJQQhWL34wj1B2q0Ie0TKU,1792
|
|
60
63
|
digitalhub/entities/_operations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
61
|
-
digitalhub/entities/_operations/processor.py,sha256=
|
|
64
|
+
digitalhub/entities/_operations/processor.py,sha256=baQA_L-NFU5uDfP4M2d3B9VLerbmjK3pFKUkZDZI2Sg,49470
|
|
62
65
|
digitalhub/entities/artifact/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
63
66
|
digitalhub/entities/artifact/crud.py,sha256=y69PyoWAYOx-Pl_wnds3M0xSLNbADpdV_xG4zElJ0aI,7824
|
|
64
67
|
digitalhub/entities/artifact/utils.py,sha256=NNzD2AcIJzmV_Jo_8k5ZcSp2arKcZ07CCIYKn2lvoKM,1320
|
|
@@ -194,25 +197,25 @@ digitalhub/stores/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
|
|
|
194
197
|
digitalhub/stores/api.py,sha256=TwD8WXVuEPwtrmcyhP9SC0i7xkVHov5fpu94nHhZRHg,994
|
|
195
198
|
digitalhub/stores/builder.py,sha256=kraGDfgV5cZ2QivXTWwbyz1FgAehgjlL9dmI6dxEc0U,5927
|
|
196
199
|
digitalhub/stores/_base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
197
|
-
digitalhub/stores/_base/store.py,sha256=
|
|
200
|
+
digitalhub/stores/_base/store.py,sha256=QscqsAAgvYX9bAsoADKs0SkLahmMvvEHZ_NsBj2ri6k,5295
|
|
198
201
|
digitalhub/stores/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
199
|
-
digitalhub/stores/local/store.py,sha256=
|
|
202
|
+
digitalhub/stores/local/store.py,sha256=_-0JEnGCEcKUrLeB2xf_CDGuSeAEXcNyX6JCvSWkU3M,6766
|
|
200
203
|
digitalhub/stores/remote/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
201
|
-
digitalhub/stores/remote/store.py,sha256=
|
|
204
|
+
digitalhub/stores/remote/store.py,sha256=g9u87NBfFBasHWSTn82ZRZ0QttpYpX_Y0g16xO1r-AQ,4234
|
|
202
205
|
digitalhub/stores/s3/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
203
|
-
digitalhub/stores/s3/store.py,sha256=
|
|
206
|
+
digitalhub/stores/s3/store.py,sha256=CzjdMyatPps8uW4HFKz-OZcOUlVlGebUX4jxrzp48Eg,16912
|
|
204
207
|
digitalhub/stores/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
205
|
-
digitalhub/stores/sql/store.py,sha256=
|
|
208
|
+
digitalhub/stores/sql/store.py,sha256=jhulvAIacR0HAomv25PCxG3mr6BKhF-9c9k0D5ruCQ8,10482
|
|
206
209
|
digitalhub/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
207
210
|
digitalhub/utils/data_utils.py,sha256=2eETyxml610Ij7uyDnNc3YiELl2WFLIkj3FL4y6EPk8,2733
|
|
208
211
|
digitalhub/utils/exceptions.py,sha256=pL2C3vTCscHsjpqDqGN3OXu0bBn7NL9ENOoM2S37CyE,1139
|
|
209
212
|
digitalhub/utils/file_utils.py,sha256=gw0ozfrobAGGpVBTPIiIRJK4K6vEkB5WHFq8Ygzl-CQ,5022
|
|
210
213
|
digitalhub/utils/generic_utils.py,sha256=5flGSQ50UyzUtlnLPyIhZ6blijPGFxPITukfH-yrTMc,4312
|
|
211
|
-
digitalhub/utils/git_utils.py,sha256=
|
|
214
|
+
digitalhub/utils/git_utils.py,sha256=air8jn73FxzSWRxpvObcdOJBWcFOqb5A7D4ISwPEs7A,3244
|
|
212
215
|
digitalhub/utils/io_utils.py,sha256=8jD4Rp_b7LZEpY5JSMxVUowZsnifKnbGpHT5Hijx9-g,3299
|
|
213
216
|
digitalhub/utils/logger.py,sha256=ml3ne6D8wuRdNZ4F6ywmvWotSxjmZWnmKgNiuHb4R5M,437
|
|
214
|
-
digitalhub/utils/s3_utils.py,sha256=
|
|
215
|
-
digitalhub/utils/uri_utils.py,sha256=
|
|
217
|
+
digitalhub/utils/s3_utils.py,sha256=kbET2FL7UXfqBMgSpky-E3wO5SMjpaLbIAFtnGe2pXU,1383
|
|
218
|
+
digitalhub/utils/uri_utils.py,sha256=G0JtZNdU1SjJkabSOygLsczYtcteWNBTJrnjSlIHqh4,3821
|
|
216
219
|
test/test_crud_functions.py,sha256=tQs_QBaPCuYVSBpbl-he5_6jr_tteCXVmohj1ZluNsA,2769
|
|
217
220
|
test/test_crud_runs.py,sha256=lkssy15UPJKymgazmi5gG6RLxyTsG-tM_CpNCowD2gQ,2220
|
|
218
221
|
test/test_crud_tasks.py,sha256=sIbY-Hq6C2N20hWHfbCMw9c-zpYS0m_UJGnPINR4Q6s,2111
|
|
@@ -223,8 +226,8 @@ test/local/CRUD/test_dataitems.py,sha256=LQqTzI59uwTGy4zoq8jL0yWVe2W9vXlatkgDU9a
|
|
|
223
226
|
test/local/CRUD/test_models.py,sha256=msosbZuRwIMbZtmi3ZaOva4TjQ4lrzkNu9AguIFhrSo,2929
|
|
224
227
|
test/local/imports/test_imports.py,sha256=W-YugO0rpJwvtWp57MXaXfEmE-f5iWuCiLY-n0ZU4z8,1271
|
|
225
228
|
test/local/instances/test_validate.py,sha256=bGPKRFR_Tb5nlzzmI_ty_6UVUvYGseE2-pkNVoGWeO0,1842
|
|
226
|
-
digitalhub-0.9.
|
|
227
|
-
digitalhub-0.9.
|
|
228
|
-
digitalhub-0.9.
|
|
229
|
-
digitalhub-0.9.
|
|
230
|
-
digitalhub-0.9.
|
|
229
|
+
digitalhub-0.9.1.dist-info/LICENSE.txt,sha256=qmrTTXPlgU0kSRlRVbjhlyGs1IXs2QPxo_Y-Mn06J0k,11589
|
|
230
|
+
digitalhub-0.9.1.dist-info/METADATA,sha256=NFHbQkbJaMWjdZioDrJ5842hmXAA3PBunHyRK3WXQgM,15318
|
|
231
|
+
digitalhub-0.9.1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
|
232
|
+
digitalhub-0.9.1.dist-info/top_level.txt,sha256=ae9pDfCF27ZoaVAxuBKONMP0lm5P-N_I-e-no1WlvD8,16
|
|
233
|
+
digitalhub-0.9.1.dist-info/RECORD,,
|
|
File without changes
|