digitalhub 0.10.0b6__py3-none-any.whl → 0.10.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of digitalhub might be problematic. Click here for more details.
- digitalhub/__init__.py +2 -3
- digitalhub/entities/_base/material/entity.py +1 -1
- digitalhub/entities/_base/material/utils.py +1 -1
- digitalhub/entities/_processors/base.py +2 -2
- digitalhub/entities/_processors/context.py +1 -1
- digitalhub/entities/_processors/utils.py +2 -2
- digitalhub/entities/dataitem/table/entity.py +1 -1
- digitalhub/entities/dataitem/utils.py +2 -2
- digitalhub/entities/model/_base/entity.py +30 -0
- digitalhub/entities/project/_base/entity.py +1 -1
- digitalhub/entities/run/_base/entity.py +30 -0
- digitalhub/{client → stores/client}/_base/client.py +3 -3
- digitalhub/{client → stores/client}/api.py +2 -2
- digitalhub/{client → stores/client}/builder.py +3 -3
- digitalhub/{client → stores/client}/dhcore/api_builder.py +1 -1
- digitalhub/{client → stores/client}/dhcore/client.py +6 -6
- digitalhub/{client → stores/client}/dhcore/configurator.py +27 -78
- digitalhub/{client → stores/client}/dhcore/key_builder.py +1 -1
- digitalhub/{client → stores/client}/dhcore/params_builder.py +6 -2
- digitalhub/{client → stores/client}/dhcore/utils.py +3 -3
- digitalhub/{client → stores/client}/local/api_builder.py +2 -2
- digitalhub/{client → stores/client}/local/client.py +5 -5
- digitalhub/{client → stores/client}/local/key_builder.py +1 -1
- digitalhub/{client → stores/client}/local/params_builder.py +1 -1
- digitalhub/{configurator → stores/configurator}/api.py +1 -1
- digitalhub/{configurator → stores/configurator}/configurator.py +5 -5
- digitalhub/stores/configurator/enums.py +12 -0
- digitalhub/{configurator → stores/configurator}/ini_module.py +1 -1
- digitalhub/stores/{_base → data/_base}/store.py +2 -2
- digitalhub/stores/{api.py → data/api.py} +2 -2
- digitalhub/stores/{builder.py → data/builder.py} +6 -6
- digitalhub/stores/{local → data/local}/store.py +2 -2
- digitalhub/stores/{remote → data/remote}/store.py +1 -1
- digitalhub/stores/data/s3/configurator.py +146 -0
- digitalhub/stores/{s3 → data/s3}/store.py +26 -10
- digitalhub/stores/{s3 → data/s3}/utils.py +1 -1
- digitalhub/stores/data/sql/configurator.py +126 -0
- digitalhub/stores/{sql → data/sql}/store.py +21 -9
- digitalhub/{readers → stores/readers}/data/_base/builder.py +1 -1
- digitalhub/{readers → stores/readers}/data/api.py +2 -2
- digitalhub/{readers → stores/readers}/data/factory.py +3 -3
- digitalhub/{readers → stores/readers}/data/pandas/builder.py +2 -2
- digitalhub/{readers → stores/readers}/data/pandas/reader.py +1 -1
- digitalhub/stores/readers/query/__init__.py +0 -0
- {digitalhub-0.10.0b6.dist-info → digitalhub-0.10.1.dist-info}/METADATA +2 -2
- {digitalhub-0.10.0b6.dist-info → digitalhub-0.10.1.dist-info}/RECORD +76 -74
- digitalhub/stores/s3/configurator.py +0 -108
- digitalhub/stores/sql/configurator.py +0 -88
- /digitalhub/{client → stores/client}/__init__.py +0 -0
- /digitalhub/{client → stores/client}/_base/__init__.py +0 -0
- /digitalhub/{client → stores/client}/_base/api_builder.py +0 -0
- /digitalhub/{client → stores/client}/_base/key_builder.py +0 -0
- /digitalhub/{client → stores/client}/_base/params_builder.py +0 -0
- /digitalhub/{client → stores/client}/dhcore/__init__.py +0 -0
- /digitalhub/{client → stores/client}/dhcore/enums.py +0 -0
- /digitalhub/{client → stores/client}/dhcore/error_parser.py +0 -0
- /digitalhub/{client → stores/client}/dhcore/models.py +0 -0
- /digitalhub/{client → stores/client}/local/__init__.py +0 -0
- /digitalhub/{client → stores/client}/local/enums.py +0 -0
- /digitalhub/{configurator → stores/configurator}/__init__.py +0 -0
- /digitalhub/{configurator → stores/configurator}/credentials_store.py +0 -0
- /digitalhub/{readers → stores/data}/__init__.py +0 -0
- /digitalhub/{readers/data → stores/data/_base}/__init__.py +0 -0
- /digitalhub/{readers/data/_base → stores/data/local}/__init__.py +0 -0
- /digitalhub/{readers/data/pandas → stores/data/remote}/__init__.py +0 -0
- /digitalhub/{readers/query → stores/data/s3}/__init__.py +0 -0
- /digitalhub/stores/{s3 → data/s3}/enums.py +0 -0
- /digitalhub/stores/{s3 → data/s3}/models.py +0 -0
- /digitalhub/stores/{_base → data/sql}/__init__.py +0 -0
- /digitalhub/stores/{sql → data/sql}/enums.py +0 -0
- /digitalhub/stores/{sql → data/sql}/models.py +0 -0
- /digitalhub/stores/{local → readers}/__init__.py +0 -0
- /digitalhub/stores/{remote → readers/data}/__init__.py +0 -0
- /digitalhub/stores/{s3 → readers/data/_base}/__init__.py +0 -0
- /digitalhub/{readers → stores/readers}/data/_base/reader.py +0 -0
- /digitalhub/stores/{sql → readers/data/pandas}/__init__.py +0 -0
- {digitalhub-0.10.0b6.dist-info → digitalhub-0.10.1.dist-info}/WHEEL +0 -0
- {digitalhub-0.10.0b6.dist-info → digitalhub-0.10.1.dist-info}/licenses/LICENSE.txt +0 -0
digitalhub/__init__.py
CHANGED
|
@@ -88,11 +88,10 @@ try:
|
|
|
88
88
|
except ImportError:
|
|
89
89
|
...
|
|
90
90
|
|
|
91
|
-
from digitalhub.client.dhcore.utils import refresh_token, set_dhcore_env
|
|
92
|
-
from digitalhub.configurator.api import get_current_env, set_current_env
|
|
93
|
-
|
|
94
91
|
# Register entities into registry
|
|
95
92
|
from digitalhub.factory.utils import register_entities, register_runtimes_entities
|
|
93
|
+
from digitalhub.stores.client.dhcore.utils import refresh_token, set_dhcore_env
|
|
94
|
+
from digitalhub.stores.configurator.api import get_current_env, set_current_env
|
|
96
95
|
|
|
97
96
|
register_entities()
|
|
98
97
|
register_runtimes_entities()
|
|
@@ -5,7 +5,7 @@ from pathlib import Path
|
|
|
5
5
|
|
|
6
6
|
from digitalhub.entities._base.versioned.entity import VersionedEntity
|
|
7
7
|
from digitalhub.entities._processors.context import context_processor
|
|
8
|
-
from digitalhub.stores.api import get_store
|
|
8
|
+
from digitalhub.stores.data.api import get_store
|
|
9
9
|
from digitalhub.utils.types import SourcesOrListOfSources
|
|
10
10
|
|
|
11
11
|
if typing.TYPE_CHECKING:
|
|
@@ -2,7 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
from pathlib import Path
|
|
4
4
|
|
|
5
|
-
from digitalhub.stores.s3.utils import get_s3_bucket_from_env
|
|
5
|
+
from digitalhub.stores.data.s3.utils import get_s3_bucket_from_env
|
|
6
6
|
from digitalhub.utils.file_utils import eval_zip_type
|
|
7
7
|
from digitalhub.utils.uri_utils import S3Schemes, has_local_scheme
|
|
8
8
|
|
|
@@ -2,16 +2,16 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import typing
|
|
4
4
|
|
|
5
|
-
from digitalhub.client.api import get_client
|
|
6
5
|
from digitalhub.context.api import delete_context
|
|
7
6
|
from digitalhub.entities._commons.enums import ApiCategories, BackendOperations
|
|
8
7
|
from digitalhub.factory.api import build_entity_from_dict, build_entity_from_params
|
|
8
|
+
from digitalhub.stores.client.api import get_client
|
|
9
9
|
from digitalhub.utils.exceptions import EntityAlreadyExistsError, EntityError, EntityNotExistsError
|
|
10
10
|
from digitalhub.utils.io_utils import read_yaml
|
|
11
11
|
|
|
12
12
|
if typing.TYPE_CHECKING:
|
|
13
|
-
from digitalhub.client._base.client import Client
|
|
14
13
|
from digitalhub.entities.project._base.entity import Project
|
|
14
|
+
from digitalhub.stores.client._base.client import Client
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
class BaseEntityOperationsProcessor:
|
|
@@ -437,7 +437,7 @@ class ContextEntityOperationsProcessor:
|
|
|
437
437
|
|
|
438
438
|
kwargs = context.client.build_parameters(
|
|
439
439
|
ApiCategories.CONTEXT.value,
|
|
440
|
-
BackendOperations.
|
|
440
|
+
BackendOperations.READ_ALL_VERSIONS.value,
|
|
441
441
|
entity_name=entity_name,
|
|
442
442
|
**kwargs,
|
|
443
443
|
)
|
|
@@ -2,16 +2,16 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import typing
|
|
4
4
|
|
|
5
|
-
from digitalhub.client.api import get_client
|
|
6
5
|
from digitalhub.context.api import get_context
|
|
7
6
|
from digitalhub.entities._commons.enums import ApiCategories, BackendOperations, EntityTypes
|
|
8
7
|
from digitalhub.entities._commons.utils import get_project_from_key, parse_entity_key
|
|
9
8
|
from digitalhub.factory.api import build_entity_from_dict
|
|
9
|
+
from digitalhub.stores.client.api import get_client
|
|
10
10
|
from digitalhub.utils.exceptions import ContextError, EntityError, EntityNotExistsError
|
|
11
11
|
|
|
12
12
|
if typing.TYPE_CHECKING:
|
|
13
|
-
from digitalhub.client._base.client import Client
|
|
14
13
|
from digitalhub.context.context import Context
|
|
14
|
+
from digitalhub.stores.client._base.client import Client
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
def parse_identifier(
|
|
@@ -6,7 +6,7 @@ from pathlib import Path
|
|
|
6
6
|
from typing import Any
|
|
7
7
|
|
|
8
8
|
from digitalhub.entities.dataitem._base.entity import Dataitem
|
|
9
|
-
from digitalhub.stores.api import get_store
|
|
9
|
+
from digitalhub.stores.data.api import get_store
|
|
10
10
|
from digitalhub.utils.uri_utils import has_sql_scheme
|
|
11
11
|
|
|
12
12
|
if typing.TYPE_CHECKING:
|
|
@@ -8,8 +8,8 @@ from digitalhub.context.api import get_context
|
|
|
8
8
|
from digitalhub.entities._base.entity._constructors.uuid import build_uuid
|
|
9
9
|
from digitalhub.entities._base.material.utils import build_log_path_from_source, eval_local_source
|
|
10
10
|
from digitalhub.entities._commons.enums import EntityKinds, EntityTypes
|
|
11
|
-
from digitalhub.
|
|
12
|
-
from digitalhub.stores.api import
|
|
11
|
+
from digitalhub.stores.data.api import get_store
|
|
12
|
+
from digitalhub.stores.readers.data.api import get_reader_by_object
|
|
13
13
|
from digitalhub.utils.enums import FileExtensions
|
|
14
14
|
from digitalhub.utils.generic_utils import slugify_string
|
|
15
15
|
from digitalhub.utils.types import SourcesOrListOfSources
|
|
@@ -101,6 +101,36 @@ class Model(MaterialEntity):
|
|
|
101
101
|
self._set_metrics(key, value, overwrite, single_value)
|
|
102
102
|
context_processor.update_metric(self.project, self.ENTITY_TYPE, self.id, key, self.status.metrics[key])
|
|
103
103
|
|
|
104
|
+
def log_metrics(
|
|
105
|
+
self,
|
|
106
|
+
metrics: dict[str, MetricType],
|
|
107
|
+
overwrite: bool = False,
|
|
108
|
+
) -> None:
|
|
109
|
+
"""
|
|
110
|
+
Log metrics into entity status. If a metric is a list, it will be logged as a list.
|
|
111
|
+
Otherwise, it will be logged as a single value.
|
|
112
|
+
|
|
113
|
+
Parameters
|
|
114
|
+
----------
|
|
115
|
+
metrics : dict[str, MetricType]
|
|
116
|
+
Dict of metrics to log.
|
|
117
|
+
overwrite : bool
|
|
118
|
+
If True, overwrite existing metrics.
|
|
119
|
+
|
|
120
|
+
Returns
|
|
121
|
+
-------
|
|
122
|
+
None
|
|
123
|
+
|
|
124
|
+
See also
|
|
125
|
+
--------
|
|
126
|
+
log_metric
|
|
127
|
+
"""
|
|
128
|
+
for key, value in metrics.items():
|
|
129
|
+
if isinstance(value, list):
|
|
130
|
+
self.log_metric(key, value, overwrite)
|
|
131
|
+
else:
|
|
132
|
+
self.log_metric(key, value, overwrite, single_value=True)
|
|
133
|
+
|
|
104
134
|
##############################
|
|
105
135
|
# Helper methods
|
|
106
136
|
##############################
|
|
@@ -4,7 +4,6 @@ import typing
|
|
|
4
4
|
from pathlib import Path
|
|
5
5
|
from typing import Any
|
|
6
6
|
|
|
7
|
-
from digitalhub.client.api import get_client
|
|
8
7
|
from digitalhub.context.api import build_context
|
|
9
8
|
from digitalhub.entities._base.entity.entity import Entity
|
|
10
9
|
from digitalhub.entities._commons.enums import EntityTypes
|
|
@@ -69,6 +68,7 @@ from digitalhub.entities.workflow.crud import (
|
|
|
69
68
|
update_workflow,
|
|
70
69
|
)
|
|
71
70
|
from digitalhub.factory.api import build_entity_from_dict
|
|
71
|
+
from digitalhub.stores.client.api import get_client
|
|
72
72
|
from digitalhub.utils.exceptions import BackendError, EntityAlreadyExistsError, EntityError
|
|
73
73
|
from digitalhub.utils.generic_utils import get_timestamp
|
|
74
74
|
from digitalhub.utils.io_utils import write_yaml
|
|
@@ -210,6 +210,36 @@ class Run(UnversionedEntity):
|
|
|
210
210
|
self._set_metrics(key, value, overwrite, single_value)
|
|
211
211
|
context_processor.update_metric(self.project, self.ENTITY_TYPE, self.id, key, self.status.metrics[key])
|
|
212
212
|
|
|
213
|
+
def log_metrics(
|
|
214
|
+
self,
|
|
215
|
+
metrics: dict[str, MetricType],
|
|
216
|
+
overwrite: bool = False,
|
|
217
|
+
) -> None:
|
|
218
|
+
"""
|
|
219
|
+
Log metrics into entity status. If a metric is a list, it will be logged as a list.
|
|
220
|
+
Otherwise, it will be logged as a single value.
|
|
221
|
+
|
|
222
|
+
Parameters
|
|
223
|
+
----------
|
|
224
|
+
metrics : dict[str, MetricType]
|
|
225
|
+
Dict of metrics to log.
|
|
226
|
+
overwrite : bool
|
|
227
|
+
If True, overwrite existing metrics.
|
|
228
|
+
|
|
229
|
+
Returns
|
|
230
|
+
-------
|
|
231
|
+
None
|
|
232
|
+
|
|
233
|
+
See also
|
|
234
|
+
--------
|
|
235
|
+
log_metric
|
|
236
|
+
"""
|
|
237
|
+
for key, value in metrics.items():
|
|
238
|
+
if isinstance(value, list):
|
|
239
|
+
self.log_metric(key, value, overwrite)
|
|
240
|
+
else:
|
|
241
|
+
self.log_metric(key, value, overwrite, single_value=True)
|
|
242
|
+
|
|
213
243
|
##############################
|
|
214
244
|
# Helpers
|
|
215
245
|
##############################
|
|
@@ -5,9 +5,9 @@ from abc import abstractmethod
|
|
|
5
5
|
from typing import Any
|
|
6
6
|
|
|
7
7
|
if typing.TYPE_CHECKING:
|
|
8
|
-
from digitalhub.client._base.api_builder import ClientApiBuilder
|
|
9
|
-
from digitalhub.client._base.key_builder import ClientKeyBuilder
|
|
10
|
-
from digitalhub.client._base.params_builder import ClientParametersBuilder
|
|
8
|
+
from digitalhub.stores.client._base.api_builder import ClientApiBuilder
|
|
9
|
+
from digitalhub.stores.client._base.key_builder import ClientKeyBuilder
|
|
10
|
+
from digitalhub.stores.client._base.params_builder import ClientParametersBuilder
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
class Client:
|
|
@@ -2,10 +2,10 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import typing
|
|
4
4
|
|
|
5
|
-
from digitalhub.client.builder import client_builder
|
|
5
|
+
from digitalhub.stores.client.builder import client_builder
|
|
6
6
|
|
|
7
7
|
if typing.TYPE_CHECKING:
|
|
8
|
-
from digitalhub.client._base.client import Client
|
|
8
|
+
from digitalhub.stores.client._base.client import Client
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
def get_client(local: bool = False, config: dict | None = None) -> Client:
|
|
@@ -2,11 +2,11 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import typing
|
|
4
4
|
|
|
5
|
-
from digitalhub.client.dhcore.client import ClientDHCore
|
|
6
|
-
from digitalhub.client.local.client import ClientLocal
|
|
5
|
+
from digitalhub.stores.client.dhcore.client import ClientDHCore
|
|
6
|
+
from digitalhub.stores.client.local.client import ClientLocal
|
|
7
7
|
|
|
8
8
|
if typing.TYPE_CHECKING:
|
|
9
|
-
from digitalhub.client._base.client import Client
|
|
9
|
+
from digitalhub.stores.client._base.client import Client
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
class ClientBuilder:
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from digitalhub.client._base.api_builder import ClientApiBuilder
|
|
4
3
|
from digitalhub.entities._commons.enums import ApiCategories, BackendOperations
|
|
4
|
+
from digitalhub.stores.client._base.api_builder import ClientApiBuilder
|
|
5
5
|
from digitalhub.utils.exceptions import BackendError
|
|
6
6
|
|
|
7
7
|
API_BASE = "/api/v1"
|
|
@@ -6,12 +6,12 @@ from typing import Any
|
|
|
6
6
|
from requests import request
|
|
7
7
|
from requests.exceptions import JSONDecodeError
|
|
8
8
|
|
|
9
|
-
from digitalhub.client._base.client import Client
|
|
10
|
-
from digitalhub.client.dhcore.api_builder import ClientDHCoreApiBuilder
|
|
11
|
-
from digitalhub.client.dhcore.configurator import ClientDHCoreConfigurator
|
|
12
|
-
from digitalhub.client.dhcore.error_parser import ErrorParser
|
|
13
|
-
from digitalhub.client.dhcore.key_builder import ClientDHCoreKeyBuilder
|
|
14
|
-
from digitalhub.client.dhcore.params_builder import ClientDHCoreParametersBuilder
|
|
9
|
+
from digitalhub.stores.client._base.client import Client
|
|
10
|
+
from digitalhub.stores.client.dhcore.api_builder import ClientDHCoreApiBuilder
|
|
11
|
+
from digitalhub.stores.client.dhcore.configurator import ClientDHCoreConfigurator
|
|
12
|
+
from digitalhub.stores.client.dhcore.error_parser import ErrorParser
|
|
13
|
+
from digitalhub.stores.client.dhcore.key_builder import ClientDHCoreKeyBuilder
|
|
14
|
+
from digitalhub.stores.client.dhcore.params_builder import ClientDHCoreParametersBuilder
|
|
15
15
|
from digitalhub.utils.exceptions import BackendError
|
|
16
16
|
from digitalhub.utils.generic_utils import dump_json
|
|
17
17
|
|
|
@@ -1,17 +1,16 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
import os
|
|
4
3
|
import typing
|
|
5
4
|
from warnings import warn
|
|
6
5
|
|
|
7
6
|
from requests import request
|
|
8
7
|
|
|
9
|
-
from digitalhub.client.dhcore.enums import AuthType, DhcoreEnvVar
|
|
10
|
-
from digitalhub.
|
|
11
|
-
from digitalhub.
|
|
12
|
-
from digitalhub.stores.
|
|
13
|
-
from digitalhub.stores.sql.enums import SqlStoreEnv
|
|
8
|
+
from digitalhub.stores.client.dhcore.enums import AuthType, DhcoreEnvVar
|
|
9
|
+
from digitalhub.stores.configurator.configurator import configurator
|
|
10
|
+
from digitalhub.stores.data.s3.enums import S3StoreEnv
|
|
11
|
+
from digitalhub.stores.data.sql.enums import SqlStoreEnv
|
|
14
12
|
from digitalhub.utils.exceptions import ClientError
|
|
13
|
+
from digitalhub.utils.generic_utils import list_enum
|
|
15
14
|
from digitalhub.utils.uri_utils import has_remote_scheme
|
|
16
15
|
|
|
17
16
|
if typing.TYPE_CHECKING:
|
|
@@ -54,37 +53,8 @@ class ClientDHCoreConfigurator:
|
|
|
54
53
|
-------
|
|
55
54
|
None
|
|
56
55
|
"""
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
self._get_auth_vars()
|
|
60
|
-
return
|
|
61
|
-
|
|
62
|
-
# Read passed config
|
|
63
|
-
# Validate and save credentials
|
|
64
|
-
if config.get("access_token") is not None:
|
|
65
|
-
config = OAuth2TokenAuth(**config)
|
|
66
|
-
for pair in [
|
|
67
|
-
(AUTH_KEY, AuthType.OAUTH2.value),
|
|
68
|
-
(DhcoreEnvVar.ENDPOINT.value, config.endpoint),
|
|
69
|
-
(DhcoreEnvVar.ISSUER.value, config.issuer),
|
|
70
|
-
(DhcoreEnvVar.ACCESS_TOKEN.value, config.access_token),
|
|
71
|
-
(DhcoreEnvVar.REFRESH_TOKEN.value, config.refresh_token),
|
|
72
|
-
(DhcoreEnvVar.CLIENT_ID.value, config.client_id),
|
|
73
|
-
]:
|
|
74
|
-
configurator.set_credential(*pair)
|
|
75
|
-
|
|
76
|
-
elif config.get("user") is not None and config.get("password") is not None:
|
|
77
|
-
config = BasicAuth(**config)
|
|
78
|
-
for pair in [
|
|
79
|
-
(AUTH_KEY, AuthType.BASIC.value),
|
|
80
|
-
(DhcoreEnvVar.ENDPOINT.value, config.endpoint),
|
|
81
|
-
(DhcoreEnvVar.USER.value, config.user),
|
|
82
|
-
(DhcoreEnvVar.PASSWORD.value, config.password),
|
|
83
|
-
]:
|
|
84
|
-
configurator.set_credential(*pair)
|
|
85
|
-
|
|
86
|
-
else:
|
|
87
|
-
raise ClientError("Invalid credentials format.")
|
|
56
|
+
self._get_core_endpoint()
|
|
57
|
+
self._get_auth_vars()
|
|
88
58
|
|
|
89
59
|
def check_core_version(self, response: Response) -> None:
|
|
90
60
|
"""
|
|
@@ -257,52 +227,32 @@ class ClientDHCoreConfigurator:
|
|
|
257
227
|
|
|
258
228
|
# Otherwise try token from file
|
|
259
229
|
if response.status_code in (400, 401, 403):
|
|
260
|
-
refresh_token = configurator.
|
|
230
|
+
refresh_token = configurator.load_from_file(DhcoreEnvVar.REFRESH_TOKEN.value)
|
|
261
231
|
response = self._call_refresh_token_endpoint(url, refresh_token)
|
|
262
232
|
|
|
263
233
|
response.raise_for_status()
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
# Set new credential in stores
|
|
271
|
-
if (access_key := dict_response.get("aws_access_key_id")) is not None:
|
|
272
|
-
configurator.set_credential(S3StoreEnv.ACCESS_KEY_ID.value, access_key)
|
|
273
|
-
os.environ[S3StoreEnv.ACCESS_KEY_ID.value] = access_key
|
|
274
|
-
if (secret_key := dict_response.get("aws_secret_access_key")) is not None:
|
|
275
|
-
configurator.set_credential(S3StoreEnv.SECRET_ACCESS_KEY.value,secret_key )
|
|
276
|
-
os.environ[S3StoreEnv.SECRET_ACCESS_KEY.value] = secret_key
|
|
277
|
-
if (db_username := dict_response.get("db_username")) is not None:
|
|
278
|
-
configurator.set_credential(SqlStoreEnv.USERNAME.value, db_username)
|
|
279
|
-
os.environ[SqlStoreEnv.USERNAME.value] = db_username
|
|
280
|
-
if (db_password := dict_response.get("db_password")) is not None:
|
|
281
|
-
configurator.set_credential(SqlStoreEnv.PASSWORD.value, db_password)
|
|
282
|
-
os.environ[SqlStoreEnv.PASSWORD.value] = db_password
|
|
283
|
-
|
|
284
|
-
# Propagate new access token to config file
|
|
285
|
-
self._write_env()
|
|
286
|
-
|
|
287
|
-
def _write_env(self) -> None:
|
|
234
|
+
|
|
235
|
+
# Read new credentials and propagate to config file
|
|
236
|
+
self._set_creds(response.json())
|
|
237
|
+
|
|
238
|
+
def _set_creds(self, response: dict) -> None:
|
|
288
239
|
"""
|
|
289
|
-
|
|
290
|
-
|
|
240
|
+
Set new credentials.
|
|
241
|
+
|
|
242
|
+
Parameters
|
|
243
|
+
----------
|
|
244
|
+
response : dict
|
|
245
|
+
Response from refresh token endpoint.
|
|
291
246
|
|
|
292
247
|
Returns
|
|
293
248
|
-------
|
|
294
249
|
None
|
|
295
250
|
"""
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
S3StoreEnv.SECRET_ACCESS_KEY.value,
|
|
302
|
-
SqlStoreEnv.USERNAME.value,
|
|
303
|
-
SqlStoreEnv.PASSWORD.value
|
|
304
|
-
]
|
|
305
|
-
)
|
|
251
|
+
keys = list_enum(DhcoreEnvVar) + list_enum(S3StoreEnv) + list_enum(SqlStoreEnv)
|
|
252
|
+
for key in keys:
|
|
253
|
+
if (value := response.get(key.lower().removeprefix("dhcore_"))) is not None:
|
|
254
|
+
configurator.set_credential(key, value)
|
|
255
|
+
configurator.write_env(keys)
|
|
306
256
|
|
|
307
257
|
def _get_refresh_endpoint(self) -> str:
|
|
308
258
|
"""
|
|
@@ -315,11 +265,10 @@ class ClientDHCoreConfigurator:
|
|
|
315
265
|
"""
|
|
316
266
|
# Get issuer endpoint
|
|
317
267
|
endpoint_issuer = configurator.load_var(DhcoreEnvVar.ISSUER.value)
|
|
318
|
-
if endpoint_issuer is
|
|
319
|
-
endpoint_issuer = self._sanitize_endpoint(endpoint_issuer)
|
|
320
|
-
configurator.set_credential(DhcoreEnvVar.ISSUER.value, endpoint_issuer)
|
|
321
|
-
else:
|
|
268
|
+
if endpoint_issuer is None:
|
|
322
269
|
raise ClientError("Issuer endpoint not set.")
|
|
270
|
+
endpoint_issuer = self._sanitize_endpoint(endpoint_issuer)
|
|
271
|
+
configurator.set_credential(DhcoreEnvVar.ISSUER.value, endpoint_issuer)
|
|
323
272
|
|
|
324
273
|
# Standard issuer endpoint path
|
|
325
274
|
url = endpoint_issuer + "/.well-known/openid-configuration"
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from digitalhub.client._base.params_builder import ClientParametersBuilder
|
|
4
3
|
from digitalhub.entities._commons.enums import ApiCategories, BackendOperations
|
|
4
|
+
from digitalhub.stores.client._base.params_builder import ClientParametersBuilder
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
class ClientDHCoreParametersBuilder(ClientParametersBuilder):
|
|
@@ -77,7 +77,11 @@ class ClientDHCoreParametersBuilder(ClientParametersBuilder):
|
|
|
77
77
|
kwargs = self._set_params(**kwargs)
|
|
78
78
|
|
|
79
79
|
# Handle read
|
|
80
|
-
if operation == BackendOperations.
|
|
80
|
+
if operation == BackendOperations.READ.value:
|
|
81
|
+
name = kwargs.pop("entity_name", None)
|
|
82
|
+
if name is not None:
|
|
83
|
+
kwargs["params"]["name"] = name
|
|
84
|
+
elif operation == BackendOperations.READ_ALL_VERSIONS.value:
|
|
81
85
|
kwargs["params"]["versions"] = "all"
|
|
82
86
|
kwargs["params"]["name"] = kwargs.pop("entity_name")
|
|
83
87
|
# Handle delete
|
|
@@ -3,11 +3,11 @@ from __future__ import annotations
|
|
|
3
3
|
import os
|
|
4
4
|
import typing
|
|
5
5
|
|
|
6
|
-
from digitalhub.client.api import get_client
|
|
7
|
-
from digitalhub.client.dhcore.enums import DhcoreEnvVar
|
|
6
|
+
from digitalhub.stores.client.api import get_client
|
|
7
|
+
from digitalhub.stores.client.dhcore.enums import DhcoreEnvVar
|
|
8
8
|
|
|
9
9
|
if typing.TYPE_CHECKING:
|
|
10
|
-
from digitalhub.client.dhcore.client import ClientDHCore
|
|
10
|
+
from digitalhub.stores.client.dhcore.client import ClientDHCore
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
def set_dhcore_env(
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from digitalhub.client._base.api_builder import ClientApiBuilder
|
|
4
|
-
from digitalhub.client.local.enums import LocalClientVar
|
|
5
3
|
from digitalhub.entities._commons.enums import ApiCategories, BackendOperations
|
|
4
|
+
from digitalhub.stores.client._base.api_builder import ClientApiBuilder
|
|
5
|
+
from digitalhub.stores.client.local.enums import LocalClientVar
|
|
6
6
|
from digitalhub.utils.exceptions import BackendError
|
|
7
7
|
|
|
8
8
|
API_BASE = "/api/v1"
|
|
@@ -4,11 +4,11 @@ from copy import deepcopy
|
|
|
4
4
|
from datetime import datetime, timezone
|
|
5
5
|
from typing import Any
|
|
6
6
|
|
|
7
|
-
from digitalhub.client._base.client import Client
|
|
8
|
-
from digitalhub.client.local.api_builder import ClientLocalApiBuilder
|
|
9
|
-
from digitalhub.client.local.enums import LocalClientVar
|
|
10
|
-
from digitalhub.client.local.key_builder import ClientLocalKeyBuilder
|
|
11
|
-
from digitalhub.client.local.params_builder import ClientLocalParametersBuilder
|
|
7
|
+
from digitalhub.stores.client._base.client import Client
|
|
8
|
+
from digitalhub.stores.client.local.api_builder import ClientLocalApiBuilder
|
|
9
|
+
from digitalhub.stores.client.local.enums import LocalClientVar
|
|
10
|
+
from digitalhub.stores.client.local.key_builder import ClientLocalKeyBuilder
|
|
11
|
+
from digitalhub.stores.client.local.params_builder import ClientLocalParametersBuilder
|
|
12
12
|
from digitalhub.utils.exceptions import BackendError
|
|
13
13
|
|
|
14
14
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from digitalhub.client._base.params_builder import ClientParametersBuilder
|
|
4
3
|
from digitalhub.entities._commons.enums import ApiCategories, BackendOperations
|
|
4
|
+
from digitalhub.stores.client._base.params_builder import ClientParametersBuilder
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
class ClientLocalParametersBuilder(ClientParametersBuilder):
|
|
@@ -2,8 +2,8 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import os
|
|
4
4
|
|
|
5
|
-
from digitalhub.configurator.credentials_store import CredentialsStore
|
|
6
|
-
from digitalhub.configurator.ini_module import
|
|
5
|
+
from digitalhub.stores.configurator.credentials_store import CredentialsStore
|
|
6
|
+
from digitalhub.stores.configurator.ini_module import load_from_file, write_config
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
class EnvConfigurator:
|
|
@@ -70,7 +70,7 @@ class EnvConfigurator:
|
|
|
70
70
|
if var is None:
|
|
71
71
|
var = self.load_from_env(var_name)
|
|
72
72
|
if var is None:
|
|
73
|
-
var = self.
|
|
73
|
+
var = self.load_from_file(var_name)
|
|
74
74
|
return var
|
|
75
75
|
|
|
76
76
|
def load_from_env(self, var: str) -> str | None:
|
|
@@ -91,7 +91,7 @@ class EnvConfigurator:
|
|
|
91
91
|
if env_var != "":
|
|
92
92
|
return env_var
|
|
93
93
|
|
|
94
|
-
def
|
|
94
|
+
def load_from_file(self, var: str) -> str | None:
|
|
95
95
|
"""
|
|
96
96
|
Load variable from config file.
|
|
97
97
|
|
|
@@ -105,7 +105,7 @@ class EnvConfigurator:
|
|
|
105
105
|
str | None
|
|
106
106
|
Environment variable value.
|
|
107
107
|
"""
|
|
108
|
-
return
|
|
108
|
+
return load_from_file(var)
|
|
109
109
|
|
|
110
110
|
def write_env(self, key_to_include: list[str] | None = None) -> None:
|
|
111
111
|
"""
|
|
@@ -6,13 +6,13 @@ from pathlib import Path
|
|
|
6
6
|
from tempfile import mkdtemp
|
|
7
7
|
from typing import Any
|
|
8
8
|
|
|
9
|
-
from digitalhub.readers.data.api import get_reader_by_engine
|
|
9
|
+
from digitalhub.stores.readers.data.api import get_reader_by_engine
|
|
10
10
|
from digitalhub.utils.exceptions import StoreError
|
|
11
11
|
from digitalhub.utils.types import SourcesOrListOfSources
|
|
12
12
|
from digitalhub.utils.uri_utils import has_local_scheme
|
|
13
13
|
|
|
14
14
|
if typing.TYPE_CHECKING:
|
|
15
|
-
from digitalhub.readers.data._base.reader import DataframeReader
|
|
15
|
+
from digitalhub.stores.readers.data._base.reader import DataframeReader
|
|
16
16
|
|
|
17
17
|
|
|
18
18
|
class Store:
|
|
@@ -2,10 +2,10 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import typing
|
|
4
4
|
|
|
5
|
-
from digitalhub.stores.builder import store_builder
|
|
5
|
+
from digitalhub.stores.data.builder import store_builder
|
|
6
6
|
|
|
7
7
|
if typing.TYPE_CHECKING:
|
|
8
|
-
from digitalhub.stores._base.store import Store
|
|
8
|
+
from digitalhub.stores.data._base.store import Store
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
def get_store(project: str, uri: str, config: dict | None = None) -> Store:
|
|
@@ -2,15 +2,15 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import typing
|
|
4
4
|
|
|
5
|
-
from digitalhub.configurator.api import get_current_env
|
|
6
|
-
from digitalhub.stores.local.store import LocalStore
|
|
7
|
-
from digitalhub.stores.remote.store import RemoteStore
|
|
8
|
-
from digitalhub.stores.s3.store import S3Store
|
|
9
|
-
from digitalhub.stores.sql.store import SqlStore
|
|
5
|
+
from digitalhub.stores.configurator.api import get_current_env
|
|
6
|
+
from digitalhub.stores.data.local.store import LocalStore
|
|
7
|
+
from digitalhub.stores.data.remote.store import RemoteStore
|
|
8
|
+
from digitalhub.stores.data.s3.store import S3Store
|
|
9
|
+
from digitalhub.stores.data.sql.store import SqlStore
|
|
10
10
|
from digitalhub.utils.uri_utils import SchemeCategory, map_uri_scheme
|
|
11
11
|
|
|
12
12
|
if typing.TYPE_CHECKING:
|
|
13
|
-
from digitalhub.stores._base.store import Store
|
|
13
|
+
from digitalhub.stores.data._base.store import Store
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
def _get_class_from_type(type: str) -> Store:
|
|
@@ -4,8 +4,8 @@ import shutil
|
|
|
4
4
|
from pathlib import Path
|
|
5
5
|
from typing import Any
|
|
6
6
|
|
|
7
|
-
from digitalhub.
|
|
8
|
-
from digitalhub.stores.
|
|
7
|
+
from digitalhub.stores.data._base.store import Store
|
|
8
|
+
from digitalhub.stores.readers.data.api import get_reader_by_object
|
|
9
9
|
from digitalhub.utils.exceptions import StoreError
|
|
10
10
|
from digitalhub.utils.file_utils import get_file_info_from_local
|
|
11
11
|
from digitalhub.utils.types import SourcesOrListOfSources
|