digitalhub 0.14.0b1__py3-none-any.whl → 0.14.0b3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of digitalhub might be problematic. Click here for more details.
- digitalhub/context/builder.py +0 -4
- digitalhub/context/context.py +12 -8
- digitalhub/entities/_base/_base/entity.py +0 -4
- digitalhub/entities/_base/context/entity.py +1 -1
- digitalhub/entities/_base/entity/entity.py +0 -8
- digitalhub/entities/_base/executable/entity.py +9 -21
- digitalhub/entities/_base/material/entity.py +5 -21
- digitalhub/entities/_base/unversioned/entity.py +1 -1
- digitalhub/entities/_base/versioned/entity.py +1 -1
- digitalhub/entities/_processors/base/__init__.py +3 -0
- digitalhub/entities/_processors/{base.py → base/crud.py} +12 -232
- digitalhub/entities/_processors/base/import_export.py +122 -0
- digitalhub/entities/_processors/base/processor.py +302 -0
- digitalhub/entities/_processors/base/special_ops.py +108 -0
- digitalhub/entities/_processors/context/__init__.py +3 -0
- digitalhub/entities/_processors/context/crud.py +654 -0
- digitalhub/entities/_processors/context/import_export.py +242 -0
- digitalhub/entities/_processors/context/material.py +123 -0
- digitalhub/entities/_processors/context/processor.py +400 -0
- digitalhub/entities/_processors/context/special_ops.py +476 -0
- digitalhub/entities/_processors/processors.py +12 -0
- digitalhub/entities/_processors/utils.py +2 -2
- digitalhub/entities/artifact/crud.py +1 -1
- digitalhub/entities/dataitem/crud.py +6 -3
- digitalhub/entities/dataitem/table/entity.py +24 -1
- digitalhub/entities/dataitem/utils.py +4 -0
- digitalhub/entities/function/_base/entity.py +3 -3
- digitalhub/entities/function/crud.py +1 -1
- digitalhub/entities/model/_base/entity.py +46 -24
- digitalhub/entities/model/crud.py +1 -1
- digitalhub/entities/project/_base/entity.py +3 -12
- digitalhub/entities/project/crud.py +1 -2
- digitalhub/entities/run/_base/builder.py +0 -4
- digitalhub/entities/run/_base/entity.py +53 -66
- digitalhub/entities/run/crud.py +5 -2
- digitalhub/entities/secret/_base/entity.py +1 -5
- digitalhub/entities/secret/crud.py +1 -1
- digitalhub/entities/task/_base/builder.py +0 -4
- digitalhub/entities/task/_base/entity.py +5 -5
- digitalhub/entities/task/crud.py +1 -1
- digitalhub/entities/trigger/_base/entity.py +1 -5
- digitalhub/entities/trigger/crud.py +1 -1
- digitalhub/entities/workflow/_base/entity.py +3 -3
- digitalhub/entities/workflow/crud.py +1 -1
- digitalhub/factory/entity.py +283 -0
- digitalhub/factory/registry.py +197 -0
- digitalhub/factory/runtime.py +44 -0
- digitalhub/runtimes/_base.py +2 -2
- digitalhub/stores/client/dhcore/client.py +0 -14
- digitalhub/stores/client/dhcore/configurator.py +5 -28
- digitalhub/stores/client/dhcore/error_parser.py +0 -4
- digitalhub/stores/credentials/configurator.py +4 -29
- digitalhub/stores/credentials/handler.py +0 -12
- digitalhub/stores/credentials/store.py +0 -4
- digitalhub/stores/data/_base/store.py +0 -16
- digitalhub/stores/data/builder.py +0 -4
- digitalhub/stores/data/remote/store.py +0 -4
- digitalhub/stores/data/s3/configurator.py +2 -10
- digitalhub/stores/data/s3/store.py +0 -12
- digitalhub/stores/data/sql/configurator.py +0 -8
- digitalhub/stores/data/sql/store.py +0 -4
- digitalhub/stores/readers/data/factory.py +0 -8
- digitalhub/stores/readers/data/pandas/reader.py +9 -19
- digitalhub/utils/io_utils.py +0 -4
- {digitalhub-0.14.0b1.dist-info → digitalhub-0.14.0b3.dist-info}/METADATA +1 -1
- {digitalhub-0.14.0b1.dist-info → digitalhub-0.14.0b3.dist-info}/RECORD +69 -57
- digitalhub/entities/_processors/context.py +0 -1499
- digitalhub/factory/factory.py +0 -460
- {digitalhub-0.14.0b1.dist-info → digitalhub-0.14.0b3.dist-info}/WHEEL +0 -0
- {digitalhub-0.14.0b1.dist-info → digitalhub-0.14.0b3.dist-info}/licenses/AUTHORS +0 -0
- {digitalhub-0.14.0b1.dist-info → digitalhub-0.14.0b3.dist-info}/licenses/LICENSE +0 -0
|
@@ -53,30 +53,20 @@ class Configurator:
|
|
|
53
53
|
def load_configs(self) -> None:
|
|
54
54
|
"""
|
|
55
55
|
Load the configuration from both environment and file sources.
|
|
56
|
-
|
|
57
|
-
Returns
|
|
58
|
-
-------
|
|
59
|
-
None
|
|
60
56
|
"""
|
|
61
57
|
self.load_env_vars()
|
|
62
58
|
self.load_file_vars()
|
|
63
59
|
|
|
64
60
|
@abstractmethod
|
|
65
|
-
def load_env_vars(self) -> None:
|
|
66
|
-
...
|
|
61
|
+
def load_env_vars(self) -> None: ...
|
|
67
62
|
|
|
68
63
|
@abstractmethod
|
|
69
|
-
def load_file_vars(self) -> None:
|
|
70
|
-
...
|
|
64
|
+
def load_file_vars(self) -> None: ...
|
|
71
65
|
|
|
72
66
|
def check_config(self) -> None:
|
|
73
67
|
"""
|
|
74
68
|
Check if the current profile has changed and reload
|
|
75
69
|
the file credentials if needed.
|
|
76
|
-
|
|
77
|
-
Returns
|
|
78
|
-
-------
|
|
79
|
-
None
|
|
80
70
|
"""
|
|
81
71
|
if (current := self._creds_handler.get_current_profile()) != self._current_profile:
|
|
82
72
|
self.load_file_vars()
|
|
@@ -121,10 +111,6 @@ class Configurator:
|
|
|
121
111
|
"""
|
|
122
112
|
Attempt to change the origin of credentials.
|
|
123
113
|
Raise error if already evaluated.
|
|
124
|
-
|
|
125
|
-
Returns
|
|
126
|
-
-------
|
|
127
|
-
None
|
|
128
114
|
"""
|
|
129
115
|
try:
|
|
130
116
|
self.change_origin()
|
|
@@ -134,24 +120,17 @@ class Configurator:
|
|
|
134
120
|
def change_origin(self) -> None:
|
|
135
121
|
"""
|
|
136
122
|
Change the origin of credentials from env to file or vice versa.
|
|
137
|
-
|
|
138
|
-
Returns
|
|
139
|
-
-------
|
|
140
|
-
None
|
|
141
123
|
"""
|
|
142
124
|
if self._changed_origin:
|
|
143
125
|
raise ConfigError("Origin has already been changed.")
|
|
144
126
|
if self._origin == self._env:
|
|
145
127
|
self.change_to_file()
|
|
146
|
-
|
|
128
|
+
else:
|
|
129
|
+
self.change_to_env()
|
|
147
130
|
|
|
148
131
|
def change_to_file(self) -> None:
|
|
149
132
|
"""
|
|
150
133
|
Set the credentials origin to file.
|
|
151
|
-
|
|
152
|
-
Returns
|
|
153
|
-
-------
|
|
154
|
-
None
|
|
155
134
|
"""
|
|
156
135
|
if self._origin == self._env:
|
|
157
136
|
self._changed_origin = True
|
|
@@ -160,10 +139,6 @@ class Configurator:
|
|
|
160
139
|
def change_to_env(self) -> None:
|
|
161
140
|
"""
|
|
162
141
|
Set the credentials origin to environment.
|
|
163
|
-
|
|
164
|
-
Returns
|
|
165
|
-
-------
|
|
166
|
-
None
|
|
167
142
|
"""
|
|
168
143
|
if self._origin == self._file:
|
|
169
144
|
self._changed_origin = True
|
|
@@ -64,10 +64,6 @@ class CredentialHandler:
|
|
|
64
64
|
----------
|
|
65
65
|
creds_set : str
|
|
66
66
|
Name of the credentials profile to set.
|
|
67
|
-
|
|
68
|
-
Returns
|
|
69
|
-
-------
|
|
70
|
-
None
|
|
71
67
|
"""
|
|
72
68
|
self._profile = creds_set
|
|
73
69
|
set_current_profile(creds_set)
|
|
@@ -127,10 +123,6 @@ class CredentialHandler:
|
|
|
127
123
|
----------
|
|
128
124
|
creds : dict
|
|
129
125
|
Credentials to write.
|
|
130
|
-
|
|
131
|
-
Returns
|
|
132
|
-
-------
|
|
133
|
-
None
|
|
134
126
|
"""
|
|
135
127
|
write_config(creds, self._profile)
|
|
136
128
|
|
|
@@ -148,10 +140,6 @@ class CredentialHandler:
|
|
|
148
140
|
The origin of the credentials ('env' or 'file').
|
|
149
141
|
creds : dict
|
|
150
142
|
Credentials to set.
|
|
151
|
-
|
|
152
|
-
Returns
|
|
153
|
-
-------
|
|
154
|
-
None
|
|
155
143
|
"""
|
|
156
144
|
self._creds_store.set_credentials(self._profile, origin, creds)
|
|
157
145
|
|
|
@@ -42,10 +42,6 @@ class CredentialsStore:
|
|
|
42
42
|
Origin of the credentials ('env' or 'file').
|
|
43
43
|
credentials : dict of str to Any
|
|
44
44
|
Dictionary of credentials to set.
|
|
45
|
-
|
|
46
|
-
Returns
|
|
47
|
-
-------
|
|
48
|
-
None
|
|
49
45
|
"""
|
|
50
46
|
if origin == CredsOrigin.ENV.value:
|
|
51
47
|
for key, value in credentials.items():
|
|
@@ -114,10 +114,6 @@ class Store:
|
|
|
114
114
|
src : str
|
|
115
115
|
The source path.
|
|
116
116
|
|
|
117
|
-
Returns
|
|
118
|
-
-------
|
|
119
|
-
None
|
|
120
|
-
|
|
121
117
|
Raises
|
|
122
118
|
------
|
|
123
119
|
StoreError
|
|
@@ -135,10 +131,6 @@ class Store:
|
|
|
135
131
|
dst : str
|
|
136
132
|
The destination path.
|
|
137
133
|
|
|
138
|
-
Returns
|
|
139
|
-
-------
|
|
140
|
-
None
|
|
141
|
-
|
|
142
134
|
Raises
|
|
143
135
|
------
|
|
144
136
|
StoreError
|
|
@@ -158,10 +150,6 @@ class Store:
|
|
|
158
150
|
overwrite : bool
|
|
159
151
|
Specify if overwrite an existing file.
|
|
160
152
|
|
|
161
|
-
Returns
|
|
162
|
-
-------
|
|
163
|
-
None
|
|
164
|
-
|
|
165
153
|
Raises
|
|
166
154
|
------
|
|
167
155
|
StoreError
|
|
@@ -179,10 +167,6 @@ class Store:
|
|
|
179
167
|
----------
|
|
180
168
|
path : str | Path
|
|
181
169
|
The path to build.
|
|
182
|
-
|
|
183
|
-
Returns
|
|
184
|
-
-------
|
|
185
|
-
None
|
|
186
170
|
"""
|
|
187
171
|
if not isinstance(path, Path):
|
|
188
172
|
path = Path(path)
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
from __future__ import annotations
|
|
6
6
|
|
|
7
|
-
from datetime import datetime, timezone
|
|
7
|
+
from datetime import datetime, timedelta, timezone
|
|
8
8
|
|
|
9
9
|
from botocore.config import Config
|
|
10
10
|
|
|
@@ -46,10 +46,6 @@ class S3StoreConfigurator(Configurator):
|
|
|
46
46
|
def load_env_vars(self) -> None:
|
|
47
47
|
"""
|
|
48
48
|
Loads the credentials from the environment variables.
|
|
49
|
-
|
|
50
|
-
Returns
|
|
51
|
-
-------
|
|
52
|
-
None
|
|
53
49
|
"""
|
|
54
50
|
env_creds = self._creds_handler.load_from_env(self.keys)
|
|
55
51
|
self._creds_handler.set_credentials(self._env, env_creds)
|
|
@@ -57,10 +53,6 @@ class S3StoreConfigurator(Configurator):
|
|
|
57
53
|
def load_file_vars(self) -> None:
|
|
58
54
|
"""
|
|
59
55
|
Loads the credentials from a file.
|
|
60
|
-
|
|
61
|
-
Returns
|
|
62
|
-
-------
|
|
63
|
-
None
|
|
64
56
|
"""
|
|
65
57
|
file_creds = self._creds_handler.load_from_file(self.keys)
|
|
66
58
|
self._creds_handler.set_credentials(self._file, file_creds)
|
|
@@ -165,5 +157,5 @@ class S3StoreConfigurator(Configurator):
|
|
|
165
157
|
return False
|
|
166
158
|
dt = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%SZ")
|
|
167
159
|
dt = dt.replace(tzinfo=timezone.utc)
|
|
168
|
-
now = datetime.now(timezone.utc) +
|
|
160
|
+
now = datetime.now(timezone.utc) + timedelta(seconds=120)
|
|
169
161
|
return dt < now
|
|
@@ -350,10 +350,6 @@ class S3Store(Store):
|
|
|
350
350
|
S3 path of the object to be downloaded (e.g., 's3://bucket
|
|
351
351
|
filename : Path
|
|
352
352
|
Local path where the downloaded object will be saved.
|
|
353
|
-
|
|
354
|
-
Returns
|
|
355
|
-
-------
|
|
356
|
-
None
|
|
357
353
|
"""
|
|
358
354
|
client, bucket = self._check_factory(src)
|
|
359
355
|
key = self._get_key(src)
|
|
@@ -575,10 +571,6 @@ class S3Store(Store):
|
|
|
575
571
|
The S3 client object.
|
|
576
572
|
bucket : str
|
|
577
573
|
The name of the S3 bucket.
|
|
578
|
-
|
|
579
|
-
Returns
|
|
580
|
-
-------
|
|
581
|
-
None
|
|
582
574
|
"""
|
|
583
575
|
extra_args = {}
|
|
584
576
|
mime_type = get_file_mime_type(src)
|
|
@@ -612,10 +604,6 @@ class S3Store(Store):
|
|
|
612
604
|
The S3 client object.
|
|
613
605
|
bucket : str
|
|
614
606
|
The name of the S3 bucket.
|
|
615
|
-
|
|
616
|
-
Returns
|
|
617
|
-
-------
|
|
618
|
-
None
|
|
619
607
|
"""
|
|
620
608
|
client.upload_fileobj(
|
|
621
609
|
Fileobj=fileobj,
|
|
@@ -56,10 +56,6 @@ class SqlStoreConfigurator(Configurator):
|
|
|
56
56
|
Retrieves SQL database connection credentials from the system
|
|
57
57
|
environment variables and stores them in the configurator's
|
|
58
58
|
credential handler for use in database connections.
|
|
59
|
-
|
|
60
|
-
Returns
|
|
61
|
-
-------
|
|
62
|
-
None
|
|
63
59
|
"""
|
|
64
60
|
env_creds = self._creds_handler.load_from_env(self.keys)
|
|
65
61
|
self._creds_handler.set_credentials(self._env, env_creds)
|
|
@@ -71,10 +67,6 @@ class SqlStoreConfigurator(Configurator):
|
|
|
71
67
|
Retrieves SQL database connection credentials from a
|
|
72
68
|
configuration file and stores them in the configurator's
|
|
73
69
|
credential handler for use in database connections.
|
|
74
|
-
|
|
75
|
-
Returns
|
|
76
|
-
-------
|
|
77
|
-
None
|
|
78
70
|
"""
|
|
79
71
|
file_creds = self._creds_handler.load_from_file(self.keys)
|
|
80
72
|
self._creds_handler.set_credentials(self._file, file_creds)
|
|
@@ -33,10 +33,6 @@ class ReaderFactory:
|
|
|
33
33
|
Reader name.
|
|
34
34
|
builder : DataframeReader
|
|
35
35
|
Builder object.
|
|
36
|
-
|
|
37
|
-
Returns
|
|
38
|
-
-------
|
|
39
|
-
None
|
|
40
36
|
"""
|
|
41
37
|
if self._engine_builders is None:
|
|
42
38
|
self._engine_builders = {}
|
|
@@ -104,10 +100,6 @@ class ReaderFactory:
|
|
|
104
100
|
----------
|
|
105
101
|
engine : str
|
|
106
102
|
Engine name.
|
|
107
|
-
|
|
108
|
-
Returns
|
|
109
|
-
-------
|
|
110
|
-
None
|
|
111
103
|
"""
|
|
112
104
|
if engine not in self._engine_builders:
|
|
113
105
|
raise BuilderError(f"Engine {engine} not found.")
|
|
@@ -104,10 +104,6 @@ class DataframeReaderPandas(DataframeReader):
|
|
|
104
104
|
The destination of the dataframe.
|
|
105
105
|
**kwargs : dict
|
|
106
106
|
Keyword arguments.
|
|
107
|
-
|
|
108
|
-
Returns
|
|
109
|
-
-------
|
|
110
|
-
None
|
|
111
107
|
"""
|
|
112
108
|
if extension == FileExtensions.CSV.value:
|
|
113
109
|
return self.write_csv(df, dst, **kwargs)
|
|
@@ -128,12 +124,10 @@ class DataframeReaderPandas(DataframeReader):
|
|
|
128
124
|
The destination of the dataframe.
|
|
129
125
|
**kwargs : dict
|
|
130
126
|
Keyword arguments.
|
|
131
|
-
|
|
132
|
-
Returns
|
|
133
|
-
-------
|
|
134
|
-
None
|
|
135
127
|
"""
|
|
136
|
-
|
|
128
|
+
if "index" not in kwargs:
|
|
129
|
+
kwargs["index"] = False
|
|
130
|
+
df.to_csv(dst, **kwargs)
|
|
137
131
|
|
|
138
132
|
@staticmethod
|
|
139
133
|
def write_parquet(df: pd.DataFrame, dst: str | BytesIO, **kwargs) -> None:
|
|
@@ -148,12 +142,10 @@ class DataframeReaderPandas(DataframeReader):
|
|
|
148
142
|
The destination of the dataframe.
|
|
149
143
|
**kwargs : dict
|
|
150
144
|
Keyword arguments.
|
|
151
|
-
|
|
152
|
-
Returns
|
|
153
|
-
-------
|
|
154
|
-
None
|
|
155
145
|
"""
|
|
156
|
-
|
|
146
|
+
if "index" not in kwargs:
|
|
147
|
+
kwargs["index"] = False
|
|
148
|
+
df.to_parquet(dst, **kwargs)
|
|
157
149
|
|
|
158
150
|
@staticmethod
|
|
159
151
|
def write_table(df: pd.DataFrame, table: str, engine: Any, schema: str | None = None, **kwargs) -> None:
|
|
@@ -172,12 +164,10 @@ class DataframeReaderPandas(DataframeReader):
|
|
|
172
164
|
The destination schema.
|
|
173
165
|
**kwargs : dict
|
|
174
166
|
Keyword arguments.
|
|
175
|
-
|
|
176
|
-
Returns
|
|
177
|
-
-------
|
|
178
|
-
None
|
|
179
167
|
"""
|
|
180
|
-
|
|
168
|
+
if "index" not in kwargs:
|
|
169
|
+
kwargs["index"] = False
|
|
170
|
+
df.to_sql(table, engine, schema=schema, **kwargs)
|
|
181
171
|
|
|
182
172
|
##############################
|
|
183
173
|
# Utils
|
digitalhub/utils/io_utils.py
CHANGED
|
@@ -80,10 +80,6 @@ class NoDatesSafeLoader(yaml.SafeLoader):
|
|
|
80
80
|
----------
|
|
81
81
|
tag_to_remove : str
|
|
82
82
|
The tag to remove.
|
|
83
|
-
|
|
84
|
-
Returns
|
|
85
|
-
-------
|
|
86
|
-
None
|
|
87
83
|
"""
|
|
88
84
|
if "yaml_implicit_resolvers" not in cls.__dict__:
|
|
89
85
|
cls.yaml_implicit_resolvers = cls.yaml_implicit_resolvers.copy()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: digitalhub
|
|
3
|
-
Version: 0.14.
|
|
3
|
+
Version: 0.14.0b3
|
|
4
4
|
Summary: Python SDK for Digitalhub
|
|
5
5
|
Project-URL: Homepage, https://github.com/scc-digitalhub/digitalhub-sdk
|
|
6
6
|
Author-email: Fondazione Bruno Kessler <digitalhub@fbk.eu>, Matteo Martini <mmartini@fbk.eu>
|