digitalhub 0.7.0b2__py3-none-any.whl → 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of digitalhub might be problematic. Click here for more details.
- digitalhub/__init__.py +63 -93
- digitalhub/client/__init__.py +0 -0
- digitalhub/client/_base/__init__.py +0 -0
- digitalhub/client/_base/client.py +56 -0
- digitalhub/client/api.py +63 -0
- digitalhub/client/builder.py +50 -0
- digitalhub/client/dhcore/__init__.py +0 -0
- digitalhub/client/dhcore/client.py +669 -0
- digitalhub/client/dhcore/env.py +21 -0
- digitalhub/client/dhcore/models.py +46 -0
- digitalhub/client/dhcore/utils.py +111 -0
- digitalhub/client/local/__init__.py +0 -0
- digitalhub/client/local/client.py +533 -0
- digitalhub/context/__init__.py +0 -0
- digitalhub/context/api.py +93 -0
- digitalhub/context/builder.py +94 -0
- digitalhub/context/context.py +136 -0
- digitalhub/datastores/__init__.py +0 -0
- digitalhub/datastores/_base/__init__.py +0 -0
- digitalhub/datastores/_base/datastore.py +85 -0
- digitalhub/datastores/api.py +37 -0
- digitalhub/datastores/builder.py +110 -0
- digitalhub/datastores/local/__init__.py +0 -0
- digitalhub/datastores/local/datastore.py +50 -0
- digitalhub/datastores/remote/__init__.py +0 -0
- digitalhub/datastores/remote/datastore.py +31 -0
- digitalhub/datastores/s3/__init__.py +0 -0
- digitalhub/datastores/s3/datastore.py +46 -0
- digitalhub/datastores/sql/__init__.py +0 -0
- digitalhub/datastores/sql/datastore.py +68 -0
- digitalhub/entities/__init__.py +0 -0
- digitalhub/entities/_base/__init__.py +0 -0
- digitalhub/entities/_base/_base/__init__.py +0 -0
- digitalhub/entities/_base/_base/entity.py +82 -0
- digitalhub/entities/_base/api_utils.py +620 -0
- digitalhub/entities/_base/context/__init__.py +0 -0
- digitalhub/entities/_base/context/entity.py +118 -0
- digitalhub/entities/_base/crud.py +468 -0
- digitalhub/entities/_base/entity/__init__.py +0 -0
- digitalhub/entities/_base/entity/_constructors/__init__.py +0 -0
- digitalhub/entities/_base/entity/_constructors/metadata.py +44 -0
- digitalhub/entities/_base/entity/_constructors/name.py +31 -0
- digitalhub/entities/_base/entity/_constructors/spec.py +33 -0
- digitalhub/entities/_base/entity/_constructors/status.py +52 -0
- digitalhub/entities/_base/entity/_constructors/uuid.py +26 -0
- digitalhub/entities/_base/entity/builder.py +175 -0
- digitalhub/entities/_base/entity/entity.py +106 -0
- digitalhub/entities/_base/entity/metadata.py +59 -0
- digitalhub/entities/_base/entity/spec.py +58 -0
- digitalhub/entities/_base/entity/status.py +43 -0
- digitalhub/entities/_base/executable/__init__.py +0 -0
- digitalhub/entities/_base/executable/entity.py +405 -0
- digitalhub/entities/_base/material/__init__.py +0 -0
- digitalhub/entities/_base/material/entity.py +214 -0
- digitalhub/entities/_base/material/spec.py +22 -0
- digitalhub/entities/_base/material/status.py +49 -0
- digitalhub/entities/_base/runtime_entity/__init__.py +0 -0
- digitalhub/entities/_base/runtime_entity/builder.py +106 -0
- digitalhub/entities/_base/unversioned/__init__.py +0 -0
- digitalhub/entities/_base/unversioned/builder.py +66 -0
- digitalhub/entities/_base/unversioned/entity.py +49 -0
- digitalhub/entities/_base/versioned/__init__.py +0 -0
- digitalhub/entities/_base/versioned/builder.py +68 -0
- digitalhub/entities/_base/versioned/entity.py +53 -0
- digitalhub/entities/artifact/__init__.py +0 -0
- digitalhub/entities/artifact/_base/__init__.py +0 -0
- digitalhub/entities/artifact/_base/builder.py +86 -0
- digitalhub/entities/artifact/_base/entity.py +39 -0
- digitalhub/entities/artifact/_base/spec.py +15 -0
- digitalhub/entities/artifact/_base/status.py +9 -0
- digitalhub/entities/artifact/artifact/__init__.py +0 -0
- digitalhub/entities/artifact/artifact/builder.py +18 -0
- digitalhub/entities/artifact/artifact/entity.py +32 -0
- digitalhub/entities/artifact/artifact/spec.py +27 -0
- digitalhub/entities/artifact/artifact/status.py +15 -0
- digitalhub/entities/artifact/crud.py +332 -0
- digitalhub/entities/builders.py +63 -0
- digitalhub/entities/dataitem/__init__.py +0 -0
- digitalhub/entities/dataitem/_base/__init__.py +0 -0
- digitalhub/entities/dataitem/_base/builder.py +86 -0
- digitalhub/entities/dataitem/_base/entity.py +75 -0
- digitalhub/entities/dataitem/_base/spec.py +15 -0
- digitalhub/entities/dataitem/_base/status.py +20 -0
- digitalhub/entities/dataitem/crud.py +372 -0
- digitalhub/entities/dataitem/dataitem/__init__.py +0 -0
- digitalhub/entities/dataitem/dataitem/builder.py +18 -0
- digitalhub/entities/dataitem/dataitem/entity.py +32 -0
- digitalhub/entities/dataitem/dataitem/spec.py +15 -0
- digitalhub/entities/dataitem/dataitem/status.py +9 -0
- digitalhub/entities/dataitem/iceberg/__init__.py +0 -0
- digitalhub/entities/dataitem/iceberg/builder.py +18 -0
- digitalhub/entities/dataitem/iceberg/entity.py +32 -0
- digitalhub/entities/dataitem/iceberg/spec.py +15 -0
- digitalhub/entities/dataitem/iceberg/status.py +9 -0
- digitalhub/entities/dataitem/table/__init__.py +0 -0
- digitalhub/entities/dataitem/table/builder.py +18 -0
- digitalhub/entities/dataitem/table/entity.py +146 -0
- digitalhub/entities/dataitem/table/models.py +62 -0
- digitalhub/entities/dataitem/table/spec.py +25 -0
- digitalhub/entities/dataitem/table/status.py +9 -0
- digitalhub/entities/function/__init__.py +0 -0
- digitalhub/entities/function/_base/__init__.py +0 -0
- digitalhub/entities/function/_base/builder.py +79 -0
- digitalhub/entities/function/_base/entity.py +98 -0
- digitalhub/entities/function/_base/models.py +118 -0
- digitalhub/entities/function/_base/spec.py +15 -0
- digitalhub/entities/function/_base/status.py +9 -0
- digitalhub/entities/function/crud.py +279 -0
- digitalhub/entities/model/__init__.py +0 -0
- digitalhub/entities/model/_base/__init__.py +0 -0
- digitalhub/entities/model/_base/builder.py +86 -0
- digitalhub/entities/model/_base/entity.py +34 -0
- digitalhub/entities/model/_base/spec.py +49 -0
- digitalhub/entities/model/_base/status.py +9 -0
- digitalhub/entities/model/crud.py +331 -0
- digitalhub/entities/model/huggingface/__init__.py +0 -0
- digitalhub/entities/model/huggingface/builder.py +18 -0
- digitalhub/entities/model/huggingface/entity.py +32 -0
- digitalhub/entities/model/huggingface/spec.py +36 -0
- digitalhub/entities/model/huggingface/status.py +9 -0
- digitalhub/entities/model/mlflow/__init__.py +0 -0
- digitalhub/entities/model/mlflow/builder.py +18 -0
- digitalhub/entities/model/mlflow/entity.py +32 -0
- digitalhub/entities/model/mlflow/models.py +26 -0
- digitalhub/entities/model/mlflow/spec.py +44 -0
- digitalhub/entities/model/mlflow/status.py +9 -0
- digitalhub/entities/model/mlflow/utils.py +81 -0
- digitalhub/entities/model/model/__init__.py +0 -0
- digitalhub/entities/model/model/builder.py +18 -0
- digitalhub/entities/model/model/entity.py +32 -0
- digitalhub/entities/model/model/spec.py +15 -0
- digitalhub/entities/model/model/status.py +9 -0
- digitalhub/entities/model/sklearn/__init__.py +0 -0
- digitalhub/entities/model/sklearn/builder.py +18 -0
- digitalhub/entities/model/sklearn/entity.py +32 -0
- digitalhub/entities/model/sklearn/spec.py +15 -0
- digitalhub/entities/model/sklearn/status.py +9 -0
- digitalhub/entities/project/__init__.py +0 -0
- digitalhub/entities/project/_base/__init__.py +0 -0
- digitalhub/entities/project/_base/builder.py +128 -0
- digitalhub/entities/project/_base/entity.py +2078 -0
- digitalhub/entities/project/_base/spec.py +50 -0
- digitalhub/entities/project/_base/status.py +9 -0
- digitalhub/entities/project/crud.py +357 -0
- digitalhub/entities/run/__init__.py +0 -0
- digitalhub/entities/run/_base/__init__.py +0 -0
- digitalhub/entities/run/_base/builder.py +94 -0
- digitalhub/entities/run/_base/entity.py +307 -0
- digitalhub/entities/run/_base/spec.py +50 -0
- digitalhub/entities/run/_base/status.py +9 -0
- digitalhub/entities/run/crud.py +219 -0
- digitalhub/entities/secret/__init__.py +0 -0
- digitalhub/entities/secret/_base/__init__.py +0 -0
- digitalhub/entities/secret/_base/builder.py +81 -0
- digitalhub/entities/secret/_base/entity.py +74 -0
- digitalhub/entities/secret/_base/spec.py +35 -0
- digitalhub/entities/secret/_base/status.py +9 -0
- digitalhub/entities/secret/crud.py +290 -0
- digitalhub/entities/task/__init__.py +0 -0
- digitalhub/entities/task/_base/__init__.py +0 -0
- digitalhub/entities/task/_base/builder.py +91 -0
- digitalhub/entities/task/_base/entity.py +136 -0
- digitalhub/entities/task/_base/models.py +208 -0
- digitalhub/entities/task/_base/spec.py +53 -0
- digitalhub/entities/task/_base/status.py +9 -0
- digitalhub/entities/task/crud.py +228 -0
- digitalhub/entities/utils/__init__.py +0 -0
- digitalhub/entities/utils/api.py +346 -0
- digitalhub/entities/utils/entity_types.py +19 -0
- digitalhub/entities/utils/state.py +31 -0
- digitalhub/entities/utils/utils.py +202 -0
- digitalhub/entities/workflow/__init__.py +0 -0
- digitalhub/entities/workflow/_base/__init__.py +0 -0
- digitalhub/entities/workflow/_base/builder.py +79 -0
- digitalhub/entities/workflow/_base/entity.py +74 -0
- digitalhub/entities/workflow/_base/spec.py +15 -0
- digitalhub/entities/workflow/_base/status.py +9 -0
- digitalhub/entities/workflow/crud.py +278 -0
- digitalhub/factory/__init__.py +0 -0
- digitalhub/factory/api.py +277 -0
- digitalhub/factory/factory.py +268 -0
- digitalhub/factory/utils.py +90 -0
- digitalhub/readers/__init__.py +0 -0
- digitalhub/readers/_base/__init__.py +0 -0
- digitalhub/readers/_base/builder.py +26 -0
- digitalhub/readers/_base/reader.py +70 -0
- digitalhub/readers/api.py +80 -0
- digitalhub/readers/factory.py +133 -0
- digitalhub/readers/pandas/__init__.py +0 -0
- digitalhub/readers/pandas/builder.py +29 -0
- digitalhub/readers/pandas/reader.py +207 -0
- digitalhub/runtimes/__init__.py +0 -0
- digitalhub/runtimes/_base.py +102 -0
- digitalhub/runtimes/builder.py +32 -0
- digitalhub/stores/__init__.py +0 -0
- digitalhub/stores/_base/__init__.py +0 -0
- digitalhub/stores/_base/store.py +189 -0
- digitalhub/stores/api.py +54 -0
- digitalhub/stores/builder.py +211 -0
- digitalhub/stores/local/__init__.py +0 -0
- digitalhub/stores/local/store.py +230 -0
- digitalhub/stores/remote/__init__.py +0 -0
- digitalhub/stores/remote/store.py +143 -0
- digitalhub/stores/s3/__init__.py +0 -0
- digitalhub/stores/s3/store.py +563 -0
- digitalhub/stores/sql/__init__.py +0 -0
- digitalhub/stores/sql/store.py +328 -0
- digitalhub/utils/__init__.py +0 -0
- digitalhub/utils/data_utils.py +127 -0
- digitalhub/utils/exceptions.py +67 -0
- digitalhub/utils/file_utils.py +204 -0
- digitalhub/utils/generic_utils.py +183 -0
- digitalhub/utils/git_utils.py +148 -0
- digitalhub/utils/io_utils.py +116 -0
- digitalhub/utils/logger.py +17 -0
- digitalhub/utils/s3_utils.py +58 -0
- digitalhub/utils/uri_utils.py +56 -0
- {digitalhub-0.7.0b2.dist-info → digitalhub-0.8.0.dist-info}/METADATA +30 -13
- digitalhub-0.8.0.dist-info/RECORD +231 -0
- {digitalhub-0.7.0b2.dist-info → digitalhub-0.8.0.dist-info}/WHEEL +1 -1
- test/local/CRUD/test_artifacts.py +96 -0
- test/local/CRUD/test_dataitems.py +96 -0
- test/local/CRUD/test_models.py +95 -0
- test/test_crud_functions.py +1 -1
- test/test_crud_runs.py +1 -1
- test/test_crud_tasks.py +1 -1
- digitalhub-0.7.0b2.dist-info/RECORD +0 -14
- test/test_crud_artifacts.py +0 -96
- test/test_crud_dataitems.py +0 -96
- {digitalhub-0.7.0b2.dist-info → digitalhub-0.8.0.dist-info}/LICENSE.txt +0 -0
- {digitalhub-0.7.0b2.dist-info → digitalhub-0.8.0.dist-info}/top_level.txt +0 -0
- /test/{test_imports.py → local/imports/test_imports.py} +0 -0
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import typing
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from digitalhub.datastores._base.datastore import Datastore
|
|
7
|
+
from digitalhub.readers.api import get_reader_by_object
|
|
8
|
+
|
|
9
|
+
if typing.TYPE_CHECKING:
|
|
10
|
+
from digitalhub.stores.sql.store import SqlStore
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class SqlDatastore(Datastore):
|
|
14
|
+
"""
|
|
15
|
+
Sql Datastore class.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
def __init__(self, store: SqlStore, **kwargs) -> None:
|
|
19
|
+
super().__init__(store, **kwargs)
|
|
20
|
+
self.store: SqlStore
|
|
21
|
+
|
|
22
|
+
def write_df(self, df: Any, dst: str, extension: str | None = None, **kwargs) -> str:
|
|
23
|
+
"""
|
|
24
|
+
Write a dataframe to a database. Kwargs are passed to df.to_sql().
|
|
25
|
+
|
|
26
|
+
Parameters
|
|
27
|
+
----------
|
|
28
|
+
df : Any
|
|
29
|
+
The dataframe to write.
|
|
30
|
+
dst : str
|
|
31
|
+
The destination of the dataframe.
|
|
32
|
+
**kwargs : dict
|
|
33
|
+
Keyword arguments.
|
|
34
|
+
|
|
35
|
+
Returns
|
|
36
|
+
-------
|
|
37
|
+
str
|
|
38
|
+
Path of written dataframe.
|
|
39
|
+
"""
|
|
40
|
+
schema = self.store._get_schema(dst)
|
|
41
|
+
table = self.store._get_table_name(dst)
|
|
42
|
+
return self._upload_table(df, schema, table, **kwargs)
|
|
43
|
+
|
|
44
|
+
def _upload_table(self, df: Any, schema: str, table: str, **kwargs) -> str:
|
|
45
|
+
"""
|
|
46
|
+
Upload a table to SQL based storage.
|
|
47
|
+
|
|
48
|
+
Parameters
|
|
49
|
+
----------
|
|
50
|
+
df : DataFrame
|
|
51
|
+
The dataframe.
|
|
52
|
+
schema : str
|
|
53
|
+
Destination schema.
|
|
54
|
+
table : str
|
|
55
|
+
Destination table.
|
|
56
|
+
**kwargs : dict
|
|
57
|
+
Keyword arguments.
|
|
58
|
+
|
|
59
|
+
Returns
|
|
60
|
+
-------
|
|
61
|
+
str
|
|
62
|
+
The SQL URI where the dataframe was saved.
|
|
63
|
+
"""
|
|
64
|
+
reader = get_reader_by_object(df)
|
|
65
|
+
engine = self.store._check_factory()
|
|
66
|
+
reader.write_table(df, table, engine, schema, **kwargs)
|
|
67
|
+
engine.dispose()
|
|
68
|
+
return f"sql://{engine.url.database}/{schema}/{table}"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from digitalhub.utils.generic_utils import dict_to_json
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Base:
|
|
7
|
+
"""
|
|
8
|
+
Base class for all entities.
|
|
9
|
+
It implements to_dict abd to_json method to represent
|
|
10
|
+
the object as a dictionary/json and an any_setter method to
|
|
11
|
+
set generic attributes coming from a constructor.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
def to_dict(self) -> dict:
|
|
15
|
+
"""
|
|
16
|
+
Return object as dict with all non private keys.
|
|
17
|
+
|
|
18
|
+
Returns
|
|
19
|
+
-------
|
|
20
|
+
dict
|
|
21
|
+
A dictionary containing the attributes of the entity instance.
|
|
22
|
+
"""
|
|
23
|
+
obj = {}
|
|
24
|
+
for key, value in self.__dict__.items():
|
|
25
|
+
if not key.startswith("_") and value is not None:
|
|
26
|
+
if hasattr(value, "to_dict"):
|
|
27
|
+
sub_dict = value.to_dict()
|
|
28
|
+
if sub_dict:
|
|
29
|
+
obj[key] = sub_dict
|
|
30
|
+
else:
|
|
31
|
+
obj[key] = value
|
|
32
|
+
return obj
|
|
33
|
+
|
|
34
|
+
def to_json(self) -> str:
|
|
35
|
+
"""
|
|
36
|
+
Return object as json with all non private keys.
|
|
37
|
+
|
|
38
|
+
Returns
|
|
39
|
+
-------
|
|
40
|
+
str
|
|
41
|
+
A json string containing the attributes of the entity instance.
|
|
42
|
+
"""
|
|
43
|
+
return dict_to_json(self.to_dict())
|
|
44
|
+
|
|
45
|
+
def _any_setter(self, **kwargs) -> None:
|
|
46
|
+
"""
|
|
47
|
+
Set any attribute of the object.
|
|
48
|
+
|
|
49
|
+
Parameters
|
|
50
|
+
----------
|
|
51
|
+
**kwargs : dict
|
|
52
|
+
Keyword arguments to be set as attributes.
|
|
53
|
+
|
|
54
|
+
Returns
|
|
55
|
+
-------
|
|
56
|
+
None
|
|
57
|
+
"""
|
|
58
|
+
for k, v in kwargs.items():
|
|
59
|
+
if k not in self.__dict__:
|
|
60
|
+
setattr(self, k, v)
|
|
61
|
+
|
|
62
|
+
def _get_private_attrs(self) -> dict:
|
|
63
|
+
"""
|
|
64
|
+
Return all private attributes of the object.
|
|
65
|
+
|
|
66
|
+
Returns
|
|
67
|
+
-------
|
|
68
|
+
dict
|
|
69
|
+
A dictionary containing the private attributes of the entity instance.
|
|
70
|
+
"""
|
|
71
|
+
return {k: v for k, v in self.__dict__.items() if k.startswith("_")}
|
|
72
|
+
|
|
73
|
+
def __repr__(self) -> str:
|
|
74
|
+
"""
|
|
75
|
+
Return string representation of the entity object.
|
|
76
|
+
|
|
77
|
+
Returns
|
|
78
|
+
-------
|
|
79
|
+
str
|
|
80
|
+
A string representing the entity instance.
|
|
81
|
+
"""
|
|
82
|
+
return str(self.to_dict())
|