bagofholding 0.0.2__tar.gz → 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bagofholding-0.1.0/.gitignore +13 -0
- {bagofholding-0.0.2 → bagofholding-0.1.0}/PKG-INFO +62 -60
- {bagofholding-0.0.2 → bagofholding-0.1.0}/bagofholding/__init__.py +8 -3
- bagofholding-0.1.0/bagofholding/_version.py +21 -0
- {bagofholding-0.0.2 → bagofholding-0.1.0}/bagofholding/content.py +10 -0
- {bagofholding-0.0.2 → bagofholding-0.1.0}/bagofholding/exceptions.py +1 -1
- {bagofholding-0.0.2 → bagofholding-0.1.0}/bagofholding/retrieve.py +2 -2
- {bagofholding-0.0.2 → bagofholding-0.1.0}/bagofholding/trie.py +6 -3
- bagofholding-0.1.0/cached-miniforge/my-env/lib/python3.1/site-packages/bagofholding/__init__.py +25 -0
- bagofholding-0.1.0/cached-miniforge/my-env/lib/python3.1/site-packages/bagofholding/_version.py +21 -0
- bagofholding-0.1.0/cached-miniforge/my-env/lib/python3.1/site-packages/bagofholding/bag.py +266 -0
- bagofholding-0.1.0/cached-miniforge/my-env/lib/python3.1/site-packages/bagofholding/content.py +907 -0
- bagofholding-0.1.0/cached-miniforge/my-env/lib/python3.1/site-packages/bagofholding/exceptions.py +44 -0
- bagofholding-0.1.0/cached-miniforge/my-env/lib/python3.1/site-packages/bagofholding/h5/__init__.py +0 -0
- bagofholding-0.1.0/cached-miniforge/my-env/lib/python3.1/site-packages/bagofholding/h5/bag.py +183 -0
- bagofholding-0.1.0/cached-miniforge/my-env/lib/python3.1/site-packages/bagofholding/h5/content.py +29 -0
- bagofholding-0.1.0/cached-miniforge/my-env/lib/python3.1/site-packages/bagofholding/h5/context.py +58 -0
- bagofholding-0.1.0/cached-miniforge/my-env/lib/python3.1/site-packages/bagofholding/h5/dtypes.py +73 -0
- bagofholding-0.1.0/cached-miniforge/my-env/lib/python3.1/site-packages/bagofholding/h5/triebag.py +322 -0
- bagofholding-0.1.0/cached-miniforge/my-env/lib/python3.1/site-packages/bagofholding/metadata.py +197 -0
- bagofholding-0.1.0/cached-miniforge/my-env/lib/python3.1/site-packages/bagofholding/retrieve.py +52 -0
- bagofholding-0.1.0/cached-miniforge/my-env/lib/python3.1/site-packages/bagofholding/trie.py +181 -0
- bagofholding-0.1.0/cached-miniforge/my-env/lib/python3.1/site-packages/bagofholding/widget.py +109 -0
- {bagofholding-0.0.2 → bagofholding-0.1.0}/docs/README.md +49 -45
- {bagofholding-0.0.2 → bagofholding-0.1.0}/pyproject.toml +20 -17
- bagofholding-0.0.2/MANIFEST.in +0 -1
- bagofholding-0.0.2/bagofholding/_version.py +0 -21
- bagofholding-0.0.2/bagofholding.egg-info/PKG-INFO +0 -204
- bagofholding-0.0.2/bagofholding.egg-info/SOURCES.txt +0 -25
- bagofholding-0.0.2/bagofholding.egg-info/dependency_links.txt +0 -1
- bagofholding-0.0.2/bagofholding.egg-info/requires.txt +0 -10
- bagofholding-0.0.2/bagofholding.egg-info/top_level.txt +0 -1
- bagofholding-0.0.2/setup.cfg +0 -4
- bagofholding-0.0.2/setup.py +0 -8
- {bagofholding-0.0.2 → bagofholding-0.1.0}/LICENSE +0 -0
- {bagofholding-0.0.2 → bagofholding-0.1.0}/bagofholding/bag.py +0 -0
- {bagofholding-0.0.2 → bagofholding-0.1.0}/bagofholding/h5/__init__.py +0 -0
- {bagofholding-0.0.2 → bagofholding-0.1.0}/bagofholding/h5/bag.py +0 -0
- {bagofholding-0.0.2 → bagofholding-0.1.0}/bagofholding/h5/content.py +0 -0
- {bagofholding-0.0.2 → bagofholding-0.1.0}/bagofholding/h5/context.py +0 -0
- {bagofholding-0.0.2 → bagofholding-0.1.0}/bagofholding/h5/dtypes.py +0 -0
- {bagofholding-0.0.2 → bagofholding-0.1.0}/bagofholding/h5/triebag.py +0 -0
- {bagofholding-0.0.2 → bagofholding-0.1.0}/bagofholding/metadata.py +0 -0
- {bagofholding-0.0.2 → bagofholding-0.1.0}/bagofholding/widget.py +0 -0
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: bagofholding
|
|
3
|
-
Version: 0.0
|
|
3
|
+
Version: 0.1.0
|
|
4
4
|
Summary: bagofholding - browsable, partially-reloadable serialization for pickleable python objects.
|
|
5
|
+
Project-URL: Homepage, https://pyiron.org/
|
|
6
|
+
Project-URL: Documentation, https://bagofholding.readthedocs.io
|
|
7
|
+
Project-URL: Repository, https://github.com/pyiron/bagofholding
|
|
5
8
|
Author-email: Liam Huber <liamhuber@greyhavensolutions.com>
|
|
6
9
|
License: BSD 3-Clause License
|
|
7
10
|
|
|
@@ -31,40 +34,47 @@ License: BSD 3-Clause License
|
|
|
31
34
|
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
32
35
|
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
33
36
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
34
|
-
|
|
35
|
-
Project-URL: Homepage, https://pyiron.org/
|
|
36
|
-
Project-URL: Documentation, https://bagofholding.readthedocs.io
|
|
37
|
-
Project-URL: Repository, https://github.com/pyiron/bagofholding
|
|
37
|
+
License-File: LICENSE
|
|
38
38
|
Keywords: pyiron
|
|
39
|
-
Classifier: Development Status ::
|
|
40
|
-
Classifier: Topic :: Scientific/Engineering
|
|
41
|
-
Classifier: License :: OSI Approved :: BSD License
|
|
39
|
+
Classifier: Development Status :: 4 - Beta
|
|
42
40
|
Classifier: Intended Audience :: Science/Research
|
|
41
|
+
Classifier: License :: OSI Approved :: BSD License
|
|
43
42
|
Classifier: Operating System :: OS Independent
|
|
44
43
|
Classifier: Programming Language :: Python :: 3.12
|
|
45
44
|
Classifier: Programming Language :: Python :: 3.13
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
License-File: LICENSE
|
|
45
|
+
Classifier: Topic :: Scientific/Engineering
|
|
46
|
+
Requires-Python: <3.14,>=3.11
|
|
49
47
|
Requires-Dist: bidict==0.23.1
|
|
50
48
|
Requires-Dist: h5py<3.15.0,>=3.14.0
|
|
51
49
|
Requires-Dist: mpi4py<4.1.0,>=4.0.1
|
|
52
50
|
Requires-Dist: numpy<2.4.0,>=2.3.0
|
|
53
51
|
Requires-Dist: pygtrie<2.6.0,>=2.5.0
|
|
54
|
-
Requires-Dist:
|
|
52
|
+
Requires-Dist: pyiron-snippets==0.2.0
|
|
55
53
|
Provides-Extra: widget
|
|
56
|
-
Requires-Dist: ipytree==0.2.2; extra ==
|
|
57
|
-
Requires-Dist: traitlets==5.14.3; extra ==
|
|
58
|
-
|
|
54
|
+
Requires-Dist: ipytree==0.2.2; extra == 'widget'
|
|
55
|
+
Requires-Dist: traitlets==5.14.3; extra == 'widget'
|
|
56
|
+
Description-Content-Type: text/markdown
|
|
59
57
|
|
|
60
58
|
# bagofholding
|
|
61
59
|
|
|
62
|
-
|
|
60
|
+
<img src="_static/bagofholding_logo.png" alt="Logo" width="190"/>
|
|
61
|
+
|
|
62
|
+
[](https://mybinder.org/v2/gh/pyiron/bagofholding/HEAD)
|
|
63
|
+
[](https://opensource.org/licenses/BSD-3-Clause)
|
|
64
|
+
[](https://codecov.io/gh/pyiron/bagofholding)
|
|
65
|
+
[](https://bagofholding.readthedocs.io/en/latest/?badge=latest)
|
|
66
|
+
|
|
67
|
+
[](https://anaconda.org/conda-forge/bagofholding)
|
|
68
|
+
[](https://anaconda.org/conda-forge/bagofholding)
|
|
69
|
+
[](https://anaconda.org/conda-forge/bagofholding)
|
|
70
|
+
[](https://anaconda.org/conda-forge/bagofholding)
|
|
71
|
+
|
|
72
|
+
`bagofholding` is designed to be an easy stand-in for `pickle` serialization for python objects that is transparent, flexible, and suitable for long-term storage.
|
|
63
73
|
|
|
64
74
|
## Advantages
|
|
65
75
|
### Drop-in replacement
|
|
66
76
|
|
|
67
|
-
`bagofholding` stores
|
|
77
|
+
`bagofholding` stores `pickle`-able python objects, and can be easily used as a drop-in replacement for `pickle` serialization:
|
|
68
78
|
|
|
69
79
|
```python
|
|
70
80
|
>>> import bagofholding as boh
|
|
@@ -78,7 +88,7 @@ Dynamic: license-file
|
|
|
78
88
|
|
|
79
89
|
### Browseable
|
|
80
90
|
|
|
81
|
-
The contents of stored objects can be browsed without
|
|
91
|
+
The contents of stored objects can be browsed without re-instantiating any of the stored data.
|
|
82
92
|
In the example above, we saw that saving is a class-method, while loading is an instance method.
|
|
83
93
|
We can grab the "bag" instance and use it to peek at what's inside!
|
|
84
94
|
|
|
@@ -132,53 +142,32 @@ With only `bagofholding` and `numpy` installed, the end user can browse through
|
|
|
132
142
|
|
|
133
143
|
### Version control
|
|
134
144
|
|
|
135
|
-
In the examples above, we saw that version (and of course package) information is part of the stored metadata.
|
|
136
|
-
This is useful post-facto for knowing what packages need to be installed to properly load your serialized data.
|
|
137
|
-
You can also specify at load-time how strict or relaxed `bagofholding` should be in re-instantiating data if a stored version does not match the currently installed version,
|
|
145
|
+
In the examples above, we saw that version (and of course package) information is part of the automatically-scraped and stored metadata.
|
|
146
|
+
This is useful post-facto for knowing what packages need to be installed to properly load your serialized data, and allows us to fail in clean and helpful ways if the loading environment does not match the saving environment.
|
|
147
|
+
You can also specify at load-time how strict or relaxed `bagofholding` should be in re-instantiating data if a stored version does not match the currently installed version, giving flexible protection from flawed re-instantiations.
|
|
138
148
|
|
|
139
149
|
`bagofholding` also provides tools to act on this data a-priori.
|
|
140
150
|
To increase the likelihood that stored data will be accessible in the future, you can outlaw any (sub)objects coming from particular modules:
|
|
141
151
|
|
|
142
152
|
```python
|
|
143
|
-
import bagofholding
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
boh.
|
|
147
|
-
...
|
|
148
|
-
|
|
149
|
-
print(e)
|
|
150
|
-
Module
|
|
151
|
-
'__main__' is forbidden as a
|
|
152
|
-
source
|
|
153
|
-
of
|
|
154
|
-
stored
|
|
155
|
-
objects.Change
|
|
156
|
-
the
|
|
157
|
-
`forbidden_modules` or move
|
|
158
|
-
this
|
|
159
|
-
object
|
|
160
|
-
to
|
|
161
|
-
an
|
|
162
|
-
allowed
|
|
163
|
-
module.
|
|
153
|
+
import bagofholding as boh
|
|
154
|
+
>>> try:
|
|
155
|
+
... boh.H5Bag.save(something, "will_fail.h5", forbidden_modules=("__main__",))
|
|
156
|
+
... except boh.ModuleForbiddenError as e:
|
|
157
|
+
... print(e)
|
|
158
|
+
Module '__main__' is forbidden as a source of stored objects. Change the `forbidden_modules` or move this object to an allowed module.
|
|
164
159
|
|
|
165
160
|
```
|
|
166
161
|
|
|
167
|
-
And/or demand that all objects have an identifiable version
|
|
162
|
+
And/or demand that all objects have an identifiable version:
|
|
168
163
|
|
|
169
164
|
```python
|
|
170
|
-
import bagofholding
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
boh.
|
|
174
|
-
...
|
|
175
|
-
|
|
176
|
-
print(e)
|
|
177
|
-
Could
|
|
178
|
-
not find
|
|
179
|
-
a
|
|
180
|
-
version
|
|
181
|
-
for __main__.Either disable `require_versions`, use `version_scraping` to find an existing version for this package, or add versioning to the unversioned package.
|
|
165
|
+
import bagofholding as boh
|
|
166
|
+
>>> try:
|
|
167
|
+
... boh.H5Bag.save(something, "will_fail.h5", require_versions=True)
|
|
168
|
+
... except boh.NoVersionError as e:
|
|
169
|
+
... print(e)
|
|
170
|
+
Could not find a version for __main__. Either disable `require_versions`, use `version_scraping` to find an existing version for this package, or add versioning to the unversioned package.
|
|
182
171
|
|
|
183
172
|
```
|
|
184
173
|
|
|
@@ -197,8 +186,21 @@ H5Info(qualname='H5Bag', module='bagofholding.h5.bag', version='...', libver_str
|
|
|
197
186
|
|
|
198
187
|
For a more in-depth look at the above features and to explore other aspects of `bagofholding`, check out [the tutorial notebook](../notebooks/tutorial.ipynb).
|
|
199
188
|
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
189
|
+
|
|
190
|
+
## Object requirements
|
|
191
|
+
|
|
192
|
+
Under-the-hood, we follow the same patterns as `pickle` by explicitly invoking many of the same method (`__reduce__`, `__setstate__`, etc).
|
|
193
|
+
_Almost_ and object which can be pickled can be stored using `bagofholding`.
|
|
194
|
+
Our requirements are that the object...
|
|
195
|
+
|
|
196
|
+
- Must be pickleable
|
|
197
|
+
- You can use the `pickle_check` method on bag classes to quickly assess this
|
|
198
|
+
- Must not depend on `pickle` protocol >4
|
|
199
|
+
- Must have a valid boolean response to `hasattr` for each of the following, and they must conform to python and `abc.collections` norms if present:
|
|
200
|
+
- `__setstate__`
|
|
201
|
+
- `__setitem__`
|
|
202
|
+
- `append`
|
|
203
|
+
- `extend`
|
|
204
|
+
- Must have a valid boolean response to `hasattr` for `__metadata__`, and this attribute must be castable to a string if present
|
|
205
|
+
|
|
206
|
+
If your object satisfies these conditions and fails to "bag", please raise a bug report on the issues page!
|
|
@@ -1,6 +1,11 @@
|
|
|
1
|
-
|
|
1
|
+
import importlib.metadata
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
try:
|
|
4
|
+
# Installed package will find its version
|
|
5
|
+
__version__ = importlib.metadata.version(__name__)
|
|
6
|
+
except importlib.metadata.PackageNotFoundError:
|
|
7
|
+
# Repository clones will register an unknown version
|
|
8
|
+
__version__ = "0.0.0+unknown"
|
|
4
9
|
|
|
5
10
|
from bagofholding.exceptions import BagMismatchError as BagMismatchError
|
|
6
11
|
from bagofholding.exceptions import BagOfHoldingError as BagOfHoldingError
|
|
@@ -14,7 +19,7 @@ from bagofholding.exceptions import NotAGroupError as NotAGroupError
|
|
|
14
19
|
from bagofholding.exceptions import NoVersionError as NoVersionError
|
|
15
20
|
from bagofholding.exceptions import PickleProtocolError as PickleProtocolError
|
|
16
21
|
from bagofholding.exceptions import (
|
|
17
|
-
|
|
22
|
+
StringNotImportableError as StringNotImportableError,
|
|
18
23
|
)
|
|
19
24
|
from bagofholding.h5.bag import H5Bag as H5Bag
|
|
20
25
|
from bagofholding.h5.triebag import TrieH5Bag as TrieH5Bag
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# file generated by setuptools-scm
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
|
|
4
|
+
__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
|
|
5
|
+
|
|
6
|
+
TYPE_CHECKING = False
|
|
7
|
+
if TYPE_CHECKING:
|
|
8
|
+
from typing import Tuple
|
|
9
|
+
from typing import Union
|
|
10
|
+
|
|
11
|
+
VERSION_TUPLE = Tuple[Union[int, str], ...]
|
|
12
|
+
else:
|
|
13
|
+
VERSION_TUPLE = object
|
|
14
|
+
|
|
15
|
+
version: str
|
|
16
|
+
__version__: str
|
|
17
|
+
__version_tuple__: VERSION_TUPLE
|
|
18
|
+
version_tuple: VERSION_TUPLE
|
|
19
|
+
|
|
20
|
+
__version__ = version = '0.1.0'
|
|
21
|
+
__version_tuple__ = version_tuple = (0, 1, 0)
|
|
@@ -35,6 +35,7 @@ from bagofholding.exceptions import (
|
|
|
35
35
|
ModuleForbiddenError,
|
|
36
36
|
NoVersionError,
|
|
37
37
|
PickleProtocolError,
|
|
38
|
+
StringNotImportableError,
|
|
38
39
|
)
|
|
39
40
|
from bagofholding.metadata import (
|
|
40
41
|
Metadata,
|
|
@@ -237,6 +238,15 @@ class Global(Item[GlobalType, Any, Packer]):
|
|
|
237
238
|
value = "builtins." + obj if "." not in obj else obj
|
|
238
239
|
else:
|
|
239
240
|
value = obj.__module__ + "." + obj.__qualname__
|
|
241
|
+
|
|
242
|
+
if "<lambda>" in value:
|
|
243
|
+
raise StringNotImportableError(
|
|
244
|
+
f"Lambda functions are not re-importable, can't pack {obj}"
|
|
245
|
+
)
|
|
246
|
+
elif "<locals>" in value:
|
|
247
|
+
raise StringNotImportableError(
|
|
248
|
+
f"Local functions are not re-importable, can't pack {obj}"
|
|
249
|
+
)
|
|
240
250
|
packer.pack_string(value, path)
|
|
241
251
|
|
|
242
252
|
@classmethod
|
|
@@ -7,7 +7,7 @@ from __future__ import annotations
|
|
|
7
7
|
from importlib import import_module
|
|
8
8
|
from typing import Any
|
|
9
9
|
|
|
10
|
-
from bagofholding.exceptions import
|
|
10
|
+
from bagofholding.exceptions import StringNotImportableError
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
def import_from_string(library_path: str) -> Any:
|
|
@@ -44,7 +44,7 @@ def get_importable_string_from_string_reduction(
|
|
|
44
44
|
try:
|
|
45
45
|
import_from_string(importable)
|
|
46
46
|
except (ModuleNotFoundError, AttributeError) as e:
|
|
47
|
-
raise
|
|
47
|
+
raise StringNotImportableError(
|
|
48
48
|
f"Couldn't import {string_reduction} after scoping it as {importable}. "
|
|
49
49
|
f"Please contact the developers so we can figure out how to handle "
|
|
50
50
|
f"this edge case."
|
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
import random
|
|
2
|
-
from typing import cast
|
|
2
|
+
from typing import TypeVar, cast
|
|
3
3
|
|
|
4
4
|
import numpy as np
|
|
5
5
|
import pygtrie
|
|
6
6
|
|
|
7
|
+
ValueType = TypeVar("ValueType") # python <3.12 compatibility
|
|
8
|
+
# def not_allowed_to_have_inline[FunctionGenerics](in_python=3.12):
|
|
7
9
|
|
|
8
|
-
|
|
10
|
+
|
|
11
|
+
def decompose_stringtrie(
|
|
9
12
|
trie: pygtrie.StringTrie, null_value: ValueType
|
|
10
13
|
) -> tuple[list[str], list[int], list[ValueType]]:
|
|
11
14
|
"""
|
|
@@ -63,7 +66,7 @@ def decompose_stringtrie[ValueType](
|
|
|
63
66
|
return segments, parents, values
|
|
64
67
|
|
|
65
68
|
|
|
66
|
-
def reconstruct_stringtrie
|
|
69
|
+
def reconstruct_stringtrie(
|
|
67
70
|
segments: list[str],
|
|
68
71
|
parents: list[int],
|
|
69
72
|
values: list[ValueType],
|
bagofholding-0.1.0/cached-miniforge/my-env/lib/python3.1/site-packages/bagofholding/__init__.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import importlib.metadata
|
|
2
|
+
|
|
3
|
+
try:
|
|
4
|
+
# Installed package will find its version
|
|
5
|
+
__version__ = importlib.metadata.version(__name__)
|
|
6
|
+
except importlib.metadata.PackageNotFoundError:
|
|
7
|
+
# Repository clones will register an unknown version
|
|
8
|
+
__version__ = "0.0.0+unknown"
|
|
9
|
+
|
|
10
|
+
from bagofholding.exceptions import BagMismatchError as BagMismatchError
|
|
11
|
+
from bagofholding.exceptions import BagOfHoldingError as BagOfHoldingError
|
|
12
|
+
from bagofholding.exceptions import EnvironmentMismatchError as EnvironmentMismatchError
|
|
13
|
+
from bagofholding.exceptions import FileAlreadyOpenError as FileAlreadyOpenError
|
|
14
|
+
from bagofholding.exceptions import FileNotOpenError as FileNotOpenError
|
|
15
|
+
from bagofholding.exceptions import FilepathError as FilepathError
|
|
16
|
+
from bagofholding.exceptions import InvalidMetadataError as InvalidMetadataError
|
|
17
|
+
from bagofholding.exceptions import ModuleForbiddenError as ModuleForbiddenError
|
|
18
|
+
from bagofholding.exceptions import NotAGroupError as NotAGroupError
|
|
19
|
+
from bagofholding.exceptions import NoVersionError as NoVersionError
|
|
20
|
+
from bagofholding.exceptions import PickleProtocolError as PickleProtocolError
|
|
21
|
+
from bagofholding.exceptions import (
|
|
22
|
+
StringNotImportableError as StringNotImportableError,
|
|
23
|
+
)
|
|
24
|
+
from bagofholding.h5.bag import H5Bag as H5Bag
|
|
25
|
+
from bagofholding.h5.triebag import TrieH5Bag as TrieH5Bag
|
bagofholding-0.1.0/cached-miniforge/my-env/lib/python3.1/site-packages/bagofholding/_version.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# file generated by setuptools-scm
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
|
|
4
|
+
__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
|
|
5
|
+
|
|
6
|
+
TYPE_CHECKING = False
|
|
7
|
+
if TYPE_CHECKING:
|
|
8
|
+
from typing import Tuple
|
|
9
|
+
from typing import Union
|
|
10
|
+
|
|
11
|
+
VERSION_TUPLE = Tuple[Union[int, str], ...]
|
|
12
|
+
else:
|
|
13
|
+
VERSION_TUPLE = object
|
|
14
|
+
|
|
15
|
+
version: str
|
|
16
|
+
__version__: str
|
|
17
|
+
__version_tuple__: VERSION_TUPLE
|
|
18
|
+
version_tuple: VERSION_TUPLE
|
|
19
|
+
|
|
20
|
+
__version__ = version = '0.1.0'
|
|
21
|
+
__version_tuple__ = version_tuple = (0, 1, 0)
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
"""
|
|
2
|
+
The core user-facing object.
|
|
3
|
+
|
|
4
|
+
Full implementations of bags should guarantee the key features promised by the package:
|
|
5
|
+
- Storage and retrieval of arbitrary pickleable python objects
|
|
6
|
+
- Metadata preservation
|
|
7
|
+
- Versioning verification
|
|
8
|
+
- Browsing without loading
|
|
9
|
+
- Partial reloading
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import abc
|
|
15
|
+
import dataclasses
|
|
16
|
+
import os.path
|
|
17
|
+
import pathlib
|
|
18
|
+
import pickle
|
|
19
|
+
from collections.abc import Iterator, Mapping
|
|
20
|
+
from typing import (
|
|
21
|
+
Any,
|
|
22
|
+
ClassVar,
|
|
23
|
+
Self,
|
|
24
|
+
SupportsIndex,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
import bidict
|
|
28
|
+
from pyiron_snippets import import_alarm
|
|
29
|
+
|
|
30
|
+
from bagofholding.content import BespokeItem, Packer, pack, unpack
|
|
31
|
+
from bagofholding.exceptions import BagMismatchError, InvalidMetadataError
|
|
32
|
+
from bagofholding.metadata import (
|
|
33
|
+
HasFieldIterator,
|
|
34
|
+
HasVersionInfo,
|
|
35
|
+
Metadata,
|
|
36
|
+
VersionScrapingMap,
|
|
37
|
+
VersionValidatorType,
|
|
38
|
+
get_version,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
try:
|
|
42
|
+
from bagofholding.widget import BagTree
|
|
43
|
+
|
|
44
|
+
alarm = import_alarm.ImportAlarm()
|
|
45
|
+
except (ImportError, ModuleNotFoundError):
|
|
46
|
+
alarm = import_alarm.ImportAlarm(
|
|
47
|
+
"The browsing widget relies on ipytree and traitlets, but this was "
|
|
48
|
+
"unavailable. You can get a text-representation of all available paths with "
|
|
49
|
+
":meth:`bagofholding.bag.Bag.list_paths`.",
|
|
50
|
+
_fail_on_warning=True,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
PATH_DELIMITER = "/"
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclasses.dataclass(frozen=True)
|
|
57
|
+
class BagInfo(HasVersionInfo, HasFieldIterator):
|
|
58
|
+
pass
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class Bag(Packer, Mapping[str, Metadata | None], abc.ABC):
|
|
62
|
+
"""
|
|
63
|
+
Bags are the user-facing object.
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
bag_info: BagInfo
|
|
67
|
+
storage_root: ClassVar[str] = "object"
|
|
68
|
+
filepath: pathlib.Path
|
|
69
|
+
|
|
70
|
+
@classmethod
|
|
71
|
+
def get_bag_info(cls) -> BagInfo:
|
|
72
|
+
return BagInfo(
|
|
73
|
+
qualname=cls.__qualname__,
|
|
74
|
+
module=cls.__module__,
|
|
75
|
+
version=cls.get_version(),
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
@classmethod
|
|
79
|
+
def _bag_info_class(cls) -> type[BagInfo]:
|
|
80
|
+
return BagInfo
|
|
81
|
+
|
|
82
|
+
@classmethod
|
|
83
|
+
def save(
|
|
84
|
+
cls,
|
|
85
|
+
obj: Any,
|
|
86
|
+
filepath: str | pathlib.Path,
|
|
87
|
+
require_versions: bool = False,
|
|
88
|
+
forbidden_modules: list[str] | tuple[str, ...] = (),
|
|
89
|
+
version_scraping: VersionScrapingMap | None = None,
|
|
90
|
+
_pickle_protocol: SupportsIndex = pickle.DEFAULT_PROTOCOL,
|
|
91
|
+
overwrite_existing: bool = True,
|
|
92
|
+
) -> None:
|
|
93
|
+
"""
|
|
94
|
+
Save a python object to file.
|
|
95
|
+
|
|
96
|
+
Args:
|
|
97
|
+
obj (Any): The (pickleble) python object to be saved.
|
|
98
|
+
filepath (str|pathlib.Path): The path to save the object to.
|
|
99
|
+
require_versions (bool): Whether to require a metadata for reduced
|
|
100
|
+
and complex objects to contain a non-None version. (Default is False,
|
|
101
|
+
objects can be stored from non-versioned packages/modules.)
|
|
102
|
+
forbidden_modules (list[str] | tuple[str, ...] | None): Do not allow saving
|
|
103
|
+
objects whose root-most modules are listed here. (Default is an empty
|
|
104
|
+
tuple, i.e. don't disallow anything.) This is particularly useful to
|
|
105
|
+
disallow `"__main__"` to improve the odds that objects will actually
|
|
106
|
+
be loadable in the future.
|
|
107
|
+
version_scraping (dict[str, Callable[[str], str]] | None): An optional
|
|
108
|
+
dictionary mapping module names to a callable that takes this name and
|
|
109
|
+
returns a version (or None). The default callable imports the module
|
|
110
|
+
string and looks for a `__version__` attribute.
|
|
111
|
+
"""
|
|
112
|
+
if os.path.exists(filepath):
|
|
113
|
+
if overwrite_existing and os.path.isfile(filepath):
|
|
114
|
+
os.remove(filepath)
|
|
115
|
+
else:
|
|
116
|
+
raise FileExistsError(f"{filepath} already exists or is not a file.")
|
|
117
|
+
bag = cls(filepath)
|
|
118
|
+
bag._pack_bag_info()
|
|
119
|
+
pack(
|
|
120
|
+
obj,
|
|
121
|
+
bag,
|
|
122
|
+
bag.storage_root,
|
|
123
|
+
bidict.bidict(),
|
|
124
|
+
[],
|
|
125
|
+
require_versions,
|
|
126
|
+
forbidden_modules,
|
|
127
|
+
version_scraping,
|
|
128
|
+
_pickle_protocol=_pickle_protocol,
|
|
129
|
+
)
|
|
130
|
+
bag._write()
|
|
131
|
+
|
|
132
|
+
@classmethod
|
|
133
|
+
def get_version(cls) -> str:
|
|
134
|
+
return str(get_version(cls.__module__, {}))
|
|
135
|
+
|
|
136
|
+
def __init__(
|
|
137
|
+
self, filepath: str | pathlib.Path, *args: object, **kwargs: Any
|
|
138
|
+
) -> None:
|
|
139
|
+
super().__init__(*args, **kwargs)
|
|
140
|
+
self.filepath = pathlib.Path(filepath)
|
|
141
|
+
if os.path.isfile(self.filepath):
|
|
142
|
+
self.bag_info = self._unpack_bag_info()
|
|
143
|
+
if not self.validate_bag_info(self.bag_info, self.get_bag_info()):
|
|
144
|
+
raise BagMismatchError(
|
|
145
|
+
f"The bag class {self.__class__} does not match the bag saved at "
|
|
146
|
+
f"{filepath}; class info is {self.get_bag_info()}, but the info saved "
|
|
147
|
+
f"is {self.bag_info}"
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
@abc.abstractmethod
|
|
151
|
+
def _pack_field(self, path: str, key: str, value: str) -> None: ...
|
|
152
|
+
|
|
153
|
+
@abc.abstractmethod
|
|
154
|
+
def _unpack_field(self, path: str, key: str) -> str | None: ...
|
|
155
|
+
|
|
156
|
+
@staticmethod
|
|
157
|
+
def validate_bag_info(bag_info: BagInfo, reference: BagInfo) -> bool:
|
|
158
|
+
return bag_info == reference
|
|
159
|
+
|
|
160
|
+
def load(
|
|
161
|
+
self,
|
|
162
|
+
path: str = storage_root,
|
|
163
|
+
version_validator: VersionValidatorType = "exact",
|
|
164
|
+
version_scraping: VersionScrapingMap | None = None,
|
|
165
|
+
) -> Any:
|
|
166
|
+
return unpack(
|
|
167
|
+
self,
|
|
168
|
+
path,
|
|
169
|
+
{},
|
|
170
|
+
version_validator=version_validator,
|
|
171
|
+
version_scraping=version_scraping,
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
def __getitem__(self, path: str) -> Metadata:
|
|
175
|
+
return self.unpack_metadata(path)
|
|
176
|
+
|
|
177
|
+
@abc.abstractmethod
|
|
178
|
+
def list_paths(self) -> list[str]:
|
|
179
|
+
"""A list of all available content paths."""
|
|
180
|
+
|
|
181
|
+
@alarm
|
|
182
|
+
def widget(self): # type: ignore[no-untyped-def]
|
|
183
|
+
return BagTree(self)
|
|
184
|
+
|
|
185
|
+
def browse(self): # type: ignore[no-untyped-def]
|
|
186
|
+
try:
|
|
187
|
+
return self.widget()
|
|
188
|
+
except ImportError:
|
|
189
|
+
return self.list_paths()
|
|
190
|
+
|
|
191
|
+
def __len__(self) -> int:
|
|
192
|
+
return len(self.list_paths())
|
|
193
|
+
|
|
194
|
+
def __iter__(self) -> Iterator[str]:
|
|
195
|
+
return iter(self.list_paths())
|
|
196
|
+
|
|
197
|
+
def join(self, *paths: str) -> str:
|
|
198
|
+
return PATH_DELIMITER.join(paths)
|
|
199
|
+
|
|
200
|
+
@staticmethod
|
|
201
|
+
def pickle_check(
|
|
202
|
+
obj: Any, raise_exceptions: bool = True, print_message: bool = False
|
|
203
|
+
) -> str | None:
|
|
204
|
+
"""
|
|
205
|
+
A simple helper to check if an object can be pickled and unpickled.
|
|
206
|
+
Useful if you run into trouble saving or loading and want to see whether the
|
|
207
|
+
underlying object is compliant with pickle-ability requirements to begin with.
|
|
208
|
+
|
|
209
|
+
Args:
|
|
210
|
+
obj: The object to test for pickling support.
|
|
211
|
+
raise_exceptions: If True, re-raise any exception encountered.
|
|
212
|
+
print_message: If True, print the exception message on failure.
|
|
213
|
+
|
|
214
|
+
Returns:
|
|
215
|
+
None if pickling is successful; otherwise, returns the exception message as a string.
|
|
216
|
+
"""
|
|
217
|
+
|
|
218
|
+
try:
|
|
219
|
+
pickle.loads(pickle.dumps(obj))
|
|
220
|
+
except Exception as e:
|
|
221
|
+
if print_message:
|
|
222
|
+
print(e)
|
|
223
|
+
if raise_exceptions:
|
|
224
|
+
raise e
|
|
225
|
+
return str(e)
|
|
226
|
+
return None
|
|
227
|
+
|
|
228
|
+
def _pack_fields(self, dataclass: HasFieldIterator, path: str) -> None:
|
|
229
|
+
for k, v in dataclass.field_items():
|
|
230
|
+
if v is not None:
|
|
231
|
+
self._pack_field(path, k, v)
|
|
232
|
+
|
|
233
|
+
def _unpack_fields(
|
|
234
|
+
self, dataclass_type: type[HasFieldIterator], path: str
|
|
235
|
+
) -> dict[str, str | None]:
|
|
236
|
+
field_values: dict[str, str | None] = {}
|
|
237
|
+
for k in dataclass_type.__dataclass_fields__:
|
|
238
|
+
field_values[k] = self._unpack_field(path, k)
|
|
239
|
+
return field_values
|
|
240
|
+
|
|
241
|
+
def _pack_bag_info(self) -> None:
|
|
242
|
+
self._pack_fields(self.get_bag_info(), PATH_DELIMITER)
|
|
243
|
+
|
|
244
|
+
def _unpack_bag_info(self) -> BagInfo:
|
|
245
|
+
return self._bag_info_class()(
|
|
246
|
+
**self._unpack_fields(self._bag_info_class(), PATH_DELIMITER)
|
|
247
|
+
)
|
|
248
|
+
|
|
249
|
+
def _write(self) -> None:
|
|
250
|
+
return
|
|
251
|
+
|
|
252
|
+
def pack_metadata(self, metadata: Metadata, path: str) -> None:
|
|
253
|
+
self._pack_fields(metadata, path)
|
|
254
|
+
return None
|
|
255
|
+
|
|
256
|
+
def unpack_metadata(self, path: str) -> Metadata:
|
|
257
|
+
metadata = self._unpack_fields(Metadata, path)
|
|
258
|
+
content_type = metadata.pop("content_type", None)
|
|
259
|
+
if content_type is None:
|
|
260
|
+
raise InvalidMetadataError(f"Metadata at {path} is missing a content type")
|
|
261
|
+
return Metadata(content_type, **metadata)
|
|
262
|
+
|
|
263
|
+
def get_bespoke_content_class(
|
|
264
|
+
self, obj: object
|
|
265
|
+
) -> type[BespokeItem[Any, Self]] | None:
|
|
266
|
+
return None
|