bagofholding 0.1.6__py3-none-any.whl → 0.1.8__py3-none-any.whl

bagofholding/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.1.6'
-__version_tuple__ = version_tuple = (0, 1, 6)
+__version__ = version = '0.1.8'
+__version_tuple__ = version_tuple = (0, 1, 8)
 
 __commit_id__ = commit_id = None

bagofholding/content.py

@@ -871,9 +871,23 @@ KNOWN_GROUP_MAP: dict[type, type[Group[Any, Any]]] = {
 }
 
 
+def is_simple_string(string: str) -> bool:
+    return string.isascii() and ("/" not in string)
+
+
+def has_surrogates(s: str) -> bool:
+    try:
+        s.encode("utf-8")
+        return False
+    except UnicodeEncodeError:
+        return True
+
+
 def get_group_content_class(obj: object) -> type[Group[Any, Any]] | None:
     t = type(obj)
-    if t is dict and all(isinstance(k, str) for k in cast(dict[str, Any], obj)):
+    if t is dict and all(
+        isinstance(k, str) and is_simple_string(k) for k in cast(dict[str, Any], obj)
+    ):
         return StrKeyDict
 
     return KNOWN_GROUP_MAP.get(t)

bagofholding/h5/bag.py

@@ -8,7 +8,7 @@ import h5py
 import numpy as np
 
 from bagofholding.bag import Bag, BagInfo
-from bagofholding.content import BespokeItem
+from bagofholding.content import BespokeItem, has_surrogates
 from bagofholding.exceptions import NotAGroupError
 from bagofholding.h5.content import Array, ArrayPacker, ArrayType
 from bagofholding.h5.context import HasH5FileContext
@@ -107,11 +107,21 @@ class H5Bag(Bag, HasH5FileContext, ArrayPacker):
         self.file.create_dataset(path, data=h5py.Empty(dtype="f"))
 
     def pack_string(self, obj: str, path: str) -> None:
-        self.file.create_dataset(
-            path, data=obj, dtype=h5py.string_dtype(encoding="utf-8")
-        )
+        if has_surrogates(obj):
+            encoded = obj.encode("utf-16", errors="surrogatepass")
+            self.file.create_dataset(path, data=np.void(encoded))
+            self.file[path].attrs["_surrogate_str"] = True
+        else:
+            self.file.create_dataset(
+                path, data=obj, dtype=h5py.string_dtype(encoding="utf-8")
+            )
 
     def unpack_string(self, path: str) -> str:
+        if self.file[path].attrs.get("_surrogate_str", False):
+            return cast(
+                str,
+                self.file[path][()].tobytes().decode("utf-16", errors="surrogatepass"),
+            )
         return cast(str, self._unpack_raw(path).decode("utf-8"))
 
     def _pack_raw(self, obj: bytearray | bool | int | float, path: str) -> None:
@@ -146,7 +156,11 @@ class H5Bag(Bag, HasH5FileContext, ArrayPacker):
         return complex(data[0], data[1])
 
     def pack_bytes(self, obj: bytes, path: str) -> None:
-        self.file.create_dataset(path, data=np.void(obj))
+        if obj == b"":
+            special = h5py.special_dtype(vlen=bytes)
+            self.file.create_dataset(path, data=b"", dtype=special)
+        else:
+            self.file.create_dataset(path, data=np.void(obj))
 
     def unpack_bytes(self, path: str) -> bytes:
         return bytes(self._unpack_raw(path))
@@ -172,7 +186,7 @@ class H5Bag(Bag, HasH5FileContext, ArrayPacker):
     def get_bespoke_content_class(
         self, obj: object
     ) -> type[BespokeItem[Any, Self]] | None:
-        if type(obj) is np.ndarray and obj.dtype in H5PY_DTYPE_WHITELIST:
+        if type(obj) is np.ndarray and obj.dtype.type in H5PY_DTYPE_WHITELIST:
             return cast(type[BespokeItem[Any, Self]], Array)
         return None
 

bagofholding/h5/dtypes.py

@@ -23,8 +23,9 @@ H5PY_DTYPE_WHITELIST = (
     np.complex64,
     np.complex128,
     np.bool_,
-    np.bytes_,
-    np.str_,
+    # np.bytes_,
+    # np.str_,
+    # h5py needs special handling for byte and string info
 )
 
 

bagofholding/h5/triebag.py

@@ -9,7 +9,7 @@ import numpy as np
 import pygtrie
 
 from bagofholding.bag import PATH_DELIMITER, Bag, BagInfo
-from bagofholding.content import BespokeItem
+from bagofholding.content import BespokeItem, has_surrogates
 from bagofholding.h5.bag import H5Info
 from bagofholding.h5.content import Array, ArrayPacker, ArrayType
 from bagofholding.h5.context import HasH5FileContext
@@ -54,6 +54,8 @@ class TrieH5Bag(Bag, HasH5FileContext, ArrayPacker):
             "array": 8,
             "empty": 9,
             "group": 10,
+            "empty_bytes": 11,
+            "surrogate_str": 12,
         }
     )
     _field_delimiter: ClassVar[str] = "::"
@@ -95,7 +97,8 @@ class TrieH5Bag(Bag, HasH5FileContext, ArrayPacker):
             list[bytes],
             list[bytearray],
             list[ArrayType],
-        ] = ([], [], [], [], [], [], [], [], [])
+            list[bytes],
+        ] = ([], [], [], [], [], [], [], [], [], [])
 
     @property
     def unpacked_trie(self) -> pygtrie.StringTrie:
@@ -133,16 +136,26 @@ class TrieH5Bag(Bag, HasH5FileContext, ArrayPacker):
         self.file.create_dataset(
             "complex_imag", data=np.array(self._packed[5], dtype=float)
         )
-        self.file.create_dataset("bytes", data=np.array(self._packed[6]))
-        self.file.create_dataset(
-            "bytearray", data=np.array(self._packed[7])
-        )  # dtype=bytearray
+        bytes_group = self.file.create_group("bytes_data")
+        for i, b in enumerate(self._packed[6]):
+            bytes_group.create_dataset(f"i{i}", data=np.void(b))
+        bytearray_group = self.file.create_group("bytearrays")
+        for i, ba in enumerate(self._packed[7]):
+            bytearray_group.create_dataset(f"i{i}", data=ba)
+        # A reasonable alternative would be to group bytes and bytearrays by their
+        # length, creating fewer overall groups. Benefit depends on how heavily weighted
+        # unique lengths are to shared lengths, so for now keep it simple
         array_group = self.file.create_group("ndarrays")
         for i, ra in enumerate(self._packed[8]):
             array_group.create_dataset(f"i{i}", data=ra)
         # Empty doesn't need to be packed -- it's always None so the meta info is enough
         # Groups don't need to be packed -- they are just holders so meta info is enough
 
+        # Some string encoding requires special treatment to play with h5py
+        surrogate_group = self.file.create_group("surrogate_strs")
+        for i, s in enumerate(self._packed[9]):
+            surrogate_group.create_dataset(f"i{i}", data=np.void(s))
+
         self.close()
 
     def _unpack_bag_info(self) -> BagInfo:
@@ -235,9 +248,21 @@ class TrieH5Bag(Bag, HasH5FileContext, ArrayPacker):
         self._pack_trie(path, type_index, len(group) - 1)
 
     def pack_string(self, obj: str, path: str) -> None:
-        self._pack_thing(obj, "str", path)
+        if has_surrogates(obj):
+            encoded = obj.encode("utf-16", errors="surrogatepass")
+            type_index = self._index_map["surrogate_str"]
+            group = self._packed[9]
+            group.append(encoded)
+            self._pack_trie(path, type_index, len(group) - 1)
+        else:
+            self._pack_thing(obj, "str", path)
 
     def unpack_string(self, path: str) -> str:
+        type_index, position_index = self._read_trie(path)
+        if self._index_map.inverse[type_index] == "surrogate_str":
+            with self:
+                data = self.file[f"surrogate_strs/i{position_index}"][()]
+            return cast(str, data.tobytes().decode("utf-16", errors="surrogatepass"))
         return self.maybe_decode(cast(str, self._read_pathlike(path)))
 
     def pack_bool(self, obj: bool, path: str) -> None:
@@ -277,16 +302,27 @@ class TrieH5Bag(Bag, HasH5FileContext, ArrayPacker):
         return value
 
     def pack_bytes(self, obj: bytes, path: str) -> None:
-        self._pack_thing(obj, "bytes", path)
+        if obj == b"":
+            self._pack_trie(path, self._index_map["empty_bytes"], -1)
+        else:
+            self._pack_thing(obj, "bytes", path)
 
     def unpack_bytes(self, path: str) -> bytes:
-        return cast(bytes, self._read_pathlike(path).tobytes())
+        type_index, position_index = self._read_trie(path)
+        if self._index_map.inverse[type_index] == "empty_bytes":
+            return b""
+        else:
+            with self:
+                return bytes(self.file[f"bytes_data/i{position_index}"][()])
 
     def pack_bytearray(self, obj: bytearray, path: str) -> None:
         self._pack_thing(obj, "bytearray", path)
 
     def unpack_bytearray(self, path: str) -> bytearray:
-        return bytearray(self._read_pathlike(path))
+        _, position_index = self._read_trie(path)
+        with self:
+            value = bytearray(self.file[f"bytearrays/i{position_index}"][()])
+        return value
 
     def create_group(self, path: str) -> None:
         self._pack_trie(path, self._index_map["group"], -1)
@@ -308,7 +344,7 @@ class TrieH5Bag(Bag, HasH5FileContext, ArrayPacker):
     def get_bespoke_content_class(
         self, obj: object
     ) -> type[BespokeItem[Any, Self]] | None:
-        if type(obj) is np.ndarray and obj.dtype in H5PY_DTYPE_WHITELIST:
+        if type(obj) is np.ndarray and obj.dtype.type in H5PY_DTYPE_WHITELIST:
             return cast(type[BespokeItem[Any, Self]], Array)
         return None
 

{bagofholding-0.1.6.dist-info → bagofholding-0.1.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bagofholding
-Version: 0.1.6
+Version: 0.1.8
 Summary: bagofholding - browsable, partially-reloadable serialization for pickleable python objects.
 Project-URL: Homepage, https://pyiron.org/
 Project-URL: Documentation, https://bagofholding.readthedocs.io
@@ -48,9 +48,11 @@ Classifier: Topic :: Scientific/Engineering
 Requires-Python: <3.15,>=3.11
 Requires-Dist: bidict==0.23.1
 Requires-Dist: h5py<3.16.0,>=3.12.1
-Requires-Dist: numpy<2.4.0,>=1.26.4
+Requires-Dist: numpy<2.5.0,>=1.26.4
 Requires-Dist: pygtrie<2.6.0,>=2.5.0
 Requires-Dist: pyiron-snippets<2.0.0,>=1.0.0
+Provides-Extra: dev
+Requires-Dist: hypothesis==6.151.5; extra == 'dev'
 Provides-Extra: widget
 Requires-Dist: ipytree==0.2.2; extra == 'widget'
 Requires-Dist: traitlets==5.14.3; extra == 'widget'

{bagofholding-0.1.6.dist-info → bagofholding-0.1.8.dist-info}/RECORD

@@ -1,18 +1,18 @@
 bagofholding/__init__.py,sha256=a8YLAb8TmNVGW-zwWOGBk1iUA4vJLYp92TklXCjV5sw,1314
-bagofholding/_version.py,sha256=riGXiVTWXmtdoju9hVCWvTxpszEMAAIK0sZZWoLKlnU,704
+bagofholding/_version.py,sha256=Zaz3s9gl_rzsS46-ymJOALojMxviW77EJq_agE8knLk,704
 bagofholding/bag.py,sha256=i4ves0M_nMR0e7WT-KQQQZOdb29LEQwc0PmXwsBtSuU,8642
-bagofholding/content.py,sha256=zezO4YDEHnuRqElFPxJkewBStQqu5A3AHogjV3blXT4,26337
+bagofholding/content.py,sha256=0eN55ytBL-1YdYnGpuXpLqqS0m8_1qbPOO4VEEEtU5s,26617
 bagofholding/exceptions.py,sha256=e6Vwc6ok7xXvdFy3k2hE7LytiVn1w0PaHH1Cwv06wu0,939
 bagofholding/metadata.py,sha256=F7Yv6RB1p_BbW3dKSodx7AcMoUWUFX8zWd9iz6owwMg,6866
 bagofholding/trie.py,sha256=rAWxR8hITeGZimSsAWp-_A2ymLNbBFLJXEtC_en__S8,6083
 bagofholding/widget.py,sha256=DCsSKxZ855lt5dgoxhT4_YoHbyqygO7D2rJxPaKtyWg,3608
 bagofholding/h5/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-bagofholding/h5/bag.py,sha256=Rt-L8vTr1acwBg3_9A6xfkOYlIWXXh3-Lf4m_TwR-f8,5801
+bagofholding/h5/bag.py,sha256=XEfGlAkKmE7vdRy_ImOApIMAovPw3f2s6mxh7Av8OTw,6445
 bagofholding/h5/content.py,sha256=xAHfeMsm6c8ZgQ0RvkFrF1ooDA3BhMPSx9e6MfG8FLM,970
 bagofholding/h5/context.py,sha256=JT70idJPUF0jEtIpZObTrQ8Xcf595dFSEGyss7dYKR8,1613
-bagofholding/h5/dtypes.py,sha256=7WKR0U-RsM2IAMPWnxDFCKgg9WLabD0j9Py72-fF8MM,1206
-bagofholding/h5/triebag.py,sha256=fyYKweuMLms3wSDFB7_DTVsoMv9I1y8kLwVxxztQcYk,12119
-bagofholding-0.1.6.dist-info/METADATA,sha256=nkMjQdLiLzZDt9pNjkmB01FJ8kgU65AnJNW8pvnHuUI,10014
-bagofholding-0.1.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-bagofholding-0.1.6.dist-info/licenses/LICENSE,sha256=bNZX-3pFSnVGhRfng8SbI2FgEKLZb2nGO5jdsbwjEtk,1542
-bagofholding-0.1.6.dist-info/RECORD,,
+bagofholding/h5/dtypes.py,sha256=trcwpQfShiarcN2Fg5HI2p18vakfUH74mms0_CeyzLc,1269
+bagofholding/h5/triebag.py,sha256=ck8hCsupiJTYj9GUWTTaadivenst1XfM0i0bLxF14TY,13932
+bagofholding-0.1.8.dist-info/METADATA,sha256=dsTzNb_rQVvjnvUSV5ZTB7tIQvnAo88bsQAU8x_6HPk,10085
+bagofholding-0.1.8.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+bagofholding-0.1.8.dist-info/licenses/LICENSE,sha256=bNZX-3pFSnVGhRfng8SbI2FgEKLZb2nGO5jdsbwjEtk,1542
+bagofholding-0.1.8.dist-info/RECORD,,

{bagofholding-0.1.6.dist-info → bagofholding-0.1.8.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: hatchling 1.28.0
 Root-Is-Purelib: true
 Tag: py3-none-any