pymodaq_data 5.2.13__tar.gz → 5.2.14__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/PKG-INFO +1 -1
  2. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/data.py +1 -1
  3. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/h5modules/backends.py +21 -3
  4. pymodaq_data-5.2.14/src/pymodaq_data/h5modules/migration.py +90 -0
  5. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/.gitignore +0 -0
  6. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/LICENSE +0 -0
  7. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/README.rst +0 -0
  8. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/pyproject.toml +0 -0
  9. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/__init__.py +0 -0
  10. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/config.py +0 -0
  11. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/h5modules/__init__.py +0 -0
  12. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/h5modules/browsing.py +0 -0
  13. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/h5modules/data_saving.py +0 -0
  14. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/h5modules/exporter.py +0 -0
  15. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/h5modules/exporters/__init__.py +0 -0
  16. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/h5modules/exporters/base.py +0 -0
  17. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/h5modules/exporters/flimj.py +0 -0
  18. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/h5modules/exporters/hyperspy.py +0 -0
  19. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/h5modules/saving.py +0 -0
  20. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/h5modules/swmr.py +0 -0
  21. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/h5modules/utils.py +0 -0
  22. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/icon.ico +0 -0
  23. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/numpy_func.py +0 -0
  24. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/plotting/__init__.py +0 -0
  25. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/plotting/plotter/plotter.py +0 -0
  26. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/plotting/plotter/plotters/__init__.py +0 -0
  27. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/plotting/plotter/plotters/matplotlib_plotters.py +0 -0
  28. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/plotting/utils.py +0 -0
  29. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/post_treatment/__init__.py +0 -0
  30. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/post_treatment/process_to_scalar.py +0 -0
  31. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/resources/__init__.py +0 -0
  32. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/resources/config_template.toml +0 -0
  33. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/slicing.py +0 -0
  34. {pymodaq_data-5.2.13 → pymodaq_data-5.2.14}/src/pymodaq_data/splash.png +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: pymodaq_data
3
- Version: 5.2.13
3
+ Version: 5.2.14
4
4
  Summary: Modular Data Acquisition with Python
5
5
  Project-URL: Homepage, http://pymodaq.cnrs.fr
6
6
  Project-URL: Source, https://github.com/PyMoDAQ/PyMoDAQ
@@ -2894,7 +2894,7 @@ class DataWithAxes(DataBase, SerializableBase):
2894
2894
  return False
2895
2895
  return True
2896
2896
  try:
2897
- if errors is not None and check_errors(data, errors):
2897
+ if errors is not None and not check_errors(data, errors):
2898
2898
  errors = None
2899
2899
  old_data = self.data
2900
2900
  self._data = None
@@ -10,11 +10,11 @@ from pathlib import Path
10
10
  import numpy as np
11
11
  import importlib
12
12
  from importlib import metadata
13
- import pickle
14
13
 
15
14
  from pymodaq_utils.logger import set_logger, get_module_name
16
15
  from pymodaq_utils.config import GlobalConfig as Config
17
16
  from pymodaq_utils.utils import capitalize, JsonConverter
17
+ from pymodaq_utils.safe_pickle import restricted_loads, warn_legacy_format
18
18
  from pymodaq_utils import utils
19
19
  from pymodaq_utils.enums import BaseEnum, enum_checker
20
20
 
@@ -476,10 +476,28 @@ class StringARRAY(VLARRAY):
476
476
  super().append(data)
477
477
 
478
478
  def array_to_string(self, array):
479
- return pickle.loads(array)
479
+ """Decode a stored string
480
+
481
+ New files hold the plain UTF-8 bytes of the string. Files written by older versions hold a pickle
482
+ (always starting with the 0x80 protocol byte, which can never start a valid UTF-8 text): these are
483
+ read by a restricted loader that only rebuilds plain data and never runs code.
484
+ """
485
+ raw = bytes(array)
486
+ if raw[:1] == b'\x80':
487
+ warn_legacy_format(f'The string node {self.path} of the file {self._file_name()}')
488
+ return restricted_loads(raw)
489
+ return raw.decode('utf-8')
480
490
 
481
491
  def string_to_array(self, string):
482
- return np.frombuffer(pickle.dumps(string), np.uint8)
492
+ return np.frombuffer(string.encode('utf-8'), np.uint8)
493
+
494
+ def _file_name(self) -> str:
495
+ try:
496
+ if self.backend == 'tables':
497
+ return str(self.node._v_file.filename)
498
+ return str(self.node.file.filename)
499
+ except Exception:
500
+ return '(unknown)'
483
501
 
484
502
 
485
503
  class Attributes(object):
@@ -0,0 +1,90 @@
1
+ """Migration of the h5 files written by older versions of PyMoDAQ
2
+
3
+ Older versions stored the strings of the string nodes (such as the logger) as pickles. They are still read, but
4
+ the migration rewrites them as plain UTF-8, as the current version does, in a copy of the file.
5
+ """
6
+ import os
7
+ import shutil
8
+ from pathlib import Path
9
+ from typing import Optional, Tuple, Union
10
+
11
+ import numpy as np
12
+
13
+ from pymodaq_utils.safe_pickle import restricted_loads, UnsafePickleError
14
+
15
+ from pymodaq_data.h5modules.backends import get_attr
16
+
17
+
18
+ def default_destination(source: Path) -> Path:
19
+ return source.with_name(f'{source.stem}_migrated{source.suffix}')
20
+
21
+
22
+ def migrate_h5_file(source: Union[str, Path], destination: Optional[Union[str, Path]] = None) -> Tuple[Path, int]:
23
+ """Write a copy of a h5 file in which the pickled strings are rewritten as UTF-8
24
+
25
+ The source file is never modified, and an existing destination file is never overwritten.
26
+
27
+ Parameters
28
+ ----------
29
+ source: str or Path
30
+ The file to migrate
31
+ destination: str or Path
32
+ The file to create, by default the name of the source followed by _migrated
33
+
34
+ Returns
35
+ -------
36
+ Path: the migrated file
37
+ int: the number of string nodes that were migrated
38
+
39
+ Raises
40
+ ------
41
+ FileExistsError
42
+ if the destination exists
43
+ UnsafePickleError
44
+ if a string node contains anything else than a string, in this case nothing is written
45
+ """
46
+ import h5py
47
+
48
+ source = Path(source)
49
+ destination = default_destination(source) if destination is None else Path(destination)
50
+ if destination.exists():
51
+ raise FileExistsError(f'{destination} already exists, it will not be overwritten')
52
+ part = destination.with_name(destination.name + '.part')
53
+ shutil.copyfile(source, part)
54
+ try:
55
+ migrated = 0
56
+ with h5py.File(part, 'r+') as h5file:
57
+ to_migrate = []
58
+ h5file.visititems(lambda name, obj: to_migrate.append(obj) if _is_string_node(obj) else None)
59
+ for dataset in to_migrate:
60
+ migrated += _migrate_string_node(dataset)
61
+ os.replace(part, destination)
62
+ finally:
63
+ if part.exists():
64
+ part.unlink()
65
+ return destination, migrated
66
+
67
+
68
+ def _is_string_node(obj) -> bool:
69
+ import h5py
70
+ if not isinstance(obj, h5py.Dataset):
71
+ return False
72
+ try:
73
+ return get_attr(obj, 'CLASS', 'h5py') == 'VLARRAY' and get_attr(obj, 'subdtype', 'h5py') == 'string'
74
+ except KeyError:
75
+ return False
76
+
77
+
78
+ def _migrate_string_node(dataset) -> int:
79
+ """Rewrite the pickled strings of a node, returns 1 if the node had some, 0 otherwise"""
80
+ new_values = {}
81
+ for index in range(len(dataset)):
82
+ raw = bytes(dataset[index])
83
+ if raw[:1] == b'\x80':
84
+ text = restricted_loads(raw)
85
+ if not isinstance(text, str):
86
+ raise UnsafePickleError(f'Refused to migrate {dataset.name}: it does not contain only strings')
87
+ new_values[index] = text
88
+ for index, text in new_values.items(): # nothing is written if a value above was refused
89
+ dataset[index] = np.frombuffer(text.encode('utf-8'), np.uint8)
90
+ return 1 if new_values else 0
File without changes
File without changes
File without changes