pymodaq_data 5.3.9__tar.gz → 5.3.10__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/PKG-INFO +12 -12
  2. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/README.rst +11 -11
  3. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/h5modules/backends.py +21 -3
  4. pymodaq_data-5.3.10/src/pymodaq_data/h5modules/migration.py +90 -0
  5. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/.gitignore +0 -0
  6. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/LICENSE +0 -0
  7. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/pyproject.toml +0 -0
  8. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/__init__.py +0 -0
  9. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/config.py +0 -0
  10. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/data.py +0 -0
  11. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/h5modules/__init__.py +0 -0
  12. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/h5modules/browsing.py +0 -0
  13. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/h5modules/data_saving.py +0 -0
  14. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/h5modules/exporter.py +0 -0
  15. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/h5modules/exporters/__init__.py +0 -0
  16. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/h5modules/exporters/base.py +0 -0
  17. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/h5modules/exporters/flimj.py +0 -0
  18. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/h5modules/exporters/hyperspy.py +0 -0
  19. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/h5modules/saving.py +0 -0
  20. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/h5modules/swmr.py +0 -0
  21. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/h5modules/utils.py +0 -0
  22. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/icon.ico +0 -0
  23. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/numpy_func.py +0 -0
  24. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/plotting/__init__.py +0 -0
  25. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/plotting/plotter/plotter.py +0 -0
  26. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/plotting/plotter/plotters/__init__.py +0 -0
  27. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/plotting/plotter/plotters/matplotlib_plotters.py +0 -0
  28. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/plotting/utils.py +0 -0
  29. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/post_treatment/__init__.py +0 -0
  30. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/post_treatment/process_to_scalar.py +0 -0
  31. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/py.typed +0 -0
  32. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/resources/__init__.py +0 -0
  33. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/resources/config_template.toml +0 -0
  34. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/slicing.py +0 -0
  35. {pymodaq_data-5.3.9 → pymodaq_data-5.3.10}/src/pymodaq_data/splash.png +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: pymodaq_data
3
- Version: 5.3.9
3
+ Version: 5.3.10
4
4
  Summary: Modular Data Acquisition with Python
5
5
  Project-URL: Homepage, http://pymodaq.cnrs.fr
6
6
  Project-URL: Source, https://github.com/PyMoDAQ/PyMoDAQ
@@ -71,11 +71,11 @@ PyMoDAQ Data
71
71
  :target: https://pypi.org/project/pymodaq_data/
72
72
  :alt: Latest Version
73
73
 
74
- .. image:: https://readthedocs.org/projects/pymodaq/badge/?version=patch-3
75
- :target: https://pymodaq.readthedocs.io/en/stable/?badge=patch-3
74
+ .. image:: https://readthedocs.org/projects/pymodaq/badge/?version=5.3.x
75
+ :target: https://pymodaq.readthedocs.io/en/stable/?badge=5.3.x
76
76
  :alt: Documentation Status
77
77
 
78
- .. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/patch-3/graph/badge.svg?token=IQNJRCQDM2
78
+ .. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/5.3.x/graph/badge.svg?token=IQNJRCQDM2
79
79
  :target: https://codecov.io/gh/PyMoDAQ/PyMoDAQ
80
80
 
81
81
  +-------------+-------------+---------------+
@@ -94,28 +94,28 @@ PyMoDAQ Data
94
94
 
95
95
 
96
96
 
97
- .. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Linux_3.10.svg
97
+ .. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.10.svg
98
98
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
99
99
 
100
- .. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Linux_3.11.svg
100
+ .. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.11.svg
101
101
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
102
102
 
103
- .. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Linux_3.12.svg
103
+ .. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.12.svg
104
104
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
105
105
 
106
- .. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Linux_3.13.svg
106
+ .. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.13.svg
107
107
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
108
108
 
109
- .. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Windows_3.10.svg
109
+ .. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.10.svg
110
110
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
111
111
 
112
- .. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Windows_3.11.svg
112
+ .. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.11.svg
113
113
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
114
114
 
115
- .. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Windows_3.12.svg
115
+ .. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.12.svg
116
116
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
117
117
 
118
- .. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Windows_3.13.svg
118
+ .. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.13.svg
119
119
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
120
120
 
121
121
 
@@ -11,11 +11,11 @@ PyMoDAQ Data
11
11
  :target: https://pypi.org/project/pymodaq_data/
12
12
  :alt: Latest Version
13
13
 
14
- .. image:: https://readthedocs.org/projects/pymodaq/badge/?version=patch-3
15
- :target: https://pymodaq.readthedocs.io/en/stable/?badge=patch-3
14
+ .. image:: https://readthedocs.org/projects/pymodaq/badge/?version=5.3.x
15
+ :target: https://pymodaq.readthedocs.io/en/stable/?badge=5.3.x
16
16
  :alt: Documentation Status
17
17
 
18
- .. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/patch-3/graph/badge.svg?token=IQNJRCQDM2
18
+ .. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/5.3.x/graph/badge.svg?token=IQNJRCQDM2
19
19
  :target: https://codecov.io/gh/PyMoDAQ/PyMoDAQ
20
20
 
21
21
  +-------------+-------------+---------------+
@@ -34,28 +34,28 @@ PyMoDAQ Data
34
34
 
35
35
 
36
36
 
37
- .. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Linux_3.10.svg
37
+ .. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.10.svg
38
38
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
39
39
 
40
- .. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Linux_3.11.svg
40
+ .. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.11.svg
41
41
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
42
42
 
43
- .. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Linux_3.12.svg
43
+ .. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.12.svg
44
44
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
45
45
 
46
- .. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Linux_3.13.svg
46
+ .. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Linux_3.13.svg
47
47
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
48
48
 
49
- .. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Windows_3.10.svg
49
+ .. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.10.svg
50
50
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
51
51
 
52
- .. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Windows_3.11.svg
52
+ .. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.11.svg
53
53
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
54
54
 
55
- .. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Windows_3.12.svg
55
+ .. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.12.svg
56
56
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
57
57
 
58
- .. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/patch-3/tests_Windows_3.13.svg
58
+ .. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.3.x/tests_Windows_3.13.svg
59
59
  :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
60
60
 
61
61
 
@@ -10,11 +10,11 @@ from pathlib import Path
10
10
  import numpy as np
11
11
  import importlib
12
12
  from importlib import metadata
13
- import pickle
14
13
 
15
14
  from pymodaq_utils.logger import set_logger, get_module_name
16
15
  from pymodaq_utils.config import GlobalConfig as Config
17
16
  from pymodaq_utils.utils import capitalize, JsonConverter
17
+ from pymodaq_utils.safe_pickle import restricted_loads, warn_legacy_format
18
18
  from pymodaq_utils import utils
19
19
  from pymodaq_utils.enums import BaseEnum, enum_checker
20
20
 
@@ -475,10 +475,28 @@ class StringARRAY(VLARRAY):
475
475
  super().append(data)
476
476
 
477
477
  def array_to_string(self, array):
478
- return pickle.loads(array)
478
+ """Decode a stored string
479
+
480
+ New files hold the plain UTF-8 bytes of the string. Files written by older versions hold a pickle
481
+ (always starting with the 0x80 protocol byte, which can never start a valid UTF-8 text): these are
482
+ read by a restricted loader that only rebuilds plain data and never runs code.
483
+ """
484
+ raw = bytes(array)
485
+ if raw[:1] == b'\x80':
486
+ warn_legacy_format(f'The string node {self.path} of the file {self._file_name()}')
487
+ return restricted_loads(raw)
488
+ return raw.decode('utf-8')
479
489
 
480
490
  def string_to_array(self, string):
481
- return np.frombuffer(pickle.dumps(string), np.uint8)
491
+ return np.frombuffer(string.encode('utf-8'), np.uint8)
492
+
493
+ def _file_name(self) -> str:
494
+ try:
495
+ if self.backend == 'tables':
496
+ return str(self.node._v_file.filename)
497
+ return str(self.node.file.filename)
498
+ except Exception:
499
+ return '(unknown)'
482
500
 
483
501
 
484
502
  class Attributes(object):
@@ -0,0 +1,90 @@
1
+ """Migration of the h5 files written by older versions of PyMoDAQ
2
+
3
+ Older versions stored the strings of the string nodes (such as the logger) as pickles. They are still read, but
4
+ the migration rewrites them as plain UTF-8, as the current version does, in a copy of the file.
5
+ """
6
+ import os
7
+ import shutil
8
+ from pathlib import Path
9
+ from typing import Optional, Tuple, Union
10
+
11
+ import numpy as np
12
+
13
+ from pymodaq_utils.safe_pickle import restricted_loads, UnsafePickleError
14
+
15
+ from pymodaq_data.h5modules.backends import get_attr
16
+
17
+
18
+ def default_destination(source: Path) -> Path:
19
+ return source.with_name(f'{source.stem}_migrated{source.suffix}')
20
+
21
+
22
+ def migrate_h5_file(source: Union[str, Path], destination: Optional[Union[str, Path]] = None) -> Tuple[Path, int]:
23
+ """Write a copy of a h5 file in which the pickled strings are rewritten as UTF-8
24
+
25
+ The source file is never modified, and an existing destination file is never overwritten.
26
+
27
+ Parameters
28
+ ----------
29
+ source: str or Path
30
+ The file to migrate
31
+ destination: str or Path
32
+ The file to create, by default the name of the source followed by _migrated
33
+
34
+ Returns
35
+ -------
36
+ Path: the migrated file
37
+ int: the number of string nodes that were migrated
38
+
39
+ Raises
40
+ ------
41
+ FileExistsError
42
+ if the destination exists
43
+ UnsafePickleError
44
+ if a string node contains anything else than a string, in this case nothing is written
45
+ """
46
+ import h5py
47
+
48
+ source = Path(source)
49
+ destination = default_destination(source) if destination is None else Path(destination)
50
+ if destination.exists():
51
+ raise FileExistsError(f'{destination} already exists, it will not be overwritten')
52
+ part = destination.with_name(destination.name + '.part')
53
+ shutil.copyfile(source, part)
54
+ try:
55
+ migrated = 0
56
+ with h5py.File(part, 'r+') as h5file:
57
+ to_migrate = []
58
+ h5file.visititems(lambda name, obj: to_migrate.append(obj) if _is_string_node(obj) else None)
59
+ for dataset in to_migrate:
60
+ migrated += _migrate_string_node(dataset)
61
+ os.replace(part, destination)
62
+ finally:
63
+ if part.exists():
64
+ part.unlink()
65
+ return destination, migrated
66
+
67
+
68
+ def _is_string_node(obj) -> bool:
69
+ import h5py
70
+ if not isinstance(obj, h5py.Dataset):
71
+ return False
72
+ try:
73
+ return get_attr(obj, 'CLASS', 'h5py') == 'VLARRAY' and get_attr(obj, 'subdtype', 'h5py') == 'string'
74
+ except KeyError:
75
+ return False
76
+
77
+
78
+ def _migrate_string_node(dataset) -> int:
79
+ """Rewrite the pickled strings of a node, returns 1 if the node had some, 0 otherwise"""
80
+ new_values = {}
81
+ for index in range(len(dataset)):
82
+ raw = bytes(dataset[index])
83
+ if raw[:1] == b'\x80':
84
+ text = restricted_loads(raw)
85
+ if not isinstance(text, str):
86
+ raise UnsafePickleError(f'Refused to migrate {dataset.name}: it does not contain only strings')
87
+ new_values[index] = text
88
+ for index, text in new_values.items(): # nothing is written if a value above was refused
89
+ dataset[index] = np.frombuffer(text.encode('utf-8'), np.uint8)
90
+ return 1 if new_values else 0
File without changes
File without changes