pymodaq_data 5.1.7__tar.gz → 5.1.15__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/PKG-INFO +19 -20
  2. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/README.rst +16 -16
  3. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/pyproject.toml +1 -2
  4. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/src/pymodaq_data/data.py +4 -4
  5. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/src/pymodaq_data/h5modules/backends.py +21 -3
  6. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/src/pymodaq_data/h5modules/data_saving.py +1 -1
  7. pymodaq_data-5.1.15/src/pymodaq_data/h5modules/migration.py +90 -0
  8. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/.gitignore +0 -0
  9. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/LICENSE +0 -0
  10. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/src/pymodaq_data/__init__.py +0 -0
  11. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/src/pymodaq_data/h5modules/__init__.py +0 -0
  12. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/src/pymodaq_data/h5modules/browsing.py +0 -0
  13. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/src/pymodaq_data/h5modules/exporter.py +0 -0
  14. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/src/pymodaq_data/h5modules/exporters/__init__.py +0 -0
  15. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/src/pymodaq_data/h5modules/exporters/base.py +0 -0
  16. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/src/pymodaq_data/h5modules/exporters/flimj.py +0 -0
  17. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/src/pymodaq_data/h5modules/exporters/hyperspy.py +0 -0
  18. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/src/pymodaq_data/h5modules/saving.py +0 -0
  19. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/src/pymodaq_data/h5modules/utils.py +0 -0
  20. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/src/pymodaq_data/icon.ico +0 -0
  21. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/src/pymodaq_data/numpy_func.py +0 -0
  22. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/src/pymodaq_data/plotting/__init__.py +0 -0
  23. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/src/pymodaq_data/plotting/plotter/plotter.py +0 -0
  24. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/src/pymodaq_data/plotting/plotter/plotters/__init__.py +0 -0
  25. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/src/pymodaq_data/plotting/plotter/plotters/matplotlib_plotters.py +0 -0
  26. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/src/pymodaq_data/post_treatment/__init__.py +0 -0
  27. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/src/pymodaq_data/post_treatment/process_to_scalar.py +0 -0
  28. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/src/pymodaq_data/slicing.py +0 -0
  29. {pymodaq_data-5.1.7 → pymodaq_data-5.1.15}/src/pymodaq_data/splash.png +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: pymodaq_data
3
- Version: 5.1.7
3
+ Version: 5.1.15
4
4
  Summary: Modular Data Acquisition with Python
5
5
  Project-URL: Homepage, http://pymodaq.cnrs.fr
6
6
  Project-URL: Source, https://github.com/PyMoDAQ/PyMoDAQ
@@ -45,8 +45,7 @@ Classifier: Topic :: Software Development :: User Interfaces
45
45
  Requires-Python: >=3.8
46
46
  Requires-Dist: pymodaq-utils>=0.0.8
47
47
  Requires-Dist: scipy
48
- Requires-Dist: tables<3.9; python_version < '3.10'
49
- Requires-Dist: tables>=3.10; python_version >= '3.10'
48
+ Requires-Dist: tables>=3.10
50
49
  Provides-Extra: dev
51
50
  Requires-Dist: flake8; extra == 'dev'
52
51
  Requires-Dist: h5py; extra == 'dev'
@@ -69,52 +68,52 @@ PyMoDAQ Data
69
68
  :target: https://pypi.org/project/pymodaq_data/
70
69
  :alt: Latest Version
71
70
 
72
- .. image:: https://readthedocs.org/projects/pymodaq/badge/?version=latest
73
- :target: https://pymodaq.readthedocs.io/en/stable/?badge=latest
71
+ .. image:: https://readthedocs.org/projects/pymodaq/badge/?version=5.1.x
72
+ :target: https://pymodaq.readthedocs.io/en/stable/?badge=5.1.x
74
73
  :alt: Documentation Status
75
74
 
76
- .. image:: https://codecov.io/gh/PyMoDAQ/pymodaq_data/branch/5.0.x_dev/graph/badge.svg?token=H32JflMEYR
77
- :target: https://codecov.io/gh/PyMoDAQ/pymodaq_data
75
+ .. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/5.1.x/graph/badge.svg?token=IQNJRCQDM2
76
+ :target: https://codecov.io/gh/PyMoDAQ/PyMoDAQ
78
77
 
79
78
  +-------------+-------------+---------------+
80
79
  | | Linux | Windows |
81
80
  +=============+=============+===============+
82
- | Python 3.9 | |39-linux| | |39-windows| |
83
- +-------------+-------------+---------------+
84
81
  | Python 3.10 | |310-linux| | |310-windows| |
85
82
  +-------------+-------------+---------------+
86
83
  | Python 3.11 | |311-linux| | |311-windows| |
87
84
  +-------------+-------------+---------------+
88
85
  | Python 3.12 | |312-linux| | |312-windows| |
89
86
  +-------------+-------------+---------------+
87
+ | Python 3.13 | |313-linux| | |313-windows| |
88
+ +-------------+-------------+---------------+
90
89
 
91
90
 
92
91
 
93
92
 
94
- .. |39-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Linux_3.9.svg
95
- :target: https://github.com/PyMoDAQ/pymodaq_data/actions/workflows/tests-data.yml
96
93
 
97
94
  .. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Linux_3.10.svg
98
- :target: https://github.com/PyMoDAQ/pymodaq_data/actions/workflows/tests-data.yml
95
+ :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
99
96
 
100
97
  .. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Linux_3.11.svg
101
- :target: https://github.com/PyMoDAQ/pymodaq_data/actions/workflows/tests-data.yml
98
+ :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
102
99
 
103
100
  .. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Linux_3.12.svg
104
- :target: https://github.com/PyMoDAQ/pymodaq_data/actions/workflows/tests-data.yml
101
+ :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
105
102
 
106
- .. |39-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Windows_3.9.svg
107
- :target: https://github.com/PyMoDAQ/pymodaq_data/actions/workflows/tests-data.yml
103
+ .. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Linux_3.13.svg
104
+ :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
108
105
 
109
106
  .. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Windows_3.10.svg
110
- :target: https://github.com/PyMoDAQ/pymodaq_data/actions/workflows/tests-data.yml
107
+ :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
111
108
 
112
109
  .. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Windows_3.11.svg
113
- :target: https://github.com/PyMoDAQ/pymodaq_data/actions/workflows/tests-data.yml
110
+ :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
114
111
 
115
112
  .. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Windows_3.12.svg
116
- :target: https://github.com/PyMoDAQ/pymodaq_data/actions/workflows/tests-data.yml
113
+ :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
117
114
 
115
+ .. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Windows_3.13.svg
116
+ :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
118
117
 
119
118
 
120
119
 
@@ -11,52 +11,52 @@ PyMoDAQ Data
11
11
  :target: https://pypi.org/project/pymodaq_data/
12
12
  :alt: Latest Version
13
13
 
14
- .. image:: https://readthedocs.org/projects/pymodaq/badge/?version=latest
15
- :target: https://pymodaq.readthedocs.io/en/stable/?badge=latest
14
+ .. image:: https://readthedocs.org/projects/pymodaq/badge/?version=5.1.x
15
+ :target: https://pymodaq.readthedocs.io/en/stable/?badge=5.1.x
16
16
  :alt: Documentation Status
17
17
 
18
- .. image:: https://codecov.io/gh/PyMoDAQ/pymodaq_data/branch/5.0.x_dev/graph/badge.svg?token=H32JflMEYR
19
- :target: https://codecov.io/gh/PyMoDAQ/pymodaq_data
18
+ .. image:: https://codecov.io/gh/PyMoDAQ/PyMoDAQ/branch/5.1.x/graph/badge.svg?token=IQNJRCQDM2
19
+ :target: https://codecov.io/gh/PyMoDAQ/PyMoDAQ
20
20
 
21
21
  +-------------+-------------+---------------+
22
22
  | | Linux | Windows |
23
23
  +=============+=============+===============+
24
- | Python 3.9 | |39-linux| | |39-windows| |
25
- +-------------+-------------+---------------+
26
24
  | Python 3.10 | |310-linux| | |310-windows| |
27
25
  +-------------+-------------+---------------+
28
26
  | Python 3.11 | |311-linux| | |311-windows| |
29
27
  +-------------+-------------+---------------+
30
28
  | Python 3.12 | |312-linux| | |312-windows| |
31
29
  +-------------+-------------+---------------+
30
+ | Python 3.13 | |313-linux| | |313-windows| |
31
+ +-------------+-------------+---------------+
32
32
 
33
33
 
34
34
 
35
35
 
36
- .. |39-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Linux_3.9.svg
37
- :target: https://github.com/PyMoDAQ/pymodaq_data/actions/workflows/tests-data.yml
38
36
 
39
37
  .. |310-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Linux_3.10.svg
40
- :target: https://github.com/PyMoDAQ/pymodaq_data/actions/workflows/tests-data.yml
38
+ :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
41
39
 
42
40
  .. |311-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Linux_3.11.svg
43
- :target: https://github.com/PyMoDAQ/pymodaq_data/actions/workflows/tests-data.yml
41
+ :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
44
42
 
45
43
  .. |312-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Linux_3.12.svg
46
- :target: https://github.com/PyMoDAQ/pymodaq_data/actions/workflows/tests-data.yml
44
+ :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
47
45
 
48
- .. |39-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Windows_3.9.svg
49
- :target: https://github.com/PyMoDAQ/pymodaq_data/actions/workflows/tests-data.yml
46
+ .. |313-linux| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Linux_3.13.svg
47
+ :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
50
48
 
51
49
  .. |310-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Windows_3.10.svg
52
- :target: https://github.com/PyMoDAQ/pymodaq_data/actions/workflows/tests-data.yml
50
+ :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
53
51
 
54
52
  .. |311-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Windows_3.11.svg
55
- :target: https://github.com/PyMoDAQ/pymodaq_data/actions/workflows/tests-data.yml
53
+ :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
56
54
 
57
55
  .. |312-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Windows_3.12.svg
58
- :target: https://github.com/PyMoDAQ/pymodaq_data/actions/workflows/tests-data.yml
56
+ :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
59
57
 
58
+ .. |313-windows| image:: https://raw.githubusercontent.com/PyMoDAQ/PyMoDAQ/badges/pymodaq_data/5.1.x/tests_Windows_3.13.svg
59
+ :target: https://github.com/PyMoDAQ/PyMoDAQ/actions/workflows/tests-data.yml
60
60
 
61
61
 
62
62
 
@@ -33,8 +33,7 @@ classifiers = [
33
33
  dependencies = [
34
34
  "pymodaq_utils>=0.0.8",
35
35
  "scipy",
36
- "tables>=3.10; python_version>=\"3.10\"", # issue with some version of required package blosc2>=2.2.8
37
- "tables<3.9; python_version<\"3.10\"", # issue with some version of required package blosc2>=2.2.8
36
+ "tables>=3.10",
38
37
  ]
39
38
 
40
39
  [project.optional-dependencies]
@@ -1536,12 +1536,12 @@ class AxesManagerBase:
1536
1536
  return axes
1537
1537
 
1538
1538
  def is_axis_signal(self, axis: Axis) -> bool:
1539
- """Check if an axis is considered signal or navigation"""
1540
- return axis.index in self._nav_indexes
1539
+ """Check if an axis is a signal one (its index is not a navigation index)"""
1540
+ return axis.index not in self._nav_indexes
1541
1541
 
1542
1542
  def is_axis_navigation(self, axis: Axis) -> bool:
1543
- """Check if an axis is considered signal or navigation"""
1544
- return axis.index not in self._nav_indexes
1543
+ """Check if an axis is a navigation one (its index is a navigation index)"""
1544
+ return axis.index in self._nav_indexes
1545
1545
 
1546
1546
  def __repr__(self):
1547
1547
  return self._get_dimension_str()
@@ -9,12 +9,12 @@ from enum import Enum
9
9
  import numpy as np
10
10
  import importlib
11
11
  from importlib import metadata
12
- import pickle
13
12
  from typing import Dict
14
13
 
15
14
  from pymodaq_utils.logger import set_logger, get_module_name
16
15
  from pymodaq_utils.config import Config
17
16
  from pymodaq_utils.utils import capitalize, JsonConverter
17
+ from pymodaq_utils.safe_pickle import restricted_loads, warn_legacy_format
18
18
  from pymodaq_utils import utils
19
19
  from pymodaq_utils.enums import BaseEnum, enum_checker
20
20
 
@@ -450,10 +450,28 @@ class StringARRAY(VLARRAY):
450
450
  super().append(data)
451
451
 
452
452
  def array_to_string(self, array):
453
- return pickle.loads(array)
453
+ """Decode a stored string
454
+
455
+ New files hold the plain UTF-8 bytes of the string. Files written by older versions hold a pickle
456
+ (always starting with the 0x80 protocol byte, which can never start a valid UTF-8 text): these are
457
+ read by a restricted loader that only rebuilds plain data and never runs code.
458
+ """
459
+ raw = bytes(array)
460
+ if raw[:1] == b'\x80':
461
+ warn_legacy_format(f'The string node {self.path} of the file {self._file_name()}')
462
+ return restricted_loads(raw)
463
+ return raw.decode('utf-8')
454
464
 
455
465
  def string_to_array(self, string):
456
- return np.frombuffer(pickle.dumps(string), np.uint8)
466
+ return np.frombuffer(string.encode('utf-8'), np.uint8)
467
+
468
+ def _file_name(self) -> str:
469
+ try:
470
+ if self.backend == 'tables':
471
+ return str(self.node._v_file.filename)
472
+ return str(self.node.file.filename)
473
+ except Exception:
474
+ return '(unknown)'
457
475
 
458
476
 
459
477
  class Attributes(object):
@@ -971,7 +971,7 @@ class DataToExportExtendedSaver(DataToExportSaver):
971
971
  self._nav_axis_saver.add_axis(nav_group, axis)
972
972
 
973
973
  def add_data(self, where: Union[Node, str], data: DataToExport, indexes: Iterable[int],
974
- distribution=DataDistribution['uniform'],
974
+ distribution=DataDistribution.uniform,
975
975
  settings_as_xml='', **kwargs):
976
976
 
977
977
  """
@@ -0,0 +1,90 @@
1
+ """Migration of the h5 files written by older versions of PyMoDAQ
2
+
3
+ Older versions stored the strings of the string nodes (such as the logger) as pickles. They are still read, but
4
+ the migration rewrites them as plain UTF-8, as the current version does, in a copy of the file.
5
+ """
6
+ import os
7
+ import shutil
8
+ from pathlib import Path
9
+ from typing import Optional, Tuple, Union
10
+
11
+ import numpy as np
12
+
13
+ from pymodaq_utils.safe_pickle import restricted_loads, UnsafePickleError
14
+
15
+ from pymodaq_data.h5modules.backends import get_attr
16
+
17
+
18
+ def default_destination(source: Path) -> Path:
19
+ return source.with_name(f'{source.stem}_migrated{source.suffix}')
20
+
21
+
22
+ def migrate_h5_file(source: Union[str, Path], destination: Optional[Union[str, Path]] = None) -> Tuple[Path, int]:
23
+ """Write a copy of a h5 file in which the pickled strings are rewritten as UTF-8
24
+
25
+ The source file is never modified, and an existing destination file is never overwritten.
26
+
27
+ Parameters
28
+ ----------
29
+ source: str or Path
30
+ The file to migrate
31
+ destination: str or Path
32
+ The file to create, by default the name of the source followed by _migrated
33
+
34
+ Returns
35
+ -------
36
+ Path: the migrated file
37
+ int: the number of string nodes that were migrated
38
+
39
+ Raises
40
+ ------
41
+ FileExistsError
42
+ if the destination exists
43
+ UnsafePickleError
44
+ if a string node contains anything else than a string, in this case nothing is written
45
+ """
46
+ import h5py
47
+
48
+ source = Path(source)
49
+ destination = default_destination(source) if destination is None else Path(destination)
50
+ if destination.exists():
51
+ raise FileExistsError(f'{destination} already exists, it will not be overwritten')
52
+ part = destination.with_name(destination.name + '.part')
53
+ shutil.copyfile(source, part)
54
+ try:
55
+ migrated = 0
56
+ with h5py.File(part, 'r+') as h5file:
57
+ to_migrate = []
58
+ h5file.visititems(lambda name, obj: to_migrate.append(obj) if _is_string_node(obj) else None)
59
+ for dataset in to_migrate:
60
+ migrated += _migrate_string_node(dataset)
61
+ os.replace(part, destination)
62
+ finally:
63
+ if part.exists():
64
+ part.unlink()
65
+ return destination, migrated
66
+
67
+
68
+ def _is_string_node(obj) -> bool:
69
+ import h5py
70
+ if not isinstance(obj, h5py.Dataset):
71
+ return False
72
+ try:
73
+ return get_attr(obj, 'CLASS', 'h5py') == 'VLARRAY' and get_attr(obj, 'subdtype', 'h5py') == 'string'
74
+ except KeyError:
75
+ return False
76
+
77
+
78
+ def _migrate_string_node(dataset) -> int:
79
+ """Rewrite the pickled strings of a node, returns 1 if the node had some, 0 otherwise"""
80
+ new_values = {}
81
+ for index in range(len(dataset)):
82
+ raw = bytes(dataset[index])
83
+ if raw[:1] == b'\x80':
84
+ text = restricted_loads(raw)
85
+ if not isinstance(text, str):
86
+ raise UnsafePickleError(f'Refused to migrate {dataset.name}: it does not contain only strings')
87
+ new_values[index] = text
88
+ for index, text in new_values.items(): # nothing is written if a value above was refused
89
+ dataset[index] = np.frombuffer(text.encode('utf-8'), np.uint8)
90
+ return 1 if new_values else 0
File without changes
File without changes