ewokstxs 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,20 @@
1
+ # MIT License
2
+
3
+ **Copyright (c) 2023 European Synchrotron Radiation Facility**
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy of
6
+ this software and associated documentation files (the "Software"), to deal in
7
+ the Software without restriction, including without limitation the rights to
8
+ use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
9
+ the Software, and to permit persons to whom the Software is furnished to do so,
10
+ subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
17
+ FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
18
+ COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
19
+ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
20
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,69 @@
1
+ Metadata-Version: 2.4
2
+ Name: ewokstxs
3
+ Version: 0.1.0
4
+ Summary: Data processing workflows for ID09
5
+ Author-email: ESRF <dau-pydev@esrf.fr>
6
+ License: # MIT License
7
+
8
+ **Copyright (c) 2023 European Synchrotron Radiation Facility**
9
+
10
+ Permission is hereby granted, free of charge, to any person obtaining a copy of
11
+ this software and associated documentation files (the "Software"), to deal in
12
+ the Software without restriction, including without limitation the rights to
13
+ use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
14
+ the Software, and to permit persons to whom the Software is furnished to do so,
15
+ subject to the following conditions:
16
+
17
+ The above copyright notice and this permission notice shall be included in all
18
+ copies or substantial portions of the Software.
19
+
20
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
22
+ FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
23
+ COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
24
+ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
25
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26
+
27
+ Project-URL: Homepage, https://gitlab.esrf.fr/workflow/ewoksapps/ewokstxs/
28
+ Project-URL: Documentation, https://ewokstxs.readthedocs.io/
29
+ Project-URL: Repository, https://gitlab.esrf.fr/workflow/ewoksapps/ewokstxs/
30
+ Project-URL: Issues, https://gitlab.esrf.fr/workflow/ewoksapps/ewokstxs/issues
31
+ Project-URL: Changelog, https://gitlab.esrf.fr/workflow/ewoksapps/ewokstxs/-/blob/main/CHANGELOG.md
32
+ Keywords: ewoks
33
+ Classifier: Intended Audience :: Science/Research
34
+ Classifier: License :: OSI Approved :: MIT License
35
+ Classifier: Programming Language :: Python :: 3
36
+ Requires-Python: >=3.10
37
+ Description-Content-Type: text/markdown
38
+ License-File: LICENSE.md
39
+ Requires-Dist: ewoks
40
+ Requires-Dist: ewokscore
41
+ Requires-Dist: ewoksutils
42
+ Requires-Dist: ewoksxrpd[online]>=1.0.0
43
+ Requires-Dist: pytxs
44
+ Requires-Dist: redis!=5.3.0b1,!=5.3.0b3
45
+ Provides-Extra: test
46
+ Requires-Dist: pytest>=7; extra == "test"
47
+ Requires-Dist: pyqt5; extra == "test"
48
+ Provides-Extra: dev
49
+ Requires-Dist: ewokstxs[test]; extra == "dev"
50
+ Requires-Dist: bandit; extra == "dev"
51
+ Requires-Dist: black>=25; extra == "dev"
52
+ Requires-Dist: flake8>=4; extra == "dev"
53
+ Requires-Dist: mypy; extra == "dev"
54
+ Provides-Extra: doc
55
+ Requires-Dist: ewokstxs[test]; extra == "doc"
56
+ Requires-Dist: sphinx>=4.5; extra == "doc"
57
+ Requires-Dist: sphinx-autodoc-typehints>=1.16; extra == "doc"
58
+ Requires-Dist: pydata_sphinx_theme; extra == "doc"
59
+ Requires-Dist: ewokssphinx; extra == "doc"
60
+ Requires-Dist: sphinx-copybutton; extra == "doc"
61
+ Dynamic: license-file
62
+
63
+ # ewokstxs
64
+
65
+ Ewoks tasks for the [txs package](https://gitlab.esrf.fr/levantin/txs).
66
+
67
+ ## Documentation
68
+
69
+ https://ewokstxs.readthedocs.io/
@@ -0,0 +1,7 @@
1
+ # ewokstxs
2
+
3
+ Ewoks tasks for the [txs package](https://gitlab.esrf.fr/levantin/txs).
4
+
5
+ ## Documentation
6
+
7
+ https://ewokstxs.readthedocs.io/
@@ -0,0 +1,81 @@
1
+ [build-system]
2
+ requires = ["setuptools>=46.4"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "ewokstxs"
7
+ version = "0.1.0"
8
+ keywords = ['ewoks']
9
+ authors = [{name = "ESRF", email = "dau-pydev@esrf.fr"}]
10
+ description = "Data processing workflows for ID09"
11
+ readme = {file = "README.md", content-type = "text/markdown"}
12
+ license = {file = "LICENSE.md"}
13
+ classifiers = [
14
+ "Intended Audience :: Science/Research",
15
+ "License :: OSI Approved :: MIT License",
16
+ "Programming Language :: Python :: 3",
17
+ ]
18
+ requires-python = ">=3.10"
19
+ dependencies = [
20
+ "ewoks",
21
+ "ewokscore",
22
+ "ewoksutils",
23
+ "ewoksxrpd[online] >=1.0.0",
24
+ "pytxs",
25
+ "redis !=5.3.0b1,!=5.3.0b3",
26
+ ]
27
+
28
+ [project.urls]
29
+ Homepage = "https://gitlab.esrf.fr/workflow/ewoksapps/ewokstxs/"
30
+ Documentation = "https://ewokstxs.readthedocs.io/"
31
+ Repository = "https://gitlab.esrf.fr/workflow/ewoksapps/ewokstxs/"
32
+ Issues = "https://gitlab.esrf.fr/workflow/ewoksapps/ewokstxs/issues"
33
+ Changelog = "https://gitlab.esrf.fr/workflow/ewoksapps/ewokstxs/-/blob/main/CHANGELOG.md"
34
+
35
+ [project.optional-dependencies]
36
+ test = [
37
+ "pytest >=7",
38
+ "pyqt5",
39
+ ]
40
+ dev = [
41
+ "ewokstxs[test]",
42
+ "bandit",
43
+ "black >=25",
44
+ "flake8 >=4",
45
+ "mypy",
46
+ ]
47
+ doc = [
48
+ "ewokstxs[test]",
49
+ "sphinx >=4.5",
50
+ "sphinx-autodoc-typehints >=1.16",
51
+ "pydata_sphinx_theme",
52
+ "ewokssphinx",
53
+ "sphinx-copybutton",
54
+ ]
55
+
56
+ [tool.setuptools]
57
+ package-dir = { "" = "src" }
58
+
59
+ [tool.setuptools.packages.find]
60
+ where = ["src"]
61
+
62
+ [tool.setuptools.package-data]
63
+ "*"= ['*.mat']
64
+
65
+ [project.entry-points."ewoks.tasks.class"]
66
+ "ewokstxs.tasks.*" = "ewokstxs"
67
+
68
+ [tool.bandit.assert_used]
69
+ skips = ["*/test_*.py"]
70
+
71
+ [tool.coverage.run]
72
+ omit = ['*/tests/*']
73
+
74
+ [tool.isort]
75
+ profile = "black"
76
+
77
+ [tool.mypy]
78
+ python_version = "3.10"
79
+ files = "src"
80
+ ignore_missing_imports = true
81
+ explicit_package_bases = true
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
File without changes
@@ -0,0 +1,109 @@
1
+ import argparse
2
+ import logging
3
+ from pprint import pprint
4
+
5
+ from ewoks import execute_graph
6
+ from ewoksutils.task_utils import task_inputs
7
+
8
+
9
+ def main():
10
+ """Run TxsTask with the provided arguments"""
11
+ parser = argparse.ArgumentParser(
12
+ description="Run TxsTask with the given arguments",
13
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
14
+ )
15
+ parser.add_argument(
16
+ "filename",
17
+ type=str,
18
+ help="Path of the HDF5 file where the scan is saved",
19
+ )
20
+ parser.add_argument(
21
+ "scan",
22
+ type=int,
23
+ help="Scan number",
24
+ )
25
+ parser.add_argument(
26
+ "output_filename",
27
+ type=str,
28
+ help="Path of the HDF5 file where to save the integrated results",
29
+ )
30
+ parser.add_argument(
31
+ "-e",
32
+ "--energy",
33
+ required=True,
34
+ type=float,
35
+ help="X-ray photon energy (eV)",
36
+ )
37
+ parser.add_argument(
38
+ "-d",
39
+ "--distance",
40
+ required=True,
41
+ type=float,
42
+ help="Sample-to-detector distance (m)",
43
+ )
44
+
45
+ # optional
46
+
47
+ parser.add_argument(
48
+ "-c",
49
+ "--center",
50
+ nargs=2,
51
+ type=float,
52
+ default=(960.0, 960.0),
53
+ help="Coordinates of the image center (hor, ver) (pixel)",
54
+ )
55
+ parser.add_argument(
56
+ "--detector",
57
+ type=str,
58
+ default="rayonix",
59
+ help="Detector name",
60
+ )
61
+ parser.add_argument(
62
+ "-b",
63
+ "--binning",
64
+ nargs=2,
65
+ type=int,
66
+ default=(2, 2),
67
+ help="Detector binning (hor, ver)",
68
+ )
69
+ parser.add_argument(
70
+ "-p",
71
+ "--pixel",
72
+ nargs=2,
73
+ type=float,
74
+ default=None,
75
+ help="Pixel size (hor, ver) (m)",
76
+ )
77
+ parser.add_argument(
78
+ "-v",
79
+ "--verbose",
80
+ action="count",
81
+ default=0,
82
+ help="Increase verbosity level (-v: INFO, -vv:DEBUG)",
83
+ )
84
+
85
+ inputs = {k: v for k, v in vars(parser.parse_args()).items() if k != "verbose"}
86
+ inputs["scan_key"] = None
87
+
88
+ print("Inputs:")
89
+ pprint(inputs)
90
+ result = execute_graph(
91
+ {
92
+ "graph": {"id": "txs"},
93
+ "nodes": [
94
+ {
95
+ "id": "txs_task",
96
+ "task_type": "class",
97
+ "task_identifier": "ewokstxs.tasks.txs.TxsTask",
98
+ },
99
+ ],
100
+ },
101
+ inputs=task_inputs(id="txs_task", inputs=inputs),
102
+ )
103
+ print("Result:")
104
+ pprint(result)
105
+
106
+
107
+ if __name__ == "__main__":
108
+ logging.basicConfig(level=logging.WARNING)
109
+ main()
File without changes
@@ -0,0 +1,33 @@
1
+ from contextlib import contextmanager
2
+
3
+ from ewoksxrpd.tasks.integrate import IntegrateBlissScan
4
+ from txs import __version__ as txs_version
5
+
6
+ from ..utils import TXS_INPUTS_KEYS
7
+ from ..worker import TxsWorker
8
+
9
+
10
+ class TxsIntegrateBlissScan(IntegrateBlissScan, optional_input_names=TXS_INPUTS_KEYS): # type: ignore[call-arg]
11
+ def _get_ewoks_pyfai_options(self) -> dict:
12
+ pyfai_options = super()._get_ewoks_pyfai_options()
13
+ txs_options = {
14
+ k: v for k, v in self.get_input_values().items() if k in TXS_INPUTS_KEYS
15
+ }
16
+
17
+ return {**pyfai_options, **txs_options}
18
+
19
+ @contextmanager
20
+ def _worker(self):
21
+ options = self._get_ewoks_pyfai_options()
22
+ yield TxsWorker(options), options
23
+
24
+ def run(self):
25
+ super().run()
26
+
27
+ nxdata_url = self.get_output_value("nxdata_url")
28
+ with self.open_h5item(nxdata_url, mode="a", create=True) as nxdata:
29
+ nxprocess = nxdata.parent
30
+ del nxprocess["program"]
31
+ nxprocess["program"] = "txs"
32
+ del nxprocess["version"]
33
+ nxprocess["version"] = txs_version
@@ -0,0 +1,152 @@
1
+ from __future__ import annotations
2
+
3
+ import os.path
4
+ from collections.abc import Generator
5
+ from typing import Any
6
+
7
+ import numpy as np
8
+ import txs
9
+ from ewokscore import Task
10
+ from pyFAI.containers import Integrate1dResult
11
+ from silx.io.url import DataUrl
12
+
13
+ from .utils import TxsResultsWriter, detector_frames
14
+
15
+
16
+ def txs_integration_results(
17
+ scan_key: str | None,
18
+ filename: str | None,
19
+ scan_number: int,
20
+ energy: float,
21
+ distance: float,
22
+ center: tuple[float, float],
23
+ detector: str,
24
+ binning: int | tuple[int, int] | None,
25
+ pixel: float | tuple[float, float] | None = None,
26
+ integrate1d_options: dict[str, Any] | None = None,
27
+ ) -> Generator[tuple[float, Integrate1dResult]]:
28
+ """Generator of txs's azimuthal integration results
29
+
30
+ At least one of scan_key and filename must not be None or an empty string
31
+
32
+ :param scan_key: blissdata scan unique identifier
33
+ :param filename: Path of the HDF5 file where the scan is saved
34
+ :param scan_number: Scan number
35
+ :param energy: X-ray photon energy (eV)
36
+ :param distance: Sample-to-detector distance (m)
37
+ :param center: Coordinates of the image center (hor, ver) (pixel)
38
+ :param detector: Detector name
39
+ :param binning: Detector binning (hor, ver). Giving one value will set both hor and ver.
40
+ :param pixel: Pixel size (hor, ver) (m)
41
+ :param integrate1d_options: Extra arguments to pass to integrate1d
42
+ """
43
+ if not scan_key and not filename:
44
+ raise ValueError(
45
+ "Both scan_key and filename are None or an empty string. At least one must be defined"
46
+ )
47
+
48
+ if pixel:
49
+ ai = txs.get_ai(energy, distance, center, binning=binning, pixel=pixel)
50
+ else:
51
+ ai = txs.get_ai(energy, distance, center, binning=binning, detector=detector)
52
+
53
+ if integrate1d_options is None:
54
+ integrate1d_options = {}
55
+ for timestamp, image in detector_frames(scan_key, filename, scan_number, detector):
56
+ yield timestamp, txs.azav.integrate1d(image, ai, **integrate1d_options)
57
+
58
+
59
+ class TxsTask( # type: ignore[call-arg]
60
+ Task,
61
+ input_names=[
62
+ "scan_key",
63
+ "filename",
64
+ "scan",
65
+ "energy",
66
+ "distance",
67
+ "center",
68
+ "detector",
69
+ "binning",
70
+ "output_filename",
71
+ ],
72
+ optional_input_names=["pixel", "integrate1d_options"],
73
+ output_names=["nxdata_url"],
74
+ ):
75
+ """txs integration task which saves the results to a HDF5 file"""
76
+
77
+ def run(self):
78
+ scan_group_name = f"{self.inputs.scan}.1"
79
+
80
+ with TxsResultsWriter(
81
+ scan_nxentry_url=DataUrl(
82
+ file_path=os.path.abspath(self.inputs.filename),
83
+ data_path=scan_group_name,
84
+ ),
85
+ results_nxentry_url=DataUrl(
86
+ file_path=os.path.abspath(self.inputs.output_filename),
87
+ data_path=scan_group_name,
88
+ ),
89
+ ) as writer:
90
+ for timestamp, result in txs_integration_results(
91
+ self.inputs.scan_key,
92
+ self.inputs.filename,
93
+ self.inputs.scan,
94
+ self.inputs.energy,
95
+ self.inputs.distance,
96
+ self.inputs.center,
97
+ self.inputs.detector,
98
+ self.inputs.binning,
99
+ self.get_input_value("pixel", None),
100
+ self.get_input_value("integrate1d_options", None),
101
+ ):
102
+ writer.add_result(result, timestamp)
103
+
104
+ nxdata_url = writer.nxdata_url
105
+ self.outputs.nxdata_url = nxdata_url.path() if nxdata_url else None
106
+
107
+
108
+ class TxsTaskWithoutSaving( # type: ignore[call-arg]
109
+ Task,
110
+ input_names=[
111
+ "scan_key",
112
+ "filename",
113
+ "scan",
114
+ "energy",
115
+ "distance",
116
+ "center",
117
+ "detector",
118
+ "binning",
119
+ ],
120
+ optional_input_names=["pixel", "integrate1d_options"],
121
+ output_names=[
122
+ "radial",
123
+ "radial_units",
124
+ "intensity",
125
+ "intensity_error",
126
+ ],
127
+ ):
128
+ """txs integration task which returns the results"""
129
+
130
+ def run(self):
131
+ results = []
132
+ for _, result in txs_integration_results(
133
+ self.inputs.scan_key,
134
+ self.inputs.filename,
135
+ self.inputs.scan,
136
+ self.inputs.energy,
137
+ self.inputs.distance,
138
+ self.inputs.center,
139
+ self.inputs.detector,
140
+ self.inputs.binning,
141
+ self.get_input_value("pixel", None),
142
+ self.get_input_value("integrate1d_options", None),
143
+ ):
144
+ results.append(result)
145
+
146
+ if not results:
147
+ raise RuntimeError("No data was processed")
148
+
149
+ self.outputs.radial = results[0].radial
150
+ self.outputs.radial_units = results[0].unit.name
151
+ self.outputs.intensity = np.array([res.intensity for res in results])
152
+ self.outputs.intensity_error = np.array([res.sigma for res in results])
@@ -0,0 +1,273 @@
1
+ from __future__ import annotations
2
+
3
+ import logging
4
+ import os.path
5
+ from collections.abc import Generator
6
+
7
+ import h5py
8
+ import numpy as np
9
+ import txs
10
+ from blissdata.beacon.data import BeaconData
11
+ from blissdata.redis_engine.exceptions import ScanNotFoundError, ScanValidationError
12
+ from blissdata.redis_engine.store import DataStore
13
+ from ewoksdata.data import nexus
14
+ from ewoksdata.data.bliss import iter_bliss_scan_data, iter_bliss_scan_data_from_memory
15
+ from pyFAI.containers import Integrate1dResult
16
+ from silx.io import h5py_utils
17
+ from silx.io.dictdump import dicttonx
18
+ from silx.io.url import DataUrl
19
+
20
+ _logger = logging.getLogger(__name__)
21
+
22
+
23
+ RETRY_OPTIONS = {"retry_timeout": 2 * 60, "retry_period": 1}
24
+ """Retry options used both for reading and writing"""
25
+
26
+
27
+ @h5py_utils.retry()
28
+ def _open_h5_file(filename: str, mode: str = "r"):
29
+ """Open a HDF5 file and retries if it fails"""
30
+ return h5py.File(filename, mode)
31
+
32
+
33
+ class TxsResultsWriter:
34
+ """Writes txs results to a HDF5 group.
35
+
36
+ :param scan_nxentry_url: URL of the HDF5 group of the scan
37
+ :param results_nxentry_url: URL of the HDF5 group where to write the results
38
+ """
39
+
40
+ FLUSH_PERIOD = 3.0
41
+ """Period at which to write to the HDF5 file (seconds)"""
42
+
43
+ def __init__(
44
+ self,
45
+ scan_nxentry_url: DataUrl,
46
+ results_nxentry_url: DataUrl,
47
+ ):
48
+ self._scan_nxentry_url = scan_nxentry_url
49
+ self._results_nxentry_url = results_nxentry_url
50
+
51
+ self._results_buffer: list[Integrate1dResult] = []
52
+ self._results_timestamp: list[float] = []
53
+ self._chunk_length: int | None = None
54
+ self._nxdata_url: DataUrl | None = None
55
+
56
+ def __del__(self):
57
+ self.flush(all=True)
58
+
59
+ def __enter__(self):
60
+ return self
61
+
62
+ def __exit__(self, exc_type, exc_val, exc_tb):
63
+ self.flush(all=True)
64
+
65
+ @property
66
+ def nxdata_url(self) -> DataUrl | None:
67
+ """URL of the HDF5 group storing the NXdata, None if not yet created"""
68
+ return self._nxdata_url
69
+
70
+ def add_result(self, result: Integrate1dResult, timestamp: float):
71
+ """Append data from txs result to the HDF5 datasets"""
72
+ self._results_buffer.append(result)
73
+ self._results_timestamp.append(timestamp)
74
+
75
+ if timestamp - self._results_timestamp[0] < self.FLUSH_PERIOD:
76
+ return
77
+
78
+ if (
79
+ self._chunk_length is None
80
+ or len(self._results_buffer) >= self._chunk_length
81
+ ):
82
+ self.flush()
83
+
84
+ def _create_result_nxentry(self, result: Integrate1dResult) -> DataUrl:
85
+ # Create the output result group and create folders and HDF5 file if needed
86
+ nexus.create_url(self._results_nxentry_url, **RETRY_OPTIONS)
87
+ result_filename = self._results_nxentry_url.file_path()
88
+
89
+ # Use a relative path for the external links
90
+ scan_filename = os.path.relpath(
91
+ self._scan_nxentry_url.file_path(),
92
+ os.path.dirname(result_filename),
93
+ )
94
+ scan_group_name = self._scan_nxentry_url.data_path()
95
+
96
+ radial_name, radial_unit = result.unit.name.split("_")
97
+
98
+ with _open_h5_file(result_filename, mode="a", **RETRY_OPTIONS) as h5file:
99
+ dicttonx(
100
+ {
101
+ # Add links to some of the scan entries
102
+ "instrument": h5py.ExternalLink(
103
+ scan_filename, f"{scan_group_name}/instrument"
104
+ ),
105
+ "measurement": h5py.ExternalLink(
106
+ scan_filename, f"{scan_group_name}/measurement"
107
+ ),
108
+ "sample": h5py.ExternalLink(
109
+ scan_filename, f"{scan_group_name}/sample"
110
+ ),
111
+ "title": h5py.ExternalLink(
112
+ scan_filename, f"{scan_group_name}/title"
113
+ ),
114
+ # NXProcess group storing txs results
115
+ "@default": "integrate",
116
+ "integrate": {
117
+ "@NX_class": "NXprocess",
118
+ # NXData group storing integrated patterns
119
+ "@default": "integrated",
120
+ "integrated": {
121
+ "@NX_class": "NXdata",
122
+ "@signal": "intensity",
123
+ "@axes": [".", radial_name],
124
+ "@interpretation": "spectrum",
125
+ radial_name: result.radial,
126
+ f"{radial_name}@units": radial_unit,
127
+ },
128
+ "program": "txs",
129
+ "version": txs.__version__,
130
+ },
131
+ },
132
+ h5file,
133
+ self._results_nxentry_url.data_path(),
134
+ update_mode="modify",
135
+ )
136
+
137
+ nxdata_path = (
138
+ f"{self._results_nxentry_url.data_path()}/integrate/integrated"
139
+ )
140
+
141
+ # Create intensity and errors datasets
142
+ npt_rad = len(result.radial)
143
+
144
+ nxdata_group = h5file[nxdata_path]
145
+ nxdata_group.create_dataset(
146
+ "intensity",
147
+ dtype=result.intensity.dtype,
148
+ shape=(0, npt_rad),
149
+ chunks=(self._chunk_length, npt_rad),
150
+ maxshape=(None, npt_rad),
151
+ compression="gzip",
152
+ )
153
+ if result.sigma is not None:
154
+ nxdata_group.create_dataset(
155
+ "intensity_errors",
156
+ dtype=result.sigma.dtype,
157
+ shape=(0, npt_rad),
158
+ chunks=(self._chunk_length, npt_rad),
159
+ maxshape=(None, npt_rad),
160
+ compression="gzip",
161
+ )
162
+
163
+ return DataUrl(file_path=result_filename, data_path=nxdata_path)
164
+
165
+ def _guess_chunk_length(self) -> int:
166
+ if len(self._results_timestamp) > 1:
167
+ # expects regular framerate
168
+ average_frame_rate = 1.0 / np.mean(np.diff(self._results_timestamp))
169
+ chunk_length = int(average_frame_rate * self.FLUSH_PERIOD)
170
+ else:
171
+ chunk_length = len(self._results_timestamp)
172
+ return max(min(chunk_length, 32), 1)
173
+
174
+ def flush(self, all: bool = False) -> None:
175
+ if len(self._results_buffer) == 0:
176
+ return
177
+
178
+ if self._chunk_length is None:
179
+ self._chunk_length = self._guess_chunk_length()
180
+
181
+ if all:
182
+ write_length = len(self._results_buffer)
183
+ else:
184
+ write_length = (
185
+ len(self._results_buffer) // self._chunk_length * self._chunk_length
186
+ )
187
+ if write_length == 0:
188
+ return
189
+
190
+ if self._nxdata_url is None:
191
+ # Creates hdf5 entry lazily from first received result
192
+ self._nxdata_url = self._create_result_nxentry(self._results_buffer[0])
193
+
194
+ with _open_h5_file(
195
+ self._nxdata_url.file_path(), mode="a", **RETRY_OPTIONS
196
+ ) as h5file:
197
+ nxdata_group = h5file[self._nxdata_url.data_path()]
198
+ intensity_group = nxdata_group["intensity"]
199
+
200
+ write_offset = len(intensity_group)
201
+ dataset_length = write_offset + write_length
202
+
203
+ intensity_group.resize(dataset_length, axis=0)
204
+ intensity_group[write_offset:dataset_length] = [
205
+ result.intensity for result in self._results_buffer[:write_length]
206
+ ]
207
+
208
+ intensity_errors_group = nxdata_group.get("intensity_errors")
209
+ if intensity_errors_group is not None:
210
+ intensity_errors_group.resize(dataset_length, axis=0)
211
+ intensity_errors_group[write_offset:dataset_length] = [
212
+ result.sigma for result in self._results_buffer[:write_length]
213
+ ]
214
+
215
+ self._results_buffer = self._results_buffer[write_length:]
216
+
217
+
218
+ def detector_frames(
219
+ scan_key: str | None,
220
+ filename: str | None,
221
+ scan_number: int,
222
+ detector: str,
223
+ ) -> Generator[tuple[float, np.ndarray]]:
224
+ """Generator of detector frames from a scan retrieved either through redis or from file
225
+
226
+ At least one of scan_key and filename must not be None or empty.
227
+
228
+ :scan_key: blissdata key of the scan
229
+ :filename: Path of the HDF5 raw data file to read
230
+ :scan_entry_name: HDF5 path in the file of the group containing the scan data
231
+ :detector: name of the detector
232
+ """
233
+ if not scan_key and not filename:
234
+ raise ValueError("At least one of scan_key and filename must not None or empty")
235
+
236
+ if scan_key: # Try using redis first
237
+ # TODO rework
238
+ try:
239
+ data_store = DataStore(BeaconData().get_redis_data_db())
240
+ except Exception:
241
+ _logger.info("Cannot connect to beacon host or redis")
242
+ _logger.debug("Backtrace", exc_info=True)
243
+ else:
244
+ try:
245
+ _ = data_store.load_scan(scan_key)
246
+ except (ScanNotFoundError, ScanValidationError):
247
+ _logger.info(f"Cannot retrieve scan from redis: {scan_key}")
248
+ _logger.debug("Backtrace", exc_info=True)
249
+ else:
250
+ _logger.info("Retrieve frames through redis")
251
+ for data in iter_bliss_scan_data_from_memory(
252
+ scan_key,
253
+ lima_names=[detector],
254
+ counter_names=["elapsed_time"],
255
+ **RETRY_OPTIONS,
256
+ ):
257
+ yield data["elapsed_time"], data[detector]
258
+ return
259
+
260
+ if not filename:
261
+ raise RuntimeError("Cannot connect to redis or retrieve the scan")
262
+
263
+ _logger.info("Read frames from file")
264
+ for data in iter_bliss_scan_data(
265
+ filename,
266
+ scan_number,
267
+ lima_names=[detector],
268
+ # One counter is needed here to make sure to iterate over the right number of frames
269
+ counter_names=["elapsed_time"],
270
+ **RETRY_OPTIONS,
271
+ ):
272
+ _logger.info(f"Read image for {data['elapsed_time']}")
273
+ yield data["elapsed_time"], data[detector]
File without changes
@@ -0,0 +1,10 @@
1
+ try:
2
+ from ewoksxrpd.tests.conftest import pytest_addoption # noqa: F401
3
+ except ImportError:
4
+ pass # only required since ewoksxrpd 1.2.1
5
+ from ewoksxrpd.tests.conftest import aiSetup1 # noqa: F401
6
+ from ewoksxrpd.tests.conftest import bliss_lab6_scan # noqa: F401
7
+ from ewoksxrpd.tests.conftest import bliss_task_inputs # noqa: F401
8
+ from ewoksxrpd.tests.conftest import imageSetup1Calibrant1 # noqa: F401
9
+ from ewoksxrpd.tests.conftest import pyfai_integration_version # noqa: F401
10
+ from ewoksxrpd.tests.conftest import setup1 # noqa: F401
@@ -0,0 +1,59 @@
1
+ import logging
2
+ from pathlib import Path
3
+
4
+ import pytest
5
+ import txs
6
+ from ewoks import execute_graph
7
+ from silx.io.url import DataUrl
8
+
9
+ # This requires txs to be installed in dev mode with pip install -e
10
+ _TXS_TEST_FILEPATH = (
11
+ Path(txs.__file__).parent.absolute()
12
+ / "tests/sample_data/bliss/dye1/dye1_0002/dye1_0002.h5"
13
+ )
14
+
15
+
16
+ @pytest.mark.skipif(not _TXS_TEST_FILEPATH.is_file(), reason="Missing test file")
17
+ def test_task(tmp_path):
18
+ workflow = {
19
+ "graph": {"id": "txs"},
20
+ "nodes": [
21
+ {
22
+ "id": "txs",
23
+ "task_type": "class",
24
+ "task_identifier": "ewokstxs.tasks.txs.TxsTask",
25
+ },
26
+ ],
27
+ }
28
+
29
+ output_filename = str(tmp_path.absolute() / "id09_azav.h5")
30
+ scan_number = 12
31
+
32
+ parameters = {
33
+ "scan_key": None,
34
+ "filename": str(_TXS_TEST_FILEPATH),
35
+ "scan": scan_number,
36
+ "energy": 18000,
37
+ "distance": 0.3,
38
+ "center": (960.0, 960.0),
39
+ "detector": "rayonix",
40
+ "binning": (2, 2),
41
+ "output_filename": output_filename,
42
+ }
43
+
44
+ inputs = [
45
+ {"id": "txs", "name": name, "value": value}
46
+ for name, value in parameters.items()
47
+ ]
48
+
49
+ logging.debug(f"worflow parameters: {parameters}")
50
+ result = execute_graph(workflow, inputs=inputs)
51
+ logging.debug(f"workflow result: {result}")
52
+ assert DataUrl(result["nxdata_url"]) == DataUrl(
53
+ file_path=output_filename, data_path=f"/{scan_number}.1/integrate/integrated"
54
+ )
55
+
56
+
57
+ if __name__ == "__main__":
58
+ logging.basicConfig(level=logging.DEBUG)
59
+ test_task(Path().absolute())
@@ -0,0 +1,27 @@
1
+ import h5py
2
+
3
+ from ewokstxs.tasks.integrate import TxsIntegrateBlissScan
4
+
5
+
6
+ def test_TxsIntegrateBlissScan_task(bliss_task_inputs, tmpdir):
7
+ npt = 700
8
+
9
+ inputs = dict(bliss_task_inputs)
10
+ output_filename = str(tmpdir / "result.h5")
11
+ inputs["output_filename"] = output_filename
12
+ inputs["integration_options"] = {
13
+ "error_model": "azimuthal",
14
+ "method": "nosplit_csr",
15
+ "npt": npt,
16
+ }
17
+ inputs["sample_thickness"] = 0.5
18
+
19
+ task = TxsIntegrateBlissScan(inputs=inputs)
20
+ task.execute()
21
+ outputs = task.get_output_values()
22
+ assert outputs["nxdata_url"] == f"{output_filename}::/2.1/p3_integrate/integrated"
23
+
24
+ with h5py.File(output_filename) as root:
25
+ axes = ["points", "2th"]
26
+ assert root["2.1/p3_integrate/integrated"].attrs["axes"].tolist() == axes
27
+ assert root["2.1/measurement/p3_integrated"].shape == (31, npt)
@@ -0,0 +1,27 @@
1
+ TXS_INPUTS_KEYS = (
2
+ "dezinger_method",
3
+ "dezinger",
4
+ "sensor_material",
5
+ "sensor_thickness",
6
+ "sensor_density",
7
+ "sample_material",
8
+ "sample_thickness",
9
+ "sample_density",
10
+ )
11
+
12
+ PROCESS_KEYS = (
13
+ "npt",
14
+ "method",
15
+ "unit",
16
+ "mask",
17
+ "dark",
18
+ "flat",
19
+ "solid_angle",
20
+ "polarization_factor",
21
+ "error_model",
22
+ "radial_range",
23
+ "dummy",
24
+ "delta_dummy",
25
+ "safe",
26
+ *TXS_INPUTS_KEYS,
27
+ )
@@ -0,0 +1,21 @@
1
+ import pyFAI
2
+ from pyFAI.io import integration_config
3
+ from txs.azav import integrate1d
4
+
5
+ from .utils import PROCESS_KEYS
6
+
7
+
8
+ class TxsWorker:
9
+ def __init__(self, options) -> None:
10
+ self.ai = pyFAI.load(integration_config.normalize(options))
11
+ self.process_options = {k: v for k, v in options.items() if k in PROCESS_KEYS}
12
+
13
+ def process(self, data, variance=None, normalization_factor=1.0, metadata=None):
14
+ return integrate1d(
15
+ img=data,
16
+ ai=self.ai,
17
+ normalization_factor=normalization_factor,
18
+ variance=variance,
19
+ metadata=metadata,
20
+ **self.process_options,
21
+ )
@@ -0,0 +1,69 @@
1
+ Metadata-Version: 2.4
2
+ Name: ewokstxs
3
+ Version: 0.1.0
4
+ Summary: Data processing workflows for ID09
5
+ Author-email: ESRF <dau-pydev@esrf.fr>
6
+ License: # MIT License
7
+
8
+ **Copyright (c) 2023 European Synchrotron Radiation Facility**
9
+
10
+ Permission is hereby granted, free of charge, to any person obtaining a copy of
11
+ this software and associated documentation files (the "Software"), to deal in
12
+ the Software without restriction, including without limitation the rights to
13
+ use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
14
+ the Software, and to permit persons to whom the Software is furnished to do so,
15
+ subject to the following conditions:
16
+
17
+ The above copyright notice and this permission notice shall be included in all
18
+ copies or substantial portions of the Software.
19
+
20
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
22
+ FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
23
+ COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
24
+ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
25
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26
+
27
+ Project-URL: Homepage, https://gitlab.esrf.fr/workflow/ewoksapps/ewokstxs/
28
+ Project-URL: Documentation, https://ewokstxs.readthedocs.io/
29
+ Project-URL: Repository, https://gitlab.esrf.fr/workflow/ewoksapps/ewokstxs/
30
+ Project-URL: Issues, https://gitlab.esrf.fr/workflow/ewoksapps/ewokstxs/issues
31
+ Project-URL: Changelog, https://gitlab.esrf.fr/workflow/ewoksapps/ewokstxs/-/blob/main/CHANGELOG.md
32
+ Keywords: ewoks
33
+ Classifier: Intended Audience :: Science/Research
34
+ Classifier: License :: OSI Approved :: MIT License
35
+ Classifier: Programming Language :: Python :: 3
36
+ Requires-Python: >=3.10
37
+ Description-Content-Type: text/markdown
38
+ License-File: LICENSE.md
39
+ Requires-Dist: ewoks
40
+ Requires-Dist: ewokscore
41
+ Requires-Dist: ewoksutils
42
+ Requires-Dist: ewoksxrpd[online]>=1.0.0
43
+ Requires-Dist: pytxs
44
+ Requires-Dist: redis!=5.3.0b1,!=5.3.0b3
45
+ Provides-Extra: test
46
+ Requires-Dist: pytest>=7; extra == "test"
47
+ Requires-Dist: pyqt5; extra == "test"
48
+ Provides-Extra: dev
49
+ Requires-Dist: ewokstxs[test]; extra == "dev"
50
+ Requires-Dist: bandit; extra == "dev"
51
+ Requires-Dist: black>=25; extra == "dev"
52
+ Requires-Dist: flake8>=4; extra == "dev"
53
+ Requires-Dist: mypy; extra == "dev"
54
+ Provides-Extra: doc
55
+ Requires-Dist: ewokstxs[test]; extra == "doc"
56
+ Requires-Dist: sphinx>=4.5; extra == "doc"
57
+ Requires-Dist: sphinx-autodoc-typehints>=1.16; extra == "doc"
58
+ Requires-Dist: pydata_sphinx_theme; extra == "doc"
59
+ Requires-Dist: ewokssphinx; extra == "doc"
60
+ Requires-Dist: sphinx-copybutton; extra == "doc"
61
+ Dynamic: license-file
62
+
63
+ # ewokstxs
64
+
65
+ Ewoks tasks for the [txs package](https://gitlab.esrf.fr/levantin/txs).
66
+
67
+ ## Documentation
68
+
69
+ https://ewokstxs.readthedocs.io/
@@ -0,0 +1,21 @@
1
+ LICENSE.md
2
+ README.md
3
+ pyproject.toml
4
+ src/ewokstxs/__init__.py
5
+ src/ewokstxs/__main__.py
6
+ src/ewokstxs/utils.py
7
+ src/ewokstxs/worker.py
8
+ src/ewokstxs.egg-info/PKG-INFO
9
+ src/ewokstxs.egg-info/SOURCES.txt
10
+ src/ewokstxs.egg-info/dependency_links.txt
11
+ src/ewokstxs.egg-info/entry_points.txt
12
+ src/ewokstxs.egg-info/requires.txt
13
+ src/ewokstxs.egg-info/top_level.txt
14
+ src/ewokstxs/tasks/__init__.py
15
+ src/ewokstxs/tasks/integrate.py
16
+ src/ewokstxs/tasks/txs.py
17
+ src/ewokstxs/tasks/utils.py
18
+ src/ewokstxs/tests/__init__.py
19
+ src/ewokstxs/tests/conftest.py
20
+ src/ewokstxs/tests/test_TxsTask.py
21
+ src/ewokstxs/tests/test_integrate.py
@@ -0,0 +1,2 @@
1
+ [ewoks.tasks.class]
2
+ ewokstxs.tasks.* = ewokstxs
@@ -0,0 +1,25 @@
1
+ ewoks
2
+ ewokscore
3
+ ewoksutils
4
+ ewoksxrpd[online]>=1.0.0
5
+ pytxs
6
+ redis!=5.3.0b1,!=5.3.0b3
7
+
8
+ [dev]
9
+ ewokstxs[test]
10
+ bandit
11
+ black>=25
12
+ flake8>=4
13
+ mypy
14
+
15
+ [doc]
16
+ ewokstxs[test]
17
+ sphinx>=4.5
18
+ sphinx-autodoc-typehints>=1.16
19
+ pydata_sphinx_theme
20
+ ewokssphinx
21
+ sphinx-copybutton
22
+
23
+ [test]
24
+ pytest>=7
25
+ pyqt5
@@ -0,0 +1 @@
1
+ ewokstxs