ewoksid02 0.0.1a0__tar.gz → 0.1.0__tar.gz

{ewoksid02-0.0.1a0/src/ewoksid02.egg-info → ewoksid02-0.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ewoksid02
-Version: 0.0.1a0
+Version: 0.1.0
 Summary: Data processing SAXS and XPCS workflows for ID02
 Author-email: ESRF <edgar.gutierrez-fernandez@esrf.fr>
 License: # MIT License
@@ -29,7 +29,7 @@ Project-URL: Documentation, https://ewoksid02.readthedocs.io/
 Project-URL: Repository, https://gitlab.esrf.fr/workflow/ewoksapps/ewoksid02/
 Project-URL: Issues, https://gitlab.esrf.fr/workflow/ewoksapps/ewoksid02/issues
 Project-URL: Changelog, https://gitlab.esrf.fr/workflow/ewoksapps/ewoksid02/-/blob/main/CHANGELOG.md
-Keywords: ewoks
+Keywords: ewoks,ID02,SAXS,XPCS
 Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
@@ -37,7 +37,6 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
 Requires-Dist: ewoks
-Requires-Dist: ewokscore
 Requires-Dist: ewoksjob[blissworker]
 Requires-Dist: ewoksjob[slurm]
 Requires-Dist: ewoksppf
@@ -63,6 +62,7 @@ Requires-Dist: sphinx-autodoc-typehints>=1.16; extra == "doc"
 Requires-Dist: pydata-sphinx-theme; extra == "doc"
 Requires-Dist: nbsphinx; extra == "doc"
 Requires-Dist: ipython; extra == "doc"
+Requires-Dist: ewokssphinx; extra == "doc"
 Provides-Extra: cupy
 Requires-Dist: cupy; extra == "cupy"
 Dynamic: license-file

{ewoksid02-0.0.1a0 → ewoksid02-0.1.0}/pyproject.toml

@@ -6,9 +6,12 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ewoksid02"
-version = "0.0.1-alpha"
+version = "0.1.0"
 keywords = [
-    'ewoks'
+    'ewoks',
+    'ID02',
+    'SAXS',
+    'XPCS',
 ]
 
 authors = [{name = "ESRF", email = "edgar.gutierrez-fernandez@esrf.fr"}]
@@ -23,7 +26,6 @@ classifiers = [
 requires-python = ">=3.8"
 dependencies = [
     "ewoks",
-    "ewokscore",
     "ewoksjob[blissworker]",
     "ewoksjob[slurm]",
     "ewoksppf",
@@ -61,6 +63,7 @@ doc = [
     'pydata-sphinx-theme',
     'nbsphinx',
     'ipython',
+    'ewokssphinx',
 ]
 cupy = [
     'cupy',

{ewoksid02-0.0.1a0 → ewoksid02-0.1.0}/src/ewoksid02/resources/saxs_loop.json

@@ -37,6 +37,11 @@
       "task_type": "class",
       "task_identifier": "ewoksid02.tasks.scalerstask.ScalersTask",
       "id": "scalers"
+    },
+    {
+      "task_type": "class",
+      "task_identifier": "ewoksid02.tasks.end.EndTask",
+      "id": "end"
     }
   ],
   "links": [
@@ -51,6 +56,17 @@
         }
       ]
     },
+    {
+      "map_all_data": true,
+      "source": "norm",
+      "target": "end",
+      "conditions": [
+        {
+          "source_output": "continue_pipeline",
+          "value": false
+        }
+      ]
+    },
     {
       "map_all_data": true,
       "source": "2scat",

ewoksid02-0.1.0/src/ewoksid02/resources/template_saxs.yaml

@@ -0,0 +1,37 @@
+###########################
+## ewoksid02 template SAXS.yaml
+###########################
+
+bliss_filenames:                        # List of RAW_DATA files (proposal / sample / dataset files)
+    - /data/visitor/xxx/id02/xxx/RAW_DATA/xxx/.../file1.h5
+    #- /data/visitor/xxx/id02/xxx/RAW_DATA/xxx/.../file2.h5
+
+detectors:                              # List of 2D detectors to integrate (Options: eiger2, waxs, eiger500k)
+    - eiger2
+    # - waxs
+
+scans:                                  # List of scan numbers (if empty, takes all)
+    - 1
+    # - 2
+    # - !slice 1:10                     # Slice option: it will iterate between scans 1 and 10 (included)
+
+tag:                                    # Additional suffix to the output files (overwritting is forbidden)
+output_root:                            # Root folder to save the output files (if empty, replace RAW_DATA -> PROCESSED_DATA)
+
+# Processing parameters
+to_process: "norm 2scat cave azim ave"  # Include in this string which processing steps you want to perform (norm, 2scat, cave, azim, ave)
+to_save: "norm 2scat cave azim ave"     # Include in this string which processing results you want to save (norm, 2scat, cave, azim, ave)
+
+# Correction parameters
+filename_flatfield:                     # Path to the file with the flat field array (if empty, it will be taken from headers)
+filename_darkcurrent:                   # Path to the file with the dark current array (if empty, it will be taken from headers)
+filename_maskgaps:                      # Path to the file with the detector mask array (if empty, it will be taken from headers)
+filename_maskbeamstop:                  # Path to the file with an additional detector mask, like beamstop (if empty, it will be taken from headers)
+filename_window_wagon:                  # Path to the file with the WAXS pattern from the wagon window
+WindowRoiSize: 120                      # Size of the window used to convolute the WAXS pattern from the wagon window
+
+# Submit parameters
+submit: True                            # If False, it will save the instructions without submitting the job
+slurm_job_parameters:                   # Some SLURM parameters to be tweak if the job fails
+    time_limit: 01:00:00
+    memory_per_cpu: 50GB

{ewoksid02-0.0.1a0 → ewoksid02-0.1.0}/src/ewoksid02/scripts/saxs/main.py

@@ -21,6 +21,7 @@ from ...utils.blissdata import LIMA_URL_TEMPLATE_ID02, get_lima_url_template_arg
 from ..utils import SLURM_JOB_PARAMETERS_SAXS
 
 logger = logging.getLogger(__name__)
+WORKER_MODULE = "scattering"
 
 
 def get_saxs_inputs(
@@ -33,17 +34,12 @@ def get_saxs_inputs(
     # Add all nodes inputs
     ###########
     inputs_dict_all_nodes = {
-        "filename_data": kwargs.get("dataset_filename"),
         "detector_name": kwargs.get("detector_name"),
+        "filename_data": kwargs.get("dataset_filename"),
         "scan_nb": kwargs.get("scan_nb"),
-        "filename_metadata": kwargs.get("filename_scalers"),
         "filename_lima": kwargs.get("filename_scan"),
-        "dummy": kwargs.get("dummy", -10),
-        "delta_dummy": kwargs.get("delta_dummy", 0.1),
-        # "subtitle": kwargs.get("tag", ""),
         "log_level": kwargs.get("log_level", "info"),
         "datatype": kwargs.get("datatype"),
-        "use_cupy": kwargs.get("use_cupy"),
         "save_variance": kwargs.get("save_variance"),
         "max_slice_size": kwargs.get("max_slice_size"),
         "lima_url_template": LIMA_URL_TEMPLATE_ID02,
@@ -69,6 +65,8 @@ def get_saxs_inputs(
         "filename_dark": kwargs.get("filename_darkcurrent"),
         "filename_flat": kwargs.get("filename_flatfield"),
         "algorithm": kwargs.get("algorithm_norm", "cython"),
+        "Dummy": kwargs.get("Dummy"),
+        "DDummy": kwargs.get("DDummy"),
     }
     inputs += validate_inputs_ewoks(
         inputs=inputs_dict_norm,
@@ -81,8 +79,8 @@ def get_saxs_inputs(
     #############
     inputs_dict_2scat = {
         "filename_window_wagon": kwargs.get("filename_window_wagon"),
-        "window_roi_size": kwargs.get("window_roi_size"),
-        "filename_mask_to_cave": kwargs.get("filename_maskgaps"),
+        "WindowRoiSize": kwargs.get("WindowRoiSize"),
+        "filename_mask_static": kwargs.get("filename_maskgaps"),
         "filename_mask_reference": kwargs.get("filename_maskbeamstop"),
         "algorithm": kwargs.get("algorithm_2scat", "numpy"),
     }
@@ -96,7 +94,7 @@ def get_saxs_inputs(
     # Add caving inputs
     #############
     inputs_dict_cave = {
-        "filename_mask_to_cave": kwargs.get("filename_maskbeamstop"),
+        "filename_mask_static": kwargs.get("filename_maskbeamstop"),
         "flip_caving": kwargs.get("flip_caving"),
         "algorithm": kwargs.get("algorithm_cave", "numpy"),
     }
@@ -141,7 +139,7 @@ def get_saxs_inputs(
     nodes = ["norm", "2scat", "cave", "azim", "ave", "scalers"]
 
     inputs += [
-        {"name": "do_processing", "id": node, "value": node in to_process}
+        {"name": "do_process", "id": node, "value": node in to_process}
         for node in nodes
     ]
     inputs += [
@@ -247,6 +245,7 @@ def main(args):
                     "engine": "ppf",
                     "pool_type": "thread",
                 },
+                worker_module=WORKER_MODULE,
             )
 
             print(

ewoksid02-0.1.0/src/ewoksid02/tasks/__init__.py

@@ -0,0 +1,7 @@
+###
+from .id02processingtask import ID02ProcessingTask  # noqa
+from .normalizationtask import NormalizationTask  # noqa
+from .secondaryscatteringtask import SecondaryScatteringTask  # noqa
+from .cavingtask import CavingBeamstopTask, CavingGapsTask, CavingTask  # noqa
+from .azimuthaltask import AzimuthalTask  # noqa
+from .averagetask import AverageTask  # noqa

{ewoksid02-0.0.1a0 → ewoksid02-0.1.0}/src/ewoksid02/tasks/averagetask.py

@@ -3,7 +3,7 @@ from contextlib import ExitStack
 from pyFAI.units import to_unit
 from silx.io.h5py_utils import open_item as open_item_silx
 
-from ewoksid02.tasks.id02processingtask import ENTRY_NAME, ID02ProcessingTask
+from ewoksid02.tasks.id02processingtask import ID02ProcessingTask
 from ewoksid02.utils.average import calculate_average, get_array_limit
 
 
@@ -46,27 +46,10 @@ class AverageTask(
 
     def run(self):
         super().run(processing_type="ave")
-        self.update_dataset_ave()
 
-    def get_processing_inputs(self, stack: ExitStack = None) -> dict:
-        if stack:
-            return self._get_processing_inputs(stack=stack)
-
-        with ExitStack() as stack:
-            return self._get_processing_inputs(
-                stack=stack,
-            )
-
-    def _get_processing_inputs(self, stack: ExitStack) -> dict:
+    def get_processing_inputs(self) -> dict:
         azimuth_range = self.get_input_value("azimuth_range", None)
         array_ranges = None
-        metadata_parameters_group = None
-        if self.filename_metadata_headers_input and self.filename_metadata_mcs_input:
-            metadata_parameters_group = self._open_metadata_h5pygroup(
-                stack,
-                self.filename_metadata_headers_input,
-                self.path_to_metadata_headers_input,
-            )
 
         if azimuth_range is None:
             self.log_info("There is no azimuth_range. Full average will be done.")
@@ -86,7 +69,8 @@ class AverageTask(
 
         params_ave = {
             "array_ranges": array_ranges,
-            "Dummy": self.get_parameter("Dummy", metadata_parameters_group),
+            "Dummy": self.get_parameter("Dummy"),
+            "azimuth_range": azimuth_range,
         }
 
         return params_ave
@@ -102,11 +86,11 @@ class AverageTask(
             )
             stack.enter_context(self.bench_process)
 
-            processing_params = self.get_processing_inputs(
-                stack=stack,
-            )
+            processing_params = self.get_processing_inputs()
             self.processing_params = processing_params
 
+            self.log_info("Performing 1D integration...")
+
             (
                 dataset_average_intensity,
                 dataset_average_signal_norm,
@@ -129,36 +113,30 @@ class AverageTask(
             self.outputs.dataset_sigma = dataset_average_sigma
             self.outputs.radial_array = self.get_input_value("radial_array", None)
 
-        self.log_benchmark(self.bench_process)
+        self._log_benchmark(self.bench_process)
 
-    def update_dataset_ave(self):
-        if not self.processing_filename or not self.do_save:
+    def save(self) -> None:
+        do_save = super().save()
+        if not do_save:
             return
 
-        with ExitStack() as stack:
-            file = stack.enter_context(
-                open_item_silx(filename=self.processing_filename, name="/", mode="a")
-            )
-            nexus_data_grp = file[self.path_to_nxdata_output]
-            metadata_parameters_group = None
-            if (
-                self.filename_metadata_headers_input
-                and self.filename_metadata_mcs_input
-            ):
-                metadata_parameters_group = self._open_metadata_h5pygroup(
-                    stack,
-                    self.filename_metadata_headers_input,
-                    self.path_to_metadata_headers_input,
-                )
-
-            self.update_dataset(
+        filename_output, h5path_nxdata = self.links["destination"]["nxdata"].split("::")
+        params = {
+            "filename": filename_output,
+            "name": h5path_nxdata,
+            "retry_timeout": 0.1,
+            "mode": "a",
+        }
+        with open_item_silx(**params) as nexus_data_grp:
+            self._update_dataset(
                 added_dataset=self.outputs.dataset_average_signal_norm,
                 h5_group=nexus_data_grp,
                 h5_dataset_name="data_signal_norm",
+                index_read=self.index_range_last,
             )
 
             # Add nexus data information (only once)
-            unit = self.get_parameter("unit", metadata_parameters_group)
+            unit = self.get_parameter("unit")
             radial_unit = to_unit(unit)
 
             if radial_unit is not None and radial_unit.short_name not in nexus_data_grp:
@@ -175,4 +153,27 @@ class AverageTask(
 
     def processing_info(self) -> list:
         azimuth_range = self.get_input_value("azimuth_range", [0, 360])
-        return [{"h5path": ENTRY_NAME, "name": "ave_limits", "value": azimuth_range}]
+        return [{"h5path": "entry_0000", "name": "ave_limits", "value": azimuth_range}]
+
+    def _save_in_gallery(self) -> str:
+        if self.loop_nb != 1:
+            return
+
+        dataset_signal = self.outputs.dataset_signal
+        if dataset_signal is None or dataset_signal.size == 0:
+            return
+
+        import matplotlib.pyplot as plt
+
+        unit = self.get_parameter("unit")
+        radial_unit = to_unit(unit)
+
+        signal = dataset_signal.mean(axis=0)
+        q = self.outputs.radial_array
+        fig, ax = plt.subplots(figsize=(15, 10))
+        ax.plot(q, signal)
+        ax.set_xlabel(f"{radial_unit.short_name} ({radial_unit.unit_symbol})")
+        ax.set_ylabel("Intensity (arb. units)")
+        ax.set_title(f"scan {self.scan_nb} - average")
+        filename_png = self._get_filename_gallery()
+        fig.savefig(filename_png)

{ewoksid02-0.0.1a0 → ewoksid02-0.1.0}/src/ewoksid02/tasks/azimuthaltask.py

@@ -8,7 +8,9 @@ from ewoksid02.tasks.id02processingtask import ID02ProcessingTask
 from ewoksid02.utils.pyfai import (
     get_gpu_method,
     get_persistent_azimuthal_integrator,
+    _get_persistent_pyfai_worker,
     process_dataset_azim,
+    guess_npt2_rad,
 )
 
 DEFAULT_NPT_RAD = 1600
@@ -88,83 +90,69 @@ class AzimuthalTask(
 
     def run(self):
         super().run(processing_type="azim")
-        self.update_dataset_azim()
 
-    def get_processing_inputs(self, stack: ExitStack = None) -> dict:
-        if stack:
-            return self._get_processing_inputs(stack=stack)
-
-        with ExitStack() as stack:
-            return self._get_processing_inputs(
-                stack=stack,
-            )
+    def get_processing_inputs(
+        self,
+    ) -> dict:
+        params_azimuthalintegrator = {
+            "Center_1": self.get_parameter("Center_1"),
+            "Center_2": self.get_parameter("Center_2"),
+            "PSize_1": self.get_parameter("PSize_1"),
+            "PSize_2": self.get_parameter("PSize_2"),
+            "SampleDistance": self.get_parameter("SampleDistance"),
+            "WaveLength": self.get_parameter("WaveLength"),
+            "BSize_1": self.get_parameter("BSize_1"),
+            "BSize_2": self.get_parameter("BSize_2"),
+            "DetectorRotation_1": self.get_parameter("DetectorRotation_1"),
+            "DetectorRotation_2": self.get_parameter("DetectorRotation_2"),
+            "DetectorRotation_3": self.get_parameter("DetectorRotation_3"),
+        }
 
-    def _get_processing_inputs(self, stack: ExitStack) -> dict:
-        metadata_parameters_group = None
-        if self.filename_metadata_headers_input and self.filename_metadata_mcs_input:
-            metadata_parameters_group = self._open_metadata_h5pygroup(
-                stack,
-                self.filename_metadata_headers_input,
-                self.path_to_metadata_headers_input,
-            )
         azimuthal_integrator = get_persistent_azimuthal_integrator(
-            dataset=self.dataset_signal,
-            Center_1=self.get_parameter("Center_1", metadata_parameters_group),
-            Center_2=self.get_parameter("Center_2", metadata_parameters_group),
-            PSize_1=self.get_parameter("PSize_1", metadata_parameters_group),
-            PSize_2=self.get_parameter("PSize_2", metadata_parameters_group),
-            SampleDistance=self.get_parameter(
-                "SampleDistance", metadata_parameters_group
-            ),
-            WaveLength=self.get_parameter("WaveLength", metadata_parameters_group),
-            BSize_1=self.get_parameter("BSize_1", metadata_parameters_group),
-            BSize_2=self.get_parameter("BSize_2", metadata_parameters_group),
-            DetectorRotation_1=self.get_parameter(
-                "DetectorRotation_1", metadata_parameters_group
-            ),
-            DetectorRotation_2=self.get_parameter(
-                "DetectorRotation_2", metadata_parameters_group
-            ),
-            DetectorRotation_3=self.get_parameter(
-                "DetectorRotation_3", metadata_parameters_group
-            ),
+            data_signal_shape=self.dataset_signal[0].shape,
+            **params_azimuthalintegrator,
         )
 
         processing_params = {
             "filename_mask": self.get_input_value(
                 "filename_mask",
-                self.get_mask_beamstop_filename(
-                    metadata_file_group=metadata_parameters_group
-                ),
+                self.get_mask_beamstop_filename(),
             ),
             "filename_dark": self.get_input_value(
                 "filename_dark",
-                self.get_dark_filename(metadata_file_group=metadata_parameters_group),
+                self.get_dark_filename(),
             ),
-            "Dummy": self.get_parameter("Dummy", metadata_parameters_group),
-            "DDummy": self.get_parameter("DDummy", metadata_parameters_group),
-            "npt2_rad": self.get_parameter("npt2_rad", metadata_parameters_group),
-            "npt2_azim": self.get_parameter("npt2_azim", metadata_parameters_group),
-            "unit": self.get_parameter("unit", metadata_parameters_group),
+            "Dummy": self.get_parameter("Dummy"),
+            "DDummy": self.get_parameter("DDummy"),
+            "npt2_rad": self.get_parameter("npt2_rad"),
+            "npt2_azim": self.get_parameter("npt2_azim"),
+            "unit": self.get_parameter("unit", defaut="q_nm^-1"),
             "method": self.get_input_value("method", get_gpu_method()),
             "integration_options": self.get_input_value("integration_options", {}),
             "do_variance_formula": self.get_input_value("do_variance_formula", False),
-            "variance_formula": self.get_parameter(
-                "variance_formula", metadata_parameters_group
-            ),
+            "variance_formula": self.get_parameter("variance_formula"),
             "datatype": self.get_input_value("datatype", "float32"),
             "binning": (
-                self.get_parameter("BSize_1", metadata_parameters_group),
-                self.get_parameter("BSize_2", metadata_parameters_group),
+                self.get_parameter(
+                    "BSize_1",
+                ),
+                self.get_parameter(
+                    "BSize_2",
+                ),
             ),
             "save_variance": self.get_input_value("save_variance", False),
             "save_sum": self.get_input_value("save_sum", False),
             "azimuthal_integrator": azimuthal_integrator,
+            **params_azimuthalintegrator,
         }
         if not processing_params.get("npt2_rad"):
-            processing_params["npt2_rad"] = DEFAULT_NPT_RAD
+            processing_params["npt2_rad"] = guess_npt2_rad(
+                azimuthal_integrator=azimuthal_integrator
+            )
         if not processing_params.get("npt2_azim"):
             processing_params["npt2_azim"] = DEFAULT_NPT_AZIM
+        if not processing_params.get("method"):
+            processing_params["method"] = get_gpu_method()
         return processing_params
 
     def process(self) -> None:
@@ -178,9 +166,7 @@ class AzimuthalTask(
             )
             stack.enter_context(self.bench_process)
 
-            processing_params = self.get_processing_inputs(
-                stack=stack,
-            )
+            processing_params = self.get_processing_inputs()
             self.processing_params = processing_params
 
             if self.dataset_variance is not None:
@@ -194,6 +180,8 @@ class AzimuthalTask(
             else:
                 dataset_variance = None
 
+            self.log_info("Performing 2D integration...")
+
             (
                 dataset_signal_azim,
                 dataset_variance_azim,
@@ -218,50 +206,43 @@ class AzimuthalTask(
             self.outputs.radial_array = array_radial
             self.outputs.azimuth_array = array_azim
 
-        self.log_benchmark(self.bench_process)
+        self._log_benchmark(self.bench_process)
 
-    def update_dataset_azim(
-        self,
-    ):
-        if not self.processing_filename or not self.do_save:
+    def save(self) -> None:
+        do_save = super().save()
+        if not do_save:
             return
 
-        with ExitStack() as stack:
-            file = stack.enter_context(
-                open_item_silx(filename=self.processing_filename, name="/", mode="a")
-            )
-            nexus_data_grp = file[self.path_to_nxdata_output]
-
-            metadata_parameters_group = None
-            if (
-                self.filename_metadata_headers_input
-                and self.filename_metadata_mcs_input
-            ):
-                metadata_parameters_group = self._open_metadata_h5pygroup(
-                    stack,
-                    self.filename_metadata_headers_input,
-                    self.path_to_metadata_headers_input,
-                )
-
-            self.update_dataset(
+        filename_output, h5path_nxdata = self.links["destination"]["nxdata"].split("::")
+        params = {
+            "filename": filename_output,
+            "name": h5path_nxdata,
+            "retry_timeout": 0.1,
+            "mode": "a",
+        }
+        with open_item_silx(**params) as nexus_data_grp:
+            self._update_dataset(
                 added_dataset=self.outputs.dataset_sum_signal,
                 h5_group=nexus_data_grp,
                 h5_dataset_name="sum_signal",
+                index_read=self.index_range_last,
             )
 
-            self.update_dataset(
+            self._update_dataset(
                 added_dataset=self.outputs.dataset_sum_normalization,
                 h5_group=nexus_data_grp,
                 h5_dataset_name="sum_normalization",
+                index_read=self.index_range_last,
             )
 
-            self.update_dataset(
+            self._update_dataset(
                 added_dataset=self.outputs.dataset_sum_variance,
                 h5_group=nexus_data_grp,
                 h5_dataset_name="sum_variance",
+                index_read=self.index_range_last,
             )
 
-            unit = self.get_parameter("unit", metadata_parameters_group)
+            unit = self.get_parameter("unit")
             radial_unit = to_unit(unit)
 
             # Update radial and azimuthal arrays only once
@@ -286,3 +267,6 @@ class AzimuthalTask(
                     chi_dset.attrs["unit"] = "deg"
 
                     nexus_data_grp.attrs["axes"] = [".", "chi", radial_unit.short_name]
+
+    def _get_last_worker(self):
+        return _get_persistent_pyfai_worker(**self.get_processing_inputs())