ewoksid02 0.1.0__py3-none-any.whl

ewoksid02/tasks/id02processingtask.py

@@ -0,0 +1,2582 @@
+import json
+import os
+import gc
+import threading
+import socket
+import time
+from importlib.metadata import version
+from contextlib import ExitStack
+from ewokscore import Task
+import psutil
+from pathlib import Path
+import h5py
+import hdf5plugin
+import numpy
+import logging
+from ewokscore import missing_data
+import matplotlib.image
+from pyFAI import version as pyFAIVersion
+from silx.io.h5py_utils import open_item as open_item_silx
+from ewoksid02.utils.blissdata import (
+    copy_group_excluding_dataset,
+)
+from ewoksid02.utils.io import (
+    KEY_BEAMSTOP_MASK_FILE,
+    KEY_BEAMSTOP_MASK_FOLDER,
+    KEY_DARK_FILE,
+    KEY_DARK_FOLDER,
+    KEY_DETECTOR_MASK_FILE,
+    KEY_DETECTOR_MASK_FOLDER,
+    KEY_FLAT_FILE,
+    KEY_FLAT_FOLDER,
+    KEY_WINDOW_FILE,
+    KEY_WINDOW_FOLDER,
+    get_isotime,
+    refactor_stream_name_raw,
+    refactor_stream_name_interpreted,
+    parse_titleextension_template,
+    serialize_h5py_task,
+    deserialize_h5py_task,
+)
+
+from ewoksid02.utils.blissdata import (
+    LIMA_URL_TEMPLATE_ID02,
+    # do_continue_pipeline,
+    load_scan,
+    _slice_dataset_online,
+    _slice_dataset_offline,
+    does_scan_contain_subscan2,
+    # get_streams_subscan1,
+    # _get_streams_subscan2,
+    _get_new_slice_limits,
+    _get_stream_names_from_acquisition_chain,
+)
+
+lock = threading.Lock()
+
+PYFAI_PROCESSES = ["norm", "gaps", "2scat", "cave", "azim", "ave", "caving"]
+TRUSAXS_PROCESSES = ["scalers", "dispatch", "debug"]
+ALL_PROCESSES = PYFAI_PROCESSES + TRUSAXS_PROCESSES
+
+PROCESSING_TYPE_TASK = {
+    "norm": "ewoksid02.tasks.normalizationtask.NormalizationTask",
+    "gaps": "ewoksid02.tasks.cavingtask.CavingGapsTask",
+    "2scat": "ewoksid02.tasks.secondaryscatteringtask.SecondaryScatteringTask",
+    "cave": "ewoksid02.tasks.cavingtask.CavingBeamstopTask",
+    "azim": "ewoksid02.tasks.azimuthaltask.AzimuthalTask",
+    "ave": "ewoksid02.tasks.averagetask.AverageTask",
+    "scalers": "ewoksid02.tasks.scalerstask.ScalersTask",
+}
+
+KEYS_FLOAT = [
+    "Center_1",
+    "Center_2",
+    "Dummy",
+    "DDummy",
+    "PSize_1",
+    "PSize_2",
+    "SampleDistance",
+    "WaveLength",
+]
+
+KEYS_INT = [
+    "BSize_1",
+    "BSize_2",
+    "Offset_1",
+    "Offset_2",
+    "RasterOrientation",
+]
+
+HEADERS_KEY_EXPOSURE_TIME = "HSTime"
+HEADERS_KEY_MONITOR_0 = "HSI0"  # Monitor for beam intensity before the sample
+HEADERS_KEY_MONITOR_0_FACTOR = "HSI0Factor"
+HEADERS_KEY_MONITOR_1 = "HSI1"  # Monitor for beam intensity after the sample
+HEADERS_KEY_MONITOR_1_FACTOR = "HSI1Factor"
+HEADERS_KEY_SOT = "ShutterOpeningTime"
+HEADERS_KEY_SCT = "ShutterClosingTime"
+MAP_DETECTORS_LIMA = {
+    "eiger2": "ESRF-ID02",
+    "waxs": "instrument",
+    "default": "ESRF-ID02",
+    "eiger500k": "ESRF-ID02",  # TODO ???
+}
+DETECTOR_LIMA_DEFAULT = MAP_DETECTORS_LIMA.get("default")
+
+SLOW_COUNTER_TIMER = "sampling_timer:epoch"
+
+
+CHUNK_SIZE_3D = (1, 200, 200)
+
+INFO_COMMON = {"h5path": "entry_0000"}
+
+MAX_SLICE_SIZE = 50
+LOG_LEVEL_DEFAULT = "warning"
+LIMA_INDEX_NUMBER_FORMAT_ID02 = "%02d"
+MEM_USAGE_START = None
+
+# Global logger at ewoksid02.tasks.id02processingtask
+logger = logging.getLogger("ewoksid02")
+logger.propagate = True
+
+
+class ID02ProcessingTask(
+    Task,
+    optional_input_names=[
+        "detector_name",
+        "scan_memory_url",
+        "beacon_host",
+        "reading_node",
+        "filename_data",  # Bliss master file for a dataset
+        "filename_lima",
+        "scan_nb",
+        "subscan",
+        "headers",
+        "dataset_signal",
+        "dataset_variance",
+        "dataset_sigma",
+        "datatype",
+        "lima_url_template",
+        "lima_url_template_args",
+        "log_level",
+        "processing_filename",
+        "processing_subtitle",
+        "subtitle",
+        "do_process",
+        "do_save",
+        "save_variance",
+        "save_sigma",
+        "save_metadata",
+        "index_range",  # Global range, do not propagate
+        "index_range_last",  # Dynamic range, propagate and change every loop
+        "max_slice_size",
+        "loop_nb",
+        "info",
+        "info_history",
+        "gc_collect",
+        "lima_index_number_format",
+        "save_in_gallery",
+    ],
+    output_names=[
+        "index_range_last",
+        "loop_nb",
+        "dataset_signal",
+        "dataset_variance",
+        "dataset_sigma",
+        "continue_pipeline",
+        "info_history",
+    ],
+):
+    """This class contains processing support methods and saving methods in the ID02 SAXS pipeline.
+    It extends the `ID02LoopTask` class and provides additional functionality for handling metadata, processing flags,
+    and saving processed data to HDF5 files.This class is designed to be used as part of the ID02 pipeline.It does not contain a process method, that has to be implemented in the child class.
+
+    Optional Inputs:
+        - detector_name (str): Name of the detector used for data acquisition. This is the only mandatory input.
+        - scan_memory_url (str): URL for accessing scan memory in online processing.
+        - beacon_host (str): Host and port to plug blissdata to the correct beacon server. Only for online processing.
+        - reading_node (bool): Flag to indicate if the task should read data from the node.
+        - filename_data (str): Path to the dataset file (Master file, Nexus writer) for offline processing.
+        - filename_lima (str): Path to the first Lima file, the only place where some detector metadata can be found.
+        - scan_nb (int): Scan number for identifying the dataset.
+        - subscan (int): Subscan number for processing. Default is `1`.
+        - headers (dict): Only for Online processing. Dictionary containing headers information.
+        - max_slice_size (int): Maximum number of frames to process in one iteration. Default is `20`.
+        - dataset_signal (numpy.ndarray): Signal dataset to be processed.
+        - dataset_variance (numpy.ndarray): Variance dataset to be processed.
+        - dataset_sigma (numpy.ndarray): Sigma dataset to be processed.
+        - datatype (str): Datatype to be used to save the 2D data. Default and recommended is float32.
+        - lima_url_template (str): Format string to locate the Lima file and the path to the data inside that file.
+        - lima_url_template_args (dict): Dictionary to format the lima_url_template.
+        - log_level (str): Logging level for the task. Default is `"warning"`.
+        - processing_filename (str): Full path to the (new) output file.
+        - processing_subtitle (str): Additional subtitle for the processing task.
+        - subtitle (str): Subtitle for the processing task to be added to the output filename.
+        - do_process (bool): Flag to enable or disable processing. Default is `True`.
+        - do_save (bool): Flag to enable or disable saving of processed data. Default is `True`.
+        - save_variance (bool): Flag to enable or disable saving of variance dataset. Default is `False`.
+        - save_sigma (bool): Flag to enable or disable saving of sigma dataset. Default is `True`.
+        - save_metadata (bool): Flag to enable or disable saving of metadata. Default is `True`.
+        - last_index_read (int): Index of the last frame read in the dataset. Default is `0`.
+        - range_index_read (list): Range of indices to read from the dataset. This parameter is not propagated to the next task.
+        - loop_nb (int): Current loop iteration number. Default is `0`.
+        - info (dict): Additional metadata to save.
+        - info_history (dict): Additional metadata to propagate and save, creating a history of processing.
+        - gc_collect (bool): Manually collect garbage at the end of every task.
+        - lima_index_number_format (str): format to find the first Lima file (02%d by default)
+    Outputs:
+        - last_index_read (int): Updated index of the last frame read.
+        - loop_nb (int): Updated loop iteration number.
+        - dataset_signal (numpy.ndarray): Processed signal dataset.
+        - dataset_variance (numpy.ndarray): Processed variance dataset.
+        - dataset_sigma (numpy.ndarray): Processed sigma dataset.
+        - continue_pipeline (bool): Flag to indicate whether the pipeline should continue.
+        - info_history (dict): Additional metadata to propagate and save, creating a history of processing.
+    """
+
+    class Benchmark:
+        """A context manager for benchmarking."""
+
+        def __init__(self, nb_frames, benchmark_name="processing"):
+            self.nb_frames = nb_frames
+            self.benchmark_name = benchmark_name
+            self.bench_total_s = 0.0
+            self.bench_per_frame_ms = 0.0
+
+        def __enter__(self):
+            self.start = time.perf_counter()
+            return self
+
+        def __exit__(self, exc_type, exc_value, traceback):
+            self.end = time.perf_counter()
+            self.bench_total_s = self.end - self.start
+            if self.nb_frames > 0:
+                self.bench_per_frame_ms = self.bench_total_s / self.nb_frames * 1000
+            else:
+                self.bench_per_frame_ms = 0
+
+    def run(self, processing_type: str = "debug"):
+        self.processing_type = processing_type
+        self._pid = os.getpid()
+        self._process = psutil.Process()
+        self._set_log_level(
+            log_level=self.get_input_value("log_level", LOG_LEVEL_DEFAULT)
+        )
+        self.detector_name = self.get_input_value("detector_name", None)
+        self.scan_memory_url = self.get_input_value("scan_memory_url", None)
+        self.beacon_host = self.get_input_value(
+            "beacon_host", os.environ.get("BEACON_HOST")
+        )
+        self.filename_data = self.get_input_value("filename_data", None)
+        self.filename_lima = self.get_input_value("filename_lima", None)
+        self.subscan = self.get_input_value("subscan", 1)
+        self.max_slice_size = self.get_input_value("max_slice_size", MAX_SLICE_SIZE)
+        self.loop_nb = self.get_input_value("loop_nb", 0)
+        self.headers = self.get_input_value("headers", None)
+        self.index_range = self.get_input_value("index_range", None)
+        self.index_range_last = self.get_input_value("index_range_last", None)
+        self.links = {"source": {}, "destination": {}}
+
+        self._set_input_parameters()
+        self._load_headers()
+        self._load_streams()
+        self._log_allocated_memory()
+        self._set_datasets()
+
+        # Check-point to avoid data overwritting (only for offline processing)
+        self.processing_filename = self.get_input_value("processing_filename", None)
+        if not self.scan_memory_url and self.processing_filename:
+            if self.loop_nb == 1 and os.path.exists(self.processing_filename):
+                self.log_error(
+                    f"Offline processing, processing file {self.processing_filename} already exist. Data will not be saved. Choose another name! Workflow is canceled!"
+                )
+                self.outputs.continue_pipeline = False
+
+        if self.outputs.continue_pipeline is False:
+            return False
+        if self.processing_type == "debug":
+            return
+
+        self.do_process = self.get_input_value("do_process", True)
+        self.do_save = self.get_input_value("do_save", True)
+        if self.do_process:
+            self.process()
+            if self.do_save and self.processing_filename:
+                self.save()
+            elif self.do_save and not self.processing_filename:
+                raise ValueError(
+                    f"Processing filename for {self.processing_type} is not set."
+                )
+            else:
+                self.log_warning("Save flag was set to False, data will not be saved")
+        else:
+            self.log_warning(msg=f"Processing {self.processing_type} will be skipped.")
+            self.outputs.info_history = self.get_input_value("info_history", [])
+
+        if self.get_input_value("save_in_gallery", False):
+            self._save_in_gallery()
+
+        if self.get_input_value("gc_collect", True):
+            gc.collect()
+
+    def _set_log_level(self, log_level="warning"):
+        if not isinstance(log_level, str):
+            return
+        if log_level.lower() == "info":
+            logger.setLevel(logging.INFO)
+        elif log_level.lower() == "warning":
+            logger.setLevel(logging.WARNING)
+        elif log_level.lower() == "error":
+            logger.setLevel(logging.ERROR)
+        elif log_level.lower() == "debug":
+            logger.setLevel(logging.DEBUG)
+
+    def log_debug(self, msg):
+        self._log(level="debug", msg=msg)
+
+    def log_info(self, msg):
+        self._log(level="info", msg=msg)
+
+    def log_warning(self, msg):
+        self._log(level="warning", msg=msg)
+
+    def log_error(self, msg):
+        self._log(level="error", msg=msg)
+
+    def _log(self, level, msg):
+        msg = f"Loop #{self.loop_nb}: {self.__class__.__name__}: (PID: {self._pid}): {msg}"
+        logger.__getattribute__(level)(msg)
+
+    def _log_allocated_memory(self):
+        memory_info = self.get_memory_info()
+        mem_usage_GB = memory_info["used"]
+        total_mem_GB = memory_info["total"]
+        available_mem_GB = memory_info["available"]
+
+        if available_mem_GB / total_mem_GB < 0.1:
+            mem_message = "Low memory available"
+            color_prefix = "\033[91m"
+        elif available_mem_GB / total_mem_GB < 0.3:
+            mem_message = "Medium memory available"
+            color_prefix = "\033[93m"
+        else:
+            mem_message = "Sufficient memory available"
+            color_prefix = "\033[92m"
+        color_suffix = "\033[0m"
+
+        global MEM_USAGE_START
+        if MEM_USAGE_START is None:
+            memory_delta = 0.0
+            MEM_USAGE_START = mem_usage_GB
+        else:
+            memory_delta = mem_usage_GB - MEM_USAGE_START
+
+        logger.info(
+            f"{color_prefix}Loop #{self.loop_nb}: {self.__class__.__name__}: (PID: {self._pid}): Memory: {mem_usage_GB:.2f}GB used, increased by {memory_delta:.2f}GB; {available_mem_GB:.2f}GB available. {mem_message}{color_suffix}"
+        )
+
+    def get_memory_info(self):
+        # Return memory info in GBs
+        return {
+            "used": self._process.memory_info().rss / 1e9,
+            "total": psutil.virtual_memory().total / 1e9,
+            "available": psutil.virtual_memory().available / 1e9,
+        }
+
+    def _save_in_gallery(self): ...
+
+    def _log_benchmark(self, bench):
+        self.log_info(
+            f"Benchmark. Total ({bench.nb_frames}). {bench.benchmark_name}: {bench.bench_total_s:.2f} s. Per frame: {bench.bench_per_frame_ms:.2f} ms"
+        )
+
+    def _set_input_parameters(self):
+        # self.outputs.last_index_read = self.last_index_read
+        # self.outputs.loop_nb = self.loop_nb
+
+        if self.scan_memory_url:
+            if self.scan_memory_url.startswith("esrf:scan"):
+                # We trust this is an online processing
+                if self.detector_name is None:
+                    raise ValueError("Online processing requires a detector_name")
+                if not self.beacon_host:
+                    raise ValueError("Online processing requires a beacon_host")
+                self.scan = load_scan(
+                    scan_memory_url=self.scan_memory_url, beacon_host=self.beacon_host
+                )
+                self.scan_info = self.scan.info
+
+            elif Path(self.scan_memory_url).is_file():
+                # We trust this is a fake online processing, recreating streams from a file
+                if self.detector_name is None:
+                    raise ValueError("Fake online processing requires a detector_name")
+                with h5py.File(self.scan_memory_url, "r") as f:
+                    self.scan_info = json.loads(f["scan_info"][()].decode())
+
+            self.filename_data = self.filename_data or self.scan_info["filename"]
+            if self.filename_lima is None:
+                lima_index_number_format = self.get_input_value(
+                    "lima_index_number_format", LIMA_INDEX_NUMBER_FORMAT_ID02
+                )
+                self.filename_lima = f"{self.scan_info['images_path'].format(img_acq_device=self.detector_name)}{lima_index_number_format % 0}.h5"
+
+            self.scan_nb = self.scan_info["scan_nb"]
+            self.links["source"].update(
+                {
+                    "datasignal": f"{self.filename_data}::{self.scan_nb}.{self.subscan}/instrument/{self.detector_name}/data",
+                    "datavariance": None,
+                    "datasigma": None,
+                    "metadata_counters": f"{self.filename_data}::/{self.scan_nb}.1/measurement",
+                    "metadata_counters_subscan2": f"{self.filename_data}::/{self.scan_nb}.2/measurement",
+                    "metadata_detector": f"{self.filename_lima}::entry_0000/{MAP_DETECTORS_LIMA.get(self.detector_name, DETECTOR_LIMA_DEFAULT)}/{self.detector_name}",
+                    "metadata_headers": f"{self.filename_lima}::entry_0000/{MAP_DETECTORS_LIMA.get(self.detector_name, DETECTOR_LIMA_DEFAULT)}/{self.detector_name}/header",
+                    "metadata_titleextension": None,
+                    "ewoks": None,
+                }
+            )
+
+        elif self.filename_data:
+            # This is offline processing, but there are two types
+            self.scan_nb = self.get_input_value("scan_nb", None)
+            self.subscan = self.get_input_value("subscan", 1)
+            self.scan = None
+            self.scan_memory_url = None
+            self.beacon_host = None
+            with open_item_silx(self.filename_data, "/", "r") as file_input:
+                if "entry_0000/PyFAI" in file_input:
+                    # We trust this is an offline processing from an already processed file
+                    self.log_info(
+                        "Looks like an offline processing from PROCESSED data"
+                    )
+                    self.detector_name = file_input["entry_0000/detector_name"][
+                        ()
+                    ].decode()
+                    self.filename_lima = None
+                    self.links["source"].update(
+                        {
+                            "datasignal": f"{self.filename_data}::/entry_0000/PyFAI/result_{self.processing_type}/data",
+                            "datavariance": f"{self.filename_data}::/entry_0000/PyFAI/result_{self.processing_type}/data_variance",
+                            "datasigma": f"{self.filename_data}::/entry_0000/PyFAI/result_{self.processing_type}/data_errors",
+                            "metadata_counters": f"{self.filename_data}::/entry_0000/PyFAI/MCS/raw/subscan_1",
+                            "metadata_counters_subscan2": f"{self.filename_data}::/entry_0000/PyFAI/MCS/raw/subscan_2",
+                            "metadata_detector": f"{self.filename_data}::/entry_0000/PyFAI/{self.detector_name}",
+                            "metadata_headers": f"{self.filename_data}::/entry_0000/PyFAI/parameters",
+                            "metadata_titleextension": f"{self.filename_data}::/entry_0000/PyFAI/parameters/TitleExtension",
+                            "ewoks": f"{self.filename_data}::/entry_0000/ewoks",
+                        }
+                    )
+                elif (
+                    self.scan_nb is not None
+                    and f"{self.scan_nb}.{self.subscan}" in file_input
+                ):
+                    # We trust this is an offline processing from a RAW_DATA file
+                    if self.detector_name is None:
+                        raise ValueError(
+                            "Offline processing from RAW_DATA requires a detector_name"
+                        )
+
+                    collection_name = file_input[
+                        f"{self.scan_nb}.{self.subscan}/sample/name"
+                    ][()].decode()
+                    self.filename_lima = (
+                        Path(self.filename_data).parent
+                        / f"{collection_name}_{self.detector_name}_{self.scan_nb:05}_00.h5"
+                    )
+                    if not self.filename_lima.exists():
+                        raise FileNotFoundError(
+                            f"Lima file could not be found: {self.filename_lima}"
+                        )
+                    self.filename_lima = str(self.filename_lima)
+                    self.links["source"].update(
+                        {
+                            "datasignal": f"{self.filename_data}::/{self.scan_nb}.{self.subscan}/instrument/{self.detector_name}/data",
+                            "datavariance": None,
+                            "datasigma": None,
+                            "metadata_counters": f"{self.filename_data}::/{self.scan_nb}.1/measurement",
+                            "metadata_counters_subscan2": f"{self.filename_data}::/{self.scan_nb}.2/measurement",
+                            "metadata_detector": f"{self.filename_lima}::/entry_0000/{MAP_DETECTORS_LIMA.get(self.detector_name, DETECTOR_LIMA_DEFAULT)}/{self.detector_name}",
+                            "metadata_headers": f"{self.filename_lima}::/entry_0000/{MAP_DETECTORS_LIMA.get(self.detector_name, DETECTOR_LIMA_DEFAULT)}/{self.detector_name}/header",
+                            "metadata_titleextension": None,
+                            "ewoks": None,
+                        }
+                    )
+                elif self.scan_nb is None:
+                    raise ValueError(
+                        "Offline processing requires a scan number: scan_nb"
+                    )
+                else:
+                    raise ValueError(
+                        f"The format in {self.filename_data} is not valid for offline processing."
+                    )
+
+    def _load_headers(self) -> None:
+        if self.headers is not None:
+            return
+
+        self.headers = {}
+        file_headers, h5path_headers = self.links["source"]["metadata_headers"].split(
+            "::"
+        )
+        with open_item_silx(file_headers, h5path_headers, "r") as headers_group:
+            for key_header in headers_group:
+                value_header = headers_group[key_header][()]
+                if isinstance(value_header, bytes):
+                    value_header = value_header.decode()
+                self.headers[key_header] = str(value_header)
+
+    def _load_streams(self) -> None:
+        self.streams_subscan1 = []
+        self.streams_subscan2 = []
+
+        if self.scan:
+            acquisition_chains = self.scan.info.get("acquisition_chain")
+            if does_scan_contain_subscan2(scan=self.scan):
+                acquisition_chain_subscan1 = acquisition_chains.get("mcs")
+                acquisition_chain_subscan2 = acquisition_chains.get("sampling_timer")
+
+                # Subscan1
+                if acquisition_chain_subscan1:
+                    stream_names_subscan1 = [
+                        s
+                        for s in _get_stream_names_from_acquisition_chain(
+                            acq_chain=acquisition_chain_subscan1, include_images=False
+                        )
+                        if s in self.scan.streams
+                    ]
+                else:
+                    stream_names_subscan1 = []
+
+                stream_names_subscan1_raw = [
+                    refactor_stream_name_raw(stream_name=s, cut_name=True)
+                    for s in stream_names_subscan1
+                ]
+                stream_names_subscan1_interpreted = [
+                    refactor_stream_name_interpreted(stream_name=s)
+                    for s in stream_names_subscan1
+                ]
+
+                for stream_name, stream_name_raw_, stream_name_interpreted in zip(
+                    stream_names_subscan1,
+                    stream_names_subscan1_raw,
+                    stream_names_subscan1_interpreted,
+                ):
+                    if stream_names_subscan1_raw.count(stream_name_raw_) > 1:
+                        stream_name_raw = refactor_stream_name_raw(
+                            stream_name=stream_name,
+                            detector_name=self.detector_name,
+                            cut_name=False,
+                        )
+                    else:
+                        stream_name_raw = stream_name_raw_
+
+                    stream_subscan1_info = {
+                        "stream": self.scan.streams[stream_name],
+                        "name": stream_name,
+                        "name_raw": stream_name_raw,
+                        "name_interpreted": stream_name_interpreted,
+                    }
+                    self.streams_subscan1.append(stream_subscan1_info)
+
+                # Subscan2
+                if acquisition_chain_subscan2:
+                    stream_names_subscan2 = [
+                        s
+                        for s in _get_stream_names_from_acquisition_chain(
+                            acq_chain=acquisition_chain_subscan2, include_images=False
+                        )
+                        if s in self.scan.streams
+                    ]
+                else:
+                    stream_names_subscan2 = []
+
+                stream_names_subscan2_raw = [
+                    refactor_stream_name_raw(stream_name=s, cut_name=True)
+                    for s in stream_names_subscan2
+                ]
+                stream_names_subscan2_interpreted = [
+                    refactor_stream_name_interpreted(stream_name=s)
+                    for s in stream_names_subscan2
+                ]
+
+                for stream_name, stream_name_raw_, stream_name_interpreted in zip(
+                    stream_names_subscan2,
+                    stream_names_subscan2_raw,
+                    stream_names_subscan2_interpreted,
+                ):
+                    if stream_names_subscan2_raw.count(stream_name_raw_) > 1:
+                        stream_name_raw = refactor_stream_name_raw(
+                            stream_name=stream_name, cut_name=False
+                        )
+                    else:
+                        stream_name_raw = stream_name_raw_
+
+                    stream_subscan2_info = {
+                        "stream": self.scan.streams[stream_name],
+                        "name": stream_name,
+                        "name_raw": stream_name_raw,
+                        "name_interpreted": stream_name_interpreted,
+                    }
+                    self.streams_subscan2.append(stream_subscan2_info)
+            else:
+                # One subscan1, no subscan2
+                acquisition_chain_subscan1 = next(iter(acquisition_chains.values()))
+                acquisition_chain_subscan2 = None
+
+                stream_names_subscan1 = [
+                    s
+                    for s in _get_stream_names_from_acquisition_chain(
+                        acq_chain=acquisition_chain_subscan1, include_images=False
+                    )
+                    if s in self.scan.streams
+                ]
+                stream_names_subscan1_raw = [
+                    refactor_stream_name_raw(stream_name=s, cut_name=True)
+                    for s in stream_names_subscan1
+                ]
+                stream_names_subscan1_interpreted = [
+                    refactor_stream_name_interpreted(stream_name=s)
+                    for s in stream_names_subscan1
+                ]
+
+                for stream_name, stream_name_raw_, stream_name_interpreted in zip(
+                    stream_names_subscan1,
+                    stream_names_subscan1_raw,
+                    stream_names_subscan1_interpreted,
+                ):
+                    if stream_names_subscan1_raw.count(stream_name_raw_) > 1:
+                        stream_name_raw = refactor_stream_name_raw(
+                            stream_name=stream_name,
+                            cut_name=False,
+                        )
+                    else:
+                        stream_name_raw = stream_name_raw_
+
+                    stream_subscan1_info = {
+                        "stream": self.scan.streams[stream_name],
+                        "name": stream_name,
+                        "name_raw": stream_name_raw,
+                        "name_interpreted": stream_name_interpreted,
+                    }
+                    self.streams_subscan1.append(stream_subscan1_info)
+
+        elif self.scan_memory_url and Path(self.scan_memory_url).is_file():
+            ...
+            # with h5py.File(self.scan_memory_url, "r") as f:
+            #     self.scan_info = json.loads(f["scan_info"][()].decode())
+            #     nb_points = self.scan_info["npoints"]
+            #     for stream_name in f["streams"]:
+            #         stream = f["streams"][stream_name]
+            #         if len(stream) == nb_points:
+            #             self.streams[stream_name] = stream[:]
+            #         else:
+            #             self.streams_subscan2[stream_name] = stream[:]
+        else:
+            # The available counters are either in the RAW master file or in the HS32 arrays in the PROCESSED file
+            link_counters = self.links["source"]["metadata_counters"]
+            if link_counters is not None:
+                filename, h5path_subscan1 = link_counters.split("::")
+                _, h5path_subscan2 = self.links["source"][
+                    "metadata_counters_subscan2"
+                ].split("::")
+                params = {
+                    "filename": filename,
+                    "name": "/",
+                    "retry_timeout": 0.1,
+                }
+
+                with open_item_silx(**params) as root:
+                    if h5path_subscan1 in root:
+                        group_subscan1 = root[h5path_subscan1]
+                        for dset_name in group_subscan1:
+                            if group_subscan1[dset_name].ndim == 3:
+                                # Skip the detector
+                                continue
+
+                            stream_info = {
+                                "stream": group_subscan1[dset_name][:],
+                                "name": dset_name,
+                                "name_raw": dset_name,
+                                "name_interpreted": dset_name,
+                            }
+                            self.streams_subscan1.append(stream_info)
+
+                    if h5path_subscan2 in root:
+                        group_subscan2 = root[h5path_subscan2]
+                        for dset_name in group_subscan2:
+                            stream_info = {
+                                "stream": group_subscan2[dset_name][:],
+                                "name": dset_name,
+                                "name_raw": dset_name,
+                                "name_interpreted": dset_name,
+                            }
+                            self.streams_subscan2.append(stream_info)
+
+    def _get_new_datasets(self) -> dict:
+        filename_data, h5path_datasignal = self.links["source"]["datasignal"].split(
+            "::"
+        )
+
+        out = {
+            "dataset_signal": numpy.array([]),
+            "dataset_variance": numpy.array([]),
+            "dataset_sigma": numpy.array([]),
+            "index_range": None,
+        }
+
+        if self.scan_memory_url and self.scan_memory_url.startswith("esrf:scan"):
+            # Reading online
+            # - dataset_signal comes from the blissdata stream
+            # - dataset_variance = empty
+            # - dataset_sigma = empty
+
+            if self.processing_type in PYFAI_PROCESSES:
+                stream_name = f"{self.detector_name}:image"
+            else:
+                stream_name = "mcs:epoch"
+
+            indew_range_new = _get_new_slice_limits(
+                stream_name=stream_name,
+                scan=self.scan,
+                scan_memory_url=self.scan_memory_url,
+                beacon_host=self.beacon_host,
+                index_range=self.index_range,
+                index_range_last=self.index_range_last,
+                max_slice_size=self.max_slice_size,
+            )
+            if indew_range_new is None:
+                return out
+
+            ptdata_signal = _slice_dataset_online(
+                stream_name=stream_name,
+                detector_name=self.detector_name,
+                scan=self.scan,
+                scan_memory_url=self.scan_memory_url,
+                beacon_host=self.beacon_host,
+                lima_url_template=self.get_input_value(
+                    "lima_url_template", LIMA_URL_TEMPLATE_ID02
+                ),
+                lima_url_template_args=self.get_input_value(
+                    "lima_url_template_args", {}
+                ),
+                subscan=self.subscan,
+                index_range=indew_range_new,
+                start_from_memory=True,
+            )
+
+            if ptdata_signal["dataset"] is not None:
+                out["dataset_signal"] = ptdata_signal["dataset"]
+                out["index_range"] = ptdata_signal["index_range"]
+
+        else:
+            # Reading offline
+            # - dataset_signal comes from the RAW_DATA or PROCESSED_DATA
+            # - dataset_variance = comes only from PROCESSED_DATA
+            # - dataset_sigma = comes only from PROCESSED_DATA
+
+            filename_data, h5path_datasignal = self.links["source"]["datasignal"].split(
+                "::"
+            )
+            indew_range_new = _get_new_slice_limits(
+                filename_data=self.filename_data,
+                h5path=h5path_datasignal,
+                index_range=self.index_range,
+                index_range_last=self.index_range_last,
+                max_slice_size=self.max_slice_size,
+            )
+            if indew_range_new is None:
+                return out
+
+            ptdata_signal = _slice_dataset_offline(
+                filename_data=filename_data,
+                h5path_to_data=h5path_datasignal,
+                index_range=indew_range_new,
+            )
+            if ptdata_signal["dataset"] is not None:
+                out["dataset_signal"] = ptdata_signal["dataset"]
+                out["index_range"] = ptdata_signal["index_range"]
+
+            if self.links["source"]["datavariance"] is not None:
+                filename_data, h5path_datavariance = self.links["source"][
+                    "datavariance"
+                ].split("::")
+                ptdata_variance = _slice_dataset_offline(
+                    filename_data=filename_data,
+                    h5path_to_data=h5path_datavariance,
+                    index_range=indew_range_new,
+                )
+                if ptdata_variance["dataset"] is not None:
+                    out["dataset_variance"] = ptdata_variance["dataset"]
+
+            if self.links["source"]["datasigma"] is not None:
+                filename_data, h5path_datasigma = self.links["source"][
+                    "datasigma"
+                ].split("::")
+                ptdata_sigma = _slice_dataset_offline(
+                    filename_data=filename_data,
+                    h5path_to_data=h5path_datasigma,
+                    index_range=indew_range_new,
+                )
+                if ptdata_sigma["dataset"] is not None:
+                    out["dataset_sigma"] = ptdata_sigma["dataset"]
+
+        nb_frames_read = len(out["dataset_signal"])
+        index_range_sliced = out["index_range"]
+        if nb_frames_read > 0:
+            self.log_info(f"""
+                    \n\tIncoming ({nb_frames_read} frames) in the range {index_range_sliced[0]} -> {index_range_sliced[-1]},
+                """)
+        else:
+            self.log_info("""
+                    \n\tNo more data to read. End of the workflow.
+                """)
+
+        return out
+
+    def _set_datasets(self) -> None:
+        reading_node = self.get_input_value("reading_node", False)
+        dataset_signal = self.get_input_value("dataset_signal", None)
+        dataset_variance = self.get_input_value("dataset_variance", None)
+        dataset_sigma = self.get_input_value("dataset_sigma", None)
+
+        self.outputs.continue_pipeline = True
+        if reading_node or dataset_signal is None:
+            # Get new incoming data
+            ptdata = self._get_new_datasets()
+            dataset_signal = ptdata["dataset_signal"]
+            dataset_variance = ptdata["dataset_variance"]
+            dataset_sigma = ptdata["dataset_sigma"]
+            index_range_new = ptdata["index_range"]
+
+            if len(dataset_signal) == 0:
+                self.outputs.continue_pipeline = False
+                self.outputs.loop_nb = self.loop_nb
+                self.outputs.index_range_last = self.index_range_last
+                return
+            self.loop_nb += 1
+
+            # We define here the actual index limits because we read the data from streams
+            self.index_range_last = index_range_new
+
+        if (
+            dataset_signal is not None
+            and len(dataset_signal) > 0
+            and self.index_range_last is None
+        ):
+            # Only possible if the data was sent as inputs without any index_range_last
+            self.index_range_last = [0, len(dataset_signal)]
+
+        self.outputs.loop_nb = self.loop_nb
+        self.outputs.index_range_last = self.index_range_last
+        self.outputs.dataset_signal = dataset_signal
+        self.outputs.dataset_variance = dataset_variance
+        self.outputs.dataset_sigma = dataset_sigma
+        self.dataset_signal = dataset_signal
+        self.dataset_variance = dataset_variance
+        self.dataset_sigma = dataset_sigma
+
+    def _new_loop(self):
+        self.loop_nb += 1
+
+        memory_used = self.get_memory_info()["used"]
+        global MEM_USAGE_START
+        if self.loop_nb == 1:
+            memory_delta = 0.0
+            MEM_USAGE_START = memory_used
+        else:
+            memory_delta = memory_used - MEM_USAGE_START
+
+        if memory_delta == 0.0:
+            color_prefix = "\033[93m"
+        elif memory_delta < 0.0:
+            color_prefix = "\033[92m"
+        else:
+            color_prefix = "\033[91m"
+        color_suffix = "\033[0m"
+
+        logger.info(
+            f"{color_prefix}Loop #{self.loop_nb}: {self.__class__.__name__}: (PID: {self._pid}): \
+                Memory Delta: {memory_delta:.2f}GB{color_suffix}"
+        )
+
+    def process(self):
+        self.bench_process = None
+        self.bench_saving = None
+        self.processing_params = {}
+        if self.dataset_signal.size == 0:
+            self.log_warning(
+                f"Skipping processing {self.processing_type} due to empty array."
+            )
+            self.do_save = False
+            return False
+        return True
+
+    def _set_output_variables(self):
+        # Write some h5 paths
+        if not self.processing_filename:
+            return
+
+        if self.processing_type in PYFAI_PROCESSES:
+            nxprocess = "PyFAI"
+        elif self.processing_type in TRUSAXS_PROCESSES:
+            nxprocess = "TRUSAXS"
+        else:
+            nxprocess = "other"
+
+        self.links["destination"].update(
+            {
+                "nxdata": f"{self.processing_filename}::/entry_0000/{nxprocess}/result_{self.processing_type}",
+                "datasignal": f"{self.processing_filename}::/entry_0000/{nxprocess}/result_{self.processing_type}/data",
+                "datavariance": f"{self.processing_filename}::/entry_0000/{nxprocess}/result_{self.processing_type}/data_variance",
+                "datasigma": f"{self.processing_filename}::/entry_0000/{nxprocess}/result_{self.processing_type}/data_errors",
+                "metadata_detector": f"{self.processing_filename}::/entry_0000/{nxprocess}/{self.detector_name}",
+                "metadata_headers": f"{self.processing_filename}::/entry_0000/{nxprocess}/parameters",
+                "metadata_mcs": f"{self.processing_filename}::/entry_0000/{nxprocess}/MCS",
+                "metadata_tfg": f"{self.processing_filename}::/entry_0000/{nxprocess}/TFG",
+                "metadata_counters": None,
+                "ewoks": f"{self.processing_filename}::/entry_0000/ewoks",
+            }
+        )
+
+    def get_parameter(self, key: str, to_integer: bool = False, defaut=None):
+        value = self.get_input_value(key=key)
+        if value == missing_data.MISSING_DATA:
+            # Try to get it from header
+            value = self.get_from_headers(
+                key=key,
+                to_integer=to_integer,
+                default=defaut,
+            )
+        return value
+
+    def get_headers_filename(self, folder_key: str, file_key: str):
+        folder = self.get_from_headers(
+            key=folder_key,
+        )
+        file_name = self.get_from_headers(
+            key=file_key,
+        )
+        if folder is None or file_name is None:
+            return
+
+        filename = os.path.join(folder, file_name)
+        if not os.path.exists(filename):
+            self.log_warning(f"{filename} from headers could not be found.")
+            return
+        return filename
+
+    def get_mask_beamstop_filename(self):
+        return self.get_headers_filename(
+            folder_key=KEY_BEAMSTOP_MASK_FOLDER,
+            file_key=KEY_BEAMSTOP_MASK_FILE,
+        )
+
+    def get_mask_gaps_filename(self):
+        return self.get_headers_filename(
+            folder_key=KEY_DETECTOR_MASK_FOLDER,
+            file_key=KEY_DETECTOR_MASK_FILE,
+        )
+
+    def get_flat_filename(self):
+        return self.get_headers_filename(
+            folder_key=KEY_FLAT_FOLDER,
+            file_key=KEY_FLAT_FILE,
+        )
+
+    def get_dark_filename(self):
+        return self.get_headers_filename(
+            folder_key=KEY_DARK_FOLDER,
+            file_key=KEY_DARK_FILE,
+        )
+
+    def get_mask_window(self):
+        return self.get_headers_filename(
+            folder_key=KEY_WINDOW_FOLDER,
+            file_key=KEY_WINDOW_FILE,
+        )
+
+    def save(self):
+        self.log_debug("Saving processed data...")
+        self._set_output_variables()
+        with ExitStack() as stack:
+            if (
+                not self.processing_filename
+                or not self.do_save
+                or self.dataset_signal.size == 0
+            ):
+                return False
+
+            # Create the file and root groups
+            self._create_processing_file()
+
+            # Append data to the nexus data group
+            stack.enter_context(lock)
+            self.bench_saving = self.Benchmark(
+                nb_frames=len(self.dataset_signal), benchmark_name="saving"
+            )
+            stack.enter_context(self.bench_saving)
+            self._update_id02_data(
+                stack=stack,
+            )
+            if self.get_input_value("save_metadata", True):
+                self._update_id02_metadata(stack)
+
+        self._log_benchmark(self.bench_saving)
+        self._save_benchmark(self.bench_process)
+        self._save_benchmark(self.bench_saving)
+        return True
+
+    def _create_processing_file(self):
+        if self.scan_memory_url:
+            # With online processing, loop_nb=1 doesnt have to be the beginning of processing
+            # Processing files are created if they dont exist, never overwrite, and only if there is self.dataset_signal
+            if os.path.exists(self.processing_filename):
+                return
+
+            if self.dataset_signal is None:
+                return
+        else:
+            # With offline processing, it will create the file in the first loop, since there will always be dataset_signal
+            # Comment for dispatching
+            if self.loop_nb == 1 and os.path.exists(self.processing_filename):
+                self.log_error(
+                    f"Offline processing, processing file {self.processing_filename} already exist. Data will not be saved. Choose another name!"
+                )
+                self.outputs.repeat = False
+                return
+
+            if os.path.exists(self.processing_filename):
+                return
+
+            if self.loop_nb > 1:
+                return
+
+        # Create directories if needed
+        os.makedirs(os.path.dirname(self.processing_filename), exist_ok=True)
+
+        self.log_info(f"Creating file: {self.processing_filename}")
+        with ExitStack() as stack:
+            root_group = stack.enter_context(
+                open_item_silx(filename=self.processing_filename, name="/", mode="w")
+            )
+            root_group = self.write_root_group(root_group=root_group)
+
+            if self.processing_type in PYFAI_PROCESSES:
+                title = self.links["destination"]["nxdata"]
+            elif self.processing_type in TRUSAXS_PROCESSES:
+                title = "TFG metadata collection"
+            else:
+                title = ""
+
+            # Entry group
+            entry_group = self.create_h5_group(
+                h5_parent_group=root_group,
+                h5_group_name="entry_0000",
+                title=title,
+                NX_class="NXentry",
+            )
+            if self.processing_type in PYFAI_PROCESSES:
+                entry_group.attrs["default"] = self.links["destination"][
+                    "nxdata"
+                ].split("::")[-1]
+
+            entry_group["start_time"] = str(get_isotime())
+            if self.processing_type in PYFAI_PROCESSES:
+                entry_group["detector_name"] = self.detector_name
+
+            # Configuration group
+            configuration_group = self.create_h5_group(
+                h5_parent_group=entry_group,
+                h5_group_name="configuration",
+                NX_class="NXnote",
+            )
+            configuration_group["type"] = "text/json"
+            configuration_group["data"] = json.dumps(
+                self.get_headers(), indent=2, separators=(",\r\n", ": ")
+            )
+
+            # PyFAI Nexus group
+            if self.processing_type in PYFAI_PROCESSES:
+                process_group = self.create_h5_group(
+                    h5_parent_group=entry_group,
+                    h5_group_name="PyFAI",
+                    NX_class="NXprocess",
+                    default=self.links["destination"]["nxdata"].split("::")[-1],
+                )
+                process_group["date"] = str(get_isotime())
+                process_group["processing_type"] = self.processing_type
+                process_group["program"] = "pyFAI"
+                process_group["version"] = pyFAIVersion
+            elif self.processing_type in TRUSAXS_PROCESSES:
+                process_group = self.create_h5_group(
+                    h5_parent_group=entry_group,
+                    h5_group_name="TRUSAXS",
+                    NX_class="NXinstrument",
+                )
+                process_group["date"] = str(get_isotime())
+                process_group["processing_type"] = self.processing_type
+                process_group["program"] = "TruSAXS"
+            else:
+                return
+
+            # MCS group
+            metadata_mcs_output = self.create_h5_group(
+                h5_parent_group=process_group,
+                h5_group_name="MCS",
+                NX_class="NXcollection",
+            )
+            metadata_mcs_output["device"] = "bliss"
+
+            # HS32 N(name), Z(zero), F(factor) arrays
+            nb_pins = self.get_HS32_number_pins()
+            HS32N_array = numpy.array(
+                [
+                    self.get_pin_name_from_index(index_pin=index_pin)
+                    for index_pin in range(nb_pins)
+                ],
+                dtype=h5py.string_dtype(encoding="utf-8"),
+            )
+            HS32Z_array = numpy.array(
+                [
+                    self.get_pin_zerovalue_from_index(index_pin=index_pin)
+                    for index_pin in range(nb_pins)
+                ]
+            )
+            HS32F_array = numpy.array(
+                [
+                    self.get_pin_factorvalue_from_index(index_pin=index_pin)
+                    for index_pin in range(nb_pins)
+                ]
+            )
+            metadata_mcs_output.create_dataset(
+                name="HS32N",
+                data=HS32N_array,
+                dtype=h5py.string_dtype(encoding="utf-8"),
+            )
+
+            metadata_mcs_output.create_dataset(
+                name="HS32Z",
+                data=HS32Z_array,
+                dtype="float64",
+            )
+            metadata_mcs_output.create_dataset(
+                name="HS32F",
+                data=HS32F_array,
+                dtype="float64",
+            )
+
+            # HSI0Factor, HSI1Factor
+            for key in ["HSI0Factor", "HSI1Factor"]:
+                value = self.get_from_headers(
+                    key=key,
+                )
+                if value is not None:
+                    metadata_mcs_output.create_dataset(
+                        name=key, data=value, dtype="float64"
+                    )
+
+            # HSI0, HSI1, HSTime
+            for key in ["HSI0", "HSI1", "HSTime"]:
+                pin_name = self.get_from_headers(key=key)
+                for index_pin in range(nb_pins):
+                    if self.get_pin_name_from_index(index_pin=index_pin) == pin_name:
+                        metadata_mcs_output.create_dataset(
+                            name=key, data=index_pin + 1, dtype="int64"
+                        )
+                        break
+
+            # ShutterTime
+            for key in ["ShutterOpeningTime", "ShutterClosingTime"]:
+                value = self.get_from_headers(key=key)
+                if value is None:
+                    value = 0.0
+                metadata_mcs_output.create_dataset(
+                    name=key, data=value, dtype="float64"
+                )
+
+            # Parameters (header) group
+            metadata_headers_output = self.create_h5_group(
+                h5_parent_group=process_group,
+                h5_group_name="parameters",
+                NX_class="NXcollection",
+            )
+            headers = self.get_headers()
+            for key, value in headers.items():
+                if key in KEYS_FLOAT:
+                    metadata_headers_output.create_dataset(
+                        name=key, data=value, dtype="float64"
+                    )
+                elif key in KEYS_INT:
+                    metadata_headers_output.create_dataset(
+                        name=key, data=value, dtype="int64"
+                    )
+                elif key == "TitleExtension":
+                    continue
+                else:
+                    value = str(value)
+                    metadata_headers_output.create_dataset(
+                        name=key,
+                        data=value,
+                        dtype=h5py.string_dtype(encoding="utf-8"),
+                    )
+
+            # TFG group
+            metadata_tfg_output = self.create_h5_group(
+                h5_parent_group=process_group,
+                h5_group_name="TFG",
+                NX_class="NXcollection",
+            )
+            metadata_tfg_output["device"] = "bliss"
+
+            for key in ["HMStartEpoch", "HMStartTime"]:
+                if key in headers:
+                    value = str(headers[key])
+                    metadata_tfg_output.create_dataset(
+                        name=key, data=value, dtype=h5py.string_dtype(encoding="utf-8")
+                    )
+                else:
+                    self.log_warning(f"Key {key} not found in headers")
+
+            # Type some static information
+            # info_list = [{**INFO_COMMON}]
+            # info_list += self.processing_info()
+            # for info_item in info_list:
+            #     h5group = info_item.get("h5path")
+            #     if h5group in root_group:
+            #         root_group[h5group][info_item["name"]] = str(info_item["value"])
+
+            if self.processing_type in PYFAI_PROCESSES:
+                self._link_ewoks_history(
+                    root_group_destination=root_group,
+                    stack=stack,
+                )
+
+    def get_HS32_number_pins(self):
+        """
+        Get the number of HS32 pins, which will determine the shape of metadata arrays
+        """
+        return self.get_from_headers(key="HS32Len", to_integer=True)
+
+    def get_pin_name_from_index(self, index_pin: int):
+        """
+        Retrieve the name of the pin that is going to be saved in the
+        array HS32V at position [:,index]
+        No need to use scalers_keys dictionary
+        index goes from 0 -> (self.get_hs32_number_pins() - 1)
+        """
+        return self.get_from_headers(key=f"HS32N{(index_pin + 1):02}")
+
+    def get_pin_zerovalue_from_index(self, index_pin: int):
+        return self.get_from_headers(key=f"HS32Z{(index_pin + 1):02}")
+
+    def get_pin_factorvalue_from_index(self, index_pin: int):
+        return self.get_from_headers(key=f"HS32F{(index_pin + 1):02}")
+
+    def _update_id02_data(
+        self,
+        stack: ExitStack,
+    ):
+        file = stack.enter_context(
+            open_item_silx(filename=self.processing_filename, name="/", mode="a")
+        )
+        # Three datasets: data, data_variance, data_errors
+        _, path_nxdata_output = self.links["destination"]["nxdata"].split("::")
+        _, path_data_output = self.links["destination"]["datasignal"].split("::")
+        if path_nxdata_output not in file:
+            nexus_data_grp = self.create_h5_group(
+                h5_parent_group=file,
+                h5_group_name=path_nxdata_output,
+                NX_class="NXdata",
+                default=path_data_output,
+                signal="data",
+            )
+        else:
+            nexus_data_grp = file[path_nxdata_output]
+        self._update_dataset(
+            added_dataset=self.outputs.dataset_signal,
+            h5_group=nexus_data_grp,
+            h5_dataset_name="data",
+            index_read=self.index_range_last,
+        )
+        if self.get_input_value("save_variance", False):
+            self._update_dataset(
+                added_dataset=self.outputs.dataset_variance,
+                h5_group=nexus_data_grp,
+                h5_dataset_name="data_variance",
+                index_read=self.index_range_last,
+            )
+        if self.get_input_value("save_sigma", True):
+            self._update_dataset(
+                added_dataset=self.outputs.dataset_sigma,
+                h5_group=nexus_data_grp,
+                h5_dataset_name="data_errors",
+                index_read=self.index_range_last,
+            )
+
+    def _update_id02_metadata(self, stack: ExitStack):
+        # Update MCS group
+        if self.processing_type not in ALL_PROCESSES:
+            self.log_error(f"Processing type {self.processing_type} not valid")
+            return
+
+        root_group_destination = stack.enter_context(
+            open_item_silx(filename=self.processing_filename, name="/", mode="a")
+        )
+
+        _, h5path_mcs_output = self.links["destination"]["metadata_mcs"].split("::")
+        _, h5path_tfg_output = self.links["destination"]["metadata_tfg"].split("::")
+        _, h5path_headers_output = self.links["destination"]["metadata_headers"].split(
+            "::"
+        )
+        _, h5path_detector = self.links["destination"]["metadata_detector"].split("::")
+        _, h5path_nxdata_output = self.links["destination"]["nxdata"].split("::")
+        _, h5path_ewoks = self.links["destination"]["ewoks"].split("::")
+
+        metadata_mcs_output = root_group_destination.require_group(h5path_mcs_output)
+        metadata_tfg_output = root_group_destination.require_group(h5path_tfg_output)
+        metadata_headers_output = root_group_destination.require_group(
+            h5path_headers_output
+        )
+        nexus_data_grp_destination = None
+        if self.processing_type in PYFAI_PROCESSES:
+            nexus_data_grp_destination = root_group_destination.require_group(
+                h5path_nxdata_output
+            )
+
+        self._update_subscan1(metadata_mcs_output)
+        self._update_subscan2(metadata_mcs_output)
+        # self._copy_streams(metadata_mcs_output)
+        self._update_HS32C(metadata_mcs_output)
+        self._update_HS32V(metadata_mcs_output)
+        self._update_exposuretime(metadata_mcs_output)
+        self._update_deltatime(metadata_tfg_output, nexus_data_grp_destination)
+        self._update_intensities(metadata_mcs_output)
+
+        # Save the processing params (only once except for normalization values)
+        if self.processing_type in PYFAI_PROCESSES:
+            self._save_processing_params(root_group=root_group_destination)
+
+        # Update TitleExtension
+        self.update_title_extension(
+            parameters_group_destination=metadata_headers_output,
+        )
+
+        # Update NexusDetector with metadata from the RAW_DATA file (has to be accesible). To be done only once
+        if (
+            self.processing_type in PYFAI_PROCESSES
+            and h5path_detector not in root_group_destination
+        ):
+            filename_input, h5path_metadata_detector_input = self.links["source"][
+                "metadata_detector"
+            ].split("::")
+
+            if not os.path.exists(filename_input):
+                return
+
+            try:
+                file_input = stack.enter_context(h5py.File(filename_input, "r"))
+            except Exception:
+                return
+
+            if h5path_metadata_detector_input in file_input:
+                metadata_detector_input = file_input[h5path_metadata_detector_input]
+                if (
+                    metadata_detector_input is not None
+                    and len(metadata_detector_input) > 0
+                ):
+                    metadata_detector_output = self.create_h5_group(
+                        h5_parent_group=root_group_destination,
+                        h5_group_name=h5path_detector,
+                        NX_class="NXdetector",
+                    )
+
+                    self.update_nexus_detector_group(
+                        nxdetector_group_destination=metadata_detector_output,
+                        nxdetector_group_source=metadata_detector_input,
+                    )
+
+    def _update_HS32C(self, metadata_mcs_output: h5py.Group):
+        HS32C_array = self.get_HS32C_array()
+        self._update_dataset(
+            added_dataset=HS32C_array,
+            h5_group=metadata_mcs_output,
+            h5_dataset_name="HS32C",
+            index_read=self.index_range_last,
+        )
+
+    def _update_HS32V(self, metadata_mcs_output: h5py.Group):
+        HS32V_array = self.get_HS32V_array()
+        self._update_dataset(
+            added_dataset=HS32V_array,
+            h5_group=metadata_mcs_output,
+            h5_dataset_name="HS32V",
+            index_read=self.index_range_last,
+        )
+
+    def _update_exposuretime(self, metadata_mcs_output: h5py.Group):
+        exposuretime_values, slice_init, slice_end = self._get_exposuretime_values()
+        self._update_dataset(
+            added_dataset=exposuretime_values,
+            h5_group=metadata_mcs_output,
+            h5_dataset_name="ExposureTime",
+            index_read=[slice_init, slice_end],
+        )
+
+    def _update_deltatime(
+        self, metadata_tfg_output: h5py.Group, nexus_data_grp_destination: h5py.Group
+    ):
+        stream_deltatime = self.get_stream_deltatime()
+        deltatime, slice_init, slice_end = self._read_from_stream(
+            stream_object=stream_deltatime.get("stream"),
+            slice_init=self.index_range_last[0],
+            slice_end=self.index_range_last[-1],
+        )
+
+        self._update_dataset(
+            added_dataset=deltatime,
+            h5_group=metadata_tfg_output,
+            h5_dataset_name="delta_time",
+            index_read=[slice_init, slice_end],
+        )
+
+        if self.processing_type in PYFAI_PROCESSES:
+            self._update_dataset(
+                added_dataset=deltatime,
+                h5_group=nexus_data_grp_destination,
+                h5_dataset_name="t",
+                unit="s",
+                index_read=[slice_init, slice_end],
+            )
+
+    def _update_intensities(self, metadata_mcs_output: h5py.Group):
+        exposuretime_values, slice_init, slice_end = self._get_exposuretime_values()
+        stream_intensity0 = self.get_stream_monitor_0()
+
+        intensity0_values, monitor0_slice_init, monitor0_slice_end = (
+            self._read_from_stream(
+                stream_object=stream_intensity0.get("stream"),
+                slice_init=slice_init,
+                slice_end=slice_end,
+            )
+        )
+
+        intensity0_factor = self.get_from_headers(key=HEADERS_KEY_MONITOR_0_FACTOR)
+        intensity0uncor = intensity0_values * intensity0_factor
+        sot = self.get_from_headers(key=HEADERS_KEY_SOT, default=0.0)
+        sct = self.get_from_headers(key=HEADERS_KEY_SCT, default=0.0)
+        intensity0shutcor = (
+            intensity0uncor
+            * (exposuretime_values - sot + sct)
+            / (exposuretime_values - sot)
+        )
+
+        self._update_dataset(
+            added_dataset=intensity0shutcor,
+            h5_group=metadata_mcs_output,
+            h5_dataset_name="Intensity0ShutCor",
+            index_read=[monitor0_slice_init, monitor0_slice_end],
+        )
+        self._update_dataset(
+            added_dataset=intensity0uncor,
+            h5_group=metadata_mcs_output,
+            h5_dataset_name="Intensity0UnCor",
+            index_read=[monitor0_slice_init, monitor0_slice_end],
+        )
+
+        stream_intensity1 = self.get_stream_monitor_1()
+        intensity1_values, monitor1_slice_init, monitor1_slice_end = (
+            self._read_from_stream(
+                stream_object=stream_intensity1.get("stream"),
+                slice_init=slice_init,
+                slice_end=slice_end,
+            )
+        )
+        intensity1_factor = self.get_from_headers(key=HEADERS_KEY_MONITOR_1_FACTOR)
+        intensity1uncor = intensity1_values * intensity1_factor
+        intensity1shutcor = (
+            intensity1uncor
+            * (exposuretime_values - sot + sct)
+            / (exposuretime_values - sot)
+        )
+
+        self._update_dataset(
+            added_dataset=intensity1shutcor,
+            h5_group=metadata_mcs_output,
+            h5_dataset_name="Intensity1ShutCor",
+            index_read=[monitor1_slice_init, monitor1_slice_end],
+        )
+
+        self._update_dataset(
+            added_dataset=intensity1uncor,
+            h5_group=metadata_mcs_output,
+            h5_dataset_name="Intensity1UnCor",
+            index_read=[monitor1_slice_init, monitor1_slice_end],
+        )
+
+    def update_nexus_detector_group(
+        self,
+        nxdetector_group_destination: h5py.Group = None,
+        nxdetector_group_source: h5py.Group = None,
+    ):
+        for name, item in nxdetector_group_source.items():
+            if isinstance(item, h5py.Group):
+                # Recursively copy subgroups
+                new_subgroup = nxdetector_group_destination.create_group(name)
+                copy_group_excluding_dataset(item, new_subgroup, "data")
+            elif isinstance(item, h5py.Dataset):
+                if name != "data":  # Skip the excluded dataset
+                    nxdetector_group_source.copy(
+                        name, nxdetector_group_destination, name=name
+                    )
+
+    def get_headers(self):
+        if not self.headers:
+            self.log_error(
+                "Headers was not initialized! Maybe after using .run() method"
+            )
+        return self.headers
+
+    def get_from_headers(self, key: str, to_integer: bool = False, default=None):
+        if not self.headers:
+            self.log_error(
+                "Headers was not initialized! Maybe after using .run() method"
+            )
+            return
+
+        value = self.headers.get(key, default)
+        if value is None:
+            return
+
+        if isinstance(value, bytes):
+            value = value.decode("UTF-8")
+        try:
+            value = float(value)
+            if to_integer:
+                return int(value)
+            return value
+        except Exception:
+            return value
+
+    def get_streams_HS32_scalers(self) -> list:
+        hs32_names = self.get_HS32_names()
+        streams_scalers = []
+
+        for key_name in hs32_names.values():
+            if self.scan_memory_url:
+                # In blissdata streams, they appear as scalers:{name}
+                stream_name = f"scalers:{key_name}"
+            else:
+                # In the file, they are saved with the raw refactored name (without scalers)
+                stream_name = key_name
+            stream_info = self.get_stream_info(
+                name=stream_name,
+            )
+            streams_scalers.append(stream_info)
+        return streams_scalers
+
+    def get_streams_HS32_raw(self) -> list:
+        hs32_names = self.get_HS32_names()
+        streams_raw = []
+
+        for key_name in hs32_names.values():
+            if self.scan_memory_url:
+                # In blissdata streams, they appear as mcs:{name}_raw
+                stream_name = f"mcs:{key_name}_raw"
+            else:
+                # In the file, they are saved with the raw refactored name (without mcs)
+                stream_name = f"{key_name}_raw"
+            stream_info = self.get_stream_info(
+                name=stream_name,
+            )
+            streams_raw.append(stream_info)
+        return streams_raw
+
+    def get_HS32_factors(self) -> dict:
+        nb_pins = self.get_HS32_number_pins()
+        factors = {}
+        for index_pin in range(nb_pins):
+            factors[index_pin] = self.get_from_headers(key=f"HS32F{(index_pin + 1):02}")
+        return factors
+
+    def get_HS32_zeros(self) -> dict:
+        nb_pins = self.get_HS32_number_pins()
+        zeros = {}
+        for index_pin in range(nb_pins):
+            zeros[index_pin] = self.get_from_headers(key=f"HS32Z{(index_pin + 1):02}")
+        return zeros
+
+    def get_HS32_names(self) -> dict:
+        nb_pins = self.get_HS32_number_pins()
+        names = {}
+        for index_pin in range(nb_pins):
+            names[index_pin] = self.get_from_headers(key=f"HS32N{(index_pin + 1):02}")
+        return names
+
+    def get_stream_slow_timer(self) -> dict:
+        for stream_info in self.streams_subscan2:
+            if "epoch" in stream_info["name"]:
+                return stream_info
+
+    def get_stream_fast_timer(self) -> dict:
+        return self.get_stream_info(name="epoch", prefix="mcs")
+
+    def get_stream_exposuretime(self) -> dict:
+        """
+        In blissdata, this stream is (normally) called scalers:time (in subscan1)
+        In the header, there is key "HSTime" pointing to the string 'time'
+        """
+        key_name = self.get_from_headers(key=HEADERS_KEY_EXPOSURE_TIME)
+        stream_time_name = f"scalers:{key_name}"
+        return self.get_stream_info(
+            name=stream_time_name,
+            subscan_2=False,
+        )
+
+    def get_stream_exposuretime_raw(self) -> dict:
+        """
+        In blissdata, this stream is (normally) called mcs:time_raw (in subscan1)
+        In the header, there is key "HSTime" pointing to the string 'time'
+        """
+        key_name = self.get_from_headers(key=HEADERS_KEY_EXPOSURE_TIME)
+        stream_time_name = f"mcs:{key_name}_raw"
+        return self.get_stream_info(
+            name=stream_time_name,
+            subscan_2=False,
+        )
+
+    def get_stream_deltatime(self) -> dict:
+        if self.scan_memory_url:
+            stream_name = "mcs:elapsed_time"
+        else:
+            stream_name = "elapsed_time"
+
+        stream_info = self.get_stream_info(
+            name=stream_name,
+        )
+        if not stream_info:
+            stream_name = "mcs_elapsed_time"
+            stream_info = self.get_stream_info(
+                name=stream_name,
+            )
+        return stream_info
+
+    def get_stream_info(
+        self,
+        index_pin: int = None,
+        name: str = None,
+        header_key_pin: str = None,
+        prefix: str = "",
+        suffix: str = "",
+        subscan_2: bool = False,
+    ) -> dict:
+        if name is None:
+            if header_key_pin is not None:
+                name = self.get_from_headers(key=header_key_pin)
+            elif index_pin is not None:
+                name = self.get_pin_name_from_index(index_pin=index_pin)
+        if name is None:
+            return
+
+        if prefix:
+            name = f"{prefix}:{name}"
+        if suffix:
+            name = f"{name}_{suffix}"
+
+        name_raw = refactor_stream_name_raw(stream_name=name, cut_name=False)
+        name_raw_cut = refactor_stream_name_raw(stream_name=name, cut_name=True)
+        name_interpreted = refactor_stream_name_interpreted(stream_name=name)
+
+        if subscan_2:
+            streams = self.streams_subscan2
+        else:
+            streams = self.streams_subscan1
+
+        for stream_info in streams:
+            if name == stream_info["name"]:
+                return stream_info
+
+        for stream_info in streams:
+            if name_raw == stream_info["name_raw"]:
+                return stream_info
+
+        for stream_info in streams:
+            if name_raw_cut == stream_info["name_raw"]:
+                return stream_info
+
+        for stream_info in streams:
+            if name_interpreted == stream_info["name_interpreted"]:
+                return stream_info
+
+    def _read_from_stream(
+        self,
+        stream_object,
+        slice_init: int,
+        slice_end: int,
+        datatype: str = "float32",
+    ) -> tuple:
+        nb_available_frames = len(stream_object)
+        if slice_init > nb_available_frames:
+            self.log_error(
+                f"Not enough frames in {stream_object} ({nb_available_frames}). Requested init: {slice_init}"
+            )
+            return
+        if slice_end > nb_available_frames:
+            self.log_warning(
+                f"Not enough frames in {stream_object} ({nb_available_frames}). Requested end: {slice_end}"
+            )
+            slice_end = nb_available_frames
+
+        try:
+            stream_values = stream_object[slice_init:slice_end]
+            return stream_values.astype(datatype), slice_init, slice_end
+        except Exception as e:
+            self.log_error(f"{stream_object} could not be sliced: {e}")
+
+    def read_from_stream_interpolate(
+        self,
+        stream_object,
+        stream_slice=None,
+        datatype: str = "float32",
+    ) -> numpy.ndarray:
+        """
+        stream_object is a sliceable object (numpy.ndarray or blissdata stream object)
+        Valid for all streams, from subscan1 and subscan2
+        """
+        if stream_slice is None:
+            slice_init = 0
+            slice_end = len(stream_object)
+        else:
+            slice_init, slice_end = stream_slice
+
+        stream_values = stream_object[slice_init:slice_end]
+        stream_values = stream_values.astype(datatype)
+
+        # 2) Read all the values in the slow epoch counter
+        slow_epoch_stream = self.get_stream_slow_timer()
+        if slow_epoch_stream is None:
+            self.log_debug("Slow epoch stream could not be found.")
+            return stream_values
+
+        slow_epoch_stream = slow_epoch_stream.get("stream")
+
+        # Both epoch must be read with double precision
+        slow_epoch_values, slice_init, slice_end = self._read_from_stream(
+            stream_object=slow_epoch_stream,
+            slice_init=0,
+            slice_end=len(slow_epoch_stream),
+            datatype="float64",
+        )
+        if slow_epoch_values is None:
+            self.log_error(f"Stream {slow_epoch_stream} could not be read.")
+            return stream_values
+
+        # 3) Match the slow streams
+        if len(slow_epoch_values) != len(stream_values):
+            nb_slow_frames = min(len(slow_epoch_values), len(stream_values))
+            slow_epoch_values = slow_epoch_values[0:nb_slow_frames]
+            stream_values = stream_values[0:nb_slow_frames]
+
+        # 4) Read the fast epoch counter
+        fast_epoch_stream = self.get_stream_fast_timer().get("stream")
+        if fast_epoch_stream is None:
+            self.log_error("Fast epoch stream could not be found.")
+            return stream_values
+
+        fast_epoch_values, slice_init, slice_end = self._read_from_stream(
+            stream_object=fast_epoch_stream,
+            slice_init=0,
+            slice_end=self.index_range_last[-1],
+            datatype="float64",
+        )
+
+        if fast_epoch_values is None:
+            self.log_error(f"Stream {fast_epoch_stream} could not be read.")
+        elif fast_epoch_values.size == 0:
+            self.log_error(f"Stream {fast_epoch_stream} is empty.")
+            return
+
+        # 5) Interpolate data, len(interpdata) = len(fast_epoch_values)
+        try:
+            return numpy.interp(fast_epoch_values, slow_epoch_values, stream_values)
+        except Exception as e:
+            self.log_error(f"Error during numpy interpolation: {e}")
+            return
+
+    def get_HS32C_array(self):
+        nb_hs32_pins = self.get_HS32_number_pins()
+        new_HS32C_array = numpy.full(
+            (self.index_range_last[1] - self.index_range_last[0], nb_hs32_pins),
+            fill_value=-1,
+            dtype="float64",
+        )
+
+        streams_raw = self.get_streams_HS32_raw()
+        for index_pin, stream_info in enumerate(streams_raw):
+            if not stream_info:
+                continue
+            stream_values, slice_init, slice_end = self._read_from_stream(
+                stream_object=stream_info.get("stream"),
+                slice_init=self.index_range_last[0],
+                slice_end=self.index_range_last[-1],
+            )
+            if stream_values is not None:
+                new_HS32C_array[0 : len(stream_values), index_pin] = stream_values
+
+        return new_HS32C_array
+
+    def get_HS32V_array(
+        self,
+    ):
+        nb_hs32_pins = self.get_HS32_number_pins()
+        new_HS32V_array = numpy.zeros(
+            (self.index_range_last[1] - self.index_range_last[0], nb_hs32_pins),
+            dtype="float64",
+        )
+
+        streams_scalers = self.get_streams_HS32_scalers()
+        for index_pin, stream_info in enumerate(streams_scalers):
+            if not stream_info:
+                continue
+            stream_values, slice_init, slice_end = self._read_from_stream(
+                stream_object=stream_info.get("stream"),
+                slice_init=self.index_range_last[0],
+                slice_end=self.index_range_last[-1],
+            )
+            if stream_values is not None:
+                new_HS32V_array[0 : len(stream_values), index_pin] = stream_values
+
+        return new_HS32V_array
+
+    def _get_exposuretime_values(
+        self,
+    ) -> tuple:
+        # Go first for the already normalized time values
+        stream_exposuretime = self.get_stream_exposuretime()
+        exposuretime, slice_init, slice_end = self._read_from_stream(
+            stream_object=stream_exposuretime.get("stream"),
+            slice_init=self.index_range_last[0],
+            slice_end=self.index_range_last[-1],
+        )
+
+        if exposuretime is None:
+            # Try to go to the raw time
+            stream_exposuretime_raw = self.get_stream_exposuretime_raw()
+            exposuretime, slice_init, slice_end = self._read_from_stream(
+                stream_object=stream_exposuretime_raw.get("stream"),
+                slice_init=self.index_range_last[0],
+                slice_end=self.index_range_last[-1],
+            )
+
+            if exposuretime is None:
+                self.log_error("Exposure time could not be read.")
+                return None, None, None
+
+            factor_exposuretime = self.get_factor_exposuretime()
+            if factor_exposuretime is None:
+                self.log_error("Default exposure time factor 1.0 will be used")
+                factor_exposuretime = 1.0
+            exposuretime *= factor_exposuretime
+        return exposuretime, slice_init, slice_end
+
+    # def _get_deltatime_values(self) -> tuple:
+    #     stream_deltatime = self.get_stream_deltatime()
+    #     deltatime = self._read_from_stream(
+    #         stream_object=stream_deltatime.get("stream"),
+    #         slice_init=self.index_range_last[-1],
+    #         slice_end=self.index_range_last[-1],
+    #     )
+    #     return deltatime
+
+    def get_stream_monitor_0(self) -> dict:
+        key_name = self.get_from_headers(key=HEADERS_KEY_MONITOR_0)
+        stream_name = f"scalers:{key_name}"
+        return self.get_stream_info(name=stream_name)
+
+    def get_stream_monitor_1(self) -> dict:
+        key_name = self.get_from_headers(key=HEADERS_KEY_MONITOR_1)
+        stream_name = f"scalers:{key_name}"
+        return self.get_stream_info(name=stream_name)
+
+    def get_factor(
+        self, index_pin: int = None, pin_name: str = None, header_key_pin: str = None
+    ):
+        if index_pin is not None:
+            return self.get_from_headers(key=f"HS32F{(index_pin + 1):02}")
+        if pin_name is None:
+            nb_pins = self.get_HS32_number_pins()
+            pin_name = self.get_from_headers(key=header_key_pin)
+        for index_pin in range(nb_pins):
+            if self.get_pin_name_from_index(index_pin=index_pin) == pin_name:
+                return self.get_from_headers(key=f"HS32F{(index_pin + 1):02}")
+
+    def get_factor_exposuretime(self):
+        return self.get_factor(header_key_pin=HEADERS_KEY_EXPOSURE_TIME)
+
+    def update_title_extension(
+        self,
+        parameters_group_destination: h5py.Group,
+    ):
+        titleextension_template = self.get_from_headers(key="TitleExtensionTemplate")
+        if not titleextension_template:
+            self.log_warning("There is no TitleExtensionTemplate in the header.")
+            return
+
+        title_extension_parsed, title_extension_formats = parse_titleextension_template(
+            titleextension_template
+        )
+        title_extension_values = {}
+
+        # We want to slice all the streams at once, and then build the array of strings
+        for titleextension_dict in title_extension_formats:
+            stream_name = titleextension_dict["stream_name"]
+            stream_values = None
+            title_extension_values[stream_name] = numpy.full(
+                shape=(self.index_range_last[1],), fill_value=numpy.nan
+            )
+
+            # Try first from subscan2
+            stream_info = self.get_stream_info(name=stream_name, subscan_2=True)
+            if stream_info:
+                stream_values = self.read_from_stream_interpolate(
+                    stream_object=stream_info.get("stream"),
+                    stream_slice=None,
+                )
+            else:
+                stream_info = stream_info or self.get_stream_info(
+                    name=stream_name, subscan_2=False
+                )
+                if stream_info is None:
+                    self.log_error(
+                        f"{stream_name} stream from TitleExtension could not be found"
+                    )
+                    continue
+                stream_values, slice_init, slice_end = self._read_from_stream(
+                    stream_object=stream_info.get("stream"),
+                    slice_init=0,
+                    slice_end=self.index_range_last[-1],
+                )
+            if stream_values is None:
+                self.log_error(f"Stream {stream_info} could not be read")
+                continue
+
+            title_extension_values[stream_name] = stream_values
+
+        # Now we go index by index, building the string cells
+        # The title extensions will always be the size of last index (from 0 -> last_frame)
+        new_title_extensions = numpy.full(
+            shape=(self.index_range_last[1],),
+            fill_value="",
+            dtype=h5py.string_dtype(encoding="utf-8"),
+        )
+        for index in range(len(new_title_extensions)):
+            format_index = {
+                stream_name: title_extension_values[stream_name][index]
+                for stream_name in title_extension_values
+            }
+
+            new_title_extensions[index] = title_extension_parsed.format(**format_index)
+
+        self._update_dataset(
+            added_dataset=new_title_extensions,
+            h5_group=parameters_group_destination,
+            h5_dataset_name="TitleExtension",
+            index_read=[0, self.index_range_last[1]],
+        )
+
+    def _update_subscan1(
+        self,
+        mcs_group_destination: h5py.Group,
+    ):
+        """
+        Updates both raw/subscan_1 and interpreted with the values of subscan_1 (equivalent)
+        """
+        raw_group = mcs_group_destination.require_group("raw")
+        subscan1 = raw_group.require_group("subscan_1")
+        interpreted_grp = mcs_group_destination.require_group("interpreted")
+
+        for stream_info in self.streams_subscan1:
+            stream_name_raw = stream_info.get("name_raw")
+            stream_name_interpreted = stream_info.get("name_interpreted")
+            stream = stream_info.get("stream")
+
+            stream_values, slice_init, slice_end = self._read_from_stream(
+                stream_object=stream,
+                slice_init=self.index_range_last[0],
+                slice_end=self.index_range_last[-1],
+            )
+
+            self._update_dataset(
+                added_dataset=stream_values,
+                h5_group=subscan1,
+                h5_dataset_name=stream_name_raw,
+                index_read=[slice_init, slice_end],
+            )
+            self._update_dataset(
+                added_dataset=stream_values,
+                h5_group=interpreted_grp,
+                h5_dataset_name=stream_name_interpreted,
+                index_read=[slice_init, slice_end],
+            )
+
+    def _update_subscan2(
+        self,
+        mcs_group: h5py.Group,
+    ):
+        if not self.streams_subscan2:
+            return
+
+        interpreted_grp = mcs_group.require_group("interpreted")
+        raw_grp = mcs_group.require_group("raw")
+
+        for stream_info in self.streams_subscan2:
+            stream_name_raw = stream_info.get("name_raw")
+            stream_name_interpreted = stream_info.get("name_interpreted")
+            stream = stream_info.get("stream")
+
+            # Read without any interpolation to send to raw-subscan2
+            stream_values, slice_init, slice_end = self._read_from_stream(
+                stream_object=stream,
+                slice_init=0,
+                slice_end=len(stream),
+            )
+            if stream_values is None:
+                continue
+
+            subscan2_grp = raw_grp.require_group("subscan_2")
+            if stream_name_raw not in subscan2_grp:
+                dset_raw_subscan2 = subscan2_grp.create_dataset(
+                    name=stream_name_raw,
+                    shape=stream_values.shape,
+                    maxshape=(None,),
+                    dtype=stream_values.dtype,
+                )
+            else:
+                dset_raw_subscan2 = subscan2_grp[stream_name_raw]
+            dset_raw_subscan2.resize((len(stream_values),))
+            dset_raw_subscan2[:] = stream_values
+
+            # Send the interpolated values to interpreted group
+            interpolated_values = self.read_from_stream_interpolate(
+                stream_object=stream,
+                stream_slice=None,
+            )
+            if interpolated_values is None:
+                continue
+
+            if stream_name_interpreted not in interpreted_grp:
+                dset_interpreted = interpreted_grp.create_dataset(
+                    name=stream_name_interpreted,
+                    shape=(0,),
+                    maxshape=(None,),
+                    dtype=interpolated_values.dtype,
+                )
+            else:
+                dset_interpreted = interpreted_grp[stream_name_interpreted]
+            dset_interpreted.resize((len(interpolated_values),))
+            dset_interpreted[:] = interpolated_values
+
+    def create_h5_group(
+        self,
+        h5_parent_group: h5py.Group,
+        h5_group_name: str,
+        title: str = None,
+        **kwargs,
+    ) -> h5py.Group:
+        """
+        Unified method to create a group in a HDF5 file with additional attributes.
+
+        h5_parent_group: h5py.Group - Parent group where the new group will be created
+        h5_group_name: str - Name of the new group
+        title: str - Title of the group
+        kwargs: dict - Additional arguments to add as attributes in the group
+        """
+        if h5_group_name in h5_parent_group:
+            self.log_warning(
+                f"Group {h5_group_name} already exists in {h5_parent_group.name}"
+            )
+            return h5_parent_group[h5_group_name]
+
+        h5_group = h5_parent_group.create_group(h5_group_name)
+        if title:
+            h5_group["title"] = title
+        for key, value in kwargs.items():
+            h5_group.attrs[key] = value
+        return h5_group
+
+    def _update_dataset(
+        self,
+        added_dataset: numpy.ndarray,
+        h5_group: h5py.Group,
+        h5_dataset_name: str,
+        index_read: tuple = None,
+        **kwargs,
+    ) -> None:
+        """
+        Update a dataset in a HDF5 file with new data.
+        It will create the dataset if it does not exist.
+
+        added_dataset: numpy.ndarray - Array with the new data. It has to contain an additional dimension to the data
+        h5_group: h5py.Group - Group in the HDF5 file where the dataset is located
+        h5_dataset_name: str - Name of the dataset in h5_group
+        kwargs: dict - Additional arguments to add as attributes in the dataset
+        """
+        if added_dataset is None:
+            return
+
+        if not isinstance(added_dataset, numpy.ndarray):
+            self.log_error(f"Added dataset is not a numpy array. {type(added_dataset)}")
+            return
+
+        nb_new_frames = len(added_dataset)
+        if nb_new_frames == 0:
+            return
+
+        if index_read is None:
+            slice_init = 0
+            slice_end = nb_new_frames
+        else:
+            slice_init, slice_end = index_read
+
+        self.log_debug(
+            f"Updating dataset {h5_dataset_name} with {nb_new_frames} new frames"
+        )
+
+        ndim = added_dataset.ndim
+        if h5_dataset_name not in h5_group:
+            if ndim == 3:
+                interpretation = "image"
+                dtype = self.get_input_value("datatype", "float32")
+                compression = hdf5plugin.Bitshuffle(cname="lz4")
+                chunks = CHUNK_SIZE_3D
+            elif ndim == 2:
+                interpretation = "spectrum"
+                dtype = "float64"
+                compression = None
+                chunks = None
+            elif ndim == 1:
+                interpretation = "scalar"
+                dtype = added_dataset.dtype
+                compression = None
+                chunks = None
+                if added_dataset.dtype.kind == "U":
+                    dtype = h5py.string_dtype(encoding="utf-8")
+                    added_dataset = added_dataset.astype(
+                        h5py.string_dtype(encoding="utf-8")
+                    )
+            dset = h5_group.create_dataset(
+                name=h5_dataset_name,
+                shape=(0,) + added_dataset.shape[1:ndim],
+                maxshape=(None,) + added_dataset.shape[1:ndim],
+                chunks=chunks,
+                dtype=dtype,
+                compression=compression,
+            )
+            dset.attrs["interpretation"] = interpretation
+            for key, value in kwargs.items():
+                dset.attrs[key] = value
+        else:
+            dset = h5_group[h5_dataset_name]
+
+        # The added dataset can be:
+        #   - New frames to append between index_range_last limits (normal loop procedure)
+        #   - Rewrite the whole dataset, posibly with more frames (subscan2 interpolation)
+        dset_current_nb_frames = len(dset)
+        if slice_end <= dset_current_nb_frames:
+            # No resize is needed
+            ...
+        else:
+            dset_new_shape = (slice_end, *dset.shape[1:ndim])
+            dset.resize(dset_new_shape)
+
+        try:
+            dset[slice_init:slice_end] = added_dataset
+            self.log_debug(
+                f"Dataset {h5_dataset_name} updated with {len(added_dataset)} frames"
+            )
+        except Exception as e:
+            self.log_error(f"""{e}: Failed while saving {h5_dataset_name}. \
+                Incoming dataset shape: {added_dataset.shape}, \
+                hdf5 dset shape={dset.shape}, {added_dataset.shape=} \
+                limits: {slice_init} -> {slice_end}
+                """)
+
+    def write_root_group(self, root_group: h5py.Group) -> h5py.Group:
+        self.log_debug("Creating root group")
+        root_group.attrs.update(
+            {
+                "HDF5_Version": h5py.version.hdf5_version,
+                "NX_class": "NXroot",
+                "creator": "ewoksid02",
+                "file_name": str(self.processing_filename),
+                "file_time": get_isotime(),
+                "default": "entry_0000",
+            }
+        )
+        return root_group
+
+    def _save_benchmark(
+        self,
+        bench,
+    ):
+        if not self.do_save:
+            return
+
+        benchmark_name = bench.benchmark_name
+        total_time = bench.bench_total_s
+        time_per_frame = bench.bench_per_frame_ms
+        nb_frames = bench.nb_frames
+
+        with ExitStack() as stack:
+            if not self.processing_filename or not self.do_save:
+                return
+
+            # Append data to the nexus data group
+            stack.enter_context(lock)
+            root_group_destination = stack.enter_context(
+                open_item_silx(filename=self.processing_filename, name="/", mode="a")
+            )
+
+            _, h5path_ewoks_destination = self.links["destination"]["ewoks"].split("::")
+            ewoks_group = root_group_destination[h5path_ewoks_destination]
+
+            if "last_process" not in ewoks_group:
+                return
+
+            title = ewoks_group["last_process/title"][()].decode()
+            h5path_ewoks_benchmark = (
+                f"{h5path_ewoks_destination}/history/{title}/benchmark"
+            )
+
+            if h5path_ewoks_benchmark not in root_group_destination:
+                bench_grp = self.create_h5_group(
+                    h5_parent_group=root_group_destination,
+                    h5_group_name=h5path_ewoks_benchmark,
+                    title="Benchmark",
+                    NX_class="NXdata",
+                    default=h5path_ewoks_benchmark,
+                    signal="data",
+                    # axes=["steps"],
+                    axes=["index"],
+                )
+                steps = bench_grp.create_dataset(
+                    name="steps",
+                    shape=(0,),
+                    maxshape=(None,),
+                    dtype=h5py.string_dtype(encoding="utf-8"),
+                )
+                indexes = bench_grp.create_dataset(
+                    name="index",
+                    shape=(0,),
+                    maxshape=(None,),
+                )
+                data = bench_grp.create_dataset(
+                    name="data",
+                    shape=(0,),
+                    maxshape=(None,),
+                )
+                data.attrs.update(
+                    {
+                        "interpretation": "spectrum",
+                    }
+                )
+                data_errors = bench_grp.create_dataset(
+                    name="data_errors",
+                    shape=(0,),
+                    maxshape=(None,),
+                )
+                data_errors.attrs.update(
+                    {
+                        "interpretation": "spectrum",
+                    }
+                )
+            else:
+                bench_grp = root_group_destination[h5path_ewoks_benchmark]
+                steps = bench_grp["steps"]
+                indexes = bench_grp["index"]
+                data = bench_grp["data"]
+                data_errors = bench_grp["data_errors"]
+
+            h5path_ewoks_benchmark_step = (
+                f"{h5path_ewoks_destination}/history/{title}/benchmark/{benchmark_name}"
+            )
+            if h5path_ewoks_benchmark_step not in root_group_destination:
+                bench_grp_step = self.create_h5_group(
+                    h5_parent_group=root_group_destination,
+                    h5_group_name=h5path_ewoks_benchmark_step,
+                    title=benchmark_name,
+                    NX_class="NXdata",
+                    default=h5path_ewoks_benchmark,
+                )
+            else:
+                bench_grp_step = root_group_destination[h5path_ewoks_benchmark_step]
+
+            if f"{benchmark_name}_loop_nb" not in bench_grp_step:
+                loop_dset = bench_grp_step.create_dataset(
+                    name=f"{benchmark_name}_loop_nb",
+                    dtype="int32",
+                    shape=(0,),
+                    maxshape=(None,),
+                )
+                perframe_dset = bench_grp_step.create_dataset(
+                    name=f"{benchmark_name}_per_frame",
+                    dtype="float32",
+                    shape=(0,),
+                    maxshape=(None,),
+                )
+                perframe_mean_dset = bench_grp_step.create_dataset(
+                    name=f"{benchmark_name}_per_frame_mean",
+                    dtype="float32",
+                    data=time_per_frame,
+                )
+                perframe_std_dset = bench_grp_step.create_dataset(
+                    name=f"{benchmark_name}_per_frame_std",
+                    dtype="float32",
+                    data=0.0,
+                )
+                nbframes_dset = bench_grp_step.create_dataset(
+                    name=f"{benchmark_name}_nb_frames",
+                    dtype="int32",
+                    shape=(0,),
+                    maxshape=(None,),
+                )
+                total_dset = bench_grp_step.create_dataset(
+                    name=f"{benchmark_name}_total",
+                    dtype="float32",
+                    shape=(0,),
+                    maxshape=(None,),
+                )
+                accumulated_dset = bench_grp_step.create_dataset(
+                    name=f"{benchmark_name}_accumulated",
+                    dtype="float32",
+                    shape=(0,),
+                    maxshape=(None,),
+                )
+                steps.resize((steps.shape[0] + 1,))
+                steps[-1] = benchmark_name
+                indexes.resize((indexes.shape[0] + 1,))
+                indexes[-1] = len(indexes)
+                data.resize((data.shape[0] + 1,))
+                data_errors.resize((data_errors.shape[0] + 1,))
+            else:
+                loop_dset = bench_grp_step[f"{benchmark_name}_loop_nb"]
+                perframe_dset = bench_grp_step[f"{benchmark_name}_per_frame"]
+                perframe_mean_dset = bench_grp_step[f"{benchmark_name}_per_frame_mean"]
+                perframe_std_dset = bench_grp_step[f"{benchmark_name}_per_frame_std"]
+                nbframes_dset = bench_grp_step[f"{benchmark_name}_nb_frames"]
+                total_dset = bench_grp_step[f"{benchmark_name}_total"]
+                accumulated_dset = bench_grp_step[f"{benchmark_name}_accumulated"]
+
+            # Append new data
+            loop_dset.resize((loop_dset.shape[0] + 1,))
+            perframe_dset.resize((perframe_dset.shape[0] + 1,))
+            nbframes_dset.resize((nbframes_dset.shape[0] + 1,))
+            total_dset.resize((total_dset.shape[0] + 1,))
+            accumulated_dset.resize((accumulated_dset.shape[0] + 1,))
+
+            loop_dset[-1] = self.loop_nb
+            perframe_dset[-1] = time_per_frame
+            nbframes_dset[-1] = nb_frames
+            total_dset[-1] = total_time
+            if len(accumulated_dset) == 1:
+                accumulated_dset[-1] = total_time
+            else:
+                accumulated_dset[-1] = accumulated_dset[-2] + total_time
+
+            mean_value = perframe_dset[:].mean()
+            std_value = perframe_dset[:].std()
+            perframe_mean_dset[()] = mean_value
+            perframe_std_dset[()] = std_value
+
+            step_names = [s.decode() for s in steps]
+            index_benchmark = next(
+                (i for i, s in enumerate(step_names) if s == benchmark_name), None
+            )
+            if index_benchmark is not None:
+                data[index_benchmark] = mean_value
+                data_errors[index_benchmark] = std_value
+
+    def _save_processing_params(self, root_group: h5py.Group):
+        # Do it once for non-numpy arrays (live parameters)
+        _, h5path_ewoks_destination = self.links["destination"]["ewoks"].split("::")
+        ewoks_group = root_group[h5path_ewoks_destination]
+
+        if "last_process" not in ewoks_group:
+            return
+
+        title = ewoks_group["last_process/title"][()].decode()
+        h5path_ewoks_info = f"{h5path_ewoks_destination}/history/{title}"
+
+        ewoks_info_group = root_group[h5path_ewoks_info]
+        params_group_name = "parameters"
+        if params_group_name in ewoks_info_group:
+            return
+
+        info_history_complete = self.outputs.info_history
+        if info_history_complete:
+            info_history = self.outputs.info_history[-1]
+        else:
+            info_history = None
+
+        if h5path_ewoks_info not in root_group:
+            self.log_error(f"{h5path_ewoks_info} not found in {root_group}")
+            return
+
+        params_group = self.create_h5_group(
+            h5_parent_group=ewoks_info_group,
+            h5_group_name=params_group_name,
+            NX_class="NXcollection",
+        )
+
+        params_info = {}
+        for key, value in self.processing_params.items():
+            if isinstance(value, numpy.ndarray):
+                continue
+                incoming_length = value.shape[0]
+                if key not in params_group:
+                    dset = params_group.create_dataset(
+                        name=key, shape=(incoming_length,), maxshape=(None,), data=value
+                    )
+                else:
+                    dset = params_group[key]
+                    current_length = len(dset)
+                    new_length = current_length + incoming_length
+                    dset.resize((new_length,))
+                    dset[current_length:new_length] = value
+                params_info[key] = dset[()]
+                continue
+
+            if isinstance(value, dict):
+                params_info[key] = json.dumps(value)
+                for k, v in value.items():
+                    params_info[k] = str(v)
+            else:
+                try:
+                    params_info[key] = str(value)
+                except Exception:
+                    params_info[key] = json.dumps(value.as_dict())
+                    for k, v in value.items():
+                        params_info[k] = v
+
+        for key, value in params_info.items():
+            params_group[key] = value
+
+        info_history["parameters"] = params_info
+        info_history_complete = self.outputs.info_history[:-1]
+        info_history_complete.append(info_history)
+        self.outputs.info_history = info_history_complete
+
+    def _link_ewoks_history(
+        self, root_group_destination: h5py.Group, stack: ExitStack = None
+    ):
+        _, h5path_ewoks_history_destination = self.links["destination"]["ewoks"].split(
+            "::"
+        )
+        if self.links["source"].get("ewoks") is not None:
+            filename_source, h5path_ewoks_source = self.links["source"]["ewoks"].split(
+                "::"
+            )
+        else:
+            filename_source = None
+            h5path_ewoks_source = None
+
+        ewoks_destination = self.create_h5_group(
+            h5_parent_group=root_group_destination,
+            h5_group_name=h5path_ewoks_history_destination,
+            NX_class="NXcollection",
+        )
+        ewoks_history = self.create_h5_group(
+            h5_parent_group=ewoks_destination,
+            h5_group_name="history",
+            NX_class="NXcollection",
+        )
+
+        info_history_complete = self.get_input_value("info_history", []).copy()
+        info_history_previous = None
+        if info_history_complete:
+            info_history_previous = info_history_complete[-1]
+
+        title_format = "{index:02} - {class_name}"
+        info_history = {
+            "index": 0,
+            "task_identifier": f"{self.__module__}.{self.__class__.__name__}",
+            "class": self.__class__.__name__,
+            "processing_type": self.processing_type,
+            "datetime": str(get_isotime()),
+            "version": version("ewoksid02"),
+            "host": socket.gethostname(),
+        }
+        info_history["title"] = title_format.format(
+            index=info_history["index"], class_name=info_history["class"]
+        )
+        if info_history_previous is not None:
+            # The previous node send the recent historic information
+            info_history["index"] = info_history_previous["index"] + 1
+            info_history["title"] = title_format.format(
+                index=info_history["index"], class_name=info_history["class"]
+            )
+        else:
+            # No previous node with historic information
+            if filename_source and h5path_ewoks_source:
+                # This is an ewoks PROCESSED file
+                root_group_source = stack.enter_context(
+                    open_item_silx(filename=filename_source, name="/", mode="r")
+                )
+                if h5path_ewoks_source not in root_group_source:
+                    # Use ewoksid02 version 1.0 onwards
+                    self.log_warning(
+                        f"{filename_source} is an ewoks PROCESSED_DATA file without ewoks group. Historical index will be set to 0"
+                    )
+                else:
+                    ewoks_source = root_group_source[h5path_ewoks_source]
+                    if "last_process" not in ewoks_source:
+                        self.log_warning(
+                            "No index could be found in ewoks group. Historical index will be set to 0"
+                        )
+                    else:
+                        # Collect all the previous history and send to the next task
+                        history_group = ewoks_source["history"]
+                        for historic_task in history_group:
+                            historic_task_serialized = serialize_h5py_task(
+                                h5py_group=history_group[historic_task]
+                            )
+                            info_history_complete.append(historic_task_serialized)
+                        info_history["index"] = info_history_complete[-1]["index"] + 1
+                        info_history["title"] = title_format.format(
+                            index=info_history["index"],
+                            class_name=info_history["class"],
+                        )
+            else:
+                # This is not an ewoks PROCESSED file (no historic information)
+                ...
+
+        # Add a new process into history group
+        info_history_complete.append(info_history)
+
+        for historic_task in info_history_complete:
+            historic_task_group = self.create_h5_group(
+                h5_parent_group=ewoks_history,
+                h5_group_name=historic_task["title"],
+                NX_class="NXcollection",
+            )
+            deserialize_h5py_task(
+                h5dict=historic_task,
+                h5py_parent=historic_task_group,
+            )
+
+        # Link to last_process group
+        title = info_history["title"]
+        ewoks_destination["last_process"] = h5py.SoftLink(f"history/{title}")
+
+        # Send to the next node
+        self.outputs.info_history = info_history_complete
+
+    def processing_info(self) -> list:
+        return []
+
+    def _get_filename_gallery(self):
+        processed_directory = Path(self.processing_filename).parent
+        gallery_directory = processed_directory / "gallery"
+        gallery_directory.mkdir(exist_ok=True)
+        filename_png = (
+            gallery_directory / Path(self.processing_filename).with_suffix(".png").name
+        )
+        return str(filename_png)
+
+    def _save_png_to_gallery(self, array: numpy.ndarray) -> str:
+        processed_directory = Path(self.processing_filename).parent
+        gallery_directory = processed_directory / "gallery"
+        gallery_directory.mkdir(exist_ok=True)
+        filename_png = (
+            gallery_directory / Path(self.processing_filename).with_suffix(".png").name
+        )
+        filename_png = str(filename_png)
+        try:
+            matplotlib.image.imsave(
+                fname=filename_png,
+                arr=array,
+                cmap="viridis",
+                dpi=100,
+                origin="lower",
+            )
+            return filename_png
+        except Exception as e:
+            self.log_error(f"Failed to save image: {e}")
+            return ""