ewoksid02 0.1.0__py3-none-any.whl

ewoksid02/tasks/metadatatask.py

@@ -0,0 +1,879 @@
+# -*- coding: utf-8 -*-
+
+import json
+import logging
+import os
+import posixpath
+from collections.abc import Mapping
+from numbers import Integral
+from string import Formatter
+
+import h5py
+import numpy as np
+import pyFAI.io
+from blissdata.h5api import dynamic_hdf5, static_hdf5
+from ewoksxrpd.tasks.data_access import TaskWithDataAccess
+
+pyFAI.io.logger.setLevel(logging.ERROR)
+if pyFAI.version_info < (0, 20):
+    from .nexus import Nexus
+else:
+    from pyFAI.io import Nexus
+
+# silence non serious error messages, which are printed
+# because we use h5py in a new thread (not in the main one)
+# this is a bug seems to be fixed on newer version !!
+# https://github.com/h5py/h5py/issues/206
+
+try:
+    h5py._errors.silence_errors()
+except Exception:
+    passStringTypes = (str, bytes)
+
+StringTypes = (str, bytes)
+logger = logging.getLogger(__name__)
+
+__authors__ = ["Jérôme Kieffer"]
+__contact__ = "Jerome.Kieffer@ESRF.eu"
+__license__ = "MIT"
+__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
+__date__ = "03/09/2020"
+__status__ = "development"
+__version__ = "1.0.0"
+
+# Deprecated, to be substituted by the Id02ProcessingTask children
+
+
+def ensure_str(junk):
+    "return a unicode string, regardless to the input"
+    if isinstance(junk, bytes):
+        return junk.decode()
+    else:
+        return str(junk)
+
+
+def channels_name_to_fullname(scan_info):
+    """
+    Return a directory with channel name in the scan file to the fullname (ensure consistency between scans, always use the fullname.)
+    """
+
+    rd = {}
+
+    for sscan_i, k in enumerate(sorted(scan_info["acquisition_chain"].keys())):
+
+        sscan = sscan_i + 1
+
+        rd[sscan] = dict()
+
+        scal = (
+            scan_info["acquisition_chain"][k]["master"]["scalars"]
+            + scan_info["acquisition_chain"][k]["scalars"]
+        )
+        # print(scal)
+        for countname in scal:
+
+            fullname = countname.replace(":", "_")
+            scnt = countname.split(":")
+
+            if len(scnt) == 2:
+                shortname = scnt[-1]
+            elif len(scnt) == 3:
+                shortname = f"{scnt[0]}_{scnt[-1]}"
+            else:
+                raise ValueError("Unknown shortname format")
+
+            if shortname not in rd[sscan]:
+                rd[sscan][shortname] = fullname
+            else:  # Name defined more than once in sscan
+                if (
+                    rd[sscan][shortname] is not None
+                ):  # Already remove the previous shortname, skip
+                    rd[sscan][rd[sscan][shortname]] = rd[sscan][
+                        shortname
+                    ]  # rd[Fullname] = Fullname
+                    # print(shortname, fullname, rd[sscan][shortname])
+                    rd[sscan][shortname] = None
+
+                if fullname in rd[sscan]:
+                    raise ValueError("Fullname appears more than once in subscan....")
+
+            rd[sscan][fullname] = fullname
+
+    # Remove Nones from dict
+    to_delete = []
+
+    for sscan, dd in rd.items():
+        for k, v in dd.items():
+            if v is None:
+                to_delete += [
+                    (sscan, k),
+                ]
+
+    for s, k in to_delete:
+        del rd[s][k]
+
+    return rd
+
+
+def get_bliss_scan_allcounters(
+    filename: str,
+    scan_nr: Integral,
+    nsubs: Integral,
+    **options,
+):
+    """Iterate over the data from one Bliss scan. The counters are assumed to have
+    many data values as scan points.
+
+    :param str filename: the Bliss dataset filename
+    :param Integral filename: the scan number in the dataset
+    :param Number retry_timeout: timeout when it cannot access the data for `retry_timeout` seconds
+    :param Number retry_period: interval in seconds between data access retries
+    :return dict: data
+    """
+
+    r = {}
+
+    # First ensure that all the subscans are finished (have end_time)
+    with dynamic_hdf5.File(filename, **options, locking=False) as root:
+
+        for subscan in range(1, nsubs + 1):
+            s = f"{scan_nr}.{subscan}"
+            _ = root[s]["end_time"]  # Ensure the subscan is finished to write
+
+    with static_hdf5.File(filename, locking=False) as root:
+        for subscan in range(1, nsubs + 1):
+
+            fullscanname = f"{scan_nr}.{subscan}"
+
+            scan = root[fullscanname]
+            measurement = scan["measurement"]
+            r[int(subscan)] = {
+                n: np.array(measurement[n])
+                for n in measurement
+                if measurement[n].ndim == 1
+            }
+
+    return r
+
+
+def safe_formatting(template, **fields):
+
+    r = ""
+
+    F = Formatter()
+
+    for lt, fn, fspec, conv in F.parse(template):
+        print((lt, fn, fspec, conv))
+        r += lt
+
+        val = fields.get(fn, None)
+
+        if val is not None:
+            try:
+                if conv is not None:
+                    val = F.convert_field(val, conv)
+
+                val = F.format_field(val, fspec)
+
+            except Exception:
+                if isinstance(val, str):
+                    val = f"!!!{val}!!!"
+                else:
+                    val = None
+
+        if val is None:
+            if fn is not None:
+                val = "{%s%s%s}" % (
+                    fn,
+                    "" if conv is None else f"!{conv}",
+                    "" if fspec is None or len(fspec) == 0 else f":{fspec}",
+                )
+            else:
+                val = ""
+
+        r += f"{val}"
+
+    return r
+
+
+class MetadataTask(
+    TaskWithDataAccess,
+    input_names=["scan_info", "hdf5_filename", "headers"],
+    output_names=["input2", "counters"],
+    optional_input_names=[
+        "entry",
+        "instrument",
+        "raw_counter_names",
+        "val_counter_names",
+    ],
+):
+    """
+    Metadata Task read the values of specific scan, and write the scalers file
+    """
+
+    TO_SKIP = ("entry", "hdf5_filename", "plugin_name")
+
+    def __del__(self):
+        self.close()
+
+    def close(self):
+        if self.nxs is not None:
+            self.nxs.close()
+            self.nxs = None
+
+    def run(self):
+
+        # Read parameters and preprocess
+        self.hdf5_filename = self.inputs["hdf5_filename"]
+        self.entry = self.get_input_value("entry", "entry")
+        self.instrument = self.get_input_value("instrument", "TRUSAXS")
+        self.raw_counter_names = self.get_input_value("raw_counter_names", None)
+        self.val_counter_names = self.get_input_value("val_counter_names", None)
+
+        self.scan_info = self.inputs["scan_info"]
+
+        scan_nr = self.scan_info["scan_nb"]
+        nsubs = len(self.scan_info["acquisition_chain"])
+        filename = self.scan_info["filename"]
+
+        self.input2 = self.preproc(self.inputs["headers"])
+        self.outputs["input2"] = self.input2
+
+        # Create HDF5 output file (from DAHU sources)
+        self.create_hdf5()
+
+        # Read counters
+        self.counters = get_bliss_scan_allcounters(
+            filename, scan_nr, nsubs, retry_timeout=1800, retry_period=10
+        )
+
+        #        print(self.counters)
+
+        self.outputs["counters"] = self.counters
+
+        self.write_counters_to_file()
+
+        # Close file
+        self.close()
+
+    def _getPin(self, val):
+        try:
+            pin = int(val)
+        except ValueError:
+            if val in self.val_counter_names.values():
+                pin = int(
+                    list(self.val_counter_names.keys())[
+                        list(self.val_counter_names.values()).index(val)
+                    ]
+                )
+            else:
+                raise
+
+        return pin
+
+    def write_counters_to_file(self):
+        """
+        Process and write the counters to the output file
+        """
+
+        fastscan = 1
+        cntcounts = dict()
+
+        fastscanchecker = self.val_counter_names.copy()
+        if fastscanchecker is None:
+            fastscanchecker = self.raw_counter_names
+        elif self.raw_counter_names is not None:
+            fastscanchecker.update(self.raw_counter_names)
+
+        if fastscanchecker is not None:
+            allthere = True
+            for subs in self.counters:
+                cntcounts[subs] = 0
+                for idx, name in self.raw_counter_names.items():
+                    if name not in self.counters[subs]:
+                        allthere = False
+                    else:
+                        cntcounts[subs] += 1
+
+                if allthere:
+                    fastscan = subs
+                    break
+            else:
+                maxc = 0
+                for subs, v in cntcounts.items():
+                    if maxc < v:
+                        maxc = v
+                        fastscan = subs
+
+                print(f"Fastscan not detected. Assuming subscan #{fastscan}.")
+        else:
+            print("no raw_counter_names nor val_counter_names option.")
+
+        nframes = None
+
+        if "npoints" in self.scan_info:
+            nframes = int(self.scan_info["npoints"])
+
+        if nframes is None:
+
+            for k in self.counters[fastscan]:
+                nframes = self.counters[fastscan][k].shape[0]
+                break
+
+        if nframes is None:
+            raise RuntimeError("Unable to determine the number of frames")
+
+        HS32C = -1 * np.ones((nframes, self.HS32Len), dtype=np.int64)
+        HS32V = np.zeros((nframes, self.HS32Len), dtype=np.float64)
+
+        # HS32C: raw counters
+        if self.raw_counter_names is not None:
+            if not isinstance(self.raw_counter_names, Mapping):
+                raise ValueError(
+                    "raw_counter_names must be a index: counter name mapping"
+                )
+
+            for idx, name in self.raw_counter_names.items():
+                try:
+                    counters_values = self.counters[fastscan][name].ravel()
+
+                    if counters_values.shape[0] > nframes:
+                        #                        raise RuntimeError(f"Counters have more values than nframes ({nframes}), this is not allowed!")
+                        counters_values = counters_values[:nframes]
+
+                    elif counters_values.shape[0] < nframes:
+                        logger.info(
+                            "There is less counters values than number of frames: filling with 0."
+                        )
+                        counters_values = np.append(
+                            counters_values,
+                            np.zeros(
+                                (nframes - counters_values.shape[0],),
+                                dtype=counters_values.dtype,
+                            ),
+                        )
+
+                    HS32C[:, int(idx) - 1] = counters_values
+                except KeyError:
+                    logger.error(f"{name} not in fastscan.")
+
+        self.mcs_grp.create_dataset(
+            "HS32C", data=HS32C, dtype=HS32C.dtype
+        ).attrs.__setitem__("interpretation", "spectrum")
+
+        # HS32V: processed counters
+        if self.val_counter_names is not None:
+            if not isinstance(self.val_counter_names, Mapping):
+                raise ValueError(
+                    "val_counter_names must be a index: counter name mapping"
+                )
+
+            for idx, name in self.val_counter_names.items():
+                try:
+                    counters_values = self.counters[fastscan][name].ravel()
+
+                    if counters_values.shape[0] > nframes:
+                        raise RuntimeError(
+                            f"Counters have more values than nframes ({nframes}), this is not allowed!"
+                        )
+
+                    elif counters_values.shape[0] < nframes:
+                        logger.info(
+                            "There is less counters values than number of frames: filling with 0."
+                        )
+                        counters_values = np.append(
+                            counters_values,
+                            np.zeros(
+                                (nframes - counters_values.shape[0],),
+                                dtype=counters_values.dtype,
+                            ),
+                        )
+
+                    HS32V[:, int(idx) - 1] = counters_values
+                except KeyError:
+                    logger.error(f"{name} not in fastscan.")
+
+        self.mcs_grp.create_dataset(
+            "HS32V", data=HS32V, dtype=HS32V.dtype
+        ).attrs.__setitem__("interpretation", "spectrum")
+
+        if "HSTime" in self.input2:
+
+            pin = self._getPin(self.input2["HSTime"])
+
+            if pin > self.HS32Len:
+                self.log_error("invalid pin number %s" % pin)
+            self.mcs_grp["HSTime"] = pin
+            self.mcs_grp["HSTime"].attrs["interpretation"] = "scalar"
+            self.mcs_grp["HSTime"].attrs["counter"] = "1-based pin number"
+            pin -= 1  # 1 based pin number
+            time_counter = HS32C[:, pin]
+            if "HS32F" in self.mcs_grp:
+                factor = self.mcs_grp["HS32F"][pin]
+            else:
+                logger.info(
+                    "No factors provided for time measurement: defaulting to 1e-6"
+                )
+                factor = 1e-6
+            measured_time = time_counter * factor
+            self.mcs_grp["ExposureTime"] = measured_time
+            self.mcs_grp["ExposureTime"].attrs["interpretation"] = "scalar"
+
+            # I0/I1
+            sot = self.input2.get("ShutterOpeningTime", 0.0)
+            sct = self.input2.get("ShutterClosingTime", 0.0)
+            for name, value in (
+                ("ShutterOpeningTime", sot),
+                ("ShutterClosingTime", sct),
+            ):
+                self.mcs_grp[name] = value
+                self.mcs_grp[name].attrs["interpretation"] = "scalar"
+            correction_time = (measured_time - sot + sct) / (measured_time - sot)
+
+            for key in ("HSI0", "HSI1"):
+                if key in self.input2:
+                    dest = "Intensity" + key[-1]
+                    pin = self._getPin(self.input2[key])
+                    if pin > self.HS32Len:
+                        self.log_error("invalid pin number %s" % pin)
+                    self.mcs_grp[key] = pin
+                    self.mcs_grp[key].attrs["interpretation"] = "scalar"
+                    self.mcs_grp[key].attrs["counter"] = "1-based pin number"
+                    pin -= 1  # 1 based pin number got 0 based.
+                    counter = HS32V[:, pin]
+                    #                     factor = self.mcs_grp["HS32F"][pin]
+                    #                     zero = self.mcs_grp["HS32Z"][pin]
+                    #                     measured = (counter - measured_time * zero) * factor
+                    I_factor = float(self.input2.get(key + "Factor", 1.0))
+                    self.mcs_grp[key + "Factor"] = I_factor
+                    self.mcs_grp[key + "Factor"].attrs["interpretation"] = "scalar"
+                    measured = counter * I_factor
+                    self.mcs_grp[dest + "UnCor"] = measured
+                    self.mcs_grp[dest + "UnCor"].attrs["interpretation"] = "scalar"
+                    self.mcs_grp[dest + "ShutCor"] = measured * correction_time
+                    self.mcs_grp[dest + "ShutCor"].attrs["interpretation"] = "scalar"
+
+        else:
+            logger.error("No HSTime pin number")
+
+        # This is needed by Saxsutilities
+        for k in ("HMStartTime", "HMStartEpoch"):
+            if k in self.inputs.headers:
+                self.tfg_grp[k] = self.inputs.headers[k]
+
+        # Write all the raw and interpreted values (All counters)
+
+        translator = channels_name_to_fullname(self.scan_info)
+
+        #        print(translator)
+
+        raw_grp = self.mcs_grp["raw"]
+        for subs, v in self.counters.items():
+            subsk = f"subscan_{subs}"
+            raw_grp.require_group(subsk)
+
+            for k, c in v.items():
+                ds = raw_grp[subsk].create_dataset(k, data=c)
+                ds.attrs.update({"interpretation": "scalar"})
+
+        # Fast scan are not interpolated
+        for k in self.counters[fastscan]:
+
+            # FIXME: Soft link are better, but not supported by DAHU...
+            # fastscankey = f'subscan_{fastscan}'
+            # self.mcs_grp['interpreted'][k] = h5py.SoftLink(f'{raw_grp.name}/{fastscankey}/{k}')
+            #            self.mcs_grp['interpreted'][translator[fastscan][k]] = self.counters[fastscan][k]
+
+            counter_val = self.counters[fastscan][k]
+
+            if counter_val.shape[0] < nframes:
+                counter_val = np.append(
+                    counter_val, np.nan * np.ones((nframes - counter_val.shape[0],))
+                )
+
+            ds = self.mcs_grp["interpreted"].create_dataset(
+                translator[fastscan][k], data=counter_val
+            )
+            ds.attrs.update(
+                {
+                    "interpretation": "scalar",
+                    "operation": "",
+                    "subscan": fastscan,
+                    "original_name": k,
+                }
+            )
+
+        master_eltime_key = self.find_elapsed_time(fastscan)
+        if master_eltime_key is not None:
+            master_elapsed_time = self.counters[fastscan][master_eltime_key]
+
+            if master_elapsed_time.shape[0] < nframes:
+                _master_elapsed_time = np.append(
+                    master_elapsed_time,
+                    np.nan * np.ones((nframes - master_elapsed_time.shape[0],)),
+                )
+            else:
+                _master_elapsed_time = master_elapsed_time
+            # FIXME: Soft link are better, but not supported by DAHU...
+            # int_grp = self.mcs_grp['interpreted']
+            # self.tfg_grp['delta_time'] = h5py.SoftLink(f'{int_grp.name}/elapsed_time')
+            self.tfg_grp["delta_time"] = _master_elapsed_time
+
+        master_epoch_key = self.find_epoch(fastscan)
+        if master_epoch_key is not None:
+            master_epoch = self.counters[fastscan][master_epoch_key]
+
+            # The main elapsed_time must be there. Create soft link if the master elaped time has different name.
+            if "epoch" not in self.mcs_grp["interpreted"]:
+                # FIXME: Soft link are better, but not supported by DAHU...
+                # self.mcs_grp['interpreted']['elapsed_time'] = h5py.SoftLink(master_eltime_key)
+
+                if master_epoch.shape[0] < nframes:
+                    _master_epoch = np.append(
+                        master_epoch,
+                        np.nan * np.ones((nframes - master_epoch.shape[0],)),
+                    )
+                else:
+                    _master_epoch = master_elapsed_time
+
+                self.mcs_grp["interpreted"]["epoch"] = _master_epoch
+
+            self.mcs_grp["interpreted"]["epoch"].attrs.update(
+                {"interpretation": "scalar", "master_epoch": True}
+            )
+
+            # Interpolate and save values from the other scans
+            for subs, v in self.counters.items():
+                if subs == fastscan:  # Skip fast scan interpolation
+                    continue
+
+                subsk = f"subscan_{subs}"
+                slave_epoch_key = self.find_epoch(subs)
+
+                if slave_epoch_key is not None:
+                    slave_epoch = self.counters[subs][slave_epoch_key]
+
+                    for k, c in v.items():
+                        # patch edgar 29-08-2025
+                        if len(slave_epoch) != len(self.counters[subs][k]):
+                            logger.error(
+                                f"Length of {slave_epoch_key} != length of counter"
+                            )
+                            continue
+
+                        # print('interp', master_elapsed_time, slave_elapsed_time, self.counters[subs][k])
+                        interpdata = np.interp(
+                            master_epoch, slave_epoch, self.counters[subs][k]
+                        )
+
+                        if interpdata.shape[0] < nframes:
+                            interpdata = np.append(
+                                interpdata,
+                                np.nan * np.ones((nframes - interpdata.shape[0],)),
+                            )
+
+                        ds = self.mcs_grp["interpreted"].create_dataset(
+                            translator[subs][k], data=interpdata
+                        )
+                        ds.attrs.update(
+                            {
+                                "interpretation": "scalar",
+                                "operation": "interpolate",
+                                "subscan": subs,
+                                "original_name": k,
+                                "slave_epoch": slave_epoch_key,
+                                "master_epoch": master_epoch_key,
+                            }
+                        )
+
+                else:
+                    print(f"Unable to find an epoch time vector for subscan {subs}")
+
+        else:
+            print(
+                "Unable to find the master epoch time vector. Give up with the interpolated values."
+            )
+
+        # Fill TitleExtension if needed
+        if "TitleExtension" in self.inputs.headers:
+            template = str(self.inputs.headers["TitleExtension"])
+            cntnames = [
+                fn for _, fn, _, _ in Formatter().parse(template) if fn is not None
+            ]
+
+            #            print(template, cntnames)
+
+            if len(cntnames) > 0:
+
+                cntval = {}
+                maxl = 0
+
+                for cnt in cntnames:
+                    if cnt not in self.mcs_grp["interpreted"]:
+                        logger.error(f"{cnt} is not a valid counter.")
+                        cntval[cnt] = None
+                    else:
+                        cntval[cnt] = np.array(self.mcs_grp["interpreted"][cnt])
+                        if len(cntval[cnt]) > maxl:
+                            maxl = len(cntval[cnt])
+
+                if maxl > 0:
+                    t_ext = []
+                    for rval in self._counters_value_iter(cntval):
+                        #                        t_ext += [ template.format(**rval), ]
+                        t_ext += [safe_formatting(template, **rval)]
+
+                    if "TitleExtension" in self.info_grp:
+                        del self.info_grp["TitleExtension"]
+
+                    self.info_grp["TitleExtension"] = t_ext
+
+    def _counters_value_iter(self, cntval):
+
+        keys = cntval.keys()
+        maxl = max(
+            [
+                0,
+            ]
+            + [len(v) for k, v in cntval.items() if v is not None]
+        )
+
+        for i in range(maxl):
+            frameval = {}
+            for k in keys:
+                if cntval[k] is None:
+                    frameval[k] = "CniS"
+                else:
+                    try:
+                        frameval[k] = cntval[k][i]
+                    except IndexError:
+                        frameval[k] = "***"
+
+            yield frameval
+
+    def find_elapsed_time(self, subs):
+
+        cnts = self.counters[subs]
+
+        known_titles_in_priority_order = (
+            "elapsed_time",
+            "mcs_elapsed_time",
+            "timer_elapsed_time",
+        )
+
+        for t in known_titles_in_priority_order:
+            if t in cnts:
+                return t
+
+        # If standard titles not found, just look for *elapsed_time
+        for k in cnts:
+            if k.endswith("elapsed_time"):
+                return k
+
+        # Give up.
+        return None
+
+    def find_epoch(self, subs):
+
+        cnts = self.counters[subs]
+
+        known_titles_in_priority_order = ("epoch", "mcs_epoch", "timer_epoch")
+
+        for t in known_titles_in_priority_order:
+            if t in cnts:
+                return t
+
+        # If standard titles not found, just look for *epoch
+        for k in cnts:
+            if k.endswith("epoch"):
+                return k
+
+        # Give up.
+        return None
+
+    # From DAHU https://github.com/kif/dahu/blob/master/plugins/id02/metadata.py
+    def preproc(self, headers):
+        """Take a dict as input and forms a metadata structure as output
+        @param: any dict
+        """
+        dd = headers.copy()
+        if "job_id" in dd:
+            dd.pop("job_id")
+        list_f = []
+        list_n = []
+        list_z = []
+        HS32Len = int(dd.get("HS32Len", 16))
+
+        self.HS32Len = HS32Len
+
+        HS32Depth = int(dd.get("HS32Depth", 32))
+        HSI0Factor = float(dd.get("HSI0Factor", 1))
+        HSI1Factor = float(dd.get("HSI1Factor", 1))
+        # "0.005 s"
+        if "ShutterOpeningTime" in dd:
+            value = dd["ShutterOpeningTime"]
+            if type(value) in StringTypes:
+                ShutterOpeningTime = float(value.split()[0])
+            else:
+                ShutterOpeningTime = float(value)
+        else:
+            ShutterOpeningTime = 0
+        if "ShutterClosingTime" in dd:
+            value = dd["ShutterClosingTime"]
+            if type(value) in StringTypes:
+                ShutterClosingTime = float(value.split()[0])
+            else:
+                ShutterClosingTime = float(value)
+        else:
+            ShutterClosingTime = 0
+        for ind in map(lambda x: "HS32F" + "{0:02d}".format(x), range(1, HS32Len + 1)):
+            list_f.append(float(dd[ind]))
+        self.HS32F = np.array(list_f)
+
+        for ind in map(lambda x: "HS32N" + "{0:02d}".format(x), range(1, HS32Len + 1)):
+            list_n.append(dd[ind])
+        self.HS32N = list_n
+
+        for ind in map(lambda x: "HS32Z" + "{0:02d}".format(x), range(1, HS32Len + 1)):
+            list_z.append(float(dd[ind]))
+        self.HS32Z = np.array(list_z)
+
+        info_dir = {}
+        for info_ind in dd:
+            if info_ind[0:2].find("HS") == 0:
+                continue
+            elif info_ind[0:2].find("HM") == 0:
+                continue
+            else:
+                info_dir[info_ind] = dd[info_ind]
+
+        final_dir = {
+            "HS32Len": HS32Len,
+            "HS32Depth": HS32Depth,
+            "HSI0Factor": HSI0Factor,
+            "HSI1Factor": HSI1Factor,
+            "ShutterOpeningTime": ShutterOpeningTime,
+            "ShutterClosingTime": ShutterClosingTime,
+            "instrument": "id02",
+            "c216": "bliss",  # DEVICE from which the meta are read
+            "HS32F": list_f,
+            "HS32Z": list_z,
+            "HS32N": list_n,
+            "Info": info_dir,
+        }
+        for key in [
+            "HMStartEpoch",
+            "HMStartTime",
+            "hdf5_filename",
+            "entry",
+            "HSTime",
+            "HSI0",
+            "HSI1",
+        ]:
+            if key in dd:
+                final_dir[key] = dd[key]
+        return final_dir
+
+    # From DAHU https://github.com/kif/dahu/blob/master/plugins/id02/metadata.py
+    def create_hdf5(self):
+        """
+        Create a HDF5 file and data-structure
+        """
+
+        # Ensure output directory exists
+
+        outdir = os.path.abspath(os.path.dirname(self.hdf5_filename))
+        print(f"Creating {outdir}")
+        print(os.environ)
+        os.makedirs(outdir, exist_ok=True)
+
+        try:
+            self.nxs = Nexus(self.hdf5_filename, mode="a", creator=__name__)
+        except IOError as error:
+            print(
+                "Unable to open %s: %s. Removing file and starting from scratch"
+                % (self.hdf5_filename, error)
+            )
+            os.unlink(self.hdf5_filename)
+            self.nxs = Nexus(self.hdf5_filename, mode="w", creator=__name__)
+
+        entry = self.nxs.new_entry(
+            self.entry, program_name=__name__, title="TFG metadata collection"
+        )
+        self.entry = entry.name
+        entry["program_name"].attrs["version"] = __version__
+
+        # configuration
+        config_grp = self.nxs.new_class(entry, "configuration", "NXnote")
+        config_grp["type"] = "text/json"
+
+        inps = dict()
+        for k, v in self.inputs._container.items():
+            try:
+                inps[k] = v.serialize()
+            except Exception:
+                pass
+
+        config_grp["data"] = json.dumps(
+            inps, indent=2, separators=(",\r\n", ": "), default=str
+        )
+
+        # Instrument
+        instrument_grp = self.nxs.new_instrument(
+            entry=entry, instrument_name=self.instrument
+        )
+        instrument_grp["name"] = "TruSAXS"
+        self.instrument = instrument_grp.name
+
+        # TimeFrameGenerator
+        self.tfg_grp = self.nxs.new_class(instrument_grp, "TFG", "NXcollection")
+        self.tfg_grp["device"] = "bliss"
+
+        # MultiCounterScaler
+        self.mcs_grp = self.nxs.new_class(instrument_grp, "MCS", "NXcollection")
+        self.mcs_grp["device"] = "bliss"
+
+        # Static metadata
+        self.info_grp = self.nxs.h5.require_group(
+            posixpath.join(self.instrument, "parameters")
+        )
+        self.info_grp.attrs["NX_class"] = "NXcollection"
+
+        for field, value in self.input2.get("Info", {}).items():
+            if field not in self.TO_SKIP and not isinstance(value, dict):
+                try:
+                    value.encode("ascii")
+                except UnicodeEncodeError:
+                    print("Unicode Error in field %s: %s, skipping" % (field, value))
+                except AttributeError:
+                    self.info_grp[field] = str(value)
+                else:
+                    self.info_grp[field] = str(value)
+
+        # Factor
+        HS32F = self.input2.get("HS32F")
+        if HS32F is not None:
+            self.mcs_grp.create_dataset("HS32F", data=HS32F).attrs.__setitem__(
+                "interpretation", "spectrum"
+            )
+        # Zero
+        HS32Z = self.input2.get("HS32Z")
+        if HS32Z is not None:
+            self.mcs_grp.create_dataset("HS32Z", data=HS32Z).attrs.__setitem__(
+                "interpretation", "spectrum"
+            )
+        # Name
+        HS32N = self.input2.get("HS32N")
+        if HS32N is not None:
+            self.mcs_grp.create_dataset(
+                "HS32N", data=[i.encode() for i in HS32N]
+            ).attrs.__setitem__("interpretation", "spectrum")
+        # Mode
+        HS32M = self.input2.get("HS32M")
+        if HS32M is not None:
+            self.mcs_grp.create_dataset("HS32M", data=HS32M).attrs.__setitem__(
+                "interpretation", "spectrum"
+            )
+
+        self.mcs_grp.require_group("interpreted")
+        self.mcs_grp.require_group("raw")