ewoksxes 0.0.1__py3-none-any.whl

ewoksxes/tasks/__init__.py

@@ -0,0 +1,19 @@
+from .calibrate_energy import CalibrateEnergy
+from .combine_spectra import CombineSpectra
+from .compute_flat_and_mask import ComputeFlatAndMask
+from .compute_roi import ComputeROI
+from .fit_polynomial_2d import FitPolynomial2D
+from .flat_field_correction import FlatFieldCorrection
+from .load_raw_data import LoadRawDataAverage
+from .save_flat_and_mask import SaveFlatAndMask
+
+__all__ = [
+    "CalibrateEnergy",
+    "CombineSpectra",
+    "ComputeFlatAndMask",
+    "ComputeROI",
+    "FitPolynomial2D",
+    "FlatFieldCorrection",
+    "LoadRawDataAverage",
+    "SaveFlatAndMask",
+]

ewoksxes/tasks/calibrate_energy.py

@@ -0,0 +1,69 @@
+import logging
+
+import numpy as np
+from ewokscore import Task
+
+logger = logging.getLogger(__name__)
+
+
+class CalibrateEnergy(
+    Task,
+    input_names=["spectra", "kb_px", "vtc_px", "e_kb", "e_vtc"],
+    output_names=["energies", "spectra", "slope", "intercept"],
+):
+    """
+    Two-point linear calibration from pixel -> energy for each ROI.
+
+    Inputs
+    ------
+    spectra : list[np.ndarray]
+        One 1D spectrum per ROI.
+    kb_px : list[float] | np.ndarray
+        Pixel index of the Kβ reference for each ROI.
+    vtc_px : list[float] | np.ndarray
+        Pixel index of the VTC reference for each ROI.
+    e_kb : float
+        Known energy of the Kβ line (eV).
+    e_vtc : float
+        Known energy of the VTC line (eV).
+
+    Outputs
+    -------
+    energies : list[np.ndarray]
+        Energy axis per ROI (same length as its spectrum).
+    spectra : list[np.ndarray]
+        Pass-through of the input spectra (pipeline convenience).
+    slope : np.ndarray
+        ΔE / Δpixel per ROI.
+    intercept : np.ndarray
+        Intercept per ROI so that E = slope * x + intercept.
+    """
+
+    def run(self):
+        spectra = list(self.inputs.spectra)
+        kb = np.asarray(self.inputs.kb_px, dtype=float).ravel()
+        vtc = np.asarray(self.inputs.vtc_px, dtype=float).ravel()
+        e_kb = float(self.inputs.e_kb)
+        e_vtc = float(self.inputs.e_vtc)
+
+        if len(spectra) != kb.size or kb.size != vtc.size:
+            raise ValueError(
+                "Lengths must match: len(spectra) == len(kb_px) == len(vtc_px)"
+            )
+
+        dv = vtc - kb
+        if np.any(dv == 0.0):
+            raise ValueError("kb_px and vtc_px must differ for every ROI")
+
+        slope = (e_vtc - e_kb) / dv  # shape (n_rois,)
+        intercept = e_kb - slope * kb  # shape (n_rois,)
+
+        energies = []
+        for s, m, b in zip(spectra, slope, intercept):
+            x = np.arange(s.shape[0], dtype=float)
+            energies.append(m * x + b)
+
+        self.outputs.energies = energies
+        self.outputs.spectra = spectra
+        self.outputs.slope = slope
+        self.outputs.intercept = intercept

ewoksxes/tasks/combine_spectra.py

@@ -0,0 +1,117 @@
+import logging
+
+import numpy as np
+from ewokscore import Task
+from scipy.interpolate import interp1d
+
+logger = logging.getLogger(__name__)
+
+
+class CombineSpectra(
+    Task,
+    input_names=["energies", "spectra", "energy_range", "n_points"],
+    optional_input_names=["normalize", "norm_range"],
+    output_names=["energy", "summed_spectrum"],
+):
+    """
+    Optionally normalize spectra, then interpolate each onto a common energy axis
+    and sum them.
+
+    Inputs
+    ------
+    - energies: list of 1D energy arrays (or lists), one per spectrum
+    - spectra:       list of 1D spectra (same length as energies)
+    - energy_range:  (emin, emax)  -> the range of the FINAL output energy axis
+    - n_points:      int >= 2      -> number of points on the FINAL output energy axis
+
+    Optional
+    --------
+    - normalize:  bool (default False). When True, each spectrum is normalized by
+                  its integral over 'norm_range' before interpolation/summing.
+    - norm_range: (emin, emax) normalization window (required when normalize=True)
+
+    Outputs
+    -------
+    - energy:           np.ndarray of length n_points, spanning [emin, emax]
+    - summed_spectrum:  np.ndarray of same length, sum of all interpolated spectra
+    """
+
+    def run(self):
+        raw_energies = self.inputs.energies
+        raw_spectra = self.inputs.spectra
+        energy_range = self.inputs.energy_range
+        n_points = self.inputs.n_points
+
+        # Validate and USE energy_range as the FINAL axis range
+        try:
+            emin, emax = energy_range
+        except (TypeError, ValueError):
+            raise ValueError(f"energy_range must be (emin, emax); got {energy_range}")
+        if emin >= emax:
+            raise ValueError(f"Invalid energy_range: {energy_range}")
+        if not isinstance(n_points, int) or n_points < 2:
+            raise ValueError(f"n_points must be integer >= 2, got {n_points}")
+
+        # Convert to arrays
+        energies_list = [np.asarray(e, dtype=float) for e in raw_energies]
+        spectra = [np.asarray(s, dtype=float) for s in raw_spectra]
+
+        if len(energies_list) != len(spectra):
+            raise ValueError("energies_list and spectra must have the same length")
+
+        # Optional normalization
+        if bool(self.get_input_value("normalize", False)):
+            norm_range = self.get_input_value("norm_range", None)
+            if norm_range is None or len(norm_range) != 2:
+                raise ValueError(
+                    "When normalize=True, norm_range=(emin, emax) must be "
+                    f"provided; got {norm_range}"
+                )
+            nmin, nmax = float(norm_range[0]), float(norm_range[1])
+
+            for idx, (energies, spectrum) in enumerate(zip(energies_list, spectra)):
+                mask = (energies >= nmin) & (energies <= nmax)
+                if not np.any(mask):
+                    logger.warning(
+                        f"Spectrum {idx}: no points in norm_range {nmin}-{nmax}; "
+                        "leaving spectrum unchanged"
+                    )
+                    continue
+                norm_factor = np.trapezoid(spectrum[mask], energies[mask])
+                if norm_factor == 0 or not np.isfinite(norm_factor):
+                    logger.warning(
+                        f"Spectrum {idx}: invalid normalization factor "
+                        f"{norm_factor}; leaving spectrum unchanged"
+                    )
+                    continue
+                spectra[idx] = spectrum / norm_factor
+            logger.info("CombineSpectra: normalization completed.")
+
+        # Build FINAL energy axis USING energy_range
+        energy = np.linspace(emin, emax, n_points, dtype=float)
+        summed = np.zeros_like(energy)
+
+        # Interpolate & sum
+        for idx, (energies, spectrum) in enumerate(zip(energies_list, spectra)):
+            if energies.shape[0] != spectrum.shape[0]:
+                common = min(energies.shape[0], spectrum.shape[0])
+                logger.warning(
+                    f"Spectrum {idx} length {spectrum.shape[0]} and energies "
+                    f"{energies.shape[0]} mismatch; trimming to first {common} "
+                    "points for interpolation."
+                )
+                energies_to_use = energies[:common]
+                spectrum_to_use = spectrum[:common]
+            else:
+                energies_to_use = energies
+                spectrum_to_use = spectrum
+
+            interp = interp1d(
+                energies_to_use, spectrum_to_use, bounds_error=False, fill_value=0.0
+            )
+            summed += interp(energy)
+            logger.info(f"CombineSpectra: added spectrum {idx} to sum.")
+
+        self.outputs.energy = energy
+        self.outputs.summed_spectrum = summed
+        logger.info("CombineSpectra completed.")

ewoksxes/tasks/compute_flat_and_mask.py

@@ -0,0 +1,176 @@
+import logging
+
+import numpy as np
+from ewokscore import Task
+
+from .utils import poly2d_eval
+
+logger = logging.getLogger(__name__)
+
+
+def _expand_columns(mask: np.ndarray, expand: int) -> np.ndarray:
+    """Binary dilate a column mask by ±expand columns (no extra deps)."""
+    if expand <= 0:
+        return mask
+    out = mask.copy()
+    for k in range(1, expand + 1):
+        out[:, k:] |= mask[:, :-k]
+        out[:, :-k] |= mask[:, k:]
+    return out
+
+
+def _expand_rows(mask: np.ndarray, expand: int) -> np.ndarray:
+    """Binary dilate a row mask by ±expand rows (no extra deps)."""
+    if expand <= 0:
+        return mask
+    out = mask.copy()
+    for k in range(1, expand + 1):
+        out[k:, :] |= mask[:-k, :]
+        out[:-k, :] |= mask[k:, :]
+    return out
+
+
+class ComputeFlatAndMask(
+    Task,
+    input_names=["image", "coeffs"],
+    optional_input_names=[
+        # base validity thresholds
+        "min_intensity",  # discard pixels with raw <= this (default: 0.0)
+        "min_fitted",  # discard where fitted surface <= this (default: 1e-6)
+        # gap detection via residual ratio
+        "ratio_threshold",  # ratio=image/fitted below this => bad (default: 0.2)
+        # fraction of bad pixels in a column to flag gap (default: 0.8)
+        "column_fraction_threshold",
+        "row_fraction_threshold",  # same for rows (default: 0.95)
+        "detect_columns",  # default True
+        "detect_rows",  # default False (usually gaps are vertical)
+        "gap_expand",  # dilate gap bands by this many pixels (default: 1)
+        # flat output
+        "invert",  # output flat = fitted/image if True (default True)
+        "clip_flat",  # optional clip for flat magnitudes, e.g. (0.25, 4.0)
+    ],
+    output_names=["flat", "mask"],
+):
+    """
+    Compute flat-field and detect gap bands *automatically* from the 2D fit residuals.
+
+    Pipeline:
+      1) Evaluate fitted surface S = poly2d(x,y; coeffs).
+      2) Build a base invalid mask:
+           - image <= min_intensity OR ~finite OR S <= min_fitted
+      3) Residual ratio R = image / S on valid base pixels; elsewhere treat as 0.
+      4) Candidate bad pixels = base_invalid OR (R <= ratio_threshold).
+      5) Gap detection (bands):
+           - per-column bad fraction; mark columns >= column_fraction_threshold as gaps.
+           - optional per-row bad fraction for horizontal bands.
+           - optional dilation by 'gap_expand' px.
+      6) Final valid mask = NOT(gap_bands) AND NOT(base_invalid)
+      7) Flat:
+           - if invert: flat = S / image on valid pixels (else flat = image / S)
+           - fill invalid pixels with 1.0 (so correction leaves them unchanged;
+             mask will zero them later).
+           - optional clipping.
+
+    Outputs:
+      - flat (float32, finite)
+      - mask (float32, 1 valid / 0 invalid). Gaps are 0 in the mask.
+    """
+
+    def run(self):
+        img = np.asarray(self.inputs.image, dtype=np.float64)
+        coeffs = np.asarray(self.inputs.coeffs, dtype=np.float64)
+
+        # --- Parameters & defaults
+        min_intensity = float(self.get_input_value("min_intensity", 0.0))
+        min_fitted = float(self.get_input_value("min_fitted", 1e-6))
+
+        ratio_thr = float(self.get_input_value("ratio_threshold", 0.2))
+        col_fr_thr = float(self.get_input_value("column_fraction_threshold", 0.8))
+        row_fr_thr = float(self.get_input_value("row_fraction_threshold", 0.95))
+        detect_columns = bool(self.get_input_value("detect_columns", True))
+        detect_rows = bool(self.get_input_value("detect_rows", False))
+        gap_expand = int(self.get_input_value("gap_expand", 1))
+
+        invert = bool(self.get_input_value("invert", True))
+        clip_flat = self.get_input_value("clip_flat", None)  # e.g., (0.25, 4.0) or None
+
+        H, W = img.shape
+        y = np.arange(H, dtype=np.float64)
+        x = np.arange(W, dtype=np.float64)
+        X, Y = np.meshgrid(x, y)
+
+        # --- Evaluate fitted surface
+        fitted = poly2d_eval((X, Y), coeffs)
+
+        # --- Base invalid mask
+        base_invalid = (
+            (img <= min_intensity)
+            | ~np.isfinite(img)
+            | (fitted <= min_fitted)
+            | ~np.isfinite(fitted)
+        )
+
+        # --- Residual ratio
+        ratio = np.zeros_like(img, dtype=np.float64)
+        good = ~base_invalid
+        ratio[good] = img[good] / np.maximum(fitted[good], min_fitted)
+        candidate_bad = base_invalid | (ratio <= ratio_thr)
+
+        # --- Gap detection (bands)
+        gap_bands = np.zeros_like(candidate_bad, dtype=bool)
+
+        if detect_columns:
+            col_bad_frac = candidate_bad.mean(axis=0)  # per-column fraction bad
+            gap_cols = col_bad_frac >= col_fr_thr
+            if gap_cols.any():
+                gap_bands[:, gap_cols] = True
+            logger.info(
+                "Detected %d gap columns (thr=%.2f)",
+                int(gap_cols.sum()),
+                col_fr_thr,
+            )
+
+        if detect_rows:
+            row_bad_frac = candidate_bad.mean(axis=1)  # per-row fraction bad
+            gap_rows = row_bad_frac >= row_fr_thr
+            if gap_rows.any():
+                gap_bands[gap_rows, :] = True
+            logger.info(
+                "Detected %d gap rows (thr=%.2f)",
+                int(gap_rows.sum()),
+                row_fr_thr,
+            )
+
+        # --- Expand/dilate gap bands to cover edges
+        if detect_columns and gap_expand > 0:
+            gap_bands = _expand_columns(gap_bands, gap_expand)
+        if detect_rows and gap_expand > 0:
+            gap_bands = _expand_rows(gap_bands, gap_expand)
+
+        # --- Final valid mask
+        valid = (~gap_bands) & (~base_invalid)
+
+        # --- Flat computation
+        # invert=True  => flat = fitted / image (typical: multiply raw by flat)
+        # invert=False => flat = image / fitted
+        eps = 1e-12
+        flat = np.ones_like(img, dtype=np.float64)
+        if invert:
+            flat[valid] = fitted[valid] / np.maximum(img[valid], eps)
+        else:
+            flat[valid] = img[valid] / np.maximum(fitted[valid], eps)
+
+        if clip_flat is not None:
+            lo, hi = clip_flat
+            flat = np.clip(flat, lo, hi)
+
+        # Fill invalids with 1.0 (so downstream (image*flat*mask) → masked zeros)
+        flat[~valid] = 1.0
+
+        self.outputs.flat = flat.astype(np.float32)
+        self.outputs.mask = valid.astype(np.float32)
+        logger.info(
+            "ComputeFlatAndMask: valid=%.2f%%, gaps=%.2f%%",
+            100.0 * valid.mean(),
+            100.0 * gap_bands.mean(),
+        )

ewoksxes/tasks/compute_roi.py

@@ -0,0 +1,50 @@
+import logging
+
+from ewokscore import Task
+
+logger = logging.getLogger(__name__)
+
+
+class ComputeROI(
+    Task,
+    input_names=["image_corrected", "roi_list"],
+    output_names=["spectra"],
+):
+    """
+    Extract mean 1D spectra for each ROI.
+
+    roi_list can be:
+      - [(y1, y2), (y1, y2), ...] for full-width horizontal stripes
+      - [(y1, y2, x1, x2), ...] for rectangular ROIs
+
+    For each ROI:
+        roi = image_corrected[y1:y2, x1:x2]
+        spectra = mean over axis=0 (vertical collapse)
+
+    The result is a list of 1D spectra.
+    """
+
+    def run(self):
+        image = self.inputs.image_corrected
+        result_spectra = []
+
+        for roi in self.inputs.roi_list:
+            if len(roi) == 2:
+                # full-width stripe
+                y1, y2 = roi
+                x1, x2 = 0, image.shape[1]
+            elif len(roi) == 4:
+                # rectangle with x-limits
+                y1, y2, x1, x2 = roi
+            else:
+                raise ValueError(
+                    f"ROI {roi} must have 2 values (y1,y2) or 4 values (y1,y2,x1,x2)"
+                )
+
+            logger.info(f"Computing spectra for ROI: y={y1}:{y2}, x={x1}:{x2}")
+            roi_data = image[y1:y2, x1:x2]
+            spectrum = roi_data.mean(axis=0)
+            result_spectra.append(spectrum)
+
+        self.outputs.spectra = result_spectra
+        logger.info(f"Extracted {len(result_spectra)} spectra")

ewoksxes/tasks/fit_polynomial_2d.py

@@ -0,0 +1,68 @@
+import logging
+
+import numpy as np
+from ewokscore import Task
+from scipy.optimize import curve_fit
+
+from .utils import poly2d_eval
+
+logger = logging.getLogger(__name__)
+
+
+class FitPolynomial2D(
+    Task,
+    input_names=["image"],
+    optional_input_names=["degree", "mask_zeros"],
+    output_names=["coeffs"],
+):
+    """
+    Fit a 2D cubic polynomial (16 coefficients) to a flatfield-like image.
+
+    - Intended for XES von Hamos detectors (gapped columns).
+    - Zeros (dead/gap pixels) can be excluded from the fit.
+
+    Inputs
+    ------
+    image : 2D float array
+    degree : int (default=3)  # only cubic supported
+    mask_zeros : bool (default=True)
+
+    Output
+    ------
+    coeffs : (16,) float64
+    """
+
+    def run(self):
+        img = np.asarray(self.inputs.image, dtype=np.float64)
+        degree = int(self.get_input_value("degree", 3))
+        mask_zeros = bool(self.get_input_value("mask_zeros", True))
+
+        if degree != 3:
+            raise NotImplementedError(
+                "Only cubic (degree=3) 2D polynomial fitting is supported"
+            )
+
+        H, W = img.shape
+        y = np.arange(H, dtype=np.float64)
+        x = np.arange(W, dtype=np.float64)
+        X, Y = np.meshgrid(x, y)
+
+        # Exclude obvious dead/gap pixels from the fit
+        mask = (img == 0.0) if mask_zeros else np.zeros_like(img, dtype=bool)
+
+        xdata = np.vstack((X.ravel()[~mask.ravel()], Y.ravel()[~mask.ravel()]))
+        zdata = img.ravel()[~mask.ravel()]
+
+        # Initial guess (all ones works fine for this basis)
+        p0 = np.ones(16, dtype=np.float64)
+
+        def model(xy, *params):
+            return poly2d_eval(xy, np.asarray(params))
+
+        logger.info(
+            "Fitting 2D cubic polynomial to flatfield (zeros masked: %s)...", mask_zeros
+        )
+        coeffs, _ = curve_fit(model, xdata, zdata, p0=p0, maxfev=200000)
+
+        self.outputs.coeffs = coeffs.astype(np.float64)
+        logger.info("FitPolynomial2D completed. Coeff count: %d", coeffs.size)

ewoksxes/tasks/flat_field_correction.py

@@ -0,0 +1,102 @@
+import logging
+import os
+
+import fabio
+import numpy as np
+from ewokscore import Task
+
+logger = logging.getLogger(__name__)
+
+
+class FlatFieldCorrection(
+    Task,
+    input_names=["image", "flat_field_path"],
+    optional_input_names=["mask_path", "i0"],
+    output_names=["image_corrected"],
+):
+    """
+    Apply flat-field and (optionally) mask corrections to the raw image,
+    then normalize by I0.
+
+    Supported file formats:
+      - Flat-field: .edf, .tif, .tiff, .npy
+      - Mask: .edf, .tif, .tiff, .msk, .npy
+
+    If no mask is provided, a mask of ones is used.
+
+    Correction:
+        corrected = ((image * flat) * mask) / i0
+
+    i0 is a scalar (incident flux) used to normalize the result.
+    If omitted, defaults to 1.
+    """
+
+    SUPPORTED_EXTENSIONS_FLAT = (".edf", ".tif", ".tiff", ".npy")
+    SUPPORTED_EXTENSIONS_MASK = (".edf", ".tif", ".tiff", ".msk", ".npy")
+
+    def _load_image_file(self, path: str, label: str, allowed_exts) -> np.ndarray:
+        """Helper to load image files with validation."""
+        if not os.path.exists(path):
+            raise FileNotFoundError(f"{label} file does not exist: {path}")
+
+        ext = os.path.splitext(path)[1].lower()
+        if ext not in allowed_exts:
+            raise ValueError(
+                f"{label} file format not supported ({path}). "
+                f"Supported extensions: {allowed_exts}"
+            )
+
+        logger.info(f"Loading {label} file: {path}")
+        if ext == ".npy":
+            data = np.load(path).astype(np.float32)
+        else:
+            data = fabio.open(path).data.astype(np.float32)
+
+        logger.info(f"{label} shape: {data.shape}")
+        return data
+
+    def run(self):
+        raw_image = self.inputs.image
+        logger.info(
+            f"Starting flat-field correction, raw image shape: {raw_image.shape}"
+        )
+
+        # Load flat-field
+        flat = self._load_image_file(
+            self.inputs.flat_field_path, "Flat-field", self.SUPPORTED_EXTENSIONS_FLAT
+        )
+
+        # Load mask if provided
+        mask_path = self.get_input_value("mask_path", None)
+        if mask_path:
+            mask = self._load_image_file(
+                mask_path, "Mask", self.SUPPORTED_EXTENSIONS_MASK
+            )
+        else:
+            logger.info("No mask file provided; using an all-ones mask.")
+            mask = np.ones_like(raw_image, dtype=np.float32)
+
+        # Validate shapes
+        if flat.shape != raw_image.shape:
+            raise ValueError(
+                f"Flat-field shape {flat.shape} does not match "
+                f"raw image shape {raw_image.shape}"
+            )
+        if mask.shape != raw_image.shape:
+            raise ValueError(
+                f"Mask shape {mask.shape} does not match "
+                f"raw image shape {raw_image.shape}"
+            )
+
+        # Apply correction
+        corrected = (raw_image * flat) * mask
+
+        # I0 normalization
+        i0_value = self.get_input_value("i0", 1.0)
+        if i0_value != 1.0:
+            logger.info(f"Normalizing by I0 = {i0_value}")
+        corrected = corrected / i0_value
+
+        self.outputs.image_corrected = corrected
+        logger.info("Flat-field correction completed successfully.")
+        logger.info(f"Corrected image shape: {corrected.shape}")

ewoksxes/tasks/load_raw_data.py

@@ -0,0 +1,66 @@
+import logging
+import os
+
+import h5py
+import numpy as np
+from ewokscore import Task
+
+logger = logging.getLogger(__name__)
+
+
+class LoadRawDataAverage(
+    Task,
+    input_names=["bliss_scan_data_url"],
+    output_names=["image"],
+):
+    """
+    Load raw detector data from a Bliss scan data URL.
+
+    The input `bliss_scan_data_url` must be of the form:
+
+        /full/path/to/sample_dataset.h5::scan.1/measurement/detector
+
+    Where:
+        - `/full/path/to/sample_dataset.h5` is the HDF5 file.
+        - `scan.1/measurement/detector` is the internal HDF5 dataset path.
+
+    The task loads the dataset. If the dataset is 3D (multiple frames),
+    it averages along the first dimension to produce a 2D image.
+    """
+
+    def run(self):
+        # Split the URL into file path and dataset path
+        url = self.inputs.bliss_scan_data_url
+        if "::" not in url:
+            raise ValueError(
+                f"Invalid bliss_scan_data_url format: {url}. "
+                "Expected file.h5::dataset/path"
+            )
+
+        file_path, dataset_path = url.split("::", 1)
+
+        logger.info(f"Opening HDF5 file: {file_path}")
+        logger.info(f"Reading dataset: {dataset_path}")
+
+        if not os.path.exists(file_path):
+            raise FileNotFoundError(f"HDF5 file does not exist: {file_path}")
+
+        with h5py.File(file_path, "r") as f:
+            if dataset_path not in f:
+                raise KeyError(f"Dataset '{dataset_path}' not found in {file_path}")
+            data = f[dataset_path][:]
+
+        logger.info(f"Raw data shape: {data.shape}")
+
+        if data.ndim == 3:
+            logger.info("Averaging along the first axis of the 3D dataset")
+            image = data.mean(axis=0)
+        elif data.ndim == 2:
+            image = data
+        else:
+            raise ValueError(
+                f"Unexpected data shape {data.shape}. Expected 2D or 3D array."
+            )
+
+        self.outputs.image = image.astype(np.float32)
+        logger.info(f"Final image shape: {self.outputs.image.shape}")