ewoksid02 0.1.0__py3-none-any.whl

ewoksid02/resources/__init__.py

@@ -0,0 +1,8 @@
+import os
+
+RESOURCES_DIR = os.path.dirname(__file__)
+WORKFLOW_SAXS_LOOP = os.path.join(RESOURCES_DIR, "saxs_loop.json")
+
+TEMPLATE_SAXS = os.path.join(RESOURCES_DIR, "template_saxs.yaml")
+
+WORKFLOW_SAXS_LINEAR = None

ewoksid02/resources/saxs_loop.json

@@ -0,0 +1,96 @@
+{
+  "directed": true,
+  "multigraph": false,
+  "graph": {
+    "id": "saxs_loop",
+    "schema_version": "1.1"
+  },
+  "nodes": [
+    {
+      "task_type": "class",
+      "task_identifier": "ewoksid02.tasks.normalizationtask.NormalizationTask",
+      "force_start_node": true,
+      "default_inputs" : [{"name": "reading_node", "value": true}],
+      "id": "norm"
+    },
+    {
+      "task_type": "class",
+      "task_identifier": "ewoksid02.tasks.secondaryscatteringtask.SecondaryScatteringTask",
+      "id": "2scat"
+    },
+    {
+      "task_type": "class",
+      "task_identifier": "ewoksid02.tasks.cavingtask.CavingBeamstopTask",
+      "id": "cave"
+    },
+    {
+      "task_type": "class",
+      "task_identifier": "ewoksid02.tasks.azimuthaltask.AzimuthalTask",
+      "id": "azim"
+    },
+    {
+      "task_type": "class",
+      "task_identifier": "ewoksid02.tasks.averagetask.AverageTask",
+      "id": "ave"
+    },
+    {
+      "task_type": "class",
+      "task_identifier": "ewoksid02.tasks.scalerstask.ScalersTask",
+      "id": "scalers"
+    },
+    {
+      "task_type": "class",
+      "task_identifier": "ewoksid02.tasks.end.EndTask",
+      "id": "end"
+    }
+  ],
+  "links": [
+    {
+      "map_all_data": true,
+      "source": "norm",
+      "target": "2scat",
+      "conditions": [
+        {
+          "source_output": "continue_pipeline",
+          "value": true
+        }
+      ]
+    },
+    {
+      "map_all_data": true,
+      "source": "norm",
+      "target": "end",
+      "conditions": [
+        {
+          "source_output": "continue_pipeline",
+          "value": false
+        }
+      ]
+    },
+    {
+      "map_all_data": true,
+      "source": "2scat",
+      "target": "cave"
+    },
+    {
+      "map_all_data": true,
+      "source": "cave",
+      "target": "azim"
+    },
+    {
+      "map_all_data": true,
+      "source": "azim",
+      "target": "ave"
+    },
+    {
+      "map_all_data": true,
+      "source": "ave",
+      "target": "scalers"
+    },
+    {
+      "map_all_data": true,
+      "source": "scalers",
+      "target": "norm"
+    }
+  ]
+}

ewoksid02/resources/template_saxs.yaml

@@ -0,0 +1,37 @@
+###########################
+## ewoksid02 template SAXS.yaml
+###########################
+
+bliss_filenames:                        # List of RAW_DATA files (proposal / sample / dataset files)
+    - /data/visitor/xxx/id02/xxx/RAW_DATA/xxx/.../file1.h5
+    #- /data/visitor/xxx/id02/xxx/RAW_DATA/xxx/.../file2.h5
+
+detectors:                              # List of 2D detectors to integrate (Options: eiger2, waxs, eiger500k)
+    - eiger2
+    # - waxs
+
+scans:                                  # List of scan numbers (if empty, takes all)
+    - 1
+    # - 2
+    # - !slice 1:10                     # Slice option: it will iterate between scans 1 and 10 (included)
+
+tag:                                    # Additional suffix to the output files (overwritting is forbidden)
+output_root:                            # Root folder to save the output files (if empty, replace RAW_DATA -> PROCESSED_DATA)
+
+# Processing parameters
+to_process: "norm 2scat cave azim ave"  # Include in this string which processing steps you want to perform (norm, 2scat, cave, azim, ave)
+to_save: "norm 2scat cave azim ave"     # Include in this string which processing results you want to save (norm, 2scat, cave, azim, ave)
+
+# Correction parameters
+filename_flatfield:                     # Path to the file with the flat field array (if empty, it will be taken from headers)
+filename_darkcurrent:                   # Path to the file with the dark current array (if empty, it will be taken from headers)
+filename_maskgaps:                      # Path to the file with the detector mask array (if empty, it will be taken from headers)
+filename_maskbeamstop:                  # Path to the file with an additional detector mask, like beamstop (if empty, it will be taken from headers)
+filename_window_wagon:                  # Path to the file with the WAXS pattern from the wagon window
+WindowRoiSize: 120                      # Size of the window used to convolute the WAXS pattern from the wagon window
+
+# Submit parameters
+submit: True                            # If False, it will save the instructions without submitting the job
+slurm_job_parameters:                   # Some SLURM parameters to be tweak if the job fails
+    time_limit: 01:00:00
+    memory_per_cpu: 50GB

ewoksid02/scripts/__main__.py

@@ -0,0 +1,70 @@
+import argparse
+import os
+import shutil
+import sys
+
+from ..utils import AVAILABLE_TEMPLATES, TEMPLATE_MESSAGE
+from .parsers import add_saxs_arguments, add_xpcs_arguments
+from .saxs.main import main as main_saxs
+from .xpcs.__main__ import main as main_xpcs
+
+TECHNIQUES = [
+    "saxs",
+    "xpcs",
+]
+
+
+def main(argv=None):
+    """Main function to run the offline scripts
+
+    Inputs:
+        - argv: command line arguments
+    """
+    if argv is None:
+        argv = sys.argv
+
+    parser = argparse.ArgumentParser(
+        prog="ewoksid02",
+        description="Run data processing pipelines for SAXS or XPCS.",
+        usage="ewoksid02 {saxs | xpcs | templates} [...]",
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    subparser_saxs = subparsers.add_parser("saxs", help="Trigger SAXS pipelines")
+    add_saxs_arguments(subparser_saxs)
+
+    subparser_xpcs = subparsers.add_parser("xpcs", help="Trigger XPCS pipelines")
+    add_xpcs_arguments(subparser_xpcs)
+
+    subparser_templates = subparsers.add_parser(  # noqa
+        "templates", help="Download available templates for ID02 offline pipelines"
+    )
+
+    if len(argv) == 1:
+        parser.print_help()
+        sys.exit(1)
+
+    if argv[1] == "templates":
+        print(TEMPLATE_MESSAGE)
+        for _, template_info in AVAILABLE_TEMPLATES.items():
+            os.makedirs(template_info["directory"], exist_ok=True)
+            shutil.copy(template_info["path"], template_info["future_path"])
+        return
+
+    args = parser.parse_args(argv[1:])
+    if args.command == "saxs":
+        if len(args.FILES) == 0:
+            subparser_saxs.print_help()
+            sys.exit(1)
+        main_saxs(args)
+    elif args.command == "xpcs":
+        main_xpcs(args)
+    elif args.command == "templates":
+        print(TEMPLATE_MESSAGE)
+        for _, template_info in AVAILABLE_TEMPLATES.items():
+            os.makedirs(template_info["directory"], exist_ok=True)
+            shutil.copy(template_info["path"], template_info["future_path"])
+        return
+    else:
+        parser.print_help()
+        sys.exit(1)

ewoksid02/scripts/parsers.py

@@ -0,0 +1,224 @@
+def add_celery_arguments(parser):
+    """Add to parser arguments related to celery queue and submit"""
+    parser.add_argument(
+        "-q",
+        "--queue",
+        dest="queue",
+        default=None,
+        help="Queue name for the job submission (default: None)",
+    )
+    parser.add_argument(
+        "--no-submit",
+        dest="submit",
+        action="store_false",
+        help="Do not submit anything",
+        default=None,
+    )
+
+
+def add_workflow_flags(parser):
+    """Add to parser arguments related to process and save flags."""
+    parser.add_argument(
+        "--to_process",
+        dest="to_process",
+        default=None,
+        help="Steps to process",
+    )
+    parser.add_argument(
+        "--to_save",
+        dest="to_save",
+        default=None,
+        help="Steps to save",
+    )
+
+
+def add_main_arguments(parser):
+    """Add to parser common arguments to run the offline pipeline."""
+    parser.add_argument(
+        "FILES",
+        metavar="FILES",
+        help="List of BLISS files read and reprocess",
+        nargs="+",
+    )
+    parser.add_argument(
+        "-n",
+        "--scan",
+        dest="scan_nb",
+        nargs="+",
+        help="Number of scan to process",
+    )
+
+    parser.add_argument(
+        "-d",
+        "--detector",
+        dest="detector_name",
+        default=None,
+        help="Name of the Lima detector which data to process",
+    )
+    parser.add_argument(
+        "-t",
+        "--tag",
+        dest="tag",
+        default=None,
+        help="Tag to be added on each processing filename. The workflow will not overwrite files",
+    )
+    parser.add_argument(
+        "--r",
+        "--output-root",
+        dest="output_root",
+        help="Root directory for the output files",
+        default=None,
+    )
+
+
+def add_common_saxs_arguments(parser):
+    """Add to parser common arguments to SAXS/WAXS pipeline"""
+    parser.add_argument(
+        "--metadata",
+        dest="filename_metadata",
+        default=None,
+        help="File with metadata used for processing",
+    )
+    parser.add_argument(
+        "--max",
+        dest="max_slice_size",
+        default=None,
+        type=int,
+        help="Maximum number of slices to process at once. Default: 100",
+    )
+    parser.add_argument(
+        "--dummy",
+        dest="dummy",
+        default=None,
+        help="Dummy value",
+    )
+    parser.add_argument(
+        "--delta-dummy",
+        dest="delta_dummy",
+        default=None,
+        help="Delta dummy value",
+    )
+    parser.add_argument(
+        "-l",
+        "--log-level",
+        dest="log_level",
+        default=None,
+        help="Logging level.",
+    )
+
+
+def add_normalization_arguments(parser):
+    """Add to parser arguments for NormalizationTask"""
+    help = "Normalization: "
+    parser.add_argument(
+        "--dark",
+        dest="dark_current_filename",
+        default=None,
+        help=help + "Filename of the dark current file",
+    )
+    parser.add_argument(
+        "--flat",
+        dest="flat_field_filename",
+        default=None,
+        help=help + "Filename of the flat field file",
+    )
+
+
+def add_masking_arguments(parser):
+    """Add to parser arguments for CavingTask"""
+    help = "Caving: "
+    parser.add_argument(
+        "--mask-gaps",
+        dest="mask_gaps_filename",
+        default=None,
+        help=help + "Filename of the mask with detector gaps",
+    )
+    parser.add_argument(
+        "--mask-beamstop",
+        dest="mask_beamstop_filename",
+        default=None,
+        help=help + "Filename of the mask with the beamstop",
+    )
+
+
+def add_secondary_scattering_arguments(parser):
+    """Add to parser arguments for SecondaryScatteringTask"""
+    help = "Secondary Scattering: "
+    parser.add_argument(
+        "--window-pattern",
+        dest="window_pattern_filename",
+        default=None,
+        help=help + "Filename of the window pattern file",
+    )
+    parser.add_argument(
+        "--window-roi",
+        dest="window_roi_size",
+        default=None,
+        help=help + "Subdata distance for secondary scattering correction",
+    )
+    parser.add_argument(
+        "--flip-vertical",
+        dest="flip_vertical",
+        action="store_true",
+        default=None,
+        help=help + "Flip the image vertically (default: False)",
+    )
+    parser.add_argument(
+        "--flip-horizontal",
+        dest="flip_horizontal",
+        action="store_true",
+        default=None,
+        help=help + "Flip the image horizontally (default: False)",
+    )
+
+
+def add_azimuthal_arguments(parser):
+    """Add to parser arguments for AzimuthalTask"""
+    help = "Azimuthal: "
+    parser.add_argument(
+        "--npt-rad",
+        dest="npt_rad",
+        default=None,
+        help=help + "Number of radial bins",
+    )
+    parser.add_argument(
+        "--npt-azim",
+        dest="npt_azim",
+        default=None,
+        help=help + "Number of azimuthal bins",
+    )
+    parser.add_argument(
+        "--unit",
+        dest="unit",
+        default=None,
+        help=help + "Unit for azimuthal averaging",
+    )
+
+
+def add_average_arguments(parser):
+    """Add to parser arguments for AverageTask"""
+    help = "Average: "
+    parser.add_argument(
+        "--azim-range",
+        dest="azimuth_range",
+        default=None,
+        help=help + "Azimuthal limits for the average",
+    )
+
+
+def add_saxs_arguments(parser):
+    """Add all arguments needed for SAXS/WAXS pipeline"""
+    add_main_arguments(parser)
+    add_common_saxs_arguments(parser)
+    add_celery_arguments(parser)
+    add_normalization_arguments(parser)
+    add_masking_arguments(parser)
+    add_secondary_scattering_arguments(parser)
+    add_azimuthal_arguments(parser)
+    add_average_arguments(parser)
+    add_workflow_flags(parser)
+
+
+def add_xpcs_arguments(parser):
+    """Add all arguments needed for XPCS pipeline"""
+    add_main_arguments(parser)

ewoksid02/scripts/saxs/main.py

@@ -0,0 +1,255 @@
+import logging
+import time
+from pprint import pprint
+
+from ewokstools.reprocess import (
+    finish_queue,
+    generate_params_from_yaml_file,
+    get_datasets_list_id02,
+    get_params_from_cli,
+    save_and_execute,
+    validate_inputs_ewoks,
+)
+
+from ...resources import WORKFLOW_SAXS_LOOP
+from ...tasks.averagetask import AverageTask
+from ...tasks.azimuthaltask import AzimuthalTask
+from ...tasks.cavingtask import CavingBeamstopTask
+from ...tasks.normalizationtask import NormalizationTask
+from ...tasks.secondaryscatteringtask import SecondaryScatteringTask
+from ...utils.blissdata import LIMA_URL_TEMPLATE_ID02, get_lima_url_template_args_id02
+from ..utils import SLURM_JOB_PARAMETERS_SAXS
+
+logger = logging.getLogger(__name__)
+WORKER_MODULE = "scattering"
+
+
+def get_saxs_inputs(
+    **kwargs,
+) -> list:
+    """Compile and return the list of inputs to be used on an ewoks SAXS/WAXS pipeline."""
+    inputs = []
+
+    ###########
+    # Add all nodes inputs
+    ###########
+    inputs_dict_all_nodes = {
+        "detector_name": kwargs.get("detector_name"),
+        "filename_data": kwargs.get("dataset_filename"),
+        "scan_nb": kwargs.get("scan_nb"),
+        "filename_lima": kwargs.get("filename_scan"),
+        "log_level": kwargs.get("log_level", "info"),
+        "datatype": kwargs.get("datatype"),
+        "save_variance": kwargs.get("save_variance"),
+        "max_slice_size": kwargs.get("max_slice_size"),
+        "lima_url_template": LIMA_URL_TEMPLATE_ID02,
+        "lima_url_template_args": get_lima_url_template_args_id02(
+            scan_number=kwargs.get("scan_nb"),
+            detector_name=kwargs.get("detector_name"),
+            collection_name=kwargs.get("collection_name"),
+        ),
+    }
+
+    inputs += validate_inputs_ewoks(
+        inputs=inputs_dict_all_nodes,
+        all=True,
+        ewoks_task=None,
+        id=None,
+    )
+
+    #############
+    # Add normalization inputs
+    #############
+    inputs_dict_norm = {
+        "filename_mask": kwargs.get("filename_maskgaps"),
+        "filename_dark": kwargs.get("filename_darkcurrent"),
+        "filename_flat": kwargs.get("filename_flatfield"),
+        "algorithm": kwargs.get("algorithm_norm", "cython"),
+        "Dummy": kwargs.get("Dummy"),
+        "DDummy": kwargs.get("DDummy"),
+    }
+    inputs += validate_inputs_ewoks(
+        inputs=inputs_dict_norm,
+        ewoks_task=NormalizationTask,
+        id="norm",
+    )
+
+    #############
+    # Add secondary scattering inputs
+    #############
+    inputs_dict_2scat = {
+        "filename_window_wagon": kwargs.get("filename_window_wagon"),
+        "WindowRoiSize": kwargs.get("WindowRoiSize"),
+        "filename_mask_static": kwargs.get("filename_maskgaps"),
+        "filename_mask_reference": kwargs.get("filename_maskbeamstop"),
+        "algorithm": kwargs.get("algorithm_2scat", "numpy"),
+    }
+    inputs += validate_inputs_ewoks(
+        inputs=inputs_dict_2scat,
+        ewoks_task=SecondaryScatteringTask,
+        id="2scat",
+    )
+
+    #############
+    # Add caving inputs
+    #############
+    inputs_dict_cave = {
+        "filename_mask_static": kwargs.get("filename_maskbeamstop"),
+        "flip_caving": kwargs.get("flip_caving"),
+        "algorithm": kwargs.get("algorithm_cave", "numpy"),
+    }
+    inputs += validate_inputs_ewoks(
+        inputs=inputs_dict_cave,
+        ewoks_task=CavingBeamstopTask,
+        id="cave",
+    )
+
+    #############
+    # Add azimuthal inputs
+    #############
+    inputs_dict_azim = {
+        "filename_mask": kwargs.get("filename_maskbeamstop"),
+        "npt_rad": kwargs.get("npt_rad"),
+        "npt_azim": kwargs.get("npt_azim"),
+        "unit": kwargs.get("unit"),
+    }
+    inputs += validate_inputs_ewoks(
+        inputs=inputs_dict_azim,
+        ewoks_task=AzimuthalTask,
+        id="azim",
+    )
+
+    #############
+    # Add average inputs
+    #############
+    inputs_dict_ave = {
+        "azimuth_range": kwargs.get("azimuth_range"),
+    }
+    inputs += validate_inputs_ewoks(
+        inputs=inputs_dict_ave,
+        ewoks_task=AverageTask,
+        id="ave",
+    )
+
+    #############
+    # Add flag inputs
+    #############
+    to_process = kwargs.get("to_process", "").split(" ")
+    to_save = kwargs.get("to_save", "").split(" ")
+    nodes = ["norm", "2scat", "cave", "azim", "ave", "scalers"]
+
+    inputs += [
+        {"name": "do_process", "id": node, "value": node in to_process}
+        for node in nodes
+    ]
+    inputs += [
+        {"name": "do_save", "id": node, "value": node in to_save} for node in nodes
+    ]
+
+    ##############
+    # Add processing filenames inputs
+    ##############
+    processing_filename_template = kwargs.get("processed_filename_scan")
+    tag = kwargs.get("tag", "")
+    if tag:
+        processing_filename_template = processing_filename_template.replace(
+            ".h5", f"_{tag}.h5"
+        )
+    inputs_dict_filenames = {
+        "norm": processing_filename_template.replace(".h5", "_norm.h5"),
+        "2scat": processing_filename_template.replace(".h5", "_2scat.h5"),
+        "cave": processing_filename_template.replace(".h5", "_cave.h5"),
+        "azim": processing_filename_template.replace(".h5", "_azim.h5"),
+        "ave": processing_filename_template.replace(".h5", "_ave.h5"),
+        "scalers": processing_filename_template.replace(".h5", "_scalers.h5"),
+    }
+    inputs += [
+        {"name": "processing_filename", "value": value, "id": task_id}
+        for task_id, value in inputs_dict_filenames.items()
+    ]
+
+    return inputs
+
+
+def main(args):
+    """Main function to trigger the SAXS/WAXS pipeline."""
+    saxs_parameters = {}
+    saxs_dataset_parameters = {}
+
+    # 2) Get parameters from .yaml files provided in the command line
+    for saxs_parameters_from_yaml in generate_params_from_yaml_file(args.FILES):
+        saxs_parameters.update(saxs_parameters_from_yaml)
+
+        # 3) Add more bliss filenames from the command line
+        saxs_parameters["bliss_filenames"] += [
+            file for file in args.FILES if file.endswith(".h5")
+        ]
+
+        # 4) Get parameters from the command line
+        reprocess_parameters_from_cli = get_params_from_cli(args)
+        saxs_parameters.update(reprocess_parameters_from_cli)
+
+        # 5) If no input/pyfai parameter was provided, try through user input
+        if not saxs_parameters.get("bliss_filenames"):
+            saxs_parameters["bliss_filenames"] = (
+                input(
+                    "No bliss filenames provided. Please enter the filenames (comma-separated): "
+                )
+                .strip()
+                .split(",")
+            )
+
+        # 6) Iterate through the bliss saving objects
+        dataset_list = get_datasets_list_id02(**saxs_parameters)
+        nb_datasets = len(dataset_list)
+        print(
+            f"\033[92mFound {nb_datasets} datasets in {saxs_parameters['bliss_filenames']}\033[0m"
+        )
+        filenames_dataset = [
+            dataset_info["dataset_filename"] for dataset_info in dataset_list
+        ]
+        print("\033[92m", end="")
+        pprint(filenames_dataset)
+        print("\033[0m", end="")
+
+        if nb_datasets > 10:
+            logger.warning(
+                "More than 10 datasets found in this file. You have 10 seconds to cancel..."
+            )
+            time.sleep(10)
+
+        for nb_submitted, dataset_info in enumerate(dataset_list, start=1):
+            saxs_dataset_parameters = {
+                **saxs_parameters,
+                **dataset_info,
+            }
+
+            tag = saxs_dataset_parameters.get("tag", "")
+            dryrun = "dryrun" if not saxs_dataset_parameters.get("submit") else ""
+            tag = "_".join(filter(None, [tag, dryrun]))
+            saxs_dataset_parameters["tag"] = tag
+
+            # Take slurm parameters
+            slurm_job_parameters = {
+                **SLURM_JOB_PARAMETERS_SAXS,
+                **saxs_dataset_parameters.pop("slurm_job_parameters", {}),
+            }
+
+            save_and_execute(
+                workflow=WORKFLOW_SAXS_LOOP,
+                inputs=get_saxs_inputs(**saxs_dataset_parameters),
+                slurm_job_parameters=slurm_job_parameters,
+                processing_name=tag,
+                **saxs_dataset_parameters,
+                execution_kwargs={
+                    "engine": "ppf",
+                    "pool_type": "thread",
+                },
+                worker_module=WORKER_MODULE,
+            )
+
+            print(
+                f"\033[92mSubmitted {nb_submitted}/{nb_datasets} datasets for reprocessing: {nb_submitted / nb_datasets * 100:.2f}%\033[0m"
+            )
+
+    finish_queue(**saxs_dataset_parameters)