ewoksid02 0.0.1a0__tar.gz

ewoksid02-0.0.1a0/LICENSE.md

@@ -0,0 +1,20 @@
+# MIT License
+
+**Copyright (c) 2024 European Synchrotron Radiation Facility**
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

ewoksid02-0.0.1a0/PKG-INFO

@@ -0,0 +1,76 @@
+Metadata-Version: 2.4
+Name: ewoksid02
+Version: 0.0.1a0
+Summary: Data processing SAXS and XPCS workflows for ID02
+Author-email: ESRF <edgar.gutierrez-fernandez@esrf.fr>
+License: # MIT License
+        
+        **Copyright (c) 2024 European Synchrotron Radiation Facility**
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy of
+        this software and associated documentation files (the "Software"), to deal in
+        the Software without restriction, including without limitation the rights to
+        use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+        the Software, and to permit persons to whom the Software is furnished to do so,
+        subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+        FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+        COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+        IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+        CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+        
+Project-URL: Homepage, https://gitlab.esrf.fr/workflow/ewoksapps/ewoksid02/
+Project-URL: Documentation, https://ewoksid02.readthedocs.io/
+Project-URL: Repository, https://gitlab.esrf.fr/workflow/ewoksapps/ewoksid02/
+Project-URL: Issues, https://gitlab.esrf.fr/workflow/ewoksapps/ewoksid02/issues
+Project-URL: Changelog, https://gitlab.esrf.fr/workflow/ewoksapps/ewoksid02/-/blob/main/CHANGELOG.md
+Keywords: ewoks
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE.md
+Requires-Dist: ewoks
+Requires-Dist: ewokscore
+Requires-Dist: ewoksjob[blissworker]
+Requires-Dist: ewoksjob[slurm]
+Requires-Dist: ewoksppf
+Requires-Dist: h5py
+Requires-Dist: numpy<2
+Requires-Dist: scipy
+Requires-Dist: silx
+Requires-Dist: pyfai
+Requires-Dist: blissdata
+Requires-Dist: numexpr!=2.8.6
+Requires-Dist: psutil
+Requires-Dist: pyyaml
+Provides-Extra: test
+Requires-Dist: pytest>=7; extra == "test"
+Provides-Extra: dev
+Requires-Dist: ewoksid02[test]; extra == "dev"
+Requires-Dist: black>=25; extra == "dev"
+Requires-Dist: flake8>=4; extra == "dev"
+Provides-Extra: doc
+Requires-Dist: ewoksid02[test]; extra == "doc"
+Requires-Dist: sphinx>=4.5; extra == "doc"
+Requires-Dist: sphinx-autodoc-typehints>=1.16; extra == "doc"
+Requires-Dist: pydata-sphinx-theme; extra == "doc"
+Requires-Dist: nbsphinx; extra == "doc"
+Requires-Dist: ipython; extra == "doc"
+Provides-Extra: cupy
+Requires-Dist: cupy; extra == "cupy"
+Dynamic: license-file
+
+# ewoksid02
+
+Data processing SAXS and XPCS workflows for ID02
+
+## Documentation
+
+https://ewoksid02.readthedocs.io/

ewoksid02-0.0.1a0/README.md

@@ -0,0 +1,7 @@
+# ewoksid02
+
+Data processing SAXS and XPCS workflows for ID02
+
+## Documentation
+
+https://ewoksid02.readthedocs.io/

ewoksid02-0.0.1a0/pyproject.toml

@@ -0,0 +1,89 @@
+[build-system]
+requires = [
+    "setuptools>=61",
+]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "ewoksid02"
+version = "0.0.1-alpha"
+keywords = [
+    'ewoks'
+]
+
+authors = [{name = "ESRF", email = "edgar.gutierrez-fernandez@esrf.fr"}]
+description = "Data processing SAXS and XPCS workflows for ID02"
+readme = {file = "README.md", content-type = "text/markdown"}
+license = {file = "LICENSE.md"}
+classifiers = [
+    'Intended Audience :: Science/Research',
+    'License :: OSI Approved :: MIT License',
+    'Programming Language :: Python :: 3',
+]
+requires-python = ">=3.8"
+dependencies = [
+    "ewoks",
+    "ewokscore",
+    "ewoksjob[blissworker]",
+    "ewoksjob[slurm]",
+    "ewoksppf",
+    "h5py",
+    "numpy<2",
+    "scipy",
+    "silx",
+    "pyfai",
+    "blissdata",
+    "numexpr !=2.8.6",
+    "psutil",
+    "pyyaml",
+]
+
+[project.urls]
+Homepage = "https://gitlab.esrf.fr/workflow/ewoksapps/ewoksid02/"
+Documentation = "https://ewoksid02.readthedocs.io/"
+Repository = "https://gitlab.esrf.fr/workflow/ewoksapps/ewoksid02/"
+Issues = "https://gitlab.esrf.fr/workflow/ewoksapps/ewoksid02/issues"
+Changelog = "https://gitlab.esrf.fr/workflow/ewoksapps/ewoksid02/-/blob/main/CHANGELOG.md"
+
+[project.optional-dependencies]
+test = [
+    'pytest >=7',
+]
+dev = [
+    "ewoksid02[test]",
+    'black >=25',
+    'flake8 >=4',
+]
+doc = [
+    "ewoksid02[test]",
+    'sphinx >=4.5',
+    'sphinx-autodoc-typehints >=1.16',
+    'pydata-sphinx-theme',
+    'nbsphinx',
+    'ipython',
+]
+cupy = [
+    'cupy',
+]
+
+[tool.setuptools]
+package-dir = {''= 'src'}
+
+[tool.setuptools.packages.find]
+where = ["src"]
+namespaces = false
+
+[tool.setuptools.package-data]
+"ewoksid02.tests.data"= ['*/*']
+"*"= ['resources/*.*']
+
+[project.entry-points."ewoks.tasks.class"]
+"ewoksid02.tasks.*" = "ewoksid02"
+
+[project.scripts]
+ewoksid02 = "ewoksid02.scripts.__main__:main"
+
+[tool.coverage.run]
+omit = [
+    "*/tests/*"
+]

ewoksid02-0.0.1a0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

ewoksid02-0.0.1a0/src/ewoksid02/resources/__init__.py

@@ -0,0 +1,8 @@
+import os
+
+RESOURCES_DIR = os.path.dirname(__file__)
+WORKFLOW_SAXS_LOOP = os.path.join(RESOURCES_DIR, "saxs_loop.json")
+
+TEMPLATE_SAXS = os.path.join(RESOURCES_DIR, "template_saxs.yaml")
+
+WORKFLOW_SAXS_LINEAR = None

ewoksid02-0.0.1a0/src/ewoksid02/resources/saxs_loop.json

@@ -0,0 +1,80 @@
+{
+  "directed": true,
+  "multigraph": false,
+  "graph": {
+    "id": "saxs_loop",
+    "schema_version": "1.1"
+  },
+  "nodes": [
+    {
+      "task_type": "class",
+      "task_identifier": "ewoksid02.tasks.normalizationtask.NormalizationTask",
+      "force_start_node": true,
+      "default_inputs" : [{"name": "reading_node", "value": true}],
+      "id": "norm"
+    },
+    {
+      "task_type": "class",
+      "task_identifier": "ewoksid02.tasks.secondaryscatteringtask.SecondaryScatteringTask",
+      "id": "2scat"
+    },
+    {
+      "task_type": "class",
+      "task_identifier": "ewoksid02.tasks.cavingtask.CavingBeamstopTask",
+      "id": "cave"
+    },
+    {
+      "task_type": "class",
+      "task_identifier": "ewoksid02.tasks.azimuthaltask.AzimuthalTask",
+      "id": "azim"
+    },
+    {
+      "task_type": "class",
+      "task_identifier": "ewoksid02.tasks.averagetask.AverageTask",
+      "id": "ave"
+    },
+    {
+      "task_type": "class",
+      "task_identifier": "ewoksid02.tasks.scalerstask.ScalersTask",
+      "id": "scalers"
+    }
+  ],
+  "links": [
+    {
+      "map_all_data": true,
+      "source": "norm",
+      "target": "2scat",
+      "conditions": [
+        {
+          "source_output": "continue_pipeline",
+          "value": true
+        }
+      ]
+    },
+    {
+      "map_all_data": true,
+      "source": "2scat",
+      "target": "cave"
+    },
+    {
+      "map_all_data": true,
+      "source": "cave",
+      "target": "azim"
+    },
+    {
+      "map_all_data": true,
+      "source": "azim",
+      "target": "ave"
+    },
+    {
+      "map_all_data": true,
+      "source": "ave",
+      "target": "scalers"
+    },
+    {
+      "map_all_data": true,
+      "source": "scalers",
+      "target": "norm"
+    }
+  ]
+}

ewoksid02-0.0.1a0/src/ewoksid02/resources/template_saxs.yaml

@@ -0,0 +1,38 @@
+###########################
+## ewoksid02 template SAXS.yaml
+###########################
+
+# File parameters
+bliss_filenames: 
+    - /data/visitor/xxx/id02/xxx/RAW_DATA/xxx/.../file1.h5
+    #- /data/visitor/xxx/id02/xxx/RAW_DATA/xxx/.../file2.h5
+detectors: 
+    - eiger2
+scans: 
+    - 1
+tag: t1
+output_root:
+
+# Processing parameters
+to_process: "norm 2scat cave azim ave"
+to_save: "norm 2scat cave azim ave"
+datatype: "float32"
+max_slice_size: 34
+save_variance: False
+algorithm_norm: cython
+algorithm_2scat: 
+algorithm_cave: 
+
+# Correction parameters
+filename_flatfield: 
+filename_darkcurrent: 
+filename_maskgaps: 
+filename_maskbeamstop: 
+filename_window_wagon: 
+window_roi_size: 120
+
+# Submit parameters
+submit: True
+slurm_job_parameters:
+    time_limit: 01:00:00
+    memory_per_cpu: 50GB

ewoksid02-0.0.1a0/src/ewoksid02/scripts/__main__.py

@@ -0,0 +1,70 @@
+import argparse
+import os
+import shutil
+import sys
+
+from ..utils import AVAILABLE_TEMPLATES, TEMPLATE_MESSAGE
+from .parsers import add_saxs_arguments, add_xpcs_arguments
+from .saxs.main import main as main_saxs
+from .xpcs.__main__ import main as main_xpcs
+
+TECHNIQUES = [
+    "saxs",
+    "xpcs",
+]
+
+
+def main(argv=None):
+    """Main function to run the offline scripts
+
+    Inputs:
+        - argv: command line arguments
+    """
+    if argv is None:
+        argv = sys.argv
+
+    parser = argparse.ArgumentParser(
+        prog="ewoksid02",
+        description="Run data processing pipelines for SAXS or XPCS.",
+        usage="ewoksid02 {saxs | xpcs | templates} [...]",
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    subparser_saxs = subparsers.add_parser("saxs", help="Trigger SAXS pipelines")
+    add_saxs_arguments(subparser_saxs)
+
+    subparser_xpcs = subparsers.add_parser("xpcs", help="Trigger XPCS pipelines")
+    add_xpcs_arguments(subparser_xpcs)
+
+    subparser_templates = subparsers.add_parser(  # noqa
+        "templates", help="Download available templates for ID02 offline pipelines"
+    )
+
+    if len(argv) == 1:
+        parser.print_help()
+        sys.exit(1)
+
+    if argv[1] == "templates":
+        print(TEMPLATE_MESSAGE)
+        for _, template_info in AVAILABLE_TEMPLATES.items():
+            os.makedirs(template_info["directory"], exist_ok=True)
+            shutil.copy(template_info["path"], template_info["future_path"])
+        return
+
+    args = parser.parse_args(argv[1:])
+    if args.command == "saxs":
+        if len(args.FILES) == 0:
+            subparser_saxs.print_help()
+            sys.exit(1)
+        main_saxs(args)
+    elif args.command == "xpcs":
+        main_xpcs(args)
+    elif args.command == "templates":
+        print(TEMPLATE_MESSAGE)
+        for _, template_info in AVAILABLE_TEMPLATES.items():
+            os.makedirs(template_info["directory"], exist_ok=True)
+            shutil.copy(template_info["path"], template_info["future_path"])
+        return
+    else:
+        parser.print_help()
+        sys.exit(1)

ewoksid02-0.0.1a0/src/ewoksid02/scripts/parsers.py

@@ -0,0 +1,224 @@
+def add_celery_arguments(parser):
+    """Add to parser arguments related to celery queue and submit"""
+    parser.add_argument(
+        "-q",
+        "--queue",
+        dest="queue",
+        default=None,
+        help="Queue name for the job submission (default: None)",
+    )
+    parser.add_argument(
+        "--no-submit",
+        dest="submit",
+        action="store_false",
+        help="Do not submit anything",
+        default=None,
+    )
+
+
+def add_workflow_flags(parser):
+    """Add to parser arguments related to process and save flags."""
+    parser.add_argument(
+        "--to_process",
+        dest="to_process",
+        default=None,
+        help="Steps to process",
+    )
+    parser.add_argument(
+        "--to_save",
+        dest="to_save",
+        default=None,
+        help="Steps to save",
+    )
+
+
+def add_main_arguments(parser):
+    """Add to parser common arguments to run the offline pipeline."""
+    parser.add_argument(
+        "FILES",
+        metavar="FILES",
+        help="List of BLISS files read and reprocess",
+        nargs="+",
+    )
+    parser.add_argument(
+        "-n",
+        "--scan",
+        dest="scan_nb",
+        nargs="+",
+        help="Number of scan to process",
+    )
+
+    parser.add_argument(
+        "-d",
+        "--detector",
+        dest="detector_name",
+        default=None,
+        help="Name of the Lima detector which data to process",
+    )
+    parser.add_argument(
+        "-t",
+        "--tag",
+        dest="tag",
+        default=None,
+        help="Tag to be added on each processing filename. The workflow will not overwrite files",
+    )
+    parser.add_argument(
+        "--r",
+        "--output-root",
+        dest="output_root",
+        help="Root directory for the output files",
+        default=None,
+    )
+
+
+def add_common_saxs_arguments(parser):
+    """Add to parser common arguments to SAXS/WAXS pipeline"""
+    parser.add_argument(
+        "--metadata",
+        dest="filename_metadata",
+        default=None,
+        help="File with metadata used for processing",
+    )
+    parser.add_argument(
+        "--max",
+        dest="max_slice_size",
+        default=None,
+        type=int,
+        help="Maximum number of slices to process at once. Default: 100",
+    )
+    parser.add_argument(
+        "--dummy",
+        dest="dummy",
+        default=None,
+        help="Dummy value",
+    )
+    parser.add_argument(
+        "--delta-dummy",
+        dest="delta_dummy",
+        default=None,
+        help="Delta dummy value",
+    )
+    parser.add_argument(
+        "-l",
+        "--log-level",
+        dest="log_level",
+        default=None,
+        help="Logging level.",
+    )
+
+
+def add_normalization_arguments(parser):
+    """Add to parser arguments for NormalizationTask"""
+    help = "Normalization: "
+    parser.add_argument(
+        "--dark",
+        dest="dark_current_filename",
+        default=None,
+        help=help + "Filename of the dark current file",
+    )
+    parser.add_argument(
+        "--flat",
+        dest="flat_field_filename",
+        default=None,
+        help=help + "Filename of the flat field file",
+    )
+
+
+def add_masking_arguments(parser):
+    """Add to parser arguments for CavingTask"""
+    help = "Caving: "
+    parser.add_argument(
+        "--mask-gaps",
+        dest="mask_gaps_filename",
+        default=None,
+        help=help + "Filename of the mask with detector gaps",
+    )
+    parser.add_argument(
+        "--mask-beamstop",
+        dest="mask_beamstop_filename",
+        default=None,
+        help=help + "Filename of the mask with the beamstop",
+    )
+
+
+def add_secondary_scattering_arguments(parser):
+    """Add to parser arguments for SecondaryScatteringTask"""
+    help = "Secondary Scattering: "
+    parser.add_argument(
+        "--window-pattern",
+        dest="window_pattern_filename",
+        default=None,
+        help=help + "Filename of the window pattern file",
+    )
+    parser.add_argument(
+        "--window-roi",
+        dest="window_roi_size",
+        default=None,
+        help=help + "Subdata distance for secondary scattering correction",
+    )
+    parser.add_argument(
+        "--flip-vertical",
+        dest="flip_vertical",
+        action="store_true",
+        default=None,
+        help=help + "Flip the image vertically (default: False)",
+    )
+    parser.add_argument(
+        "--flip-horizontal",
+        dest="flip_horizontal",
+        action="store_true",
+        default=None,
+        help=help + "Flip the image horizontally (default: False)",
+    )
+
+
+def add_azimuthal_arguments(parser):
+    """Add to parser arguments for AzimuthalTask"""
+    help = "Azimuthal: "
+    parser.add_argument(
+        "--npt-rad",
+        dest="npt_rad",
+        default=None,
+        help=help + "Number of radial bins",
+    )
+    parser.add_argument(
+        "--npt-azim",
+        dest="npt_azim",
+        default=None,
+        help=help + "Number of azimuthal bins",
+    )
+    parser.add_argument(
+        "--unit",
+        dest="unit",
+        default=None,
+        help=help + "Unit for azimuthal averaging",
+    )
+
+
+def add_average_arguments(parser):
+    """Add to parser arguments for AverageTask"""
+    help = "Average: "
+    parser.add_argument(
+        "--azim-range",
+        dest="azimuth_range",
+        default=None,
+        help=help + "Azimuthal limits for the average",
+    )
+
+
+def add_saxs_arguments(parser):
+    """Add all arguments needed for SAXS/WAXS pipeline"""
+    add_main_arguments(parser)
+    add_common_saxs_arguments(parser)
+    add_celery_arguments(parser)
+    add_normalization_arguments(parser)
+    add_masking_arguments(parser)
+    add_secondary_scattering_arguments(parser)
+    add_azimuthal_arguments(parser)
+    add_average_arguments(parser)
+    add_workflow_flags(parser)
+
+
+def add_xpcs_arguments(parser):
+    """Add all arguments needed for XPCS pipeline"""
+    add_main_arguments(parser)