httomo-backends 2.1__py3-none-any.whl

httomo_backends/scripts/yaml_templates_generator.py

@@ -0,0 +1,249 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ---------------------------------------------------------------------------
+# Copyright 2022 Diamond Light Source Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+# Created By  : Tomography Team <scientificsoftware@diamond.ac.uk>
+# Created Date: 14/October/2022
+# version ='0.3'
+# ---------------------------------------------------------------------------
+"""Script that exposes all functions of a given software package as YAML templates.
+
+Please run the generator as:
+    python -m yaml_templates_generator -i /path/to/modules.yml -o /path/to/output/
+"""
+import argparse
+import importlib
+import inspect
+import os
+import re
+from typing import Any, List, Dict
+
+import yaml
+
+
+def yaml_generator(path_to_modules: str, output_folder: str) -> int:
+    """function that exposes all method of a given software package as YAML templates
+
+    Args:
+        path_to_modules: path to the list of modules yaml file
+        output_folder: path to output folder with saved templates
+
+    Returns:
+        returns zero if the processing is succesfull
+    """
+    discard_keys = _get_discard_keys()
+    no_data_out_modules = _get_discard_data_out()
+
+    # open YAML file with modules to inspect
+    with open(path_to_modules, "r") as stream:
+        try:
+            modules_list = yaml.safe_load(stream)
+        except yaml.YAMLError as exc:
+            print(exc)
+
+    # a loop over modules in the file
+    modules_no = len(modules_list)
+    for i in range(modules_no):
+        module_name = modules_list[i]
+        try:
+            imported_module = importlib.import_module(str(module_name))
+        except NameError:
+            print(
+                "Import of the module {} has failed, check if software installed".format(
+                    module_name
+                )
+            )
+        methods_list = imported_module.__all__  # get all the methods in the module
+        methods_no = len(methods_list)
+
+        # a loop over all methods in the module
+        for m in range(methods_no):
+            method_name = methods_list[m]
+            print("Inspecting the signature of the {} method".format(method_name))
+            get_method_params = inspect.signature(
+                getattr(imported_module, methods_list[m])
+            )
+            # get method docstrings
+            get_method_docs = inspect.getdoc(getattr(imported_module, methods_list[m]))
+
+            # put the parameters in the dictionary
+            params_list: List = []
+            params_dict: Dict = {}
+            for name, value in get_method_params.parameters.items():
+                if value is not None:
+                    append = True
+                    for x in discard_keys:
+                        if name == x:
+                            append = False
+                            break
+                    if append:
+                        _set_param_value(name, value, params_dict)
+            method_dict = {
+                "method": method_name,
+                "module_path": module_name,
+                "parameters": params_dict,
+            }
+            _set_dict_special_cases(method_dict, method_name)
+            params_list = [method_dict]
+            _save_yaml(module_name, method_name, params_list)
+    return 0
+
+
+def _set_param_value(name: str, value: inspect.Parameter, params_dict: Dict[str, Any]):
+    """Set param value for method inside dictionary
+    Args:
+        name: Parameter name
+        value: Parameter value
+        params_dict: Dict containing method's parameter names and values
+    """
+    if value.default is inspect.Parameter.empty and name != "kwargs":
+        if name in ["proj1", "proj2"]:
+            params_dict[name] = "auto"
+        else:
+            params_dict[name] = "REQUIRED"
+    elif name == "kwargs":
+        # params_dict["#additional parameters"] = "AVAILABLE"
+        # parsing hashtag to yaml comes with quotes, for now we simply ignore the field
+        pass
+    elif name == "axis":
+        params_dict[name] = "auto"
+    elif name == "asynchronous":
+        params_dict[name] = True
+    elif name == "center":
+        # Temporary value
+        params_dict[name] = "${{centering.side_outputs.centre_of_rotation}}"
+    elif name == "glob_stats":
+        params_dict[name] = "${{statistics.side_outputs.glob_stats}}"
+    elif name == "overlap":
+        params_dict[name] = "${{centering.side_outputs.overlap}}"
+    else:
+        params_dict[name] = value.default
+
+
+def _save_yaml(module_name: str, method_name: str, params_list: List[str]):
+    """Save the list as a YAML file
+    Args:
+        module_name: Name of module
+        method_name: Name of method
+        params_list: List of parameters
+    """
+    path_dir = output_folder + "/" + module_name
+    path_file = path_dir + "/" + str(method_name) + ".yaml"
+
+    if not os.path.exists(path_dir):
+        os.makedirs(path_dir)
+
+    with open(path_file, "w") as file:
+        outputs = yaml.dump(params_list, file, sort_keys=False)
+
+
+def _set_dict_special_cases(method_dict: Dict, method_name: str):
+    """Dealing with special cases for "data_out"
+
+    Args:
+        method_dict: Dictionary of modules and parameters
+        method_name: Name of method
+    """
+    if method_name in ["find_center_vo", "find_center_pc"]:
+        method_dict["id"] = "centering"
+        method_dict["side_outputs"] = {"cor": "centre_of_rotation"}
+    if method_name in "find_center_360":
+        method_dict["id"] = "centering"
+        method_dict["side_outputs"] = {
+            "cor": "centre_of_rotation",
+            "overlap": "overlap",
+            "side": "side",
+            "overlap_position": "overlap_position",
+        }
+    if method_name in "calculate_stats":
+        method_dict["id"] = "statistics"
+        method_dict["side_outputs"] = {"glob_stats": "glob_stats"}
+
+
+def _get_discard_data_out() -> List[str]:
+    """Discard data_out from certain modules
+
+    Returns: list of data_out to discard
+    """
+    discard_data_out = ["save_to_images"]
+    return discard_data_out
+
+
+def _get_discard_keys() -> List[str]:
+    """Can work with any software in principle,
+    but for TomoPy and httomolib there are additional keys
+    that needed to be discarded in templates in order to let
+    httomo work smoothly.
+
+    Returns: List of keys to discard
+    """
+    discard_keys = [
+        "in_file",
+        "data_in",
+        "tomo",
+        "arr",
+        "prj",
+        "data",
+        "ncore",
+        "nchunk",
+        "flats",
+        "flat",
+        "dark",
+        "darks",
+        "theta",
+        "out",
+        "ang",
+        "comm_rank",
+        "out_dir",
+        "angles",
+        "gpu_id",
+        "comm",
+        "offset",
+    ]
+    return discard_keys
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        description="Script that exposes all functions "
+        "of a given software package as YAML templates."
+    )
+    parser.add_argument(
+        "-i",
+        "--input",
+        type=str,
+        default=None,
+        help="A path to the list of modules yaml file"
+        "which is needed to be inspected and functions extracted.",
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        type=str,
+        default="./",
+        help="Directory to save the yaml templates in.",
+    )
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    current_dir = os.path.basename(os.path.abspath(os.curdir))
+    args = get_args()
+    path_to_modules = args.input
+    output_folder = args.output
+    return_val = yaml_generator(path_to_modules, output_folder)
+    if return_val == 0:
+        print("The methods as YAML templates have been successfully generated!")

httomo_backends/scripts/yaml_unsupported_tomopy_remove.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ---------------------------------------------------------------------------
+# Copyright 2022 Diamond Light Source Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+# Created By  : Daniil Kazantsev <scientificsoftware@diamond.ac.uk>
+# Created Date: 13/April/2023
+# version ='0.1'
+# ---------------------------------------------------------------------------
+"""After _all_ templates have been generated for TomoPy we need to remove the ones
+that are not currently supported by httomo. We do that by looking into
+the library file for TomoPy."""
+
+import argparse
+import importlib
+import inspect
+import os
+import re
+import shutil
+from pathlib import Path
+
+import yaml
+
+
+def templates_filter(path_to_modules: str, library_file: str) -> int:
+    """function that removes unsupported by httomo YAML templates in TomoPy
+
+    Args:
+        path_to_modules (str): path to the list of modules yaml file
+        library_file (str): path to the library with the supported functions of TomoPy
+
+    Returns:
+        int: returns zero if the processing is succesfull
+    """
+    software_name = "tomopy"
+    yaml_info_path = Path(library_file)
+    if not yaml_info_path.exists():
+        err_str = f"The YAML file {yaml_info_path} doesn't exist."
+        raise ValueError(err_str)
+
+    with open(yaml_info_path, "r") as f:
+        yaml_library = yaml.safe_load(f)
+
+    methods_list: list = []
+    for module, module_dict in yaml_library.items():
+        for module2, module_dict2 in module_dict.items():
+            for method_name in module_dict2:
+                methods_list.append(method_name)
+
+    subfolders = [f.path for f in os.scandir(path_to_modules) if f.is_dir()]
+    for folder in subfolders:
+        for filename in os.listdir(folder):
+            filename_short = filename.split(".")
+            if filename_short[0] not in methods_list:
+                print(f"Removed template: {filename_short[0]}")
+                file_path = os.path.join(folder, filename)
+                try:
+                    if os.path.isfile(file_path) or os.path.islink(file_path):
+                        os.unlink(file_path)
+                except Exception as e:
+                    print("Failed to delete %s. Reason: %s" % (file_path, e))
+    return 0
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        description="Removes unsupported by httomo templates in TomoPy."
+    )
+    parser.add_argument(
+        "-t",
+        "--templates",
+        type=str,
+        default=None,
+        help="A path to the folder where generated templates stored.",
+    )
+    parser.add_argument(
+        "-l",
+        "--library",
+        type=str,
+        default=None,
+        help="A path to the library YAML file with the supported functions.",
+    )
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    current_dir = os.path.basename(os.path.abspath(os.curdir))
+    args = get_args()
+    path_to_modules = args.templates
+    library_file = args.library
+    return_val = templates_filter(path_to_modules, library_file)
+    if return_val == 0:
+        print("The templates have been filtered!")

httomo_backends/yaml_templates/httomo/httomo.data.hdf.loaders/standard_tomo.yaml

@@ -0,0 +1,13 @@
+- method: standard_tomo
+  module_path: httomo.data.hdf.loaders
+  parameters:
+    name: REQUIRED
+    data_path: REQUIRED
+    dimension: REQUIRED
+    preview: REQUIRED
+    pad: REQUIRED
+    image_key_path: null
+    rotation_angles:
+      data_path: /entry1/tomo_entry/data/rotation_angle
+    ignore_darks: false
+    ignore_flats: false

httomo_backends/yaml_templates/httomo/httomo.methods/calculate_stats.yaml

@@ -0,0 +1,6 @@
+- method: calculate_stats
+  module_path: httomo.methods
+  parameters: {}
+  id: statistics
+  side_outputs:
+    glob_stats: glob_stats

httomo_backends/yaml_templates/httomo/httomo.methods/save_intermediate_data.yaml

@@ -0,0 +1,9 @@
+- method: save_intermediate_data
+  module_path: httomo.methods
+  parameters:
+    global_shape: REQUIRED
+    global_index: REQUIRED
+    file: REQUIRED
+    path: REQUIRED
+    detector_x: REQUIRED
+    detector_y: REQUIRED

httomo_backends/yaml_templates/tomopy/tomopy.misc.corr/adjust_range.yaml

@@ -0,0 +1,5 @@
+- method: adjust_range
+  module_path: tomopy.misc.corr
+  parameters:
+    dmin: null
+    dmax: null

httomo_backends/yaml_templates/tomopy/tomopy.misc.corr/circ_mask.yaml

@@ -0,0 +1,6 @@
+- method: circ_mask
+  module_path: tomopy.misc.corr
+  parameters:
+    axis: REQUIRED
+    ratio: 1
+    val: 0.0

httomo_backends/yaml_templates/tomopy/tomopy.misc.corr/gaussian_filter.yaml

@@ -0,0 +1,6 @@
+- method: gaussian_filter
+  module_path: tomopy.misc.corr
+  parameters:
+    sigma: 3
+    order: 0
+    axis: auto

httomo_backends/yaml_templates/tomopy/tomopy.misc.corr/median_filter.yaml

@@ -0,0 +1,5 @@
+- method: median_filter
+  module_path: tomopy.misc.corr
+  parameters:
+    size: 3
+    axis: auto

httomo_backends/yaml_templates/tomopy/tomopy.misc.corr/median_filter3d.yaml

@@ -0,0 +1,4 @@
+- method: median_filter3d
+  module_path: tomopy.misc.corr
+  parameters:
+    size: 3

httomo_backends/yaml_templates/tomopy/tomopy.misc.corr/median_filter_nonfinite.yaml

@@ -0,0 +1,5 @@
+- method: median_filter_nonfinite
+  module_path: tomopy.misc.corr
+  parameters:
+    size: 3
+    callback: null

httomo_backends/yaml_templates/tomopy/tomopy.misc.corr/remove_nan.yaml

@@ -0,0 +1,4 @@
+- method: remove_nan
+  module_path: tomopy.misc.corr
+  parameters:
+    val: 0.0

httomo_backends/yaml_templates/tomopy/tomopy.misc.corr/remove_neg.yaml

@@ -0,0 +1,4 @@
+- method: remove_neg
+  module_path: tomopy.misc.corr
+  parameters:
+    val: 0.0

httomo_backends/yaml_templates/tomopy/tomopy.misc.corr/remove_outlier.yaml

@@ -0,0 +1,6 @@
+- method: remove_outlier
+  module_path: tomopy.misc.corr
+  parameters:
+    dif: REQUIRED
+    size: 3
+    axis: auto

httomo_backends/yaml_templates/tomopy/tomopy.misc.corr/remove_outlier1d.yaml

@@ -0,0 +1,6 @@
+- method: remove_outlier1d
+  module_path: tomopy.misc.corr
+  parameters:
+    dif: REQUIRED
+    size: 3
+    axis: auto

httomo_backends/yaml_templates/tomopy/tomopy.misc.corr/remove_outlier3d.yaml

@@ -0,0 +1,5 @@
+- method: remove_outlier3d
+  module_path: tomopy.misc.corr
+  parameters:
+    dif: REQUIRED
+    size: 3

httomo_backends/yaml_templates/tomopy/tomopy.misc.corr/remove_ring.yaml

@@ -0,0 +1,12 @@
+- method: remove_ring
+  module_path: tomopy.misc.corr
+  parameters:
+    rec: REQUIRED
+    center_x: null
+    center_y: null
+    thresh: 300.0
+    thresh_max: 300.0
+    thresh_min: -100.0
+    theta_min: 30
+    rwidth: 30
+    int_mode: WRAP

httomo_backends/yaml_templates/tomopy/tomopy.misc.corr/sobel_filter.yaml

@@ -0,0 +1,4 @@
+- method: sobel_filter
+  module_path: tomopy.misc.corr
+  parameters:
+    axis: auto