httomo-backends 0.5__tar.gz → 0.5.2__tar.gz

{httomo_backends-0.5 → httomo_backends-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: httomo-backends
-Version: 0.5
+Version: 0.5.2
 Summary: Supplementary files for HTTomo backends.
 Author: Garry ODonnell, Jacob Williamson
 Author-email: Daniil Kazantsev <daniil.kazantsev@diamond.ac.uk>, Yousef Moazzam <yousef.moazzam@diamond.ac.uk>, Jessica Verschoyle <jessica.verschoyle@diamond.ac.uk>, Naman Gera <naman.gera@diamond.ac.uk>, scientificsoftware@diamond.ac.uk

{httomo_backends-0.5 → httomo_backends-0.5.2}/httomo_backends/methods_database/packages/backends/httomolibgpu/supporting_funcs/recon/algorithm.py

@@ -159,7 +159,6 @@ def _calc_memory_bytes_FBP3d_tomobar(
     return (tot_memory_bytes, fixed_amount)
 
 
-
 def _calc_memory_bytes_LPRec3d_tomobar(
     non_slice_dims_shape: Tuple[int, int],
     dtype: np.dtype,
@@ -170,6 +169,7 @@ def _calc_memory_bytes_LPRec3d_tomobar(
     angles_tot = non_slice_dims_shape[0]
     DetectorsLengthH = non_slice_dims_shape[1]
     SLICES = 200  # dummy multiplier+divisor to pass large batch size threshold
+    _CENTER_SIZE_MIN = 192  # must be divisible by 8
 
     n = DetectorsLengthH
 
@@ -180,7 +180,11 @@ def _calc_memory_bytes_LPRec3d_tomobar(
 
     eps = 1e-4  # accuracy of usfft
     mu = -np.log(eps) / (2 * n * n)
-    m = int(np.ceil(2 * n * 1 / np.pi * np.sqrt(-mu * np.log(eps) + (mu * n) * (mu * n) / 4)))
+    m = int(
+        np.ceil(
+            2 * n * 1 / np.pi * np.sqrt(-mu * np.log(eps) + (mu * n) * (mu * n) / 4)
+        )
+    )
 
     center_size = 6144
     center_size = min(center_size, n * 2 + m * 2)
@@ -192,77 +196,153 @@ def _calc_memory_bytes_LPRec3d_tomobar(
     if "angles" in kwargs:
         angles = kwargs["angles"]
         sorted_theta_cpu = np.sort(angles)
-        theta_full_range = abs(sorted_theta_cpu[angles_tot-1] - sorted_theta_cpu[0])
+        theta_full_range = abs(sorted_theta_cpu[angles_tot - 1] - sorted_theta_cpu[0])
         angle_range_pi_count = 1 + int(np.ceil(theta_full_range / math.pi))
+        angle_range_pi_count += 1  # account for difference from actual algorithm
     else:
-        angle_range_pi_count = 1 + int(np.ceil(2)) # assume a 2 * PI projection angle range
+        angle_range_pi_count = 1 + int(
+            np.ceil(2)
+        )  # assume a 2 * PI projection angle range
+
+    chunk_count = 4
 
     output_dims = __calc_output_dim_recon(non_slice_dims_shape, **kwargs)
     if odd_horiz:
         output_dims = tuple(x + 1 for x in output_dims)
 
     in_slice_size = np.prod(non_slice_dims_shape) * dtype.itemsize
-    padded_in_slice_size = np.prod(non_slice_dims_shape) * np.float32().itemsize
+    padded_in_slice_size = angles_tot * n * np.float32().itemsize
+
     theta_size = angles_tot * np.float32().itemsize
-    sorted_theta_indices_size = angles_tot * np.int64().itemsize
-    sorted_theta_size = angles_tot * np.float32().itemsize
-    recon_output_size = (n + 1) * (n + 1) * np.float32().itemsize if odd_horiz else n * n * np.float32().itemsize    # 264
-    linspace_size = n * np.float32().itemsize
-    meshgrid_size = 2 * n * n * np.float32().itemsize
-    phi_size = 6 * n * n * np.float32().itemsize
-    angle_range_size = center_size * center_size * 1 + angle_range_pi_count * 2 * np.int32().itemsize
-    c1dfftshift_size = n * np.int8().itemsize
-    c2dfftshift_slice_size = 4 * n * n * np.int8().itemsize
     filter_size = (n // 2 + 1) * np.float32().itemsize
     rfftfreq_size = filter_size
     scaled_filter_size = filter_size
-    tmp_p_input_slice = np.prod(non_slice_dims_shape) * np.float32().itemsize
-    padded_tmp_p_input_slice = angles_tot * (n + padding_m * 2) * dtype.itemsize
-    rfft_result_size = padded_tmp_p_input_slice
-    filtered_rfft_result_size = rfft_result_size
-    rfft_plan_slice_size = cufft_estimate_1d(nx=(n + padding_m * 2),fft_type=CufftType.CUFFT_R2C,batch=angles_tot * SLICES) / SLICES
-    irfft_result_size = filtered_rfft_result_size
-    irfft_scratch_memory_size = filtered_rfft_result_size
-    irfft_plan_slice_size = cufft_estimate_1d(nx=(n + padding_m * 2),fft_type=CufftType.CUFFT_C2R,batch=angles_tot * SLICES) / SLICES
-    conversion_to_complex_size = np.prod(non_slice_dims_shape) * np.complex64().itemsize / 2
-    datac_size = np.prod(non_slice_dims_shape) * np.complex64().itemsize / 2
-    fde_size = (2 * m + 2 * n) * (2 * m + 2 * n) * np.complex64().itemsize / 2
-    shifted_datac_size = datac_size
-    fft_result_size = datac_size
-    backshifted_datac_size = datac_size
-    scaled_backshifted_datac_size = datac_size
-    fft_plan_slice_size = cufft_estimate_1d(nx=n,fft_type=CufftType.CUFFT_C2C,batch=angles_tot * SLICES) / SLICES
-    fde_view_size = 4 * n * n * np.complex64().itemsize / 2
-    shifted_fde_view_size = fde_view_size
-    ifft2_scratch_memory_size = fde_view_size
-    ifft2_plan_slice_size = cufft_estimate_2d(nx=(2 * n),ny=(2 * n),fft_type=CufftType.CUFFT_C2C) / 2
-    fde2_size = n * n * np.complex64().itemsize / 2
-    concatenate_size = fde2_size
-    circular_mask_size = np.prod(output_dims) / 2 * np.int64().itemsize * 4
 
-    after_recon_swapaxis_slice = np.prod(non_slice_dims_shape) * np.float32().itemsize
+    tmp_p_input_slice = angles_tot * n * np.float32().itemsize
 
-    tot_memory_bytes = int(
-        max(
-            in_slice_size + padded_in_slice_size + recon_output_size + rfft_plan_slice_size + irfft_plan_slice_size + tmp_p_input_slice + padded_tmp_p_input_slice + rfft_result_size + filtered_rfft_result_size + irfft_result_size + irfft_scratch_memory_size
-            , in_slice_size + padded_in_slice_size + recon_output_size + rfft_plan_slice_size + irfft_plan_slice_size + tmp_p_input_slice + datac_size + conversion_to_complex_size
-            , in_slice_size + padded_in_slice_size + recon_output_size + rfft_plan_slice_size + irfft_plan_slice_size + fft_plan_slice_size + fde_size + datac_size + shifted_datac_size + fft_result_size + backshifted_datac_size + scaled_backshifted_datac_size
-            , in_slice_size + padded_in_slice_size + recon_output_size + rfft_plan_slice_size + irfft_plan_slice_size + fft_plan_slice_size + ifft2_plan_slice_size + shifted_fde_view_size + ifft2_scratch_memory_size
-            , in_slice_size + padded_in_slice_size + recon_output_size + rfft_plan_slice_size + irfft_plan_slice_size + fft_plan_slice_size + ifft2_plan_slice_size + fde2_size + concatenate_size
-            , in_slice_size + padded_in_slice_size + recon_output_size + rfft_plan_slice_size + irfft_plan_slice_size + fft_plan_slice_size + ifft2_plan_slice_size + after_recon_swapaxis_slice
+    padded_tmp_p_input_slice = angles_tot * (n + padding_m * 2) * np.float32().itemsize
+    rfft_plan_slice_size = (
+        cufft_estimate_1d(
+            nx=n + padding_m * 2,
+            fft_type=CufftType.CUFFT_R2C,
+            batch=angles_tot * SLICES,
         )
+        / SLICES
     )
-
-    fixed_amount = int(
-        max(
-            theta_size + phi_size + linspace_size + meshgrid_size
-            , theta_size + sorted_theta_indices_size + sorted_theta_size + phi_size + angle_range_size + c1dfftshift_size + c2dfftshift_slice_size + filter_size + rfftfreq_size + scaled_filter_size
-            , theta_size + sorted_theta_indices_size + sorted_theta_size + phi_size + circular_mask_size
+    rfft_result_size = angles_tot * (n + padding_m * 2) * np.complex64().itemsize
+    filtered_rfft_result_size = rfft_result_size
+    irfft_plan_slice_size = (
+        cufft_estimate_1d(
+            nx=(n + padding_m * 2),
+            fft_type=CufftType.CUFFT_C2R,
+            batch=angles_tot * SLICES,
         )
+        / SLICES
     )
+    irfft_scratch_memory_size = filtered_rfft_result_size * 2
+    irfft_result_size = angles_tot * (n + padding_m * 2) * np.float32().itemsize
 
-    return (tot_memory_bytes * 1.1, fixed_amount)
+    datac_size = angles_tot * n * np.complex64().itemsize / 2
+    fde_size = (2 * m + 2 * n) * (2 * m + 2 * n) * np.complex64().itemsize / 2
+    fft_plan_slice_size = (
+        cufft_estimate_1d(nx=n, fft_type=CufftType.CUFFT_C2C, batch=angles_tot * SLICES)
+        / SLICES
+    )
+    fft_result_size = datac_size
 
+    sorted_theta_indices_size = angles_tot * np.int64().itemsize
+    sorted_theta_size = angles_tot * np.float32().itemsize
+    angle_range_size = (
+        center_size * center_size * (1 + angle_range_pi_count * 2) * np.int16().itemsize
+    )
+
+    recon_output_size = DetectorsLengthH * DetectorsLengthH * np.float32().itemsize
+    ifft2_plan_slice_size = (
+        cufft_estimate_2d(
+            nx=(2 * m + 2 * n), ny=(2 * m + 2 * n), fft_type=CufftType.CUFFT_C2C
+        )
+        / 2
+    )
+    circular_mask_size = np.prod(output_dims) / 2 * np.int64().itemsize * 4
+    after_recon_swapaxis_slice = recon_output_size
+
+    tot_memory_bytes = 0
+    current_tot_memory_bytes = 0
+
+    fixed_amount = 0
+    current_fixed_amount = 0
+
+    def add_to_memory_counters(amount, per_slice: bool):
+        nonlocal tot_memory_bytes
+        nonlocal current_tot_memory_bytes
+        nonlocal fixed_amount
+        nonlocal current_fixed_amount
+
+        if per_slice:
+            current_tot_memory_bytes += amount
+            tot_memory_bytes = max(tot_memory_bytes, current_tot_memory_bytes)
+        else:
+            current_fixed_amount += amount
+            fixed_amount = max(fixed_amount, current_fixed_amount)
+
+    add_to_memory_counters(in_slice_size, True)
+    add_to_memory_counters(padded_in_slice_size, True)
+
+    add_to_memory_counters(theta_size, False)
+    if center_size >= _CENTER_SIZE_MIN:
+        add_to_memory_counters(sorted_theta_indices_size, False)
+        add_to_memory_counters(sorted_theta_size, False)
+        add_to_memory_counters(angle_range_size, False)
+    add_to_memory_counters(filter_size, False)
+    add_to_memory_counters(rfftfreq_size, False)
+    add_to_memory_counters(scaled_filter_size, False)
+
+    add_to_memory_counters(tmp_p_input_slice, True)
+
+    add_to_memory_counters(rfft_plan_slice_size / chunk_count * 2, True)
+    add_to_memory_counters(irfft_plan_slice_size / chunk_count * 2, True)
+    # add_to_memory_counters(irfft_scratch_memory_size / chunk_count, True)
+    for _ in range(0, chunk_count):
+        add_to_memory_counters(padded_tmp_p_input_slice / chunk_count, True)
+
+        add_to_memory_counters(rfft_result_size / chunk_count, True)
+        add_to_memory_counters(filtered_rfft_result_size / chunk_count, True)
+        add_to_memory_counters(-rfft_result_size / chunk_count, True)
+        add_to_memory_counters(-padded_tmp_p_input_slice / chunk_count, True)
+
+        add_to_memory_counters(irfft_scratch_memory_size / chunk_count, True)
+        add_to_memory_counters(-irfft_scratch_memory_size / chunk_count, True)
+        add_to_memory_counters(irfft_result_size / chunk_count, True)
+        add_to_memory_counters(-filtered_rfft_result_size / chunk_count, True)
+
+        add_to_memory_counters(-irfft_result_size / chunk_count, True)
+
+    add_to_memory_counters(-padded_in_slice_size, True)
+    add_to_memory_counters(-filter_size, False)
+    add_to_memory_counters(-rfftfreq_size, False)
+    add_to_memory_counters(-scaled_filter_size, False)
+
+    add_to_memory_counters(datac_size, True)
+    add_to_memory_counters(fde_size, True)
+    add_to_memory_counters(-tmp_p_input_slice, True)
+    add_to_memory_counters(fft_plan_slice_size, True)
+    add_to_memory_counters(fft_result_size, True)
+    add_to_memory_counters(-datac_size, True)
+
+    add_to_memory_counters(-fft_result_size, True)
+
+    add_to_memory_counters(ifft2_plan_slice_size / chunk_count * 2, True)
+    for _ in range(0, chunk_count):
+        add_to_memory_counters(fde_size / chunk_count, True)
+        add_to_memory_counters(-fde_size / chunk_count, True)
+
+    add_to_memory_counters(recon_output_size, True)
+    add_to_memory_counters(-fde_size, True)
+    add_to_memory_counters(circular_mask_size, False)
+    add_to_memory_counters(after_recon_swapaxis_slice, True)
+
+    return (tot_memory_bytes * 1.05, fixed_amount + 250 * 1024 * 1024)
+    # return (tot_memory_bytes, fixed_amount)
 
 
 def _calc_memory_bytes_SIRT3d_tomobar(

{httomo_backends-0.5 → httomo_backends-0.5.2}/httomo_backends/methods_database/packages/backends/httomolibgpu/supporting_funcs/recon/peak_memory_line_profile_hook.py

@@ -6,7 +6,7 @@ from cupy.cuda import memory_hook
 
 
 class PeakMemoryLineProfileHook(memory_hook.MemoryHook):
-    name = 'LineProfileHook'
+    name = "LineProfileHook"
 
     def __init__(self, running_peak_root_file_names, max_depth=0):
         self._memory_frames = {}
@@ -45,15 +45,11 @@ class PeakMemoryLineProfileHook(memory_hook.MemoryHook):
     def _extract_stackframes(self):
         stackframes = traceback.extract_stack()
         stackframes = [StackFrame(st) for st in stackframes]
-        stackframes = [
-            st for st in stackframes if st.filename != self._filename]
+        stackframes = [st for st in stackframes if st.filename != self._filename]
         return stackframes
 
     def _key_frame(self, parent, stackframe):
-        return (parent,
-                stackframe.filename,
-                stackframe.lineno,
-                stackframe.name)
+        return (parent, stackframe.filename, stackframe.lineno, stackframe.name)
 
     def _add_frame(self, parent, stackframe):
         key = self._key_frame(parent, stackframe)
@@ -66,17 +62,26 @@ class PeakMemoryLineProfileHook(memory_hook.MemoryHook):
 
     def print_report(self, file=sys.stdout):
         """Prints a report of line memory profiling."""
-        line = '_root (%s, %s, %s)\n' % self._root.humanized_bytes()
+        line = "_root (%s, %s, %s)\n" % self._root.humanized_bytes()
         file.write(line)
 
         running_peak_bytes = [0]
         running_used_bytes = [0]
         for child in self._root.children:
-            self._print_frame(child, running_peak_bytes, running_used_bytes, depth=1, file=file)
+            self._print_frame(
+                child, running_peak_bytes, running_used_bytes, depth=1, file=file
+            )
         file.flush()
 
-    def _print_frame(self, memory_frame, running_peak_bytes, running_used_bytes, depth=0, file=sys.stdout):
-        indent = ' ' * (depth * 2)
+    def _print_frame(
+        self,
+        memory_frame,
+        running_peak_bytes,
+        running_used_bytes,
+        depth=0,
+        file=sys.stdout,
+    ):
+        indent = " " * (depth * 2)
         st = memory_frame.stackframe
         used_bytes, acquired_bytes, freed_bytes = memory_frame.humanized_bytes()
 
@@ -87,15 +92,33 @@ class PeakMemoryLineProfileHook(memory_hook.MemoryHook):
             running_peak_bytes[0] = max(running_peak_bytes[0], running_used_bytes[0])
             running_used_bytes[0] -= memory_frame.freed_bytes
 
-            humanized_running_peak_bytes = MemoryFrame.humanized_size(running_peak_bytes[0])
-            humanized_running_used_bytes = MemoryFrame.humanized_size(running_used_bytes[0])
-
-        line = '%s%s:%s:%s (%s, %s, %s, %s, %s)\n' % (
-            indent, st.filename, st.lineno, st.name,
-            used_bytes, acquired_bytes, freed_bytes, humanized_running_peak_bytes, humanized_running_used_bytes)
+            humanized_running_peak_bytes = MemoryFrame.humanized_size(
+                running_peak_bytes[0]
+            )
+            humanized_running_used_bytes = MemoryFrame.humanized_size(
+                running_used_bytes[0]
+            )
+
+        line = "%s%s:%s:%s (%s, %s, %s, %s, %s)\n" % (
+            indent,
+            st.filename,
+            st.lineno,
+            st.name,
+            used_bytes,
+            acquired_bytes,
+            freed_bytes,
+            humanized_running_peak_bytes,
+            humanized_running_used_bytes,
+        )
         file.write(line)
         for child in memory_frame.children:
-            self._print_frame(child, running_peak_bytes, running_used_bytes, depth=depth + 1, file=file)
+            self._print_frame(
+                child,
+                running_peak_bytes,
+                running_used_bytes,
+                depth=depth + 1,
+                file=file,
+            )
 
 
 class StackFrame(object):
@@ -147,11 +170,11 @@ class MemoryFrame(object):
 
     @staticmethod
     def humanized_size(size):
-        for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E']:
+        for unit in ["", "K", "M", "G", "T", "P", "E"]:
             if size < 1024.0:
-                return '%3.2f%sB' % (size, unit)
+                return "%3.2f%sB" % (size, unit)
             size /= 1024.0
-        return '%.2f%sB' % (size, 'Z')
+        return "%.2f%sB" % (size, "Z")
 
     def _set_parent(self, parent):
         if parent and parent not in parent.children:

{httomo_backends-0.5 → httomo_backends-0.5.2}/httomo_backends/scripts/json_pipelines_generator.py

@@ -20,29 +20,33 @@ result = process_all_yaml_files()
 """
 
 # Directory containing YAML files relative to this script
-PIPELINES_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "pipelines_full")
+PIPELINES_DIR = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)), "..", "pipelines_full"
+)
+
+ignored_methods = ["standard_tomo", "calculate_stats"]
 
-ignored_methods = ["standard_tomo","calculate_stats"]
 
 def get_yaml_path(yaml_filename: str) -> str:
     """
     Get the full path to a YAML file in the pipelines_full directory.
-    
+
     Args:
         yaml_filename: Name of the YAML file (e.g., "example.yaml")
-    
+
     Returns:
         Full path to the YAML file
     """
     return os.path.join(PIPELINES_DIR, yaml_filename)
 
+
 def import_module_safely(module_name: str):
     """
     Safely import a module handling potential import errors
-    
+
     Args:
         module_name: The name of the module to import
-    
+
     Returns:
         The imported module or None if import fails
     """
@@ -50,21 +54,24 @@ def import_module_safely(module_name: str):
         return importlib.import_module(module_name)
     except ImportError as e:
         missing_module = str(e).split("'")[1] if "'" in str(e) else str(e)
-        print(f"Warning: Could not import dependency '{missing_module}' for module '{module_name}'")
+        print(
+            f"Warning: Could not import dependency '{missing_module}' for module '{module_name}'"
+        )
         print(f"You may need to install it using: pip install {missing_module}")
         return None
     except Exception as e:
         print(f"Warning: Error importing module '{module_name}': {str(e)}")
         return None
 
+
 def inspect_method_parameters(module_name: str, method_name: str) -> Dict[str, Any]:
     """
     Inspect a method's parameters from a given module
-    
+
     Args:
         module_name: The full path to the module
         method_name: Name of the method to inspect
-    
+
     Returns:
         Dictionary of parameter names and their default values
     """
@@ -72,26 +79,46 @@ def inspect_method_parameters(module_name: str, method_name: str) -> Dict[str, A
     module = import_module_safely(module_name)
     if module is None:
         # Return empty parameters if module import failed
-        return {}    
+        return {}
     try:
         # Get the method
         method = getattr(module, method_name)
-        
+
         # Get method signature
         signature = inspect.signature(method)
-        
+
         # Create parameters dictionary
         parameters = {}
-        
+
         # List of parameters to skip
         skip_params = [
-            "in_file", "data_in", "tomo", "arr", "prj", "data",
-            "ncore", "nchunk", "flats", "flat", "dark", "darks",
-            "theta", "out", "ang", "comm_rank", "out_dir", "angles",
-            "gpu_id", "comm", "offset", "shift_xy", "step_xy", 
-            "jpeg_quality", "watermark_vals"
+            "in_file",
+            "data_in",
+            "tomo",
+            "arr",
+            "prj",
+            "data",
+            "ncore",
+            "nchunk",
+            "flats",
+            "flat",
+            "dark",
+            "darks",
+            "theta",
+            "out",
+            "ang",
+            "comm_rank",
+            "out_dir",
+            "angles",
+            "gpu_id",
+            "comm",
+            "offset",
+            "shift_xy",
+            "step_xy",
+            "jpeg_quality",
+            "watermark_vals",
         ]
-        
+
         # Process parameters
         for param_name, param in signature.parameters.items():
             if param_name not in skip_params:
@@ -100,120 +127,144 @@ def inspect_method_parameters(module_name: str, method_name: str) -> Dict[str, A
                 elif param_name == "asynchronous":
                     parameters[param_name] = True
                 elif param_name == "center":
-                    parameters[param_name] = "${centering.side_outputs.centre_of_rotation}"
+                    parameters[param_name] = (
+                        "${centering.side_outputs.centre_of_rotation}"
+                    )
                 elif param_name == "glob_stats":
                     parameters[param_name] = "${statistics.side_outputs.glob_stats}"
                 elif param_name == "overlap":
                     parameters[param_name] = "${centering.side_outputs.overlap}"
                 else:
                     # Get default value if it exists, otherwise mark as REQUIRED
-                    default = param.default if param.default != inspect.Parameter.empty else "REQUIRED"
+                    default = (
+                        param.default
+                        if param.default != inspect.Parameter.empty
+                        else "REQUIRED"
+                    )
                     parameters[param_name] = default
-        
+
         return parameters
-        
+
     except Exception as e:
-        print(f"Warning: Error inspecting method '{method_name}' in module '{module_name}': {str(e)}")
+        print(
+            f"Warning: Error inspecting method '{method_name}' in module '{module_name}': {str(e)}"
+        )
         return {}
 
+
 def json_pipeline_generator(input_yaml: str) -> Dict[str, Any]:
     """
     Generate JSON pipeline from YAML directive
-    
+
     Args:
         input_yaml: Path to input YAML directive file
-    
+
     Returns:
         Dictionary with methods information if successful, or empty dictionary if error occurred
     """
     try:
         # Read input YAML
-        with open(input_yaml, 'r') as file:
+        with open(input_yaml, "r") as file:
             pipeline = yaml.safe_load(file)
-        
+
         # Dictionary to store methods information
         methods_info = {}
-        
+
         # Process each method in pipeline
         for item in pipeline:
-            method_name = item['method']
-            module_path = item['module_path']
+            method_name = item["method"]
+            module_path = item["module_path"]
             # Skip ignored methods
-            if method_name not in ignored_methods:  
+            if method_name not in ignored_methods:
                 print(f"Processing method: {method_name} from module: {module_path}")
                 # Get method parameters
                 parameters = inspect_method_parameters(module_path, method_name)
                 # Add to methods info even if parameters are empty
                 methods_info[method_name] = {
-                    'module_path': module_path,
-                    'parameters': parameters
+                    "module_path": module_path,
+                    "parameters": parameters,
                 }
         return methods_info
-        
+
     except Exception as e:
         print(f"Error occurred: {str(e)}")
         return {}
 
+
 import json
 import os
 
 # Path to the priority configuration file
 PRIORITY_FILE = os.path.join(PIPELINES_DIR, "pipeline_priority.json")
 
+
 def load_priority_order() -> list:
     """
     Load the pipeline priority order from the JSON configuration file in the pipelines_full directory.
-    
+
     Returns:
         List of pipeline titles in priority order.
     """
     try:
-        with open(PRIORITY_FILE, 'r') as file:
+        with open(PRIORITY_FILE, "r") as file:
             config = json.load(file)
             return config.get("pipeline_order", [])
     except FileNotFoundError:
-        print(f"Warning: Priority file '{PRIORITY_FILE}' not found. Using default order.")
+        print(
+            f"Warning: Priority file '{PRIORITY_FILE}' not found. Using default order."
+        )
         return []
     except Exception as e:
-        print(f"Warning: Could not load priority file '{PRIORITY_FILE}'. Using default order. Error: {str(e)}")
+        print(
+            f"Warning: Could not load priority file '{PRIORITY_FILE}'. Using default order. Error: {str(e)}"
+        )
         return []
 
+
 def process_all_yaml_files() -> Dict[str, Any]:
     """
     Process all YAML files in the pipelines_full directory in the order of priority.
-    
+
     Returns:
         Dictionary where keys are YAML file names and values are the JSON outputs
         from the json_pipeline_generator function.
     """
     # Load the priority order
     priority_order = load_priority_order()
-    
+
     # Dictionary to store results
     results = {}
-    
+
     # List all YAML files in the pipelines_full directory
-    yaml_files = [f for f in os.listdir(PIPELINES_DIR) if f.endswith(".yaml") or f.endswith(".yml")]
-    
-     # Exclude the priority file from processing
+    yaml_files = [
+        f
+        for f in os.listdir(PIPELINES_DIR)
+        if f.endswith(".yaml") or f.endswith(".yml")
+    ]
+
+    # Exclude the priority file from processing
     yaml_files = [f for f in yaml_files if f != "pipeline_priority.yaml"]
 
     # Sort YAML files based on priority order
     if priority_order:
         # Remove the "_directive.yaml" suffix for matching
-        yaml_files.sort(key=lambda x: priority_order.index(x.removesuffix("_directive.yaml")) 
-                        if x.removesuffix("_directive.yaml") in priority_order 
-                        else len(priority_order))
-    
+        yaml_files.sort(
+            key=lambda x: (
+                priority_order.index(x.removesuffix("_directive.yaml"))
+                if x.removesuffix("_directive.yaml") in priority_order
+                else len(priority_order)
+            )
+        )
+
     # Process the YAML files in the sorted order
     for yaml_file in yaml_files:
         # Get the full path to the YAML file
         yaml_path = get_yaml_path(yaml_file)
-        
+
         # Process the YAML file
         json_output = json_pipeline_generator(yaml_path)
-        
+
         # Add to results (remove the "_directive.yaml" suffix for the key)
         results[yaml_file.removesuffix("_directive.yaml")] = json_output
-    
-    return results
+
+    return results

{httomo_backends-0.5 → httomo_backends-0.5.2}/httomo_backends/scripts/yaml_pipelines_generator.py

@@ -187,10 +187,6 @@ def yaml_pipelines_generator(
                     indent=0,
                 )
                 pipeline_full += yaml_template_method
-                pipeline_full[i]["parameters"].yaml_add_eol_comment(
-                    key="rotation",
-                    comment="'left' if rotation center is close to the left of the field-of-view, 'right' otherwise.",
-                )
             elif "normalize" in module_name:
                 pipeline_full.yaml_set_comment_before_after_key(
                     i,