httomo-backends 0.6__tar.gz → 0.6.1__tar.gz

{httomo_backends-0.6 → httomo_backends-0.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: httomo-backends
-Version: 0.6
+Version: 0.6.1
 Summary: Supplementary files for HTTomo backends.
 Author: Garry ODonnell, Jacob Williamson
 Author-email: Daniil Kazantsev <daniil.kazantsev@diamond.ac.uk>, Yousef Moazzam <yousef.moazzam@diamond.ac.uk>, Jessica Verschoyle <jessica.verschoyle@diamond.ac.uk>, Naman Gera <naman.gera@diamond.ac.uk>, scientificsoftware@diamond.ac.uk

{httomo_backends-0.6 → httomo_backends-0.6.1}/httomo_backends/methods_database/packages/backends/httomolibgpu/httomolibgpu.yaml

@@ -7,7 +7,7 @@ misc:
       save_result_default: False
       padding: True
       memory_gpu:
-        multiplier: 2.1
+        multiplier: 2.5
         method: direct
     remove_outlier:
       pattern: all
@@ -16,7 +16,7 @@ misc:
       save_result_default: False
       padding: True
       memory_gpu:
-        multiplier: 2.1
+        multiplier: 2.5
         method: direct
   denoise:
     total_variation_ROF:
@@ -87,15 +87,6 @@ prep:
       memory_gpu:
         multiplier: None
         method: module
-    paganin_filter_savu:
-      pattern: projection
-      output_dims_change: False
-      implementation: gpu_cupy
-      save_result_default: False
-      padding: False
-      memory_gpu:
-        multiplier: None
-        method: module
   alignment:
     distortion_correction_proj_discorpy:
       pattern: projection
@@ -207,7 +198,7 @@ recon:
       save_result_default: False
       padding: False
       memory_gpu:
-        multiplier: 0
+        multiplier: 2
         method: direct
     find_center_pc:
       pattern: projection

{httomo_backends-0.6 → httomo_backends-0.6.1}/httomo_backends/methods_database/packages/backends/httomolibgpu/supporting_funcs/prep/phase.py

@@ -27,70 +27,10 @@ import numpy as np
 from httomo_backends.cufft import CufftType, cufft_estimate_2d
 
 __all__ = [
-    "_calc_memory_bytes_paganin_filter_savu",
     "_calc_memory_bytes_paganin_filter_tomopy",
 ]
 
 
-def _calc_memory_bytes_paganin_filter_savu(
-    non_slice_dims_shape: Tuple[int, int],
-    dtype: np.dtype,
-    **kwargs,
-) -> Tuple[int, int]:
-    pad_x = kwargs["pad_x"]
-    pad_y = kwargs["pad_y"]
-
-    # Input (unpadded)
-    unpadded_in_slice_size = np.prod(non_slice_dims_shape) * dtype.itemsize
-
-    # Padded input
-    padded_non_slice_dims_shape = (
-        non_slice_dims_shape[0] + 2 * pad_y,
-        non_slice_dims_shape[1] + 2 * pad_x,
-    )
-    padded_in_slice_size = (
-        padded_non_slice_dims_shape[0] * padded_non_slice_dims_shape[1] * dtype.itemsize
-    )
-
-    # Padded input cast to `complex64`
-    complex_slice = padded_in_slice_size / dtype.itemsize * np.complex64().nbytes
-
-    # Plan size for 2D FFT
-    fftplan_slice_size = cufft_estimate_2d(
-        nx=padded_non_slice_dims_shape[1],
-        ny=padded_non_slice_dims_shape[0],
-        fft_type=CufftType.CUFFT_C2C,
-    )
-
-    # Shape of 2D filter is the same as the padded `complex64` slice shape, so the size will be
-    # the same
-    filter_size = complex_slice
-
-    # Size of cropped/unpadded + cast to float32 result of 2D IFFT
-    cropped_float32_res_slice = np.prod(non_slice_dims_shape) * np.float32().nbytes
-
-    # If the FFT plan size is negligible for some reason, this changes where the peak GPU
-    # memory usage occurs. Hence, the if/else branching below for calculating the total bytes.
-    NEGLIGIBLE_FFT_PLAN_SIZE = 16
-    if fftplan_slice_size < NEGLIGIBLE_FFT_PLAN_SIZE:
-        tot_memory_bytes = int(
-            unpadded_in_slice_size + padded_in_slice_size + complex_slice
-        )
-    else:
-        tot_memory_bytes = int(
-            unpadded_in_slice_size
-            + padded_in_slice_size
-            + complex_slice
-            # The padded float32 array is deallocated when a copy is made when casting to complex64
-            # and the variable `padded_tomo` is reassigned to the complex64 version
-            - padded_in_slice_size
-            + fftplan_slice_size
-            + cropped_float32_res_slice
-        )
-
-    return (tot_memory_bytes, filter_size)
-
-
 def _calc_memory_bytes_paganin_filter_tomopy(
     non_slice_dims_shape: Tuple[int, int],
     dtype: np.dtype,

{httomo_backends-0.6 → httomo_backends-0.6.1}/httomo_backends/methods_database/packages/backends/httomolibgpu/supporting_funcs/recon/algorithm.py

@@ -354,6 +354,7 @@ def _calc_memory_bytes_LPRec3d_tomobar(
 
     return (tot_memory_bytes * 1.05, fixed_amount + 250 * 1024 * 1024)
 
+
 def _calc_memory_bytes_SIRT3d_tomobar(
     non_slice_dims_shape: Tuple[int, int],
     dtype: np.dtype,

httomo_backends-0.6.1/httomo_backends/pipelines_full/sweep_paganin_FBP3d_tomobar_directive.yaml

@@ -0,0 +1,12 @@
+- method: standard_tomo
+  module_path: httomo.data.hdf.loaders
+- method: find_center_vo
+  module_path: httomolibgpu.recon.rotation  
+- method: normalize
+  module_path: httomolibgpu.prep.normalize
+- method: paganin_filter_tomopy
+  module_path: httomolibgpu.prep.phase
+  sweep_parameter: alpha
+  sweep_values: [0.01, 0.001, 0.0001]
+- method: FBP3d_tomobar
+  module_path: httomolibgpu.recon.algorithm

{httomo_backends-0.6 → httomo_backends-0.6.1}/httomo_backends/scripts/yaml_pipelines_generator.py

@@ -31,6 +31,8 @@ import ruamel.yaml
 import httomo_backends
 import yaml
 
+from httomo import __version__ as httomo_version
+
 CS = ruamel.yaml.comments.CommentedSeq  # defaults to block style
 
 
@@ -55,6 +57,20 @@ def __sweeprange_representer(
     )
 
 
+class SweepManual:
+    """SweepManual class."""
+
+    def __init__(self, lst):
+        self._lst = lst
+
+
+def __sweepmanual_representer(
+    dumper: yaml.SafeDumper, swp: SweepManual
+) -> yaml.nodes.SequenceNode:
+    """Represent a sweepmanual as a YAML sequence node."""
+    return dumper.represent_sequence("!Sweep", swp._lst)
+
+
 def __represent_none(self, data):
     return self.represent_scalar("tag:yaml.org,2002:null", "null")
 
@@ -86,17 +102,22 @@ def yaml_pipelines_generator(
         # a loop over methods in the high-level pipeline file (directive)
         methods_no = len(pipeline_file_content)
         pipeline_full = CS()
-        sweep_enabled = False
+        sweep_enabled_range = False
+        sweep_enabled_value = False
         for i in range(methods_no):
             method_content = pipeline_file_content[i]
             method_name = method_content["method"]
             module_name = method_content["module_path"]
             if "sweep_parameter" in method_content:
                 sweep_parameter = method_content["sweep_parameter"]
-                sweep_start = method_content["sweep_start"]
-                sweep_stop = method_content["sweep_stop"]
-                sweep_step = method_content["sweep_step"]
-                sweep_enabled = True
+                if "sweep_start" in method_content:
+                    sweep_start = method_content["sweep_start"]
+                    sweep_stop = method_content["sweep_stop"]
+                    sweep_step = method_content["sweep_step"]
+                    sweep_enabled_range = True
+                else:
+                    sweep_values = method_content["sweep_values"]
+                    sweep_enabled_value = True
 
             # get the corresponding yaml template from httomo-backends
             backend_name = module_name[0 : module_name.find(".")]
@@ -116,6 +137,13 @@ def yaml_pipelines_generator(
                 except OSError as e:
                     print("loading yaml template failed", e)
 
+            version_split = httomo_version.split(".")
+            major = version_split[0]
+            minor = version_split[1]
+            pipeline_full.yaml_set_start_comment(
+                f"This pipeline is supported by HTTomo ver. {major}.{minor}"
+            )            
+
             if "loaders" in module_name:
                 # should be the first method in the list
                 pipeline_full.yaml_set_comment_before_after_key(
@@ -149,7 +177,7 @@ def yaml_pipelines_generator(
                 )
                 pipeline_full[i]["side_outputs"].yaml_add_eol_comment(
                     key="cor",
-                    comment="A side output of the method, here a CoR scalar value",
+                    comment="An estimated CoR value provided as a side output",
                 )
                 pipeline_full[i]["side_outputs"].yaml_add_eol_comment(
                     key="overlap",
@@ -158,17 +186,22 @@ def yaml_pipelines_generator(
             elif "corr" in module_name and "remove_outlier" in method_name:
                 pipeline_full.yaml_set_comment_before_after_key(
                     i,
-                    "--- Removing dead pixels in the data, aka zingers. Use if sharp streaks are present in reconstruction. Please use before normalisation. ---",
+                    "--- Removing unresponsive pixels in the data, aka zingers. Use if sharp streaks are present in the reconstruction. To be applied before normalisation. ---",
                     indent=0,
                 )
                 pipeline_full += yaml_template_method
                 if pipeline_full[i]["parameters"]["dif"] == "REQUIRED":
                     # fix for the absent parameter in TomoPy's algorithm
                     pipeline_full[i]["parameters"]["dif"] = 0.1
+                pipeline_full[i]["parameters"].yaml_add_eol_comment(
+                    key="kernel_size",
+                    comment="The size of the 3D neighbourhood surrounding the voxel. Odd integer.",
+                )
                 pipeline_full[i]["parameters"].yaml_add_eol_comment(
                     key="dif",
-                    comment="A difference between the outlier value and the median value of neighboring pixels.",
+                    comment="A difference between the outlier value and the median value of neighbouring pixels.",
                 )
+
             elif "distortion" in method_name:
                 pipeline_full.yaml_set_comment_before_after_key(
                     i,
@@ -225,7 +258,11 @@ def yaml_pipelines_generator(
                 pipeline_full += yaml_template_method
                 pipeline_full[i]["parameters"].yaml_add_eol_comment(
                     key="center",
-                    comment="Reference to center of rotation side output OR an integer.",
+                    comment="Reference to center of rotation side output above OR a float number.",
+                )
+                pipeline_full[i]["parameters"].yaml_add_eol_comment(
+                    key="detector_pad",
+                    comment="Horizontal detector padding to minimise circle/arc-type artifacts in the reconstruction",
                 )
                 pipeline_full[i]["parameters"].yaml_add_eol_comment(
                     key="recon_mask_radius",
@@ -282,13 +319,20 @@ def yaml_pipelines_generator(
                 )
                 pipeline_full += yaml_template_method
 
-            if sweep_enabled:
+            if sweep_enabled_range:
                 pipeline_full[i]["parameters"][sweep_parameter] = SweepRange(
                     start=sweep_start, stop=sweep_stop, step=sweep_step
                 )
+                yaml.representer.add_representer(SweepRange, __sweeprange_representer)
+                sweep_enabled_range = False
+            if sweep_enabled_value:
+                pipeline_full[i]["parameters"][sweep_parameter] = SweepManual(
+                    list(sweep_values)
+                )
+                yaml.representer.add_representer(SweepManual, __sweepmanual_representer)
+                sweep_enabled_value = False
 
         yaml.representer.add_representer(type(None), __represent_none)
-        yaml.representer.add_representer(SweepRange, __sweeprange_representer)
         yaml.dump(pipeline_full, f)
 
     return 0

{httomo_backends-0.6 → httomo_backends-0.6.1}/httomo_backends.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: httomo-backends
-Version: 0.6
+Version: 0.6.1
 Summary: Supplementary files for HTTomo backends.
 Author: Garry ODonnell, Jacob Williamson
 Author-email: Daniil Kazantsev <daniil.kazantsev@diamond.ac.uk>, Yousef Moazzam <yousef.moazzam@diamond.ac.uk>, Jessica Verschoyle <jessica.verschoyle@diamond.ac.uk>, Naman Gera <naman.gera@diamond.ac.uk>, scientificsoftware@diamond.ac.uk

{httomo_backends-0.6 → httomo_backends-0.6.1}/httomo_backends.egg-info/SOURCES.txt

@@ -49,6 +49,7 @@ httomo_backends/pipelines_full/__init__.py
 httomo_backends/pipelines_full/deg360_distortion_FBP3d_tomobar_directive.yaml
 httomo_backends/pipelines_full/deg360_paganin_FBP3d_tomobar_directive.yaml
 httomo_backends/pipelines_full/sweep_center_FBP3d_tomobar_directive.yaml
+httomo_backends/pipelines_full/sweep_paganin_FBP3d_tomobar_directive.yaml
 httomo_backends/pipelines_full/tomopy_gridrec_directive.yaml
 httomo_backends/scripts/__init__.py
 httomo_backends/scripts/json_pipelines_generator.py

{httomo_backends-0.6 → httomo_backends-0.6.1}/pyproject.toml

@@ -24,7 +24,7 @@ dev_template = "{tag}"
 
 [project]
 name = "httomo-backends"
-version = "0.6"
+version = "0.6.1"
 description = "Supplementary files for HTTomo backends."
 readme = "README.rst"
 license = {text = "BSD-3-Clause"}

{httomo_backends-0.6 → httomo_backends-0.6.1}/tests/test_httomolibgpu.py

@@ -15,7 +15,7 @@ from httomo_backends.methods_database.query import MethodsDatabaseQuery
 
 from httomolibgpu.misc.morph import data_resampler, sino_360_to_180
 from httomolibgpu.prep.normalize import normalize
-from httomolibgpu.prep.phase import paganin_filter_tomopy, paganin_filter_savu
+from httomolibgpu.prep.phase import paganin_filter_tomopy
 from httomolibgpu.prep.alignment import distortion_correction_proj_discorpy
 from httomolibgpu.prep.stripe import (
     remove_stripe_based_sorting,
@@ -113,13 +113,11 @@ def test_normalize_memoryhook(flats, darks, ensure_clean_memory, dtype, slices):
 
 
 @pytest.mark.parametrize("dtype", ["uint16"])
-@pytest.mark.parametrize("slices", [151, 321])
+@pytest.mark.parametrize("slices", [10, 20])
 @pytest.mark.cupy
 def test_remove_outlier_memoryhook(flats, ensure_clean_memory, dtype, slices):
     hook = MaxMemoryHook()
-    data = cp.random.random_sample(
-        (slices, flats.shape[1], flats.shape[2]), dtype=np.float32
-    )
+    data = cp.random.random_sample((slices, 2560, 2560), dtype=np.float32)
     if dtype == "uint16":
         data = data.astype(np.uint16)
     with hook:
@@ -148,7 +146,7 @@ def test_remove_outlier_memoryhook(flats, ensure_clean_memory, dtype, slices):
     # the estimated_memory_mb should be LARGER or EQUAL to max_mem_mb
     # the resulting percent value should not deviate from max_mem on more than 20%
     assert estimated_memory_mb >= max_mem_mb
-    assert percents_relative_maxmem <= 20
+    assert percents_relative_maxmem <= 25
 
 
 @pytest.mark.parametrize("slices", [3, 10, 20])
@@ -257,48 +255,6 @@ def test_paganin_filter_tomopy_memoryhook(slices, dim_x, dim_y, ensure_clean_mem
     assert percents_relative_maxmem <= 20
 
 
-@pytest.mark.cupy
-@pytest.mark.parametrize("slices", [64, 128])
-@pytest.mark.parametrize("dim_x", [81, 260, 320])
-@pytest.mark.parametrize("dim_y", [340, 135, 96])
-def test_paganin_filter_savu_memoryhook(slices, dim_x, dim_y, ensure_clean_memory):
-    data = cp.random.random_sample((slices, dim_x, dim_y), dtype=np.float32)
-    kwargs = {}
-    kwargs["ratio"] = 250.0
-    kwargs["energy"] = 53.0
-    kwargs["distance"] = 1.0
-    kwargs["resolution"] = 1.28
-    kwargs["pad_x"] = 20
-    kwargs["pad_y"] = 20
-    kwargs["pad_method"] = "edge"
-    kwargs["increment"] = 0.0
-    hook = MaxMemoryHook()
-    with hook:
-        data_filtered = paganin_filter_savu(data, **kwargs).get()
-
-    # The amount of bytes used by the method's processing according to the memory hook, plus
-    # the amount of bytes needed to hold the method's input in GPU memory
-    max_mem = hook.max_mem + data.nbytes
-
-    # now we estimate how much of the total memory required for this data
-    (estimated_memory_bytes, subtract_bytes) = _calc_memory_bytes_paganin_filter_savu(
-        (dim_x, dim_y), np.float32(), **kwargs
-    )
-    estimated_memory_mb = round(slices * estimated_memory_bytes / (1024**2), 2)
-    max_mem -= subtract_bytes
-    max_mem_mb = round(max_mem / (1024**2), 2)
-
-    # now we compare both memory estimations
-    #
-    # make sure estimator function is within range (80% min, 100% max)
-    difference_mb = abs(estimated_memory_mb - max_mem_mb)
-    percents_relative_maxmem = round((difference_mb / max_mem_mb) * 100)
-    # the estimated_memory_mb should be LARGER or EQUAL to max_mem_mb
-    # the resulting percent value should not deviate from max_mem on more than 20%
-    assert estimated_memory_mb >= max_mem_mb
-    assert percents_relative_maxmem <= 20
-
-
 @pytest.mark.cupy
 @pytest.mark.parametrize("slices", [128, 190, 256])
 def test_distortion_correction_memoryhook(
@@ -719,12 +675,7 @@ def __test_recon_LPRec3d_tomobar_memoryhook_common(
     # the estimated_memory_mb should be LARGER or EQUAL to max_mem_mb
     # the resulting percent value should not deviate from max_mem on more than 20%
     assert estimated_memory_mb >= max_mem_mb
-    if slices <= 3:
-        assert percents_relative_maxmem <= 75
-    elif slices <= 5:
-        assert percents_relative_maxmem <= 63
-    else:
-        assert percents_relative_maxmem <= 50
+    assert percents_relative_maxmem <= 80
 
 
 @pytest.mark.cupy

{httomo_backends-0.6 → httomo_backends-0.6.1}/httomo_backends/pipelines_full/deg360_distortion_FBP3d_tomobar_directive.yaml

@@ -1,11 +1,11 @@
 - method: standard_tomo
   module_path: httomo.data.hdf.loaders
+- method: find_center_360
+  module_path: httomolibgpu.recon.rotation
 - method: normalize
   module_path: httomolibgpu.prep.normalize
 - method: distortion_correction_proj_discorpy
   module_path: httomolibgpu.prep.alignment
-- method: find_center_360
-  module_path: httomolibgpu.recon.rotation
 - method: sino_360_to_180
   module_path: httomolibgpu.misc.morph
 - method: remove_stripe_based_sorting

{httomo_backends-0.6 → httomo_backends-0.6.1}/httomo_backends/pipelines_full/deg360_paganin_FBP3d_tomobar_directive.yaml

@@ -1,9 +1,9 @@
 - method: standard_tomo
   module_path: httomo.data.hdf.loaders
-- method: normalize
-  module_path: httomolibgpu.prep.normalize  
 - method: find_center_360
   module_path: httomolibgpu.recon.rotation
+- method: normalize
+  module_path: httomolibgpu.prep.normalize  
 - method: sino_360_to_180
   module_path: httomolibgpu.misc.morph
 - method: remove_stripe_based_sorting