httomo-backends 0.9.0__tar.gz → 1.0__tar.gz

{httomo_backends-0.9.0 → httomo_backends-1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: httomo-backends
-Version: 0.9.0
+Version: 1.0
 Summary: Supplementary files for HTTomo backends.
 Author: Garry ODonnell, Jacob Williamson
 Author-email: Daniil Kazantsev <daniil.kazantsev@diamond.ac.uk>, Yousef Moazzam <yousef.moazzam@diamond.ac.uk>, Jessica Verschoyle <jessica.verschoyle@diamond.ac.uk>, Naman Gera <naman.gera@diamond.ac.uk>, scientificsoftware@diamond.ac.uk

{httomo_backends-0.9.0 → httomo_backends-1.0}/httomo_backends/methods_database/packages/backends/httomolibgpu/httomolibgpu.yaml

@@ -105,7 +105,7 @@ prep:
       padding: False
       memory_gpu:
         multiplier: None
-        method: module
+        method: iterative
     paganin_filter_savu_legacy:
       pattern: projection
       output_dims_change: False
@@ -114,7 +114,7 @@ prep:
       padding: False
       memory_gpu:
         multiplier: None
-        method: module       
+        method: iterative       
   alignment:
     distortion_correction_proj_discorpy:
       pattern: projection
@@ -135,6 +135,15 @@ prep:
       memory_gpu:
         multiplier: 1.17
         method: direct
+    remove_stripe_fw:
+      pattern: sinogram
+      output_dims_change: False
+      implementation: gpu_cupy
+      save_result_default: False
+      padding: False
+      memory_gpu:
+        multiplier: None
+        method: iterative
     remove_stripe_ti:
       pattern: sinogram
       output_dims_change: False

httomo_backends-1.0/httomo_backends/methods_database/packages/backends/httomolibgpu/supporting_funcs/prep/phase.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ---------------------------------------------------------------------------
+# Copyright 2022 Diamond Light Source Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+# Created By  : Tomography Team at DLS <scientificsoftware@diamond.ac.uk>
+# Created Date: 21 September 2023
+# ---------------------------------------------------------------------------
+"""Modules for memory estimation for phase retrieval and phase-contrast enhancement"""
+
+from typing import Tuple
+import numpy as np
+
+from httomolibgpu.prep.phase import paganin_filter, paganin_filter_savu_legacy
+
+__all__ = [
+    "_calc_memory_bytes_for_slices_paganin_filter",
+    "_calc_memory_bytes_for_slices_paganin_filter_savu_legacy",
+]
+
+
+def _calc_memory_bytes_for_slices_paganin_filter(
+    dims_shape: Tuple[int, int, int],
+    dtype: np.dtype,
+    **kwargs,
+) -> int:
+    return paganin_filter(dims_shape, calc_peak_gpu_mem=True, **kwargs)
+
+
+def _calc_memory_bytes_for_slices_paganin_filter_savu_legacy(
+    dims_shape: Tuple[int, int, int],
+    dtype: np.dtype,
+    **kwargs,
+) -> int:
+    return paganin_filter_savu_legacy(dims_shape, calc_peak_gpu_mem=True, **kwargs)

{httomo_backends-0.9.0 → httomo_backends-1.0}/httomo_backends/methods_database/packages/backends/httomolibgpu/supporting_funcs/prep/stripe.py

@@ -25,9 +25,11 @@ from typing import Tuple
 import numpy as np
 
 from httomo_backends.cufft import CufftType, cufft_estimate_1d
+from httomolibgpu.prep.stripe import remove_stripe_fw
 
 
 __all__ = [
+    "_calc_memory_bytes_for_slices_remove_stripe_fw",
     "_calc_memory_bytes_remove_stripe_ti",
     "_calc_memory_bytes_remove_all_stripe",
     "_calc_memory_bytes_raven_filter",
@@ -53,6 +55,14 @@ def _calc_memory_bytes_remove_stripe_ti(
     return (tot_memory_bytes, gamma_mem)
 
 
+def _calc_memory_bytes_for_slices_remove_stripe_fw(
+    dims_shape: Tuple[int, int, int],
+    dtype: np.dtype,
+    **kwargs,
+) -> int:
+    return remove_stripe_fw(dims_shape, calc_peak_gpu_mem=True, **kwargs)
+
+
 def _calc_memory_bytes_remove_all_stripe(
     non_slice_dims_shape: Tuple[int, int],
     dtype: np.dtype,

{httomo_backends-0.9.0 → httomo_backends-1.0}/httomo_backends/methods_database/query.py

@@ -143,6 +143,16 @@ class MethodsDatabaseQuery:
         )
         return memory_bytes
 
+    def calculate_memory_bytes_for_slices(
+        self, dims_shape: Tuple[int, int, int], dtype: np.dtype, **kwargs
+    ) -> int:
+        smodule = self._import_supporting_funcs_module()
+        module_mem: Callable = getattr(
+            smodule, "_calc_memory_bytes_for_slices_" + self.method_name
+        )
+        memory_bytes: int = module_mem(dims_shape, dtype, **kwargs)
+        return memory_bytes
+
     def calculate_output_dims(
         self, non_slice_dims_shape: Tuple[int, int], **kwargs
     ) -> Tuple[int, int]:

{httomo_backends-0.9.0 → httomo_backends-1.0}/httomo_backends/scripts/yaml_templates_generator.py

@@ -225,6 +225,7 @@ def _get_discard_keys() -> List[str]:
         "power_of_2_cropping",
         "min_mem_usage_filter",
         "min_mem_usage_ifft2",
+        "calc_peak_gpu_mem",
     ]
     return discard_keys
 

httomo_backends-1.0/httomo_backends/yaml_templates/httomolibgpu/httomolibgpu.prep.stripe/remove_stripe_fw.yaml

@@ -0,0 +1,6 @@
+- method: remove_stripe_fw
+  module_path: httomolibgpu.prep.stripe
+  parameters:
+    sigma: 2
+    wname: db5
+    level: null

{httomo_backends-0.9.0 → httomo_backends-1.0}/httomo_backends.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: httomo-backends
-Version: 0.9.0
+Version: 1.0
 Summary: Supplementary files for HTTomo backends.
 Author: Garry ODonnell, Jacob Williamson
 Author-email: Daniil Kazantsev <daniil.kazantsev@diamond.ac.uk>, Yousef Moazzam <yousef.moazzam@diamond.ac.uk>, Jessica Verschoyle <jessica.verschoyle@diamond.ac.uk>, Naman Gera <naman.gera@diamond.ac.uk>, scientificsoftware@diamond.ac.uk

{httomo_backends-0.9.0 → httomo_backends-1.0}/httomo_backends.egg-info/SOURCES.txt

@@ -86,6 +86,7 @@ httomo_backends/yaml_templates/httomolibgpu/httomolibgpu.prep.phase/paganin_filt
 httomo_backends/yaml_templates/httomolibgpu/httomolibgpu.prep.stripe/raven_filter.yaml
 httomo_backends/yaml_templates/httomolibgpu/httomolibgpu.prep.stripe/remove_all_stripe.yaml
 httomo_backends/yaml_templates/httomolibgpu/httomolibgpu.prep.stripe/remove_stripe_based_sorting.yaml
+httomo_backends/yaml_templates/httomolibgpu/httomolibgpu.prep.stripe/remove_stripe_fw.yaml
 httomo_backends/yaml_templates/httomolibgpu/httomolibgpu.prep.stripe/remove_stripe_ti.yaml
 httomo_backends/yaml_templates/httomolibgpu/httomolibgpu.recon.algorithm/CGLS3d_tomobar.yaml
 httomo_backends/yaml_templates/httomolibgpu/httomolibgpu.recon.algorithm/FBP2d_astra.yaml

{httomo_backends-0.9.0 → httomo_backends-1.0}/pyproject.toml

@@ -24,7 +24,7 @@ dev_template = "{tag}"
 
 [project]
 name = "httomo-backends"
-version = "0.9.0"
+version = "1.0"
 description = "Supplementary files for HTTomo backends."
 readme = "README.rst"
 license = {text = "BSD-3-Clause"}

{httomo_backends-0.9.0 → httomo_backends-1.0}/tests/test_httomolibgpu.py

@@ -15,10 +15,11 @@ from httomo_backends.methods_database.query import MethodsDatabaseQuery
 
 from httomolibgpu.misc.morph import data_resampler, sino_360_to_180
 from httomolibgpu.prep.normalize import dark_flat_field_correction, minus_log
-from httomolibgpu.prep.phase import paganin_filter
+from httomolibgpu.prep.phase import paganin_filter, paganin_filter_savu_legacy
 from httomolibgpu.prep.alignment import distortion_correction_proj_discorpy
 from httomolibgpu.prep.stripe import (
     remove_stripe_based_sorting,
+    remove_stripe_fw,
     remove_stripe_ti,
     remove_all_stripe,
     raven_filter,
@@ -277,11 +278,10 @@ def test_paganin_filter_memoryhook(slices, dim_x, dim_y, ensure_clean_memory):
     )  # the amount of memory in bytes needed for the method according to memoryhook
 
     # now we estimate how much of the total memory required for this data
-    (estimated_memory_bytes, subtract_bytes) = _calc_memory_bytes_paganin_filter(
-        (dim_x, dim_y), dtype=np.float32()
+    estimated_memory_bytes = _calc_memory_bytes_for_slices_paganin_filter(
+        (slices, dim_x, dim_y), dtype=np.float32()
     )
-    estimated_memory_mb = round(slices * estimated_memory_bytes / (1024**2), 2)
-    max_mem -= subtract_bytes
+    estimated_memory_mb = round(estimated_memory_bytes / (1024**2), 2)
     max_mem_mb = round(max_mem / (1024**2), 2)
 
     # now we compare both memory estimations
@@ -290,7 +290,40 @@ def test_paganin_filter_memoryhook(slices, dim_x, dim_y, ensure_clean_memory):
     # the estimated_memory_mb should be LARGER or EQUAL to max_mem_mb
     # the resulting percent value should not deviate from max_mem on more than 20%
     assert estimated_memory_mb >= max_mem_mb
-    assert percents_relative_maxmem <= 20
+    assert percents_relative_maxmem <= 1
+
+
+@pytest.mark.cupy
+@pytest.mark.parametrize("slices", [64, 128])
+@pytest.mark.parametrize("dim_x", [81, 260, 320])
+@pytest.mark.parametrize("dim_y", [340, 135, 96])
+def test_paganin_filter_savu_legacy_memoryhook(
+    slices, dim_x, dim_y, ensure_clean_memory
+):
+    data = cp.random.random_sample((slices, dim_x, dim_y), dtype=np.float32)
+    hook = MaxMemoryHook()
+    with hook:
+        data_filtered = paganin_filter_savu_legacy(cp.copy(data)).get()
+
+    # make sure estimator function is within range (80% min, 100% max)
+    max_mem = (
+        hook.max_mem
+    )  # the amount of memory in bytes needed for the method according to memoryhook
+
+    # now we estimate how much of the total memory required for this data
+    estimated_memory_bytes = _calc_memory_bytes_for_slices_paganin_filter_savu_legacy(
+        (slices, dim_x, dim_y), dtype=np.float32()
+    )
+    estimated_memory_mb = round(estimated_memory_bytes / (1024**2), 2)
+    max_mem_mb = round(max_mem / (1024**2), 2)
+
+    # now we compare both memory estimations
+    difference_mb = abs(estimated_memory_mb - max_mem_mb)
+    percents_relative_maxmem = round((difference_mb / max_mem_mb) * 100)
+    # the estimated_memory_mb should be LARGER or EQUAL to max_mem_mb
+    # the resulting percent value should not deviate from max_mem on more than 20%
+    assert estimated_memory_mb >= max_mem_mb
+    assert percents_relative_maxmem <= 1
 
 
 @pytest.mark.cupy
@@ -413,6 +446,35 @@ def test_remove_stripe_ti_memoryhook(slices, ensure_clean_memory):
     assert percents_relative_maxmem <= 20
 
 
+@pytest.mark.cupy
+@pytest.mark.parametrize("slices", [64, 129])
+def test_remove_stripe_fw_memoryhook(slices, ensure_clean_memory):
+    dim_x = 156
+    dim_y = 216
+    data = cp.random.random_sample((slices, dim_x, dim_y), dtype=np.float32)
+    hook = MaxMemoryHook()
+    with hook:
+        remove_stripe_fw(cp.copy(data)).get()
+    max_mem = (
+        hook.max_mem
+    )  # the amount of memory in bytes needed for the method according to memoryhook
+
+    # now we estimate how much of the total memory required for this data
+    estimated_memory_bytes = _calc_memory_bytes_for_slices_remove_stripe_fw(
+        (slices, dim_x, dim_y), dtype=np.float32()
+    )
+    estimated_memory_mb = round(estimated_memory_bytes / (1024**2), 2)
+    max_mem_mb = round(max_mem / (1024**2), 2)
+
+    # now we compare both memory estimations
+    difference_mb = abs(estimated_memory_mb - max_mem_mb)
+    percents_relative_maxmem = round((difference_mb / max_mem_mb) * 100)
+    # the estimated_memory_mb should be LARGER or EQUAL to max_mem_mb
+    # the resulting percent value should not deviate from max_mem on more than 1%
+    assert estimated_memory_mb >= max_mem_mb
+    assert percents_relative_maxmem <= 20
+
+
 @pytest.mark.cupy
 @pytest.mark.parametrize("projections", [180, 360, 720, 1080, 1440])
 def test_raven_filter_memoryhook(projections, ensure_clean_memory):

httomo_backends-0.9.0/httomo_backends/methods_database/packages/backends/httomolibgpu/supporting_funcs/prep/phase.py

@@ -1,120 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# ---------------------------------------------------------------------------
-# Copyright 2022 Diamond Light Source Ltd.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ---------------------------------------------------------------------------
-# Created By  : Tomography Team at DLS <scientificsoftware@diamond.ac.uk>
-# Created Date: 21 September 2023
-# ---------------------------------------------------------------------------
-"""Modules for memory estimation for phase retrieval and phase-contrast enhancement"""
-
-from typing import Tuple
-import numpy as np
-
-from httomo_backends.cufft import CufftType, cufft_estimate_2d
-
-__all__ = [
-    "_calc_memory_bytes_paganin_filter",
-    "_calc_memory_bytes_paganin_filter_savu_legacy",
-]
-
-
-def _calc_memory_bytes_paganin_filter_savu_legacy(
-    non_slice_dims_shape: Tuple[int, int],
-    dtype: np.dtype,
-    **kwargs,
-) -> Tuple[int, int]:
-    return _calc_memory_bytes_paganin_filter(non_slice_dims_shape, dtype, **kwargs)
-
-
-def _calc_memory_bytes_paganin_filter(
-    non_slice_dims_shape: Tuple[int, int],
-    dtype: np.dtype,
-    **kwargs,
-) -> Tuple[int, int]:
-
-    # Input (unpadded)
-    unpadded_in_slice_size = np.prod(non_slice_dims_shape) * dtype.itemsize
-
-    # estimate padding size here based on non_slice dimensions
-    pad_tup = []
-    for dim_len in non_slice_dims_shape:
-        diff = __shift_bit_length(dim_len + 1) - dim_len
-        if dim_len % 2 == 0:
-            pad_width = diff // 2
-            pad_width = (pad_width, pad_width)
-        else:
-            # need an uneven padding for odd-number lengths
-            left_pad = diff // 2
-            right_pad = diff - left_pad
-            pad_width = (left_pad, right_pad)
-        pad_tup.append(pad_width)
-
-    # Padded input
-    padded_in_slice_size = (
-        (non_slice_dims_shape[0] + pad_tup[0][0] + pad_tup[0][1])
-        * (non_slice_dims_shape[1] + pad_tup[1][0] + pad_tup[1][1])
-        * dtype.itemsize
-    )
-
-    # Padded input cast to `complex64`
-    complex_slice = padded_in_slice_size / dtype.itemsize * np.complex64().nbytes
-
-    # Plan size for 2D FFT
-    ny = non_slice_dims_shape[0] + pad_tup[0][0] + pad_tup[0][1]
-    nx = non_slice_dims_shape[1] + pad_tup[1][0] + pad_tup[1][1]
-    fftplan_slice_size = cufft_estimate_2d(
-        nx=nx,
-        ny=ny,
-        fft_type=CufftType.CUFFT_C2C,
-    )
-
-    # Size of "reciprocal grid" generated, based on padded projections shape
-    grid_size = np.prod((ny, nx)) * np.float32().nbytes
-    filter_size = grid_size
-
-    # Size of cropped/unpadded + cast to float32 result of 2D IFFT
-    cropped_float32_res_slice = np.prod(non_slice_dims_shape) * np.float32().nbytes
-
-    # Size of negative log of cropped float32 result of 2D IFFT
-    negative_log_slice = cropped_float32_res_slice
-
-    # If the FFT plan size is negligible for some reason, this changes where the peak GPU
-    # memory usage occurs. Hence, the if/else branching below for calculating the total bytes.
-    NEGLIGIBLE_FFT_PLAN_SIZE = 16
-    if fftplan_slice_size < NEGLIGIBLE_FFT_PLAN_SIZE:
-        tot_memory_bytes = int(
-            unpadded_in_slice_size + padded_in_slice_size + complex_slice
-        )
-    else:
-        tot_memory_bytes = int(
-            unpadded_in_slice_size
-            + padded_in_slice_size
-            + complex_slice
-            # The padded float32 array is deallocated when a copy is made when casting to complex64
-            # and the variable `padded_tomo` is reassigned to the complex64 version
-            - padded_in_slice_size
-            + fftplan_slice_size
-            + cropped_float32_res_slice
-            + negative_log_slice
-        )
-
-    subtract_bytes = int(filter_size + grid_size)
-
-    return (tot_memory_bytes, subtract_bytes)
-
-
-def __shift_bit_length(x: int) -> int:
-    return 1 << (x - 1).bit_length()