deepliif 1.1.14__py3-none-any.whl → 1.1.15__py3-none-any.whl

deepliif/models/__init__.py

@@ -219,13 +219,14 @@ def compute_overlap(img_size, tile_size):
     return tile_size // 4
 
 
-def run_torchserve(img, model_path=None, nets=None, eager_mode=False, opt=None, seg_only=False, use_dask=True, output_tensor=False):
+def run_torchserve(img, model_path=None, nets=None, eager_mode=False, opt=None, seg_only=False, seg_weights=None, use_dask=True, output_tensor=False):
     """
     eager_mode: not used in this function; put in place to be consistent with run_dask
            so that run_wrapper() could call either this function or run_dask with
            same syntax
     opt: same as eager_mode
     seg_only: same as eager_mode
+    seg_weights: same as eager_mode
     nets: same as eager_mode
     """
     buffer = BytesIO()
@@ -245,7 +246,7 @@ def run_torchserve(img, model_path=None, nets=None, eager_mode=False, opt=None,
     return {k: tensor_to_pil(deserialize_tensor(v)) for k, v in res.json().items()}
 
 
-def run_dask(img, model_path=None, nets=None, eager_mode=False, opt=None, seg_only=False, use_dask=True, output_tensor=False):
+def run_dask(img, model_path=None, nets=None, eager_mode=False, opt=None, seg_only=False, seg_weights=None, use_dask=True, output_tensor=False):
     """
     Provide either the model path or the networks object.
     
@@ -280,20 +281,22 @@ def run_dask(img, model_path=None, nets=None, eager_mode=False, opt=None, seg_on
                 return model(input.to(next(model.parameters()).device))
     
     if opt.model in ['DeepLIIF','DeepLIIFKD']:
-        #weights = {
-        #    'G51': 0.25, # IHC
-        #    'G52': 0.25, # Hema
-        #    'G53': 0.25, # DAPI
-        #    'G54': 0.00, # Lap2
-        #    'G55': 0.25, # Marker
-        #}
-        weights = {
-            'G51': 0.5, # IHC
-            'G52': 0.0, # Hema
-            'G53': 0.0, # DAPI
-            'G54': 0.0, # Lap2
-            'G55': 0.5, # Marker
-        }
+        if seg_weights is None:
+            weights = {
+                'G51': 0.25, # IHC
+                'G52': 0.25, # Hema
+                'G53': 0.25, # DAPI
+                'G54': 0.00, # Lap2
+                'G55': 0.25, # Marker
+            }
+        else:
+            weights = {
+                'G51': seg_weights[0], # IHC
+                'G52': seg_weights[1], # Hema
+                'G53': seg_weights[2], # DAPI
+                'G54': seg_weights[3], # Lap2
+                'G55': seg_weights[4], # Marker
+            }
 
         seg_map = {'G1': 'G52', 'G2': 'G53', 'G3': 'G54', 'G4': 'G55'}
         if seg_only:
@@ -357,12 +360,12 @@ def run_dask(img, model_path=None, nets=None, eager_mode=False, opt=None, seg_on
 def is_empty(tile):
     thresh = 15
     if isinstance(tile, list): # for pair of tiles, only mark it as empty / no need for prediction if ALL tiles are empty
-        return all([True if np.max(image_variance_rgb(t)) < thresh else False for t in tile])
+        return all([True if image_variance_gray(t) < thresh else False for t in tile])
     else:
-        return True if np.max(image_variance_rgb(tile)) < thresh else False
+        return True if image_variance_gray(tile) < thresh else False
 
 
-def run_wrapper(tile, run_fn, model_path=None, nets=None, eager_mode=False, opt=None, seg_only=False, use_dask=True, output_tensor=False):
+def run_wrapper(tile, run_fn, model_path=None, nets=None, eager_mode=False, opt=None, seg_only=False, seg_weights=None, use_dask=True, output_tensor=False):
     if opt.model in ['DeepLIIF','DeepLIIFKD']:
         if is_empty(tile):
             if seg_only:
@@ -384,7 +387,7 @@ def run_wrapper(tile, run_fn, model_path=None, nets=None, eager_mode=False, opt=
                     'G55': Image.new(mode='RGB', size=(512, 512), color=(0, 0, 0)),
                 }
         else:
-            return run_fn(tile, model_path, None, eager_mode, opt, seg_only)
+            return run_fn(tile, model_path, None, eager_mode, opt, seg_only, seg_weights)
     elif opt.model in ['DeepLIIFExt', 'SDG']:
         if is_empty(tile):
             res = {'G_' + str(i): Image.new(mode='RGB', size=(512, 512)) for i in range(1, opt.modalities_no + 1)}
@@ -405,7 +408,7 @@ def run_wrapper(tile, run_fn, model_path=None, nets=None, eager_mode=False, opt=
 
 def inference(img, tile_size, overlap_size, model_path, use_torchserve=False,
               eager_mode=False, color_dapi=False, color_marker=False, opt=None,
-              return_seg_intermediate=False, seg_only=False, opt_args={}):
+              return_seg_intermediate=False, seg_only=False, seg_weights=None, opt_args={}):
     """
     opt_args: a dictionary of key and values to add/overwrite to opt
     """
@@ -430,7 +433,7 @@ def inference(img, tile_size, overlap_size, model_path, use_torchserve=False,
 
     tiler = InferenceTiler(orig, tile_size, overlap_size)
     for tile in tiler:
-        tiler.stitch(run_wrapper(tile, run_fn, model_path, None, eager_mode, opt, seg_only))
+        tiler.stitch(run_wrapper(tile, run_fn, model_path, None, eager_mode, opt, seg_only, seg_weights))
         
     results = tiler.results()
 
@@ -515,7 +518,7 @@ def postprocess(orig, images, tile_size, model, seg_thresh=150, size_thresh='def
 
 def infer_modalities(img, tile_size, model_dir, eager_mode=False,
                      color_dapi=False, color_marker=False, opt=None,
-                     return_seg_intermediate=False, seg_only=False):
+                     return_seg_intermediate=False, seg_only=False, seg_weights=None):
     """
     This function is used to infer modalities for the given image using a trained model.
     :param img: The input image.
@@ -543,7 +546,8 @@ def infer_modalities(img, tile_size, model_dir, eager_mode=False,
         color_marker=color_marker,
         opt=opt,
         return_seg_intermediate=return_seg_intermediate,
-        seg_only=seg_only
+        seg_only=seg_only,
+        seg_weights=seg_weights,
     )
 
     if not hasattr(opt,'seg_gen') or (hasattr(opt,'seg_gen') and opt.seg_gen): # the first condition accounts for old settings of deepliif; the second refers to deepliifext models
@@ -558,7 +562,7 @@ def infer_modalities(img, tile_size, model_dir, eager_mode=False,
         return images, None
 
 
-def infer_results_for_wsi(input_dir, filename, output_dir, model_dir, tile_size, region_size=20000, color_dapi=False, color_marker=False, seg_intermediate=False, seg_only=False):
+def infer_results_for_wsi(input_dir, filename, output_dir, model_dir, tile_size, region_size=20000, color_dapi=False, color_marker=False, seg_intermediate=False, seg_only=False, seg_weights=None):
     """
     This function infers modalities and segmentation mask for the given WSI image. It
 
@@ -592,7 +596,7 @@ def infer_results_for_wsi(input_dir, filename, output_dir, model_dir, tile_size,
                 region = reader.read(XYWH=region_XYWH, rescale=rescale)
                 img = Image.fromarray((region * 255).astype(np.uint8)) if rescale else Image.fromarray(region)
 
-                region_modalities, region_scoring = infer_modalities(img, tile_size, model_dir, color_dapi=color_dapi, color_marker=color_marker, return_seg_intermediate=seg_intermediate, seg_only=seg_only)
+                region_modalities, region_scoring = infer_modalities(img, tile_size, model_dir, color_dapi=color_dapi, color_marker=color_marker, return_seg_intermediate=seg_intermediate, seg_only=seg_only, seg_weights=seg_weights)
                 if region_scoring is not None:
                     if scoring is None:
                         scoring = {
@@ -624,8 +628,6 @@ def infer_results_for_wsi(input_dir, filename, output_dir, model_dir, tile_size,
         with open(os.path.join(results_dir, f'{basename}.json'), 'w') as f:
             json.dump(scoring, f, indent=2)
 
-    javabridge.kill_vm()
-
 
 def get_wsi_resolution(filename):
     """
@@ -633,9 +635,6 @@ def get_wsi_resolution(filename):
     the corresponding tile size to use by default for DeepLIIF.
     If it cannot be found, return (None, None) instead.
 
-    Note: This will start the javabridge VM, but not kill it.
-          It must be killed elsewhere.
-
     Parameters
     ----------
     filename : str
@@ -649,11 +648,10 @@ def get_wsi_resolution(filename):
         Corresponding tile size for DeepLIIF.
     """
 
-    # make sure javabridge is already set up from with call to get_information()
-    size_x, size_y, size_z, size_c, size_t, pixel_type = get_information(filename)
+    init_javabridge_bioformats()
+    metadata = bioformats.get_omexml_metadata(filename)
 
     mag = None
-    metadata = bioformats.get_omexml_metadata(filename)
     try:
         omexml = bioformats.OMEXML(metadata)
         mag = omexml.instrument().Objective.NominalMagnification
@@ -686,7 +684,7 @@ def get_wsi_resolution(filename):
         return None, None
 
 
-def infer_cells_for_wsi(filename, model_dir, tile_size, region_size=20000, version=3, print_log=False):
+def infer_cells_for_wsi(filename, model_dir, tile_size, region_size=20000, version=3, print_log=False, seg_weights=None):
     """
     Perform inference on a slide and get the results individual cell data.
 
@@ -704,6 +702,9 @@ def infer_cells_for_wsi(filename, model_dir, tile_size, region_size=20000, versi
         Version of cell data to return (3 or 4).
     print_log : bool
         Whether or not to print updates while processing.
+    seg_weights : list | tuple
+        Optional five weights to use for creating the combined Seg image.
+        If None, then the default weights are used.
 
     Returns
     -------
@@ -717,22 +718,21 @@ def infer_cells_for_wsi(filename, model_dir, tile_size, region_size=20000, versi
 
     resolution = '40x' if tile_size > 384 else ('20x' if tile_size > 192 else '10x')
 
-    size_x, size_y, size_z, size_c, size_t, pixel_type = get_information(filename)
-    rescale = (pixel_type != 'uint8')
-    print_info('Info:', size_x, size_y, size_z, size_c, size_t, pixel_type)
-
-    num_regions_x = math.ceil(size_x / region_size)
-    num_regions_y = math.ceil(size_y / region_size)
-    stride_x = math.ceil(size_x / num_regions_x)
-    stride_y = math.ceil(size_y / num_regions_y)
-    print_info('Strides:', stride_x, stride_y)
-
     data = None
     default_marker_thresh, count_marker_thresh = 0, 0
     default_size_thresh, count_size_thresh = 0, 0
 
-    # javabridge already set up from previous call to get_information()
-    with bioformats.ImageReader(filename) as reader:
+    with WSIReader(filename) as reader:
+        size_x = reader.width
+        size_y = reader.height
+        print_info('Info:', size_x, size_y)
+
+        num_regions_x = math.ceil(size_x / region_size)
+        num_regions_y = math.ceil(size_y / region_size)
+        stride_x = math.ceil(size_x / num_regions_x)
+        stride_y = math.ceil(size_y / num_regions_y)
+        print_info('Strides:', stride_x, stride_y)
+
         start_x, start_y = 0, 0
 
         while start_y < size_y:
@@ -740,9 +740,9 @@ def infer_cells_for_wsi(filename, model_dir, tile_size, region_size=20000, versi
                 region_XYWH = (start_x, start_y, min(stride_x, size_x-start_x), min(stride_y, size_y-start_y))
                 print_info('Region:', region_XYWH)
 
-                region = reader.read(XYWH=region_XYWH, rescale=rescale)
+                region = reader.read(region_XYWH)
                 print_info(region.shape, region.dtype)
-                img = Image.fromarray((region * 255).astype(np.uint8)) if rescale else Image.fromarray(region)
+                img = Image.fromarray(region)
                 print_info(img.size, img.mode)
                 del region
 
@@ -757,6 +757,7 @@ def infer_cells_for_wsi(filename, model_dir, tile_size, region_size=20000, versi
                     opt=None,
                     return_seg_intermediate=False,
                     seg_only=True,
+                    seg_weights=seg_weights,
                 )
                 del img
 
@@ -795,8 +796,6 @@ def infer_cells_for_wsi(filename, model_dir, tile_size, region_size=20000, versi
             start_x = 0
             start_y += stride_y
 
-    javabridge.kill_vm()
-
     if count_marker_thresh == 0:
         count_marker_thresh = 1
     if count_size_thresh == 0:
@@ -804,6 +803,10 @@ def infer_cells_for_wsi(filename, model_dir, tile_size, region_size=20000, versi
     data['settings']['default_marker_thresh'] = round(default_marker_thresh / count_marker_thresh)
     data['settings']['default_size_thresh'] = round(default_size_thresh / count_size_thresh)
 
+    data['settings']['tile_size'] = tile_size
+    data['settings']['region_size'] = region_size
+    data['settings']['seg_weights'] = seg_weights
+
     try:
         data['deepliifVersion'] = importlib.metadata.version('deepliif')
     except Exception as e:

deepliif/util/__init__.py

@@ -2,6 +2,10 @@
 import os
 import collections
 
+import atexit
+import functools
+import threading
+
 import torch
 import numpy as np
 from PIL import Image, ImageOps
@@ -22,6 +26,9 @@ import javabridge
 import bioformats.omexml as ome
 import tifffile as tf
 
+from tifffile import TiffFile
+import zarr
+
 
 excluding_names = ['Hema', 'DAPI', 'DAPILap2', 'Ki67', 'Seg', 'Marked', 'SegRefined', 'SegOverlaid', 'Marker', 'Lap2']
 # Image extensions to consider
@@ -392,6 +399,30 @@ def image_variance_rgb(img):
     return var
 
 
+def init_javabridge_bioformats():
+    """
+    Initialize javabridge for use with bioformats.
+    Run as daemon so no need to explicitly call kill_vm.
+    This function will only run once; repeat calls do nothing.
+    """
+
+    if not hasattr(init_javabridge_bioformats, 'called'):
+        # https://github.com/LeeKamentsky/python-javabridge/issues/155
+        old_init = threading.Thread.__init__
+        threading.Thread.__init__ = functools.partialmethod(old_init, daemon=True)
+        javabridge.start_vm(class_path=bioformats.JARS)
+        threading.Thread.__init__ = old_init
+        atexit.register(javabridge.kill_vm)
+
+        rootLoggerName = javabridge.get_static_field("org/slf4j/Logger", "ROOT_LOGGER_NAME", "Ljava/lang/String;")
+        rootLogger = javabridge.static_call("org/slf4j/LoggerFactory", "getLogger",
+                                            "(Ljava/lang/String;)Lorg/slf4j/Logger;", rootLoggerName)
+        logLevel = javabridge.get_static_field("ch/qos/logback/classic/Level", "WARN", "Lch/qos/logback/classic/Level;")
+        javabridge.call(rootLogger, "setLevel", "(Lch/qos/logback/classic/Level;)V", logLevel)
+
+        init_javabridge_bioformats.called = True
+
+
 def read_bioformats_image_with_reader(path, channel=0, region=(0, 0, 0, 0)):
     """
     Using this function, you can read a specific region of a large image by giving the region bounding box (XYWH format)
@@ -402,14 +433,7 @@ def read_bioformats_image_with_reader(path, channel=0, region=(0, 0, 0, 0)):
     :param region: The bounding box around the region of interest (XYWH format).
     :return: The specified region of interest image (numpy array).
     """
-    javabridge.start_vm(class_path=bioformats.JARS)
-
-    rootLoggerName = javabridge.get_static_field("org/slf4j/Logger", "ROOT_LOGGER_NAME", "Ljava/lang/String;")
-    rootLogger = javabridge.static_call("org/slf4j/LoggerFactory", "getLogger",
-                                        "(Ljava/lang/String;)Lorg/slf4j/Logger;", rootLoggerName)
-    logLevel = javabridge.get_static_field("ch/qos/logback/classic/Level", "WARN", "Lch/qos/logback/classic/Level;")
-    javabridge.call(rootLogger, "setLevel", "(Lch/qos/logback/classic/Level;)V", logLevel)
-
+    init_javabridge_bioformats()
     with bioformats.ImageReader(path) as reader:
         return reader.read(t=channel, XYWH=region)
 
@@ -421,14 +445,7 @@ def get_information(filename):
     :param filename: The address to the ome image.
     :return: size_x, size_y, size_z, size_c, size_t, pixel_type
     """
-    javabridge.start_vm(class_path=bioformats.JARS)
-
-    rootLoggerName = javabridge.get_static_field("org/slf4j/Logger", "ROOT_LOGGER_NAME", "Ljava/lang/String;")
-    rootLogger = javabridge.static_call("org/slf4j/LoggerFactory", "getLogger",
-                                        "(Ljava/lang/String;)Lorg/slf4j/Logger;", rootLoggerName)
-    logLevel = javabridge.get_static_field("ch/qos/logback/classic/Level", "WARN", "Lch/qos/logback/classic/Level;")
-    javabridge.call(rootLogger, "setLevel", "(Lch/qos/logback/classic/Level;)V", logLevel)
-
+    init_javabridge_bioformats()
     metadata = bioformats.get_omexml_metadata(filename)
     omexml = bioformats.OMEXML(metadata)
     size_x, size_y, size_z, size_c, size_t, pixel_type = omexml.image().Pixels.SizeX, \
@@ -441,6 +458,76 @@ def get_information(filename):
     return size_x, size_y, size_z, size_c, size_t, pixel_type
 
 
+class WSIReader:
+    """
+    Assumes the file is a single image (e.g., not a stacked
+    OME TIFF) and will always return uint8 pixel type data.
+    """
+
+    def __init__(self, path):
+        init_javabridge_bioformats()
+        metadata = bioformats.get_omexml_metadata(path)
+        omexml = bioformats.OMEXML(metadata)
+
+        self._path = path
+        self._width = omexml.image().Pixels.SizeX
+        self._height = omexml.image().Pixels.SizeY
+        self._pixel_type = omexml.image().Pixels.PixelType
+
+        self._tif = None
+        if self._pixel_type == 'uint8':
+            try:
+                self._file = None
+                self._file = open(path, 'rb')
+                self._tif = TiffFile(self._file)
+                self._zarr = zarr.open(self._tif.pages[0].aszarr(), mode='r')
+            except Exception as e:
+                if self._tif is not None:
+                    self._tif.close()
+                    self._tif = None
+                if self._file is not None:
+                    self._file.close()
+
+        self._bfreader = None
+        if self._tif is None:
+            self._rescale = (self._pixel_type != 'uint8')
+            self._bfreader = bioformats.ImageReader(path)
+
+        if self._tif is None and self._bfreader is None:
+            raise Exception('Cannot read WSI file.')
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()
+
+    def close(self):
+        if self._tif is not None:
+            self._tif.close()
+            self._file.close()
+        if self._bfreader is not None:
+            self._bfreader.close()
+
+    @property
+    def width(self):
+        return self._width
+
+    @property
+    def height(self):
+        return self._height
+
+    def read(self, xywh):
+        if self._tif is not None:
+            x, y, w, h = xywh
+            return self._zarr[y:y+h, x:x+w]
+
+        px = self._bfreader.read(XYWH=xywh, rescale=self._rescale)
+        if self._rescale:
+            px = (px * 255).astype(np.uint8)
+        return px
+
+
 
 
 def write_results_to_pickle_file(output_addr, results):

deepliif/util/util - modified tensor_to_pil.py

@@ -0,0 +1,255 @@
+"""This module contains simple helper functions """
+import os
+from time import time
+from functools import wraps
+
+import torch
+import numpy as np
+from PIL import Image
+import cv2
+from skimage.metrics import structural_similarity as ssim
+
+
+def timeit(f):
+    @wraps(f)
+    def wrap(*args, **kwargs):
+        ts = time()
+        result = f(*args, **kwargs)
+        print(f'{f.__name__} {time() - ts}')
+
+        return result
+
+    return wrap
+
+
+def diagnose_network(net, name='network'):
+    """Calculate and print the mean of average absolute(gradients)
+
+    Parameters:
+        net (torch network) -- Torch network
+        name (str) -- the name of the network
+    """
+    mean = 0.0
+    count = 0
+    for param in net.parameters():
+        if param.grad is not None:
+            mean += torch.mean(torch.abs(param.grad.data))
+            count += 1
+    if count > 0:
+        mean = mean / count
+    print(name)
+    print(mean)
+
+
+def save_image(image_numpy, image_path, aspect_ratio=1.0):
+    """Save a numpy image to the disk
+
+    Parameters:
+        image_numpy (numpy array) -- input numpy array
+        image_path (str)          -- the path of the image
+    """
+    x, y, nc = image_numpy.shape
+    
+    if nc > 3:
+        if nc % 3 == 0:
+            nc_img = 3
+            no_img = nc // nc_img
+            
+        elif nc % 2 == 0:
+            nc_img = 2
+            no_img = nc // nc_img
+        else:
+            nc_img = 1
+            no_img = nc // nc_img
+        print(f'image (numpy) has {nc}>3 channels, inferred to have {no_img} images each with {nc_img} channel(s)')
+        l_image_numpy = np.dsplit(image_numpy,[nc_img*i for i in range(1,no_img)])
+        image_numpy = np.concatenate(l_image_numpy, axis=1) # stack horizontally
+        
+    image_pil = Image.fromarray(image_numpy)
+    h, w, _ = image_numpy.shape
+
+    if aspect_ratio > 1.0:
+        image_pil = image_pil.resize((h, int(w * aspect_ratio)), Image.BICUBIC)
+    if aspect_ratio < 1.0:
+        image_pil = image_pil.resize((int(h / aspect_ratio), w), Image.BICUBIC)
+    image_pil.save(image_path)
+
+
+def print_numpy(x, val=True, shp=False):
+    """Print the mean, min, max, median, std, and size of a numpy array
+
+    Parameters:
+        val (bool) -- if print the values of the numpy array
+        shp (bool) -- if print the shape of the numpy array
+    """
+    x = x.astype(np.float64)
+    if shp:
+        print('shape,', x.shape)
+    if val:
+        x = x.flatten()
+        print('mean = %3.3f, min = %3.3f, max = %3.3f, median = %3.3f, std=%3.3f' % (
+            np.mean(x), np.min(x), np.max(x), np.median(x), np.std(x)))
+
+
+def mkdirs(paths):
+    """create empty directories if they don't exist
+
+    Parameters:
+        paths (str list) -- a list of directory paths
+    """
+    if isinstance(paths, list) and not isinstance(paths, str):
+        for path in paths:
+            mkdir(path)
+    else:
+        mkdir(paths)
+
+
+def mkdir(path):
+    """create a single empty directory if it didn't exist
+
+    Parameters:
+        path (str) -- a single directory path
+    """
+    if not os.path.exists(path):
+        os.makedirs(path, exist_ok=True)
+
+
+import time
+time_tensor = 0
+time_convert = 0
+time_transpose = 0
+time_astype = 0
+time_topil = 0
+time_scale = 0
+def print_times():
+    print('Time to get tensor:', round(time_tensor, 1), flush=True)
+    print('Time to convert:', round(time_convert, 1), flush=True)
+    print('Time to transpose:', round(time_transpose, 1), flush=True)
+    print('Time to scale:', round(time_scale, 1), flush=True)
+    print('Time for astype:', round(time_transpose, 1), flush=True)
+    print('Time to pil:', round(time_topil, 1), flush=True)
+
+def tensor2im(input_image, imtype=np.uint8):
+    """"Converts a Tensor array into a numpy image array.
+
+    Parameters:
+        input_image (tensor) --  the input image tensor array
+        imtype (type)        --  the desired type of the converted numpy array
+    """
+    if not isinstance(input_image, np.ndarray):
+        if isinstance(input_image, torch.Tensor):  # get the data from a variable
+            ts = time.time()
+            image_tensor = input_image.data
+            te = time.time()
+            global time_tensor
+            time_tensor += (te - ts)
+        else:
+            return input_image
+        ts = time.time()
+        image_numpy = image_tensor[0].cpu().float().numpy()  # convert it into a numpy array
+        te = time.time()
+        global time_convert
+        time_convert += (te - ts)
+        if image_numpy.shape[0] == 1:  # grayscale to RGB
+            image_numpy = np.tile(image_numpy, (3, 1, 1))
+        ts = time.time()
+        #image_numpy = (np.transpose(image_numpy, (1, 2, 0)) + 1) / 2.0 * 255.0  # post-processing: tranpose and scaling
+        image_numpy = np.transpose(image_numpy, (1, 2, 0))
+        te = time.time()
+        global time_transpose
+        time_transpose += (te - ts)
+        ts = time.time()
+        image_numpy = cv2.resize(image_numpy, (256, 256), interpolation=cv2.INTER_AREA)
+        image_numpy = (image_numpy + 1) / 2.0 * 255.0
+        te = time.time()
+        global time_scale
+        time_scale += (te - ts)
+    else:  # if it is a numpy array, do nothing
+        image_numpy = input_image
+    return image_numpy.astype(imtype)
+    ts = time.time()
+    image_numpy = image_numpy.astype(imtype)
+    te = time.time()
+    global time_astype
+    time_astype += (te - ts)
+    return image_numpy
+
+
+def tensor_to_pil(t):
+    #return Image.fromarray(tensor2im(t))
+    arr = tensor2im(t)
+    ts = time.time()
+    #arr = cv2.resize(arr, (256, 256), interpolation=cv2.INTER_CUBIC)
+    im = Image.fromarray(arr)
+    te = time.time()
+    global time_topil
+    time_topil += (te - ts)
+    return im
+
+
+def calculate_ssim(img1, img2):
+    return ssim(img1, img2, data_range=img2.max() - img2.min())
+
+
+def check_multi_scale(img1, img2):
+    img1 = np.array(img1)
+    img2 = np.array(img2)
+    max_ssim = (512, 0)
+    for tile_size in range(100, 1000, 100):
+        image_ssim = 0
+        tile_no = 0
+        for i in range(0, img2.shape[0], tile_size):
+            for j in range(0, img2.shape[1], tile_size):
+                if i + tile_size <= img2.shape[0] and j + tile_size <= img2.shape[1]:
+                    tile = img2[i: i + tile_size, j: j + tile_size]
+                    tile = cv2.resize(tile, (img1.shape[0], img1.shape[1]))
+                    tile_ssim = calculate_ssim(img1, tile)
+                    image_ssim += tile_ssim
+                    tile_no += 1
+        if tile_no > 0:
+            image_ssim /= tile_no
+            if max_ssim[1] < image_ssim:
+                max_ssim = (tile_size, image_ssim)
+    return max_ssim[0]
+
+
+import subprocess
+import os
+from threading import Thread , Timer
+import sched, time
+
+# modified from https://stackoverflow.com/questions/67707828/how-to-get-every-seconds-gpu-usage-in-python
+def get_gpu_memory(gpu_id=0):
+    """
+    Currently collects gpu memory info for a given gpu id.
+    """
+    output_to_list = lambda x: x.decode('ascii').split('\n')[:-1]
+    ACCEPTABLE_AVAILABLE_MEMORY = 1024
+    COMMAND = "nvidia-smi --query-gpu=memory.used --format=csv"
+    try:
+        memory_use_info = output_to_list(subprocess.check_output(COMMAND.split(),stderr=subprocess.STDOUT))[1:]
+    except subprocess.CalledProcessError as e:
+        raise RuntimeError("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output))
+    memory_use_values = [int(x.split()[0]) for i, x in enumerate(memory_use_info)]
+    
+    #assert len(memory_use_values)==1, f"get_gpu_memory::memory_use_values should have only 1 value, now has {len(memory_use_values)} (memory_use_values)"
+    return memory_use_values[gpu_id]
+
+class HardwareStatus():
+    def __init__(self):
+        self.gpu_mem = []
+        self.timer = None
+    
+    def get_status_every_sec(self, gpu_id=0):
+        """
+            This function calls itself every 1 sec and appends the gpu_memory.
+        """
+        self.timer = Timer(1.0, self.get_status_every_sec)
+        self.timer.start()
+        self.gpu_mem.append(get_gpu_memory(gpu_id))
+        # print('self.gpu_mem',self.gpu_mem)
+    
+    def stop_timer(self):
+        self.timer.cancel()
+
+