deepliif 1.1.10__py3-none-any.whl → 1.1.12__py3-none-any.whl

deepliif/util/__init__.py

@@ -14,17 +14,18 @@ from .visualizer import Visualizer
 from ..postprocessing import imadjust
 import cv2
 
+import pickle
+import sys
+
 import bioformats
 import javabridge
 import bioformats.omexml as ome
 import tifffile as tf
 
-import pickle
-import sys
 
 excluding_names = ['Hema', 'DAPI', 'DAPILap2', 'Ki67', 'Seg', 'Marked', 'SegRefined', 'SegOverlaid', 'Marker', 'Lap2']
 # Image extensions to consider
-image_extensions = ['.png', '.jpg', '.tif', '.jpeg', '.svs']
+image_extensions = ['.png', '.jpg', '.tif', '.jpeg']
 
 
 def allowed_file(filename):
@@ -118,6 +119,211 @@ def stitch_tile(img, tile, tile_size, overlap_size, i, j):
     img.paste(tile, (i * tile_size, j * tile_size))
 
 
+class InferenceTiler:
+    """
+    Iterable class to tile image(s) and stitch result tiles together.
+
+    To perform inference on a large image, that image will need to be
+    tiled into smaller tiles that can be run individually and then
+    stitched back together. This class wraps the functionality as an
+    iterable object that can accept a single image or list of images
+    if multiple images are taken as input for inference.
+
+    An overlap size can be specified so that neighboring tiles will
+    overlap at the edges, helping to reduce seams or other artifacts
+    near the edge of a tile. Padding of a solid color around the
+    perimeter of the tile is also possible, if needed. The specified
+    tile size includes this overlap and pad sizes, so a tile size of
+    512 with an overlap size of 32 and pad size of 16 would have a
+    central area of 416 pixels that are stitched into the result image.
+
+    Example Usage
+    -------------
+    tiler = InferenceTiler(img, 512, 32)
+    for tile in tiler:
+        result_tiles = infer(tile)
+        tiler.stitch(result_tiles)
+    images = tiler.results()
+    """
+
+    def __init__(self, orig, tile_size, overlap_size=0, pad_size=0, pad_color=(255, 255, 255)):
+        """
+        Initialize for tiling an image or list of images.
+
+        Parameters
+        ----------
+        orig : Image | list(Image)
+            Original image or list of images to be tiled.
+        tile_size: int
+            Size (width and height) of the tiles to be generated.
+        overlap_size: int [default: 0]
+            Amount of overlap on each side of the tile.
+        pad_size: int [default: 0]
+            Amount of solid color padding around perimeter of tile.
+        pad_color: tuple(int, int, int) [default: (255,255,255)]
+            RGB color to use for padding.
+        """
+
+        if tile_size <= 0:
+            raise ValueError('InfereneTiler input tile_size must be positive and non-zero')
+        if overlap_size < 0:
+            raise ValueError('InfereneTiler input overlap_size must be positive or zero')
+        if pad_size < 0:
+            raise ValueError('InfereneTiler input pad_size must be positive or zero')
+
+        self.single_orig = not type(orig) is list
+        if self.single_orig:
+            orig = [orig]
+
+        for i in range(1, len(orig)):
+            if orig[i].size != orig[0].size:
+                raise ValueError('InferenceTiler input images do not have the same size.')
+        self.orig_width = orig[0].width
+        self.orig_height = orig[0].height
+
+        # patch size to extract from input image, which is then padded to tile size
+        patch_size = tile_size - (2 * pad_size)
+
+        # make sure width and height are both at least patch_size
+        if orig[0].width < patch_size:
+            for i in range(len(orig)):
+                while orig[i].width < patch_size:
+                    mirrored = ImageOps.mirror(orig[i])
+                    orig[i] = ImageOps.expand(orig[i], (0, 0, orig[i].width, 0))
+                    orig[i].paste(mirrored, (mirrored.width, 0))
+                orig[i] = orig[i].crop((0, 0, patch_size, orig[i].height))
+        if orig[0].height < patch_size:
+            for i in range(len(orig)):
+                while orig[i].height < patch_size:
+                    flipped = ImageOps.flip(orig[i])
+                    orig[i] = ImageOps.expand(orig[i], (0, 0, 0, orig[i].height))
+                    orig[i].paste(flipped, (0, flipped.height))
+                orig[i] = orig[i].crop((0, 0, orig[i].width, patch_size))
+        self.image_width = orig[0].width
+        self.image_height = orig[0].height
+
+        overlap_width = 0 if patch_size >= self.image_width else overlap_size
+        overlap_height = 0 if patch_size >= self.image_height else overlap_size
+        center_width = patch_size - (2 * overlap_width)
+        center_height = patch_size - (2 * overlap_height)
+        if center_width <= 0 or center_height <= 0:
+            raise ValueError('InferenceTiler combined overlap_size and pad_size are too large')
+
+        self.c0x = pad_size                               # crop offset for left of non-pad content in result tile
+        self.c0y = pad_size                               # crop offset for top of non-pad content in result tile
+        self.c1x = overlap_width + pad_size               # crop offset for left of center region in result tile
+        self.c1y = overlap_height + pad_size              # crop offset for top of center region in result tile
+        self.c2x = patch_size - overlap_width + pad_size  # crop offset for right of center region in result tile
+        self.c2y = patch_size - overlap_height + pad_size # crop offset for bottom of center region in result tile
+        self.c3x = patch_size + pad_size                  # crop offset for right of non-pad content in result tile
+        self.c3y = patch_size + pad_size                  # crop offset for bottom of non-pad content in result tile
+        self.p1x = overlap_width               # paste offset for left of center region w.r.t (x,y) coord
+        self.p1y = overlap_height              # paste offset for top of center region w.r.t (x,y) coord
+        self.p2x = patch_size - overlap_width  # paste offset for right of center region w.r.t (x,y) coord
+        self.p2y = patch_size - overlap_height # paste offset for bottom of center region w.r.t (x,y) coord
+
+        self.overlap_width = overlap_width
+        self.overlap_height = overlap_height
+        self.patch_size = patch_size
+        self.center_width = center_width
+        self.center_height = center_height
+
+        self.orig = orig
+        self.tile_size = tile_size
+        self.pad_size = pad_size
+        self.pad_color = pad_color
+        self.res = {}
+
+    def __iter__(self):
+        """
+        Generate the tiles as an iterable.
+
+        Tiles are created and iterated over from top left to bottom
+        right, going across the rows. The yielded tile(s) match the
+        type of the original input when initialized (either a single
+        image or a list of images in the same order as initialized).
+        The (x, y) coordinate of the current tile is maintained
+        internally for use in the stitch function.
+        """
+
+        for y in range(0, self.image_height, self.center_height):
+            for x in range(0, self.image_width, self.center_width):
+                if x + self.patch_size > self.image_width:
+                    x = self.image_width - self.patch_size
+                if y + self.patch_size > self.image_height:
+                    y = self.image_height - self.patch_size
+                self.x = x
+                self.y = y
+                tiles = [im.crop((x, y, x + self.patch_size, y + self.patch_size)) for im in self.orig]
+                if self.pad_size != 0:
+                    tiles = [ImageOps.expand(t, self.pad_size, self.pad_color) for t in tiles]
+                yield tiles[0] if self.single_orig else tiles
+
+    def stitch(self, result_tiles):
+        """
+        Stitch result tiles into the result images.
+
+        The key names for the dictionary of result tiles are used to
+        stitch each tile into its corresponding final image in the
+        results attribute. If a result image does not exist for a
+        result tile key name, then it will be created. The result tiles
+        are stitched at the location from which the list iterated tile
+        was extracted.
+
+        Parameters
+        ----------
+        result_tiles : dict(str: Image)
+            Dictionary of result tiles from the inference.
+        """
+
+        for k, tile in result_tiles.items():
+            if k not in self.res:
+                self.res[k] = Image.new('RGB', (self.image_width, self.image_height))
+            if tile.size != (self.tile_size, self.tile_size):
+                tile = tile.resize((self.tile_size, self.tile_size))
+            self.res[k].paste(tile.crop((self.c1x, self.c1y, self.c2x, self.c2y)), (self.x + self.p1x, self.y + self.p1y))
+
+            # top left corner
+            if self.x == 0 and self.y == 0:
+                self.res[k].paste(tile.crop((self.c0x, self.c0y, self.c1x, self.c1y)), (self.x, self.y))
+            # top row
+            if self.y == 0:
+                self.res[k].paste(tile.crop((self.c1x, self.c0y, self.c2x, self.c1y)), (self.x + self.p1x, self.y))
+            # top right corner
+            if self.x == self.image_width - self.patch_size and self.y == 0:
+                self.res[k].paste(tile.crop((self.c2x, self.c0y, self.c3x, self.c1y)), (self.x + self.p2x, self.y))
+            # left column
+            if self.x == 0:
+                self.res[k].paste(tile.crop((self.c0x, self.c1y, self.c1x, self.c2y)), (self.x, self.y + self.p1y))
+            # right column
+            if self.x == self.image_width - self.patch_size:
+                self.res[k].paste(tile.crop((self.c2x, self.c1y, self.c3x, self.c2y)), (self.x + self.p2x, self.y + self.p1y))
+            # bottom left corner
+            if self.x == 0 and self.y == self.image_height - self.patch_size:
+                self.res[k].paste(tile.crop((self.c0x, self.c2y, self.c1x, self.c3y)), (self.x, self.y + self.p2y))
+            # bottom row
+            if self.y == self.image_height - self.patch_size:
+                self.res[k].paste(tile.crop((self.c1x, self.c2y, self.c2x, self.c3y)), (self.x + self.p1x, self.y + self.p2y))
+            # bottom right corner
+            if self.x == self.image_width - self.patch_size and self.y == self.image_height - self.patch_size:
+                self.res[k].paste(tile.crop((self.c2x, self.c2y, self.c3x, self.c3y)), (self.x + self.p2x, self.y + self.p2y))
+
+    def results(self):
+        """
+        Return a dictionary of result images.
+
+        The keys for the result images are the same as those used for
+        the result tiles in the stitch function. This function should
+        only be called once, since the stitched images will be cropped
+        if the original image size was less than the patch size.
+        """
+
+        if self.orig_width != self.image_width or self.orig_height != self.image_height:
+            return {k: im.crop((0, 0, self.orig_width, self.orig_height)) for k, im in self.res.items()}
+        else:
+            return {k: im for k, im in self.res.items()}
+
+
 def calculate_background_mean_value(img):
     img = cv2.fastNlMeansDenoisingColored(np.array(img), None, 10, 10, 7, 21)
     img = np.array(img, dtype=float)
@@ -235,6 +441,87 @@ def get_information(filename):
     return size_x, size_y, size_z, size_c, size_t, pixel_type
 
 
+
+
+def write_results_to_pickle_file(output_addr, results):
+    """
+    This function writes data into the pickle file.
+    :param output_addr: The address of the pickle file to write data into.
+    :param results: The data to be written into the pickle file.
+    :return:
+    """
+    pickle_obj = open(output_addr, "wb")
+    pickle.dump(results, pickle_obj)
+    pickle_obj.close()
+
+
+def read_results_from_pickle_file(input_addr):
+    """
+    This function reads data from a pickle file and returns it.
+    :param input_addr: The address to the pickle file.
+    :return: The data inside pickle file.
+    """
+    pickle_obj = open(input_addr, "rb")
+    results = pickle.load(pickle_obj)
+    pickle_obj.close()
+    return results
+
+def test_diff_original_serialized(model_original,model_serialized,example,verbose=0):
+    threshold = 10
+
+    orig_res = model_original(example)
+    if verbose > 0:
+        print('Original:')
+        print(orig_res.shape)
+        print(orig_res[0, 0:10])
+        print('min abs value:{}'.format(torch.min(torch.abs(orig_res))))
+
+    ts_res = model_serialized(example)
+    if verbose > 0:
+        print('Torchscript:')
+        print(ts_res.shape)
+        print(ts_res[0, 0:10])
+        print('min abs value:{}'.format(torch.min(torch.abs(ts_res))))
+
+    abs_diff = torch.abs(orig_res-ts_res)
+    if verbose > 0:
+        print('Dif sum:')
+        print(torch.sum(abs_diff))
+        print('max dif:{}'.format(torch.max(abs_diff)))
+
+    assert torch.sum(abs_diff) <= threshold, f"Sum of difference in predicted values {torch.sum(abs_diff)} is larger than threshold {threshold}"
+
+def disable_batchnorm_tracking_stats(model):
+    # https://discuss.pytorch.org/t/performance-highly-degraded-when-eval-is-activated-in-the-test-phase/3323/16
+    # https://discuss.pytorch.org/t/performance-highly-degraded-when-eval-is-activated-in-the-test-phase/3323/67
+    # https://github.com/pytorch/pytorch/blob/ca39c5b04e30a67512589cafbd9d063cc17168a5/torch/nn/modules/batchnorm.py#L158
+    for m in model.modules():
+        for child in m.children():
+            if type(child) == torch.nn.BatchNorm2d:
+                child.track_running_stats = False
+                child.running_mean_backup = child.running_mean
+                child.running_mean = None
+                child.running_var_backup = child.running_var
+                child.running_var = None
+    return model
+
+def enable_batchnorm_tracking_stats(model):
+    """
+    This is needed during training when val set loss/metrics calculation is enabled.
+    In this case, we need to switch to eval mode for inference, which triggers
+    disable_batchnorm_tracking_stats(). After the evaluation, the model should be
+    set back to train mode, where running stats are restored for batchnorm layers.
+    """
+    for m in model.modules():
+        for child in m.children():
+            if type(child) == torch.nn.BatchNorm2d:
+                child.track_running_stats = True
+                assert hasattr(child, 'running_mean_backup') and hasattr(child, 'running_var_backup'), 'enable_batchnorm_tracking_stats() is supposed to be executed after disable_batchnorm_tracking_stats() is applied'
+                child.running_mean = child.running_mean_backup
+                child.running_var = child.running_var_backup
+    return model
+
+
 def write_big_tiff_file(output_addr, img, tile_size):
     """
     This function write the image into a big tiff file using the tiling and compression.
@@ -376,64 +663,3 @@ def write_ome_tiff_file_array(results_array, output_addr, size_t, size_z, size_c
                         output_addr,
                         SizeT=size_t, SizeZ=size_z, SizeC=len(channel_names), SizeX=size_x, SizeY=size_y,
                         channel_names=channel_names)
-
-
-def write_results_to_pickle_file(output_addr, results):
-    """
-    This function writes data into the pickle file.
-    :param output_addr: The address of the pickle file to write data into.
-    :param results: The data to be written into the pickle file.
-    :return:
-    """
-    pickle_obj = open(output_addr, "wb")
-    pickle.dump(results, pickle_obj)
-    pickle_obj.close()
-
-
-def read_results_from_pickle_file(input_addr):
-    """
-    This function reads data from a pickle file and returns it.
-    :param input_addr: The address to the pickle file.
-    :return: The data inside pickle file.
-    """
-    pickle_obj = open(input_addr, "rb")
-    results = pickle.load(pickle_obj)
-    pickle_obj.close()
-    return results
-
-def test_diff_original_serialized(model_original,model_serialized,example,verbose=0):
-    threshold = 10
-
-    orig_res = model_original(example)
-    if verbose > 0:
-        print('Original:')
-        print(orig_res.shape)
-        print(orig_res[0, 0:10])
-        print('min abs value:{}'.format(torch.min(torch.abs(orig_res))))
-
-    ts_res = model_serialized(example)
-    if verbose > 0:
-        print('Torchscript:')
-        print(ts_res.shape)
-        print(ts_res[0, 0:10])
-        print('min abs value:{}'.format(torch.min(torch.abs(ts_res))))
-
-    abs_diff = torch.abs(orig_res-ts_res)
-    if verbose > 0:
-        print('Dif sum:')
-        print(torch.sum(abs_diff))
-        print('max dif:{}'.format(torch.max(abs_diff)))
-
-    assert torch.sum(abs_diff) <= threshold, f"Sum of difference in predicted values {torch.sum(abs_diff)} is larger than threshold {threshold}"
-
-def disable_batchnorm_tracking_stats(model):
-    # https://discuss.pytorch.org/t/performance-highly-degraded-when-eval-is-activated-in-the-test-phase/3323/16
-    # https://discuss.pytorch.org/t/performance-highly-degraded-when-eval-is-activated-in-the-test-phase/3323/67
-    # https://github.com/pytorch/pytorch/blob/ca39c5b04e30a67512589cafbd9d063cc17168a5/torch/nn/modules/batchnorm.py#L158
-    for m in model.modules():
-        for child in m.children():
-            if type(child) == torch.nn.BatchNorm2d:
-                child.track_running_stats = False
-                child.running_mean = None
-                child.running_var = None
-    return model

deepliif/util/visualizer.py

@@ -165,6 +165,7 @@ class Visualizer():
                 if ncols > 0:        # show all the images in one visdom panel
                     ncols = min(ncols, len(visuals))
                     h, w = next(iter(visuals.values())).shape[:2]
+
                     table_css = """<style>
                             table {border-collapse: separate; border-spacing: 4px; white-space: nowrap; text-align: center}
                             table td {width: % dpx; height: % dpx; padding: 4px; outline: 4px solid black}
@@ -176,13 +177,16 @@ class Visualizer():
                     images = []
                     idx = 0
                     for label, image in visuals.items():
-                        image_numpy = util.tensor2im(image)
-                        label_html_row += '<td>%s</td>' % label
-                        images.append(image_numpy.transpose([2, 0, 1]))
-                        idx += 1
-                        if idx % ncols == 0:
-                            label_html += '<tr>%s</tr>' % label_html_row
-                            label_html_row = ''
+                        if image.shape[1] != 3:
+                            pass
+                        else:
+                            image_numpy = util.tensor2im(image)
+                            label_html_row += '<td>%s</td>' % label
+                            images.append(image_numpy.transpose([2, 0, 1]))
+                            idx += 1
+                            if idx % ncols == 0:
+                                label_html += '<tr>%s</tr>' % label_html_row
+                                label_html_row = ''
 
                     white_image = np.ones_like(image_numpy.transpose([2, 0, 1])) * 255
                     while idx % ncols != 0:
@@ -191,6 +195,7 @@ class Visualizer():
                         idx += 1
                     if label_html_row != '':
                         label_html += '<tr>%s</tr>' % label_html_row
+                    
 
                     try:
                         self.vis.images(images, nrow=ncols, win=self.display_id + 1,
@@ -248,6 +253,10 @@ class Visualizer():
         # if having 2 processes, each process obtains 50% of the data (effective dataset_size divided by half), the effective counter ratio shall multiply by 2 to compensate that
         n_proc = int(os.getenv('WORLD_SIZE',1))
         counter_ratio = counter_ratio * n_proc
+
+        self.plot_data_update_train = False
+        self.plot_data_update_val = False
+        self.plot_data_update_metrics = False
         
         if self.remote:
             fn = 'plot_current_losses.pickle'
@@ -263,20 +272,98 @@ class Visualizer():
                     exec(f'{self.remote_transfer_cmd_function}("{path_source}")')
         else:
             if not hasattr(self, 'plot_data'):
-                self.plot_data = {'X': [], 'Y': [], 'legend': list(losses.keys())}
-            self.plot_data['X'].append(epoch + counter_ratio)
-            self.plot_data['Y'].append([losses[k] for k in self.plot_data['legend']])
+                self.plot_data = {'X': [], 'X_val':[], 'X_metrics':[],
+                                  'Y': [], 'Y_val':[], 'Y_metrics':[],
+                                  'legend': [], 'legend_val': [], 'legend_metrics':[]}
+                for k in list(losses.keys()):
+                    if k.endswith('_val'):
+                        self.plot_data['legend_val'].append(k)
+                    elif k.startswith(('G_','D_')):
+                        self.plot_data['legend'].append(k)
+                    else:
+                        self.plot_data['legend_metrics'].append(k)
+            
+            # check if all names in losses dict have been seen
+            # currently we assume the three types of metrics (train loss, val loss, other metrics) can come into the losses dict
+            # at any step, but each type will join or leave the dict as a whole (i.e., train loss metrics will either all appear or all be missing)
+            for k in list(losses.keys()):
+                    if k.endswith('_val'):
+                        if k not in self.plot_data['legend_val']:
+                            self.plot_data['legend_val'].append(k)
+                    elif k.startswith(('G_','D_')):
+                        if k not in self.plot_data['legend']:
+                            self.plot_data['legend'].append(k)
+                    else:
+                        if k not in self.plot_data['legend_metrics']:
+                            self.plot_data['legend_metrics'].append(k)
+
+            # update training loss
+            print('update training loss')
+            if len(self.plot_data['legend']) > 0:
+                if self.plot_data['legend'][0] in losses:
+                    self.plot_data['X'].append(epoch + counter_ratio)
+                    self.plot_data['Y'].append([losses[k] for k in self.plot_data['legend']])
+                    self.plot_data_update_train = True
+
+            # update validation loss
+            print('update validation loss')
+            if len(self.plot_data['legend_val']) > 0:
+                if self.plot_data['legend_val'][0] in losses:
+                    self.plot_data['X_val'].append(epoch + counter_ratio)
+                    self.plot_data['Y_val'].append([losses[k] for k in self.plot_data['legend_val']])
+                    self.plot_data_update_val = True
+
+            # update other calculated metrics
+            print('update other metrics')
+            if len(self.plot_data['legend_metrics']) > 0:
+                if self.plot_data['legend_metrics'][0] in losses:
+                    self.plot_data['X_metrics'].append(epoch + counter_ratio)
+                    self.plot_data['Y_metrics'].append([losses[k] for k in self.plot_data['legend_metrics']])
+                    self.plot_data_update_metrics = True
 
             try:
-                self.vis.line(
-                    X=np.stack([np.array(self.plot_data['X'])] * len(self.plot_data['legend']), 1),
-                    Y=np.array(self.plot_data['Y']),
-                    opts={
-                        'title': self.name + ' loss over time',
-                        'legend': self.plot_data['legend'],
-                        'xlabel': 'epoch',
-                        'ylabel': 'loss'},
-                    win=self.display_id)
+                if self.plot_data_update_train:
+                    print('plotting train loss')
+                    self.vis.line(
+                        X=np.stack([np.array(self.plot_data['X'])] * len(self.plot_data['legend']), 1),
+                        Y=np.array(self.plot_data['Y']),
+                        opts={
+                            'title': self.name + ' train loss over time',
+                            'legend': self.plot_data['legend'],
+                            'xlabel': 'epoch',
+                            'ylabel': 'loss'},
+                        win = 'train',
+                        #env=self.display_id
+                        )
+                
+                if self.plot_data_update_val:
+                    print('plotting val loss')
+                    self.vis.line(
+                        X=np.stack([np.array(self.plot_data['X_val'])] * len(self.plot_data['legend_val']), 1),
+                        Y=np.array(self.plot_data['Y_val']),
+                        opts={
+                            'title': self.name + ' val loss over time',
+                            'legend': self.plot_data['legend_val'],
+                            'xlabel': 'epoch',
+                            'ylabel': 'loss'},
+                        win = 'val',
+                        #env=self.display_id
+                        )
+                
+                if self.plot_data_update_metrics:
+                    print('plotting other metrics')
+                    self.vis.line(
+                        X=np.stack([np.array(self.plot_data['X_metrics'])] * len(self.plot_data['legend_metrics']), 1),
+                        Y=np.array(self.plot_data['Y_metrics']),
+                        opts={
+                            'title': self.name + ' metrics over time',
+                            'legend': self.plot_data['legend_metrics'],
+                            'xlabel': 'epoch',
+                            'ylabel': 'metrics'},
+                        win = 'metrics',
+                        #env=self.display_id
+                        )
+                
             except VisdomExceptionBase:
                 self.create_visdom_connections()