deepliif 1.1.9__py3-none-any.whl → 1.1.11__py3-none-any.whl

deepliif/models/__init__.py

@@ -133,6 +133,10 @@ def load_eager_models(opt, devices):
                 net.eval()
                 net = disable_batchnorm_tracking_stats(net)
             
+            # SDG models when loaded are still DP.. not sure why
+            if isinstance(net, torch.nn.DataParallel):
+                net = net.module
+
             nets[name] = net
             nets[name].to(devices[name])
             
@@ -161,7 +165,7 @@ def init_nets(model_dir, eager_mode=False, opt=None, phase='test'):
             ('G4', 'G55'),
             ('G51',)
         ]
-    elif opt.model == 'DeepLIIFExt':
+    elif opt.model in ['DeepLIIFExt','SDG']:
         if opt.seg_gen:
             net_groups = [(f'G_{i+1}',f'GS_{i+1}') for i in range(opt.modalities_no)]
         else:
@@ -172,6 +176,7 @@ def init_nets(model_dir, eager_mode=False, opt=None, phase='test'):
     number_of_gpus_all = torch.cuda.device_count()
     number_of_gpus = len(opt.gpu_ids)
     #print(number_of_gpus)
+
     if number_of_gpus > 0:
         mapping_gpu_ids = {i:idx for i,idx in enumerate(opt.gpu_ids)}
         chunks = [itertools.chain.from_iterable(c) for c in chunker(net_groups, number_of_gpus)]
@@ -206,10 +211,7 @@ def run_torchserve(img, model_path=None, eager_mode=False, opt=None):
     opt: same as eager_mode
     """
     buffer = BytesIO()
-    if opt.model == 'DeepLIIFExt':
-        torch.save(transform(img.resize((1024, 1024))), buffer)
-    else:
-        torch.save(transform(img.resize((512, 512))), buffer)
+    torch.save(transform(img.resize((opt.scale_size, opt.scale_size))), buffer)
 
     torchserve_host = os.getenv('TORCHSERVE_HOST', 'http://localhost')
     res = requests.post(
@@ -229,10 +231,12 @@ def run_dask(img, model_path, eager_mode=False, opt=None):
     model_dir = os.getenv('DEEPLIIF_MODEL_DIR', model_path)
     nets = init_nets(model_dir, eager_mode, opt)
     
-    if opt.model == 'DeepLIIFExt':
-        ts = transform(img.resize((1024, 1024)))
+    if opt.input_no > 1 or opt.model == 'SDG':
+        l_ts = [transform(img_i.resize((opt.scale_size,opt.scale_size))) for img_i in img]
+        ts = torch.cat(l_ts, dim=1)
     else:
-        ts = transform(img.resize((512, 512)))
+        ts = transform(img.resize((opt.scale_size, opt.scale_size)))
+    
 
     @delayed
     def forward(input, model):
@@ -249,14 +253,20 @@ def run_dask(img, model_path, eager_mode=False, opt=None):
         lazy_segs['G51'] = forward(ts, nets['G51']).to(torch.device('cpu'))
         segs = compute(lazy_segs)[0]
     
-        seg_weights = [0.25, 0.25, 0.25, 0, 0.25]
-        seg = torch.stack([torch.mul(n, w) for n, w in zip(segs.values(), seg_weights)]).sum(dim=0)
+        weights = {
+            'G51': 0.25, # IHC
+            'G52': 0.25, # Hema
+            'G53': 0.25, # DAPI
+            'G54': 0.00, # Lap2
+            'G55': 0.25, # Marker
+        }
+        seg = torch.stack([torch.mul(segs[k], weights[k]) for k in segs.keys()]).sum(dim=0)
     
         res = {k: tensor_to_pil(v) for k, v in gens.items()}
         res['G5'] = tensor_to_pil(seg)
     
         return res
-    elif opt.model == 'DeepLIIFExt':
+    elif opt.model in ['DeepLIIFExt','SDG']:
         seg_map = {'G_' + str(i): 'GS_' + str(i) for i in range(1, opt.modalities_no + 1)}
         
         lazy_gens = {k: forward(ts, nets[k]) for k in seg_map}
@@ -271,14 +281,23 @@ def run_dask(img, model_path, eager_mode=False, opt=None):
     
         return res
     else:
-        raise Exception(f'run_dask() not implemented for {opt.model}')
+        raise Exception(f'run_dask() not fully implemented for {opt.model}')
 
-    
+
+def is_empty_old(tile):
+    # return True if np.mean(np.array(tile) - np.array(mean_background_val)) < 40 else False
+    if isinstance(tile, list): # for pair of tiles, only mark it as empty / no need for prediction if ALL tiles are empty
+        return all([True if calculate_background_area(t) > 98 else False for t in tile])
+    else:
+        return True if calculate_background_area(tile) > 98 else False
 
 
 def is_empty(tile):
-    # return True if np.mean(np.array(tile) - np.array(mean_background_val)) < 40 else False
-    return True if calculate_background_area(tile) > 98 else False
+    thresh = 15
+    if isinstance(tile, list): # for pair of tiles, only mark it as empty / no need for prediction if ALL tiles are empty
+        return all([True if np.max(image_variance_rgb(t)) < thresh else False for t in tile])
+    else:
+        return True if np.max(image_variance_rgb(tile)) < thresh else False
 
 
 def run_wrapper(tile, run_fn, model_path, eager_mode=False, opt=None):
@@ -293,7 +312,7 @@ def run_wrapper(tile, run_fn, model_path, eager_mode=False, opt=None):
             }
         else:
             return run_fn(tile, model_path, eager_mode, opt)
-    elif opt.model == 'DeepLIIFExt':
+    elif opt.model in ['DeepLIIFExt', 'SDG']:
         if is_empty(tile):
             res = {'G_' + str(i): Image.new(mode='RGB', size=(512, 512)) for i in range(1, opt.modalities_no + 1)}
             res.update({'GS_' + str(i): Image.new(mode='RGB', size=(512, 512)) for i in range(1, opt.modalities_no + 1)})
@@ -350,8 +369,8 @@ def inference_old(img, tile_size, overlap_size, model_path, use_torchserve=False
     return images
 
 
-def inference(img, tile_size, overlap_size, model_path, use_torchserve=False, eager_mode=False,
-              color_dapi=False, color_marker=False, opt=None):
+def inference_old2(img, tile_size, overlap_size, model_path, use_torchserve=False, eager_mode=False,
+                   color_dapi=False, color_marker=False, opt=None):
     if not opt:
         opt = get_opt(model_path)
         #print_options(opt)
@@ -362,28 +381,25 @@ def inference(img, tile_size, overlap_size, model_path, use_torchserve=False, ea
         run_fn = run_torchserve if use_torchserve else run_dask
     
         images = {}
-        images['Hema'] = create_image_for_stitching(tile_size, rows, cols)
-        images['DAPI'] = create_image_for_stitching(tile_size, rows, cols)
-        images['Lap2'] = create_image_for_stitching(tile_size, rows, cols)
-        images['Marker'] = create_image_for_stitching(tile_size, rows, cols)
-        images['Seg'] = create_image_for_stitching(tile_size, rows, cols)
+        d_modality2net = {'Hema':'G1',
+                          'DAPI':'G2',
+                          'Lap2':'G3',
+                          'Marker':'G4',
+                          'Seg':'G5'}
+        
+        for k in d_modality2net.keys():
+            images[k] = create_image_for_stitching(tile_size, rows, cols)
     
         for i in range(cols):
             for j in range(rows):
                 tile = extract_tile(rescaled, tile_size, overlap_size, i, j)
                 res = run_wrapper(tile, run_fn, model_path, eager_mode, opt)
-    
-                stitch_tile(images['Hema'], res['G1'], tile_size, overlap_size, i, j)
-                stitch_tile(images['DAPI'], res['G2'], tile_size, overlap_size, i, j)
-                stitch_tile(images['Lap2'], res['G3'], tile_size, overlap_size, i, j)
-                stitch_tile(images['Marker'], res['G4'], tile_size, overlap_size, i, j)
-                stitch_tile(images['Seg'], res['G5'], tile_size, overlap_size, i, j)
-    
-        images['Hema'] = images['Hema'].resize(img.size)
-        images['DAPI'] = images['DAPI'].resize(img.size)
-        images['Lap2'] = images['Lap2'].resize(img.size)
-        images['Marker'] = images['Marker'].resize(img.size)
-        images['Seg'] = images['Seg'].resize(img.size)
+                
+                for modality_name, net_name in d_modality2net.items():
+                    stitch_tile(images[modality_name], res[net_name], tile_size, overlap_size, i, j)
+        
+        for modality_name, output_img in images.items():
+            images[modality_name] = output_img.resize(img.size)
     
         if color_dapi:
             matrix = (       0,        0,        0, 0,
@@ -403,30 +419,133 @@ def inference(img, tile_size, overlap_size, model_path, use_torchserve=False, ea
         #param_dict = read_train_options(model_path)
         #modalities_no = int(param_dict['modalities_no']) if param_dict else 4
         #seg_gen = (param_dict['seg_gen'] == 'True') if param_dict else True
-    
-        tiles = list(generate_tiles(img, tile_size, overlap_size))
-    
+        
+        
+        rescaled, rows, cols = format_image_for_tiling(img, tile_size, overlap_size)
         run_fn = run_torchserve if use_torchserve else run_dask
-        res = [Tile(t.i, t.j, run_wrapper(t.img, run_fn, model_path, eager_mode, opt)) for t in tiles]
     
         def get_net_tiles(n):
             return [Tile(t.i, t.j, t.img[n]) for t in res]
     
         images = {}
-    
-        for i in range(1, opt.modalities_no + 1):
-            images['mod' + str(i)] = stitch(get_net_tiles('G_' + str(i)), tile_size, overlap_size).resize(img.size)
-    
+        d_modality2net = {f'mod{i}':f'G_{i}' for i in range(1, opt.modalities_no + 1)}
         if opt.seg_gen:
-            for i in range(1, opt.modalities_no + 1):
-                images['Seg' + str(i)] = stitch(get_net_tiles('GS_' + str(i)), tile_size, overlap_size).resize(img.size)
+            d_modality2net.update({f'Seg{i}':f'GS_{i}' for i in range(1, opt.modalities_no + 1)})
+        
+        for k in d_modality2net.keys():
+            images[k] = create_image_for_stitching(tile_size, rows, cols)
     
+        for i in range(cols):
+            for j in range(rows):
+                tile = extract_tile(rescaled, tile_size, overlap_size, i, j)
+                res = run_wrapper(tile, run_fn, model_path, eager_mode, opt)
+                
+                for modality_name, net_name in d_modality2net.items():
+                    stitch_tile(images[modality_name], res[net_name], tile_size, overlap_size, i, j)
+        
+        for modality_name, output_img in images.items():
+            images[modality_name] = output_img.resize(img.size)
+            
+        return images
+        
+    elif opt.model == 'SDG':
+        # SDG could have multiple input images / modalities
+        # the input hence could be a rectangle
+        # we split the input to get each modality image one by one
+        # then create tiles for each of the modality images
+        # tile_pair is a list that contains the tiles at the given location for each modality image
+        # l_tile_pair is a list of tile_pair that covers all locations
+        # for inference, each tile_pair is used to get the output at the given location
+        w, h = img.size
+        w2 = int(w / opt.input_no)
+        
+        l_img = []        
+        for i in range(opt.input_no):
+            img_i = img.crop((w2 * i, 0, w2 * (i+1), h))
+            rescaled_img_i, rows, cols = format_image_for_tiling(img_i, tile_size, overlap_size)
+            l_img.append(rescaled_img_i)
+        
+        run_fn = run_torchserve if use_torchserve else run_dask
+        
+        images = {}
+        d_modality2net = {f'mod{i}':f'G_{i}' for i in range(1, opt.modalities_no + 1)}
+        for k in d_modality2net.keys():
+            images[k] = create_image_for_stitching(tile_size, rows, cols)
+        
+        for i in range(cols):
+            for j in range(rows):
+                tile_pair = [extract_tile(rescaled, tile_size, overlap_size, i, j) for rescaled in l_img]
+                res = run_wrapper(tile_pair, run_fn, model_path, eager_mode, opt)
+                
+                for modality_name, net_name in d_modality2net.items():
+                    stitch_tile(images[modality_name], res[net_name], tile_size, overlap_size, i, j)
+        
+        for modality_name, output_img in images.items():
+            images[modality_name] = output_img.resize((w2,w2))
+        
         return images
     
     else:
         raise Exception(f'inference() not implemented for model {opt.model}')
 
 
+def inference(img, tile_size, overlap_size, model_path, use_torchserve=False,
+              eager_mode=False, color_dapi=False, color_marker=False, opt=None):
+    if not opt:
+        opt = get_opt(model_path)
+        #print_options(opt)
+
+    run_fn = run_torchserve if use_torchserve else run_dask
+
+    if opt.model == 'SDG':
+        # SDG could have multiple input images/modalities, hence the input could be a rectangle.
+        # We split the input to get each modality image then create tiles for each set of input images.
+        w, h = int(img.width / opt.input_no), img.height
+        orig = [img.crop((w * i, 0, w * (i+1), h)) for i in range(opt.input_no)]
+    else:
+        # Otherwise expect a single input image, which is used directly.
+        orig = img
+
+    tiler = InferenceTiler(orig, tile_size, overlap_size)
+    for tile in tiler:
+        tiler.stitch(run_wrapper(tile, run_fn, model_path, eager_mode, opt))
+    results = tiler.results()
+
+    if opt.model == 'DeepLIIF':
+        images = {
+            'Hema': results['G1'],
+            'DAPI': results['G2'],
+            'Lap2': results['G3'],
+            'Marker': results['G4'],
+            'Seg': results['G5'],
+        }
+        if color_dapi:
+            matrix = (       0,        0,        0, 0,
+                      299/1000, 587/1000, 114/1000, 0,
+                      299/1000, 587/1000, 114/1000, 0)
+            images['DAPI'] = images['DAPI'].convert('RGB', matrix)
+        if color_marker:
+            matrix = (299/1000, 587/1000, 114/1000, 0,
+                      299/1000, 587/1000, 114/1000, 0,
+                             0,        0,        0, 0)
+            images['Marker'] = images['Marker'].convert('RGB', matrix)
+        return images
+
+    elif opt.model == 'DeepLIIFExt':
+        images = {f'mod{i}': results[f'G_{i}'] for i in range(1, opt.modalities_no + 1)}
+        if opt.seg_gen:
+            images.update({f'Seg{i}': results[f'GS_{i}'] for i in range(1, opt.modalities_no + 1)})
+        return images
+
+    elif opt.model == 'SDG':
+        images = {f'mod{i}': results[f'G_{i}'] for i in range(1, opt.modalities_no + 1)}
+        return images
+
+    else:
+        #raise Exception(f'inference() not implemented for model {opt.model}')
+        return results # return result images with default key names (i.e., net names)
+
+
 def postprocess(orig, images, tile_size, model, seg_thresh=150, size_thresh='auto', marker_thresh='auto', size_thresh_upper=None):
     if model == 'DeepLIIF':
         resolution = '40x' if tile_size > 384 else ('20x' if tile_size > 192 else '10x')
@@ -476,11 +595,16 @@ def infer_modalities(img, tile_size, model_dir, eager_mode=False,
         tile_size = check_multi_scale(Image.open('./images/target.png').convert('L'),
                                       img.convert('L'))
     tile_size = int(tile_size)
+    
+    # for those with multiple input modalities, find the correct size to calculate overlap_size
+    input_no = opt.input_no if hasattr(opt, 'input_no') else 1
+    img_size = (img.size[0] / input_no, img.size[1]) # (width, height)
 
     images = inference(
         img,
         tile_size=tile_size,
-        overlap_size=compute_overlap(img.size, tile_size),
+        #overlap_size=compute_overlap(img_size, tile_size),
+        overlap_size=tile_size//16,
         model_path=model_dir,
         eager_mode=eager_mode,
         color_dapi=color_dapi,

deepliif/options/__init__.py

@@ -13,7 +13,6 @@ def read_model_params(file_addr):
             key = line.split(':')[0].strip()
             val = line.split(':')[1].split('[')[0].strip()
             param_dict[key] = val
-    print(param_dict)
     return param_dict
 
 class Options:
@@ -32,19 +31,7 @@ class Options:
                 else:
                     setattr(self,k,v)
             except:
-                setattr(self,k,v)
-        
-        if mode != 'train':
-            # to account for old settings where gpu_ids value is an integer, not a tuple
-            if isinstance(self.gpu_ids,int):
-                self.gpu_ids = (self.gpu_ids,)
-            
-            # to account for old settings before modalities_no was introduced
-            if not hasattr(self,'modalities_no') and hasattr(self,'targets_no'):
-                self.modalities_no = self.targets_no - 1
-                del self.targets_no
-            
-            
+                setattr(self,k,v)            
 
         if mode == 'train':
             self.is_train = True
@@ -71,32 +58,78 @@ class Options:
             self.checkpoints_dir = str(model_dir.parent)
             self.name = str(model_dir.name)
             
-            self.gpu_ids = [] # gpu_ids is only used by eager mode, set to empty / cpu to be the same as the old settings; non-eager mode will use all gpus
+            #self.gpu_ids = [] # gpu_ids is only used by eager mode, set to empty / cpu to be the same as the old settings; non-eager mode will use all gpus
+            
+            # to account for old settings where gpu_ids value is an integer, not a tuple
+            if isinstance(self.gpu_ids,int):
+                self.gpu_ids = (self.gpu_ids,)
+            
+            # to account for old settings before modalities_no was introduced
+            if not hasattr(self,'modalities_no') and hasattr(self,'targets_no'):
+                self.modalities_no = self.targets_no - 1
+                del self.targets_no
+            
+            # to account for old settings: same as in cli.py train
+            if not hasattr(self,'seg_no'):
+              if self.model == 'DeepLIIF':
+                  self.seg_no = 1
+              elif self.model == 'DeepLIIFExt':
+                  if self.seg_gen:
+                      self.seg_no = self.modalities_no
+                  else:
+                      self.seg_no = 0
+              elif self.model == 'SDG':
+                  self.seg_no = 0
+                  self.seg_gen = False
+              else:
+                  raise Exception(f'seg_gen cannot be automatically determined for {opt.model}')
+            
+            # to account for old settings: prior to SDG, our models only have 1 input image
+            if not hasattr(self,'input_no'):
+                self.input_no = 1
+            
+            # to account for old settings: before adding scale_size
+            if not hasattr(self, 'scale_size'):
+                if self.model in ['DeepLIIF','SDG']:
+                    self.scale_size = 512
+                elif self.model == 'DeepLIIFExt':
+                    self.scale_size = 1024
+                else:
+                  raise Exception(f'scale_size cannot be automatically determined for {opt.model}')
+
+            
     
     def _get_kwargs(self):
         common_attr = ['__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__']
         l_args = [x for x in dir(self) if x not in common_attr]
         return {k:getattr(self,k) for k in l_args}
-            
-def print_options(opt):
-    """Print and save options
 
-    It will print both current options and default values(if different).
-    It will save options into a text file / [checkpoints_dir] / opt.txt
-    """
+def format_options(opt):
     message = ''
     message += '----------------- Options ---------------\n'
     for k, v in sorted(vars(opt).items()):
         comment = ''
         message += '{:>25}: {:<30}{}\n'.format(str(k), str(v), comment)
     message += '----------------- End -------------------'
-    print(message)
+    return message
+    
+def print_options(opt, save=False):
+    """Print (and save) options
 
+    It will print both current options and default values(if different).
+    If save=True, it will save options into a text file / [checkpoints_dir] / opt.txt
+    """
+    message = format_options(opt)
+    print(message)
+    
     # save to the disk
-    if opt.phase == 'train':
-        expr_dir = os.path.join(opt.checkpoints_dir, opt.name)
-        mkdirs(expr_dir)
-        file_name = os.path.join(expr_dir, '{}_opt.txt'.format(opt.phase))
-        with open(file_name, 'wt') as opt_file:
-            opt_file.write(message)
-        opt_file.write('\n')
+    if save:
+        save_options(opt)
+
+def save_options(opt):
+    message = format_options(opt)
+    expr_dir = os.path.join(opt.checkpoints_dir, opt.name)
+    mkdirs(expr_dir)
+    file_name = os.path.join(expr_dir, '{}_opt.txt'.format(opt.phase))
+    with open(file_name, 'wt') as opt_file:
+        opt_file.write(message+'\n')