cell-data-loader 0.0.5__tar.gz → 0.0.7__tar.gz

{cell_data_loader-0.0.5 → cell_data_loader-0.0.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: cell_data_loader
-Version: 0.0.5
+Version: 0.0.7
 Summary: Converts general images of cells into formats and labels for deep learning pipelines
 Project-URL: Homepage, https://github.com/mleming/CellDataLoader
 Author-email: Matt Leming <mleming@mgh.harvard.edu>
@@ -86,6 +86,20 @@ for label,image in dataloader:
 	...
 ~~~
 
+Boxes
+-----
+
+In cases where you need to cut out individual cells from an image and have the coordinates file, cell_data_loader.py accepts an argument, cell_box_filelist, which is a list of files corresponding to the inputs that mark out the coordinates of labels on the cells. The format of the csv is as follows:
+
+| X  | Y   | W  | H | Label |
+| -  | -   | -  | - | ----- |
+| 14 | 13  | 5  | 6 | 0     |
+| 20 | 25  | 15 | 5 | 1     |
+
+
+~~~python
+dataloader = CellDataloader('/path/to/file.svs',cell_box_filelist=['/path/to/boxfile.csv'])
+~~~
 
 Arguments
 ---------

{cell_data_loader-0.0.5 → cell_data_loader-0.0.7}/README.md

@@ -66,6 +66,20 @@ for label,image in dataloader:
 	...
 ~~~
 
+Boxes
+-----
+
+In cases where you need to cut out individual cells from an image and have the coordinates file, cell_data_loader.py accepts an argument, cell_box_filelist, which is a list of files corresponding to the inputs that mark out the coordinates of labels on the cells. The format of the csv is as follows:
+
+| X  | Y   | W  | H | Label |
+| -  | -   | -  | - | ----- |
+| 14 | 13  | 5  | 6 | 0     |
+| 20 | 25  | 15 | 5 | 1     |
+
+
+~~~python
+dataloader = CellDataloader('/path/to/file.svs',cell_box_filelist=['/path/to/boxfile.csv'])
+~~~
 
 Arguments
 ---------

{cell_data_loader-0.0.5 → cell_data_loader-0.0.7}/example_numpy.py

@@ -1,5 +1,4 @@
 #!/usr/bin/python3
-
 from src.cell_data_loader import CellDataloader
 
 try:
@@ -10,7 +9,7 @@ except:
 	except:
 		raise Exception("Need valid tensorflow")
 
-import numpy
+import numpy as np
 import os
 
 def example_numpy(verbose=True):
@@ -20,13 +19,13 @@ def example_numpy(verbose=True):
 	the test/train folders in this script are the same -- you would need to 
 	have a separate train/test set in your own implementation
 	"""
-	wd = os.path.dirname(os.path.realpath(__file__))
+	wd = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 	imfolder1 = os.path.join(wd,'data',
 		'3368914_4_non_tumor')
 	imfolder2 = os.path.join(wd,'data',
 		'4173633_5')
 
-	model = resnet50()
+	#model = resnet50()
 	model = tf.keras.applications.resnet50.ResNet50(
 		include_top=True,
 		weights='imagenet',
@@ -42,13 +41,13 @@ def example_numpy(verbose=True):
 		dtype="numpy",verbose=False)
 
 	z = None
-	for epoch in range(100):
+	for epoch in range(1):
 		for image,y in dataloader_train:
 			if z is None:
 				z_dim = list(y.shape)
 				z_dim[1] = 1000 - z_dim[1]
 				z = np.zeros(z_dim)
-			y = np.concatenate(y,z,axis=1)
+			y = np.concatenate((y,z),axis=1)
 			model.fit(image,y)
 
 	# Test

{cell_data_loader-0.0.5 → cell_data_loader-0.0.7}/example_torch.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python3
 
-from cell_data_loader import CellDataloader
+from src.cell_data_loader import CellDataloader
 from torchvision.models import resnet50 #, ResNet50_Weights
 import torch
 import os
@@ -48,7 +48,7 @@ def example_torch(gpu_ids = None,verbose=True):
 		dtype="torch",
 		verbose=False, gpu_ids=gpu_ids)
 	if verbose: print("Beginning training")
-	n_epochs = 0
+	n_epochs = 1
 	for epoch in range(n_epochs):
 		l = 0
 		c = 0

{cell_data_loader-0.0.5 → cell_data_loader-0.0.7}/pyproject.toml

@@ -2,7 +2,7 @@
 
 name = "cell_data_loader"
 
-version = "0.0.5"
+version = "0.0.7"
 
 dependencies = [
   "numpy",

{cell_data_loader-0.0.5 → cell_data_loader-0.0.7}/src/cell_data_loader/cell_data_loader.py

@@ -12,6 +12,7 @@ from scipy.ndimage.interpolation import map_coordinates
 from scipy.ndimage.filters import gaussian_filter
 from .base_dataset import BaseDataset
 import torch,torchvision
+torchvision.disable_beta_transforms_warning()
 from .util import *
 import warnings
 
@@ -204,7 +205,8 @@ class CellDataloader():#BaseDataset):
 		cell_box_filelist = None,
 		n_channels = None,
 		channels_first = True,
-		match_labels=False):
+		match_labels=False,
+		normalize=True):
 		
 		self.verbose = verbose
 		self.label_balance = label_balance
@@ -221,19 +223,20 @@ class CellDataloader():#BaseDataset):
 		self.n_channels = n_channels
 		self.channels_first = channels_first
 		self.augment_image = augment_image
+		self.normalize = normalize
 		if self.augment_image and self.dtype == "torch":
-			from torchvision import transforms
+			import torchvision.transforms.v2 as transforms
 			
 			self.augment = transforms.Compose([
-				#transforms.ElasticTransform(),
 				transforms.RandomHorizontalFlip(0.5),
 				transforms.RandomVerticalFlip(0.5),
 				transforms.GaussianBlur(5),
 				transforms.ColorJitter(brightness=(0.5,1.5),
 					contrast=(1), saturation=(0.5,1.5), hue=(-0.1,0.1)),
-				transforms.RandomResizedCrop((self.dim))])#,
+				transforms.RandomResizedCrop(self.dim, antialias=True)])#,
 				#transforms.Normalize((0.5,0.5,0.5),(0.5,0.5,0.5))])
-		
+			self.augment2 = transforms.Compose([
+				transforms.ElasticTransform()])
 		"""
 		If true, outputs given labels equally. This may repeat images if the
 		counts of each don't line up.
@@ -304,12 +307,6 @@ class CellDataloader():#BaseDataset):
 				self.label_input_format = "Cell_Box_Regex"
 			elif self.cell_box_filelist is not None:
 				assert(len(all_filename_lists) == 1)
-				if self.image_folders is not None:
-					raise Exception(" To input cell boxes, files must be "+\
-						"input individually as a list and not as a folder, "+\
-						"to ensure one-to-one correspondence between the "+\
-						"box file and the image. Use the filelists parameter "+\
-						"instead of the image_folders parameter")
 				if isinstance(self.cell_box_filelist,str): # One image
 					self.cell_box_filelist = [[self.cell_box_filelist]]
 				elif isinstance(self.cell_box_filelist,list):
@@ -379,6 +376,11 @@ class CellDataloader():#BaseDataset):
 			self.n_channels = sample.get_n_channels()
 		else:
 			self.n_channels = n_channels
+		if self.normalize and self.dtype == "torch":
+			from torchvision.transforms.v2 import Normalize
+			self.normalizer = Normalize(
+				tuple([0.5 for _ in range(self.n_channels)]),
+				tuple([0.5 for _ in range(self.n_channels)]))
 		for image_object in self.image_objects:
 			image_object.n_channels = self.n_channels
 		if self.verbose:
@@ -471,8 +473,14 @@ class CellDataloader():#BaseDataset):
 			self.im_index = 0
 			self.index += 1
 			if self.index >= len(self.image_objects): break
-		if self.augment_image and self.dtype == "torch":
-			im = self.augment(im)
+		#if self.augment_image and self.dtype == "torch":
+			#imdim = im.size()
+			#print("imdim: %s" % str(imdim))
+			#im = torch.permute(im,[len(imdim)-1]+list(range(0,len(imdim)-1)))
+			#print("imdim: %s" % str(im.size()))
+			#im = self.augment(im)
+			#im = torch.permute(im,list(range(1,len(imdim))) + [0])
+			#print("imdim: %s"%str(im.size()))
 		if self.return_labels():
 			return im,label
 		else:
@@ -508,6 +516,22 @@ class CellDataloader():#BaseDataset):
 					b = torch.moveaxis(self.batch,-1,1)
 				else:
 					b = self.batch
+				if (self.augment_image and self.n_channels <= 3) or self.normalize:
+					if not self.channels_first:
+						b = torch.permute(b,[0,-1] + list(range(1,len(b.size())-1)))
+					#print("b.size(): %s" % str(b.size()))
+					if self.n_channels <= 3 and self.augment:
+						b = self.augment(b)
+						if self.dim[0] > 32 and self.dim[1] > 32:
+							b = self.augment2(b)
+					#print("b.size(): %s" % str(b.size()))
+					#print([0]+list(range(2,len(b.size()))) + [1])
+					if self.normalize:
+						b = self.normalizer(b)
+					if not self.channels_first:
+						b = torch.permute(b,[0]+list(range(2,len(b.size()))) + [1])
+					#print("b.size(): %s" % str(b.size()))
+					#print("sss")
 			elif self.dtype == "numpy":
 				if len(im.shape) == 2 and self.n_channels == 1:
 					im = np.expand_dims(im,axis=2)

{cell_data_loader-0.0.5 → cell_data_loader-0.0.7}/tests/unit_tests.py

@@ -8,7 +8,7 @@ import numpy as np
 import torch
 
 wd         = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-im_root    = os.path.join(wd,'data')
+im_root    = os.path.join(os.path.dirname(wd),'data')
 #imfile_svs = os.path.join(im_root,'10447627_1.svs')
 imfolder1  = os.path.join(im_root,'4173633_5')
 imfolder2  = os.path.join(im_root,'3368914_4_non_tumor')