cell-data-loader 0.0.26__tar.gz → 0.0.28__tar.gz

{cell_data_loader-0.0.26 → cell_data_loader-0.0.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cell_data_loader
-Version: 0.0.26
+Version: 0.0.28
 Summary: Converts general images of cells into formats and labels for deep learning pipelines
 Project-URL: Homepage, https://github.com/mleming/CellDataLoader
 Author-email: Matt Leming <mleming@mgh.harvard.edu>
@@ -8,7 +8,7 @@ Maintainer-email: Matt Leming <mleming@mgh.harvard.edu>
 License: MIT License
 Keywords: biomedical,cell,cell image,cellpose,csz,dataloader
 Requires-Python: >=3.8
-Requires-Dist: cell-data-loader
+Requires-Dist: hashlib
 Requires-Dist: numpy==1.22.0
 Requires-Dist: opencv-python>=4.5.4
 Requires-Dist: pillow

{cell_data_loader-0.0.26 → cell_data_loader-0.0.28}/pyproject.toml

@@ -2,7 +2,7 @@
 
 name = "cell_data_loader"
 
-version = "0.0.26"
+version = "0.0.28"
 
 dependencies = [
   "numpy==1.22.0",
@@ -13,7 +13,7 @@ dependencies = [
   "slideio==2.4.1",
   "scikit-image",
   "pillow",
-  "cell_data_loader"
+  "hashlib"
 ]
 requires-python = ">=3.8"
 

{cell_data_loader-0.0.26 → cell_data_loader-0.0.28}/src/cell_data_loader/__init__.py

@@ -1,3 +1,3 @@
-__version__ = "0.0.26"
+__version__ = "0.0.28"
 
 from .cell_data_loader import *

{cell_data_loader-0.0.26 → cell_data_loader-0.0.28}/src/cell_data_loader/cell_data_loader.py

@@ -15,6 +15,7 @@ import torch,torchvision
 #torchvision.disable_beta_transforms_warning()
 from .util import *
 import torchvision.transforms as transforms
+import hashlib
 
 #import openslide
 #import czifile
@@ -209,7 +210,9 @@ class CellDataloader():#BaseDataset):
 		n_channels = None,
 		channels_first = True,
 		match_labels=False,
-		normalize=True):
+		normalize=True,
+		split = None,
+		return_filenames = False):
 		
 		self.verbose = verbose
 		self.label_balance = label_balance
@@ -227,6 +230,7 @@ class CellDataloader():#BaseDataset):
 		self.channels_first = channels_first
 		self.augment_image = augment_image
 		self.normalize = normalize
+		self.return_filenames = return_filenames
 		if self.augment_image and self.dtype == "torch":
 			self.augment = transforms.Compose([
 				transforms.RandomHorizontalFlip(0.5),
@@ -340,6 +344,18 @@ class CellDataloader():#BaseDataset):
 		for j,all_filename_list in enumerate(all_filename_lists):
 			for i,filename in enumerate(all_filename_list):
 					if is_image_file(filename):
+						if split is not None:
+							s1 = split[0] # List of numbers
+							s2 = split[1] # Max number
+							assert isinstance(s1,list)
+							assert isinstance(s2,int)
+							assert all([isinstance(_,int) for _ in s1])
+							assert all([ _ < s2 for _ in s1])
+							assert all([ _ >= 0 for _ in s1])
+							h = hashlib.sha256(filename.encode()).hexdigest()
+							h = int(h,16)
+							if h % s2 not in s1:
+								continue
 						skip = False
 						imlabel = ImageLabelObject(filename,
 									gpu_ids=self.gpu_ids,
@@ -484,7 +500,8 @@ class CellDataloader():#BaseDataset):
 			raise StopIteration
 		#assert(len(self.image_objects[self.index]) > 0)
 		im = self.image_objects[self.index][self.im_index]
-		if self.return_labels(): label = self.image_objects[self.index].label
+		label = self.image_objects[self.index].label
+		fname = self.image_objects[self.index].filename
 		self.im_index += 1
 		while self.im_index >= len(self.image_objects[self.index]) or \
 				(len(self.image_objects[self.index]) == 0):
@@ -500,19 +517,19 @@ class CellDataloader():#BaseDataset):
 			#im = self.augment(im)
 			#im = torch.permute(im,list(range(1,len(imdim))) + [0])
 			#print("imdim: %s"%str(im.size()))
-		if self.return_labels():
-			return im,label
-		else:
-			return im
+		return im,label,fname
 	def __next__(self):
 		"""
 		Returns the next batch of images
 		"""
+		if self.return_filenames:
+			fnames = []
 		for i in range(self.batch_size):
 			if self.return_labels():
-				im,y = self.next_im()
+				im,y,fname = self.next_im()
+				
 				while isinstance(im,int) and im == -1:
-					im,y = self.next_im()
+					im,y,fname = self.next_im()
 				if self.n_labels == 0:
 					raise Exception(
 						"Cannot return labels with self.n_labels as 0")
@@ -523,9 +540,10 @@ class CellDataloader():#BaseDataset):
 						self.label_batch[i,j] = 0
 					self.label_batch[i,y] = 1
 			else:
-				im = self.next_im()
+				im,y,fname = self.next_im()
 				while isinstance(im,int) and im == -1:
-					im = self.next_im()
+					im,y,fname = self.next_im()
+			if self.return_filenames: fnames.append(fname)
 			assert(not isinstance(im,int))
 			if self.dtype == "torch":
 				if len(im.size()) == 2 and self.n_channels == 1:
@@ -561,7 +579,13 @@ class CellDataloader():#BaseDataset):
 					b = self.batch
 			else:
 				raise Exception("Unimplemented dtype: %s" % self.dtype)
+		if self.return_filenames: assert len(fnames) == self.batch_size
+		r = [b]
+		
 		if self.return_labels():
-			return b,self.label_batch
-		else:
-			return b
+			r.append(self.label_batch)
+		if self.return_filenames:
+			r.append(fnames)
+		if len(r) == 1: r = r[0]
+		else: r = tuple(r)
+		return r

{cell_data_loader-0.0.26 → cell_data_loader-0.0.28}/tests/unit_tests.py

@@ -100,6 +100,36 @@ class TestSimple(unittest.TestCase):
 				self.assertEqual(s[1],dim[0])
 				self.assertEqual(s[2],dim[1])
 				if i > 10: break
+
+	def test_split(self):
+		batch_size = 5
+		dim = (6,7)
+		for dtype in ["torch","numpy"]:
+			dataset = CellDataloader(imfolder3,dim=dim,batch_size=batch_size,
+				segment_image="cell",verbose=False,dtype=dtype,
+				channels_first=False,split=[[1,2,3],6])
+			for i,xy in enumerate(dataset):
+				s = xy.shape
+				self.assertEqual(s[0],batch_size)
+				self.assertEqual(s[1],dim[0])
+				self.assertEqual(s[2],dim[1])
+				if i > 10: break
+
+	def test_fname(self):
+		batch_size = 5
+		dim = (6,7)
+		for dtype in ["torch","numpy"]:
+			dataset = CellDataloader(imfolder3,dim=dim,batch_size=batch_size,
+				segment_image="whole",verbose=False,dtype=dtype,
+				channels_first=False,return_filenames=True)
+			for i,(xy,fname) in enumerate(dataset):
+				s = xy.shape
+				self.assertEqual(s[0],batch_size)
+				self.assertEqual(s[1],dim[0])
+				self.assertEqual(s[2],dim[1])
+				self.assertEqual(all([os.path.isfile(f) for f in fname]),True)
+				if i > 10: break
+
 	def test_channel_resample(self):
 		batch_size = 5
 		dim = (6,7)