cell-data-loader 0.0.1__tar.gz → 0.0.3__tar.gz

{cell_data_loader-0.0.1 → cell_data_loader-0.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cell_data_loader
-Version: 0.0.1
+Version: 0.0.3
 Summary: Converts general images of cells into formats and labels for deep learning pipelines
 Project-URL: Homepage, https://github.com/mleming/CellDataLoader
 Author-email: Matt Leming <mleming@mgh.harvard.edu>
@@ -10,6 +10,8 @@ Keywords: biomedical,cell,cell image,cellpose,csz,dataloader
 Requires-Python: >=3.8
 Requires-Dist: numpy
 Requires-Dist: opencv-python>=4.5.4
+Requires-Dist: pillow
+Requires-Dist: scikit-image
 Requires-Dist: scipy
 Requires-Dist: slideio==2.4.1
 Requires-Dist: torch
@@ -23,7 +25,7 @@ Cell Data Loader is a simple AI support tool in Python that can take in images o
 
 To install Cell Data Loader, simply type into a standard UNIX terminal
 
-    pip install cell_data_loader
+    pip install cell-data-loader
 
 The simplest way to use Cell Data Loader is to instantiate a dataloader as such:
 
@@ -90,8 +92,10 @@ imfolder = '/path/to/folder'
 dataloader = CellDataloader(imfolder,
 			dim = (64,64),
 			batch_size = 32,
-			dtype = "numpy",
-			n_channels = 3, #This is detected in the first read image by default, if not provided; it re-samples all images to force this number of channels
+			dtype = "numpy", # Can also be "torch"
+			label_regex = None,
+			n_channels = 3, # This is detected in the first read image by default, if not provided; it re-samples all images to force this number of channels
+			match_labels = False, # Outputs proportional amounts of each label in the dataset
 			)
 ~~~
 
@@ -101,13 +105,22 @@ Dependencies
 
 Strict dependencies:
 
-> scipy
-> numpy
-> pytorch
+	numpy
+	torch
+	torchvision
+	opencv-python>=4.5.4
+	slideio==2.4.1
+	scipy
+	scikit-image
+	pillow
 
 Soft dependencies:
-> CellPose # For cell segmentation support
-> slideio # For reading .SVS and .CZI files
 
+	CellPose # For cell segmentation support
+	Tensorflow
 
-Note that some of the dependencies are not strict and vary depending on usage. Numpy and 
+Note that some of the dependencies are not strict and vary depending on usage. Numpy is a hard requirement, but Tensorflow is not if the user only uses the Torch capabilities. If the user attempts to load cell images in "cell" mode without a working Cellpose installation, CellDataLoader will throw an error. Cellpose needs be be installed separately to use "cell" mode:
+
+	pip install cellpose
+
+And GPU integration is a separate matter.

{cell_data_loader-0.0.1 → cell_data_loader-0.0.3}/README.md

@@ -5,7 +5,7 @@ Cell Data Loader is a simple AI support tool in Python that can take in images o
 
 To install Cell Data Loader, simply type into a standard UNIX terminal
 
-    pip install cell_data_loader
+    pip install cell-data-loader
 
 The simplest way to use Cell Data Loader is to instantiate a dataloader as such:
 
@@ -72,8 +72,10 @@ imfolder = '/path/to/folder'
 dataloader = CellDataloader(imfolder,
 			dim = (64,64),
 			batch_size = 32,
-			dtype = "numpy",
-			n_channels = 3, #This is detected in the first read image by default, if not provided; it re-samples all images to force this number of channels
+			dtype = "numpy", # Can also be "torch"
+			label_regex = None,
+			n_channels = 3, # This is detected in the first read image by default, if not provided; it re-samples all images to force this number of channels
+			match_labels = False, # Outputs proportional amounts of each label in the dataset
 			)
 ~~~
 
@@ -83,13 +85,22 @@ Dependencies
 
 Strict dependencies:
 
-> scipy
-> numpy
-> pytorch
+	numpy
+	torch
+	torchvision
+	opencv-python>=4.5.4
+	slideio==2.4.1
+	scipy
+	scikit-image
+	pillow
 
 Soft dependencies:
-> CellPose # For cell segmentation support
-> slideio # For reading .SVS and .CZI files
 
+	CellPose # For cell segmentation support
+	Tensorflow
 
-Note that some of the dependencies are not strict and vary depending on usage. Numpy and 
+Note that some of the dependencies are not strict and vary depending on usage. Numpy is a hard requirement, but Tensorflow is not if the user only uses the Torch capabilities. If the user attempts to load cell images in "cell" mode without a working Cellpose installation, CellDataLoader will throw an error. Cellpose needs be be installed separately to use "cell" mode:
+
+	pip install cellpose
+
+And GPU integration is a separate matter.

{cell_data_loader-0.0.1 → cell_data_loader-0.0.3}/example_numpy.py

@@ -1,16 +1,23 @@
 #!/usr/bin/python3
 
-from src.cell_loader_MLEMING import CellDataloader
+from src.cell_data_loader import CellDataloader
 from keras import tensorflow as tf
 import numpy
 import os
 
 def example_numpy():
+
+	"""
+	Replace whese with folders locally on your computer -- script will not work
+	without new folders.
+	"""
 	wd = os.path.dirname(os.path.realpath(__file__))
-	imfolder_test = os.path.join(wd,'data','3368914_4_non_tumor')
-	imfolder_train = os.path.join(wd,'data','4173633_5')
-	model = resnet50()
+	imfolder_test = os.path.join(os.path.dirname(wd),'cell_data_loader_data',
+		'3368914_4_non_tumor')
+	imfolder_train = os.path.join(os.path.dirname(wd),'cell_data_loader_data',
+		'4173633_5')
 
+	model = resnet50()
 	model = tf.keras.applications.resnet50.ResNet50(
 		include_top=True,
 		weights='imagenet',
@@ -36,7 +43,6 @@ def example_numpy():
 	# Test
 
 	model.eval()
-
 	dataloader_test = CellDataloader(imfolder_test,imfolder_train,dtype="torch",
 		verbose=False)
 	total_images = 0

{cell_data_loader-0.0.1 → cell_data_loader-0.0.3}/example_torch.py

@@ -1,14 +1,21 @@
 #!/usr/bin/python3
 
-from src.cell_loader_MLEMING import CellDataloader
+from src.cell_data_loader import CellDataloader
 from torchvision.models import resnet50
 import torch
 import os
 
 def example_torch():
+
+	"""
+	Replace whese with folders locally on your computer -- script will not work
+	without new folders.
+	"""
 	wd = os.path.dirname(os.path.realpath(__file__))
-	imfolder_test = os.path.join(wd,'data','3368914_4_non_tumor')
-	imfolder_train = os.path.join(wd,'data','4173633_5')
+	imfolder_test = os.path.join(os.path.dirname(wd),'cell_data_loader_data',
+		'3368914_4_non_tumor')
+	imfolder_train = os.path.join(os.path.dirname(wd),'cell_data_loader_data',
+		'4173633_5')
 	model = resnet50()
 
 	# Train

{cell_data_loader-0.0.1 → cell_data_loader-0.0.3}/github_push.sh

@@ -2,6 +2,7 @@
 
 #git init
 git add README.md example_numpy.py example_torch.py src tests requirements.txt
+git rm src/cell_data_loader/slice_images_out.py src/cell_data_loader/cell_segmenter.py
 git commit -m "First commit"
 git branch -M main
 git remote add origin https://github.com/mleming/CellDataLoader.git

cell_data_loader-0.0.3/pypi_push.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+rm -r dist
+python3 -m build --sdist
+twine upload dist/*

{cell_data_loader-0.0.1 → cell_data_loader-0.0.3}/pyproject.toml

@@ -2,7 +2,7 @@
 
 name = "cell_data_loader"
 
-version = "0.0.1"
+version = "0.0.3"
 
 dependencies = [
   "numpy",
@@ -10,9 +10,11 @@ dependencies = [
   "torch",
   "torchvision",
   "opencv-python>=4.5.4",
-  "slideio==2.4.1"
+  "slideio==2.4.1",
+  "scikit-image",
+  "pillow"
 ]
-requires-python = ">= 3.8"
+requires-python = ">=3.8"
 
 authors = [
   {name = "Matt Leming", email = "mleming@mgh.harvard.edu"}

{cell_data_loader-0.0.1 → cell_data_loader-0.0.3}/requirements.txt

@@ -4,3 +4,5 @@ torchvision
 opencv-python>=4.5.4
 slideio==2.4.1
 scipy
+scikit-image
+pillow

{cell_data_loader-0.0.1 → cell_data_loader-0.0.3}/src/cell_data_loader/cell_data_loader.py

@@ -174,9 +174,9 @@ class ImageLabelObject(BaseDataset):
 			return imslice
 		elif self.mode == "cell":
 			if len(self.get_cell_box_label()) == 0:
-				print("len self: %d" % len(self))
+				#print("len self: %d" % len(self))
 				return -1
-				raise Exception("No cells in this image — should not call")
+				#raise Exception("No cells in this image — should not call")
 				#warnings.warn("No cells found in %s" % self.filename)
 				#return -1
 			index = index % len(self.get_cell_box_label())
@@ -283,10 +283,8 @@ class CellDataloader():#BaseDataset):
 			try:
 				from cellpose import models,io
 			except:
-				raise Exception("""
-					Cellpose import failed -- need valid cellpose version to use
-					cell segmentation option.
-				""")
+				raise Exception("Cellpose import failed -- need valid "+\
+					"cellpose version to use cell segmentation option.")
 			if self.cell_box_regex is not None:
 				assert(len(all_filename_lists) == 1)
 				self.cell_box_regex = re.compile(self.cell_box_regex)
@@ -294,13 +292,11 @@ class CellDataloader():#BaseDataset):
 			elif self.cell_box_filelist is not None:
 				assert(len(all_filename_lists) == 1)
 				if self.image_folders is not None:
-					raise Exception("""
-						To input cell boxes, files must be input individually
-						as a list and not as a folder, to ensure one-to-one
-						correspondence between the box file and the image. Use
-						the filelists parameter instead of the image_folders
-						parameter
-					""")
+					raise Exception(" To input cell boxes, files must be "+\
+						"input individually as a list and not as a folder, "+\
+						"to ensure one-to-one correspondence between the "+\
+						"box file and the image. Use the filelists parameter "+\
+						"instead of the image_folders parameter")
 				if isinstance(self.cell_box_filelist,str): # One image
 					self.cell_box_filelist = [[self.cell_box_filelist]]
 				elif isinstance(self.cell_box_filelist,list):
@@ -312,9 +308,8 @@ class CellDataloader():#BaseDataset):
 						assert(len(all_filenames_list[i]) == len(l))
 						for filename in l:
 							if not os.path.isfile(filename):
-								raise Exception("""
-									File doesn't exist: %s
-								""" % filename)
+								raise Exception(
+									"File doesn't exist: %s" % filename)
 				self.label_input_format = "Cell_Box_Filelist"
 		
 		"""
@@ -364,7 +359,8 @@ class CellDataloader():#BaseDataset):
 			if self.dtype == "numpy":
 				self.label_batch = np.zeros((self.batch_size,self.n_labels))
 			elif self.dtype == "torch":
-				self.label_batch = torch.zeros(self.batch_size,self.n_labels)
+				self.label_batch = torch.zeros(self.batch_size,self.n_labels,
+					device = self.gpu_ids)
 		sample = self.image_objects[0]
 		if n_channels is None:
 			self.n_channels = sample.get_n_channels()
@@ -420,9 +416,9 @@ class CellDataloader():#BaseDataset):
 			for i,reg in enumerate(self.label_regex):
 				if bool(reg.search(image_file)):
 					if m > 0:
-						warnings.warn("""
-						Image file %s matches at least two regular expressions
-						""" % image_file)
+						warnings.warn(
+							("Image file %s matches at"+\
+							" least two regular expressions") % image_file)
 					m = i + 1
 			return m
 		else:
@@ -507,20 +503,3 @@ class CellDataloader():#BaseDataset):
 			return b,self.label_batch
 		else:
 			return b
-
-if __name__ == "__main__":
-	wd = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-	imfolder = '/home/mleming/Dropbox (Partners HealthCare)/9. DROSE'
-	dataset = CellDataloader(
-				[
-					os.path.join(imfolder,'raw_Test'),
-					os.path.join(imfolder,'raw_Train, Val')
-				],
-				segment_image = "whole")
-	#dataset = CellDataloader(imfolder,label_regex=["Test","Train"])
-	
-	for i,(x,y) in enumerate(dataset):
-		print(sys.getsizeof(x))
-		#print(i.label)
-		#print(i.filename)
-

{cell_data_loader-0.0.1 → cell_data_loader-0.0.3}/src/cell_data_loader/util.py

@@ -1,9 +1,7 @@
 #!/usr/bin/python
 
 import os,sys,json,csv,cv2,glob,random,re
-import xml.etree.ElementTree as ET
 import numpy as np
-import pickle
 from numpy.random import choice,shuffle
 from math import floor,ceil,sqrt
 from PIL import Image,ImageEnhance
@@ -18,28 +16,11 @@ import warnings
 try:
 	from cellpose import models, io
 except:
-	warnings.warn("""
-		Working Cellpose not installed -- cell segmentation not possible
-	""")
-#from deepcell.applications import CytoplasmSegmentation
-#import numpy as np
-#from PIL import Image
-#from shutil import copyfile
-#from model import *
-#from scipy import ndimage
-import skimage
-#import itertools
-#import torch
-#import skimage.measure
-#from torchvision import transforms
+	warnings.warn("Working Cellpose not installed -- cell segmentation not"+\
+	" possible")
 
+import skimage
 # DEFINE CELLPOSE MODEL
-# model_type='cyto' or model_type='nuclei'
-imagename = '/home/mleming/Desktop/Malaria_Cell_Project/data/AI_DATA_SET_UNZIPPED/TE201016063147_Mal_Cls2_201019/0258_21100_54800_005_23368.jpg'
-#imagename = '/home/mleming/Desktop/Malaria_Cell_Project/data/new_update_011421_UNZIPPED/TE201231032353/0533.jpg'
-#imagename = '/home/mleming/Desktop/Malaria_Cell_Project/data/new_update_011421_UNZIPPED/M2009090043/0833.jpg'
-imagename = '/home/mleming/Desktop/Malaria_Cell_Project/data/AI_DATA_SET_UNZIPPED/M2009091344/0583.jpg'
-imagename = '/home/mleming/Desktop/Malaria_Cell_Project/data/AI_DATA_SET_UNZIPPED/TE201231032353/0509.jpg'
 
 @contextmanager
 def suppress_stdout():
@@ -178,7 +159,8 @@ def slice_and_augment(im,x,y,l,w,
 	#new_image = PIL.ImageEnhance.Contrast(image).enhance(1.2)
 	if out_size is not None:
 		if dtype == "torch":
-			imslice = torch.permute(torchvision.transforms.Resize(out_size)(torch.permute(imslice,(2,0,1))),(1,2,0))
+			imslice = torch.permute(torchvision.transforms.Resize(
+				out_size)(torch.permute(imslice,(2,0,1))),(1,2,0))
 		elif dtype == "numpy":
 			imslice = cv2.resize(imslice,out_size[::-1])
 	return imslice
@@ -338,8 +320,8 @@ def remove_unsmooth(mask,thresh=25):
 	elif len(mask.shape) == 3:
 		for z in range(mask.shape[2]):			
 			smoothnesses[z] = get_smoothness(crop2(np.squeeze(mask[:,:,z])))
-	max_smoothness = smoothnesses.max()#max([smoothnesses[s] for s in smoothnesses])
-	min_smoothness = smoothnesses.min()#min([smoothnesses[s] for s in smoothnesses])
+	max_smoothness = smoothnesses.max()
+	min_smoothness = smoothnesses.min()
 	for i in range(mask.max()):
 		if smoothnesses[i] > thresh:
 			mask[mask == i+1] = 0

{cell_data_loader-0.0.1 → cell_data_loader-0.0.3}/tests/unit_tests.py

@@ -7,13 +7,12 @@ import os,sys,time
 import numpy as np
 import torch
 
-wd         = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
-im_root    = os.path.join(wd,'cell_data_loader_data')
+wd         = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+im_root    = os.path.join(os.path.dirname(wd),'cell_data_loader_data')
 #imfile_svs = os.path.join(im_root,'10447627_1.svs')
 imfolder1  = os.path.join(im_root,'4173633_5')
 imfolder2  = os.path.join(im_root,'3368914_4_non_tumor')
 imfolder3  = os.path.join(im_root,'H6-13_Mars1+dapi')
-
 sys.path.insert(0,os.path.join(wd))
 from src.cell_data_loader import *
 
@@ -152,6 +151,29 @@ class TestSimple(unittest.TestCase):
 			counts[int(np.argmax(y))] += 1
 		self.assertEqual(counts[0],counts[1])
 		self.assertEqual(counts[1],counts[2])
+	def test_gpu(self):
+		batch_size = 5
+		dim = (6,7)
+		for dtype in ["torch","numpy"]:
+			if dtype == "torch":
+				if not torch.cuda.is_available():
+					print("No available GPU for torch -- skipping")
+					continue
+				device = torch.cuda.current_device()
+			elif dtype == "numpy":
+				continue
+			dataset = CellDataloader(imfolder1,imfolder2,imfolder3,dim=dim,
+				batch_size=batch_size,dtype=dtype,match_labels=True,
+				gpu_ids = device)
+			for x,y in dataset:
+				if dtype == "torch":
+					self.assertEquals(device,x.device.index)
+					self.assertEquals(device,y.device.index)
+					break
+				elif dtype == "numpy":
+					break
+				else:
+					break
 
 if __name__ == "__main__":
 	unittest.main()