cell-data-loader 0.0.4__tar.gz → 0.0.6__tar.gz

{cell_data_loader-0.0.4 → cell_data_loader-0.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: cell_data_loader
-Version: 0.0.4
+Version: 0.0.6
 Summary: Converts general images of cells into formats and labels for deep learning pipelines
 Project-URL: Homepage, https://github.com/mleming/CellDataLoader
 Author-email: Matt Leming <mleming@mgh.harvard.edu>

{cell_data_loader-0.0.4 → cell_data_loader-0.0.6}/example_numpy.py

@@ -1,5 +1,4 @@
 #!/usr/bin/python3
-
 from src.cell_data_loader import CellDataloader
 
 try:
@@ -10,7 +9,7 @@ except:
 	except:
 		raise Exception("Need valid tensorflow")
 
-import numpy
+import numpy as np
 import os
 
 def example_numpy(verbose=True):
@@ -20,13 +19,13 @@ def example_numpy(verbose=True):
 	the test/train folders in this script are the same -- you would need to 
 	have a separate train/test set in your own implementation
 	"""
-	wd = os.path.dirname(os.path.realpath(__file__))
+	wd = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 	imfolder1 = os.path.join(wd,'data',
 		'3368914_4_non_tumor')
 	imfolder2 = os.path.join(wd,'data',
 		'4173633_5')
 
-	model = resnet50()
+	#model = resnet50()
 	model = tf.keras.applications.resnet50.ResNet50(
 		include_top=True,
 		weights='imagenet',
@@ -42,13 +41,13 @@ def example_numpy(verbose=True):
 		dtype="numpy",verbose=False)
 
 	z = None
-	for epoch in range(100):
+	for epoch in range(1):
 		for image,y in dataloader_train:
 			if z is None:
 				z_dim = list(y.shape)
 				z_dim[1] = 1000 - z_dim[1]
 				z = np.zeros(z_dim)
-			y = np.concatenate(y,z,axis=1)
+			y = np.concatenate((y,z),axis=1)
 			model.fit(image,y)
 
 	# Test

{cell_data_loader-0.0.4 → cell_data_loader-0.0.6}/example_torch.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python3
 
-from cell_data_loader import CellDataloader
+from src.cell_data_loader import CellDataloader
 from torchvision.models import resnet50 #, ResNet50_Weights
 import torch
 import os
@@ -48,7 +48,7 @@ def example_torch(gpu_ids = None,verbose=True):
 		dtype="torch",
 		verbose=False, gpu_ids=gpu_ids)
 	if verbose: print("Beginning training")
-	n_epochs = 500
+	n_epochs = 1
 	for epoch in range(n_epochs):
 		l = 0
 		c = 0
@@ -65,7 +65,7 @@ def example_torch(gpu_ids = None,verbose=True):
 		if verbose:
 			print(
 				"Epoch {epoch:d}/{n_epochs:d}: loss: {loss:.5f}".format(
-					epoch=epoch,n_epochs=e_epochs,loss=l/c)
+					epoch=epoch,n_epochs=n_epochs,loss=l/c)
 			)
 	# Test
 	

{cell_data_loader-0.0.4 → cell_data_loader-0.0.6}/github_push.sh

@@ -3,7 +3,7 @@
 #git init
 git add README.md example_numpy.py example_torch.py src tests requirements.txt pyproject.toml
 #git rm src/cell_data_loader/slice_images_out.py src/cell_data_loader/cell_segmenter.py
-git commit -m "Updated example torch, added toml"
+git commit -m "Added augmentation"
 git branch -M main
 #git remote add origin https://github.com/mleming/CellDataLoader.git
 git push -u origin main

{cell_data_loader-0.0.4 → cell_data_loader-0.0.6}/pyproject.toml

@@ -2,7 +2,7 @@
 
 name = "cell_data_loader"
 
-version = "0.0.4"
+version = "0.0.6"
 
 dependencies = [
   "numpy",

{cell_data_loader-0.0.4 → cell_data_loader-0.0.6}/src/cell_data_loader/cell_data_loader.py

@@ -12,6 +12,7 @@ from scipy.ndimage.interpolation import map_coordinates
 from scipy.ndimage.filters import gaussian_filter
 from .base_dataset import BaseDataset
 import torch,torchvision
+torchvision.disable_beta_transforms_warning()
 from .util import *
 import warnings
 
@@ -204,7 +205,8 @@ class CellDataloader():#BaseDataset):
 		cell_box_filelist = None,
 		n_channels = None,
 		channels_first = True,
-		match_labels=False):
+		match_labels=False,
+		normalize=True):
 		
 		self.verbose = verbose
 		self.label_balance = label_balance
@@ -220,6 +222,21 @@ class CellDataloader():#BaseDataset):
 		self.dtype = dtype
 		self.n_channels = n_channels
 		self.channels_first = channels_first
+		self.augment_image = augment_image
+		self.normalize = normalize
+		if self.augment_image and self.dtype == "torch":
+			import torchvision.transforms.v2 as transforms
+			
+			self.augment = transforms.Compose([
+				transforms.RandomHorizontalFlip(0.5),
+				transforms.RandomVerticalFlip(0.5),
+				transforms.GaussianBlur(5),
+				transforms.ColorJitter(brightness=(0.5,1.5),
+					contrast=(1), saturation=(0.5,1.5), hue=(-0.1,0.1)),
+				transforms.RandomResizedCrop(self.dim, antialias=True)])#,
+				#transforms.Normalize((0.5,0.5,0.5),(0.5,0.5,0.5))])
+			self.augment2 = transforms.Compose([
+				transforms.ElasticTransform()])
 		"""
 		If true, outputs given labels equally. This may repeat images if the
 		counts of each don't line up.
@@ -365,6 +382,11 @@ class CellDataloader():#BaseDataset):
 			self.n_channels = sample.get_n_channels()
 		else:
 			self.n_channels = n_channels
+		if self.normalize and self.dtype == "torch":
+			from torchvision.transforms.v2 import Normalize
+			self.normalizer = Normalize(
+				tuple([0.5 for _ in range(self.n_channels)]),
+				tuple([0.5 for _ in range(self.n_channels)]))
 		for image_object in self.image_objects:
 			image_object.n_channels = self.n_channels
 		if self.verbose:
@@ -457,6 +479,14 @@ class CellDataloader():#BaseDataset):
 			self.im_index = 0
 			self.index += 1
 			if self.index >= len(self.image_objects): break
+		#if self.augment_image and self.dtype == "torch":
+			#imdim = im.size()
+			#print("imdim: %s" % str(imdim))
+			#im = torch.permute(im,[len(imdim)-1]+list(range(0,len(imdim)-1)))
+			#print("imdim: %s" % str(im.size()))
+			#im = self.augment(im)
+			#im = torch.permute(im,list(range(1,len(imdim))) + [0])
+			#print("imdim: %s"%str(im.size()))
 		if self.return_labels():
 			return im,label
 		else:
@@ -492,6 +522,22 @@ class CellDataloader():#BaseDataset):
 					b = torch.moveaxis(self.batch,-1,1)
 				else:
 					b = self.batch
+				if (self.augment_image and self.n_channels <= 3) or self.normalize:
+					if not self.channels_first:
+						b = torch.permute(b,[0,-1] + list(range(1,len(b.size())-1)))
+					#print("b.size(): %s" % str(b.size()))
+					if self.n_channels <= 3 and self.augment:
+						b = self.augment(b)
+						if self.dim[0] > 32 and self.dim[1] > 32:
+							b = self.augment2(b)
+					#print("b.size(): %s" % str(b.size()))
+					#print([0]+list(range(2,len(b.size()))) + [1])
+					if self.normalize:
+						b = self.normalizer(b)
+					if not self.channels_first:
+						b = torch.permute(b,[0]+list(range(2,len(b.size()))) + [1])
+					#print("b.size(): %s" % str(b.size()))
+					#print("sss")
 			elif self.dtype == "numpy":
 				if len(im.shape) == 2 and self.n_channels == 1:
 					im = np.expand_dims(im,axis=2)

{cell_data_loader-0.0.4 → cell_data_loader-0.0.6}/tests/unit_tests.py

@@ -8,7 +8,7 @@ import numpy as np
 import torch
 
 wd         = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-im_root    = os.path.join(wd,'data')
+im_root    = os.path.join(os.path.dirname(wd),'data')
 #imfile_svs = os.path.join(im_root,'10447627_1.svs')
 imfolder1  = os.path.join(im_root,'4173633_5')
 imfolder2  = os.path.join(im_root,'3368914_4_non_tumor')