cell-data-loader 0.0.25__tar.gz → 0.0.27__tar.gz

{cell_data_loader-0.0.25 → cell_data_loader-0.0.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cell_data_loader
-Version: 0.0.25
+Version: 0.0.27
 Summary: Converts general images of cells into formats and labels for deep learning pipelines
 Project-URL: Homepage, https://github.com/mleming/CellDataLoader
 Author-email: Matt Leming <mleming@mgh.harvard.edu>
@@ -8,7 +8,7 @@ Maintainer-email: Matt Leming <mleming@mgh.harvard.edu>
 License: MIT License
 Keywords: biomedical,cell,cell image,cellpose,csz,dataloader
 Requires-Python: >=3.8
-Requires-Dist: cell-data-loader
+Requires-Dist: hashlib
 Requires-Dist: numpy==1.22.0
 Requires-Dist: opencv-python>=4.5.4
 Requires-Dist: pillow
@@ -147,29 +147,3 @@ dataloader = CellDataloader(imfolder,
 			)
 ~~~
 
-
-Dependencies
-------------
-
-Note that the strict dependencies are automatically downloaded just with
-
-	pip install cell-data-loader
-
-However, to get support with cell-segmentation-specific images (i.e., segment="cell"), [CellPose](https://github.com/MouseLand/cellpose) needs to be installed. GPU integration with CellPose would also need to be handled separately.
-
-Strict dependencies:
-
-	numpy
-	torch
-	torchvision
-	opencv-python>=4.5.4
-	slideio==2.4.1
-	scipy
-	scikit-image
-	pillow
-
-Soft dependencies:
-
-	CellPose # For cell segmentation support
-	Tensorflow
-

{cell_data_loader-0.0.25 → cell_data_loader-0.0.27}/README.md

@@ -126,29 +126,3 @@ dataloader = CellDataloader(imfolder,
 			)
 ~~~
 
-
-Dependencies
-------------
-
-Note that the strict dependencies are automatically downloaded just with
-
-	pip install cell-data-loader
-
-However, to get support with cell-segmentation-specific images (i.e., segment="cell"), [CellPose](https://github.com/MouseLand/cellpose) needs to be installed. GPU integration with CellPose would also need to be handled separately.
-
-Strict dependencies:
-
-	numpy
-	torch
-	torchvision
-	opencv-python>=4.5.4
-	slideio==2.4.1
-	scipy
-	scikit-image
-	pillow
-
-Soft dependencies:
-
-	CellPose # For cell segmentation support
-	Tensorflow
-

{cell_data_loader-0.0.25 → cell_data_loader-0.0.27}/example/CellDataLoaderPlayground.ipynb

@@ -107,7 +107,8 @@
     "                model_file = None,\n",
     "                epochs = 50,\n",
     "                gpu_ids = None,\n",
-    "                verbose = True):\n",
+    "                verbose = True,\n",
+    "                loss_break=None):\n",
     "\n",
     "    if gpu_ids is not None:\n",
     "        model.to(gpu_ids)\n",
@@ -115,16 +116,17 @@
     "    # Train\n",
     "\n",
     "    model.train()\n",
-    "    #loss_fn = torch.nn.CrossEntropyLoss()\n",
-    "    loss_fn = torch.nn.NLLLoss()\n",
+    "    loss_fn = torch.nn.CrossEntropyLoss()\n",
+    "    #loss_fn = torch.nn.NLLLoss()\n",
     "    optimizer = torch.optim.Adam(model.parameters(), lr=1e-5)\n",
     "    if verbose: print(\"Beginning training\")\n",
     "    for epoch in range(epochs):\n",
     "        loss_total,count = 0,0\n",
-    "        for image,y in dataloader_train:\n",
+    "        for image,y in dataloader:\n",
     "            y_pred = model(image)\n",
-    "            y_pred = y_pred[:,:y.size()[1]]\n",
-    "            loss = loss_fn(y_pred, torch.argmax(y,1))\n",
+    "            y = torch.nn.functional.pad(y,(0,y_pred.size()[1]-y.size()[1]))\n",
+    "            assert(all(y.sum(1) == 1))\n",
+    "            loss = loss_fn(y_pred, y)#torch.argmax(y,1))\n",
     "            loss_total += float(loss)\n",
     "            count += 1\n",
     "            optimizer.zero_grad()\n",
@@ -137,6 +139,9 @@
     "                \"Epoch {epoch:d}/{epochs:d}: loss: {loss:.5f}\".format(\n",
     "                    epoch=epoch,epochs=epochs,loss=loss_total/count)\n",
     "            )\n",
+    "        if loss_break is not None and loss_total/count < loss_break:\n",
+    "            if verbose: print(\"Loss break - exiting\")\n",
+    "            return\n",
     "\n",
     "def test_torch(model,\n",
     "               dataloader,\n",
@@ -252,7 +257,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.0"
+   "version": "3.8.10"
   }
  },
  "nbformat": 4,

{cell_data_loader-0.0.25 → cell_data_loader-0.0.27}/pyproject.toml

@@ -2,7 +2,7 @@
 
 name = "cell_data_loader"
 
-version = "0.0.25"
+version = "0.0.27"
 
 dependencies = [
   "numpy==1.22.0",
@@ -13,7 +13,7 @@ dependencies = [
   "slideio==2.4.1",
   "scikit-image",
   "pillow",
-  "cell_data_loader"
+  "hashlib"
 ]
 requires-python = ">=3.8"
 

{cell_data_loader-0.0.25 → cell_data_loader-0.0.27}/src/cell_data_loader/__init__.py

@@ -1,3 +1,3 @@
-__version__ = "0.0.25"
+__version__ = "0.0.27"
 
 from .cell_data_loader import *

{cell_data_loader-0.0.25 → cell_data_loader-0.0.27}/src/cell_data_loader/cell_data_loader.py

@@ -14,7 +14,8 @@ from .base_dataset import BaseDataset
 import torch,torchvision
 #torchvision.disable_beta_transforms_warning()
 from .util import *
-
+import torchvision.transforms as transforms
+import hashlib
 
 #import openslide
 #import czifile
@@ -164,7 +165,8 @@ class ImageLabelObject(BaseDataset):
 			raise Exception("Invalid mode: %s" % self.mode)
 	def __getitem__(self,index):
 		if self.mode == "whole":
-			return self.get_image()
+			im = self.get_image()
+			return im
 		elif self.mode == "sliced":
 			x_dim,y_dim = self.get_scaled_dims()
 			x = index % (x_dim)
@@ -186,6 +188,9 @@ class ImageLabelObject(BaseDataset):
 			raise Exception("Unimplemented")
 		else:
 			raise Exception("Invalid mode: %s" % self.mode)
+	def clear(self):
+		del self.image
+		self.image = None
 
 class CellDataloader():#BaseDataset):
 	def __init__(self,
@@ -205,7 +210,8 @@ class CellDataloader():#BaseDataset):
 		n_channels = None,
 		channels_first = True,
 		match_labels=False,
-		normalize=True):
+		normalize=True,
+		split = None):
 		
 		self.verbose = verbose
 		self.label_balance = label_balance
@@ -224,8 +230,6 @@ class CellDataloader():#BaseDataset):
 		self.augment_image = augment_image
 		self.normalize = normalize
 		if self.augment_image and self.dtype == "torch":
-			import torchvision.transforms as transforms
-			
 			self.augment = transforms.Compose([
 				transforms.RandomHorizontalFlip(0.5),
 				transforms.RandomVerticalFlip(0.5),
@@ -338,6 +342,16 @@ class CellDataloader():#BaseDataset):
 		for j,all_filename_list in enumerate(all_filename_lists):
 			for i,filename in enumerate(all_filename_list):
 					if is_image_file(filename):
+						if split is not None:
+							s1 = split[0]
+							s2 = split[1]
+							assert isinstance(s1,list)
+							assert isinstance(s2,int)
+							assert all([isinstance(_,int) for _ in s1])
+							assert all([ _ < s2 for _ in s1])
+							assert all([ _ >= 0 for _ in s1])
+							if hashlib.sha256(filename.encode()) % s2 not in s1:
+								continue
 						skip = False
 						imlabel = ImageLabelObject(filename,
 									gpu_ids=self.gpu_ids,
@@ -486,6 +500,7 @@ class CellDataloader():#BaseDataset):
 		self.im_index += 1
 		while self.im_index >= len(self.image_objects[self.index]) or \
 				(len(self.image_objects[self.index]) == 0):
+			self.image_objects[self.index].clear()
 			self.im_index = 0
 			self.index += 1
 			if self.index >= len(self.image_objects): break
@@ -536,7 +551,7 @@ class CellDataloader():#BaseDataset):
 					if not self.channels_first:
 						b = torch.permute(b,[0,-1] + list(range(1,len(b.size())-1)))
 					#print("b.size(): %s" % str(b.size()))
-					if self.n_channels <= 3 and self.augment:
+					if self.n_channels <= 3 and self.augment_image:
 						b = self.augment(b)
 						if self.dim[0] > 32 and self.dim[1] > 32:
 							b = self.augment2(b)

{cell_data_loader-0.0.25 → cell_data_loader-0.0.27}/tests/unit_tests.py

@@ -81,7 +81,7 @@ class TestSimple(unittest.TestCase):
 			break
 		dataset = CellDataloader([imfolder2],imfolder3,verbose=False)
 		self.assertEqual(dataset.label_input_format,"Folder")
-		dataset = CellDataloader(imfolder1,label_regex=["Test","Train"],
+		dataset = CellDataloader(imfolder1,label_regex=["(?<!non\s)tumor","non tumor"],
 			verbose=False)
 		self.assertEqual(dataset.label_input_format,"Regex")
 		for xy in dataset:
@@ -121,26 +121,29 @@ class TestSimple(unittest.TestCase):
 					self.assertEqual(s[2],dim[1])
 					self.assertEqual(s[3],n_channels)
 					break
-	def test_example_numpy(self):
-		try:
-			import tensorflow
-			valid_tensorflow = True
-		except:
-			print("Tensorflow import failed -- skipping test")
-			valid_tensorflow = False
-		if valid_tensorflow:
-			from example_numpy import example_numpy
-			example_numpy()
-	def test_example_torch(self):
-		try:
-			import torch
-			valid_torch = True
-		except:
-			print("Torch import failed -- skipping test")
-			valid_torch = False
-		if valid_torch:
-			from example.example_torch import example_torch
-			example_torch()
+
+	"""	def test_example_numpy(self):
+			try:
+				import tensorflow
+				valid_tensorflow = True
+			except:
+				print("Tensorflow import failed -- skipping test")
+				valid_tensorflow = False
+			if valid_tensorflow:
+				from example_numpy import example_numpy
+				example_numpy()
+		def test_example_torch(self):
+			try:
+				import torch
+				valid_torch = True
+			except:
+				print("Torch import failed -- skipping test")
+				valid_torch = False
+			if valid_torch:
+				from example.example_torch import example_torch
+				example_torch()
+	"""
+
 	def test_label_match(self):
 		batch_size = 1
 		dim = (6,7)