cell-data-loader 0.0.23__tar.gz → 0.0.25__tar.gz

{cell_data_loader-0.0.23 → cell_data_loader-0.0.25}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: cell_data_loader
-Version: 0.0.23
+Version: 0.0.25
 Summary: Converts general images of cells into formats and labels for deep learning pipelines
 Project-URL: Homepage, https://github.com/mleming/CellDataLoader
 Author-email: Matt Leming <mleming@mgh.harvard.edu>

{cell_data_loader-0.0.23 → cell_data_loader-0.0.25}/example/CellDataLoaderPlayground.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/"
@@ -10,23 +10,7 @@
     "id": "RcU4j0yvYQo5",
     "outputId": "bf38507e-b5ca-4b15-f07f-c399ae53a388"
    },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Requirement already satisfied: cell-data-loader in /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages (0.0.18)\n",
-      "\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
-      "\u001b[31mERROR: Could not find a version that satisfies the requirement tkinter (from versions: none)\u001b[0m\u001b[31m\n",
-      "\u001b[0m\u001b[31mERROR: No matching distribution found for tkinter\u001b[0m\u001b[31m\n",
-      "\u001b[0m\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "!pip3 install --no-deps cell-data-loader\n",
     "!pip3 install tkinter"
@@ -34,7 +18,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 33,
+   "execution_count": null,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/",
@@ -54,13 +38,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 34,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
+      "Select your working directory to save the model\n",
+      "Select the directory with the train cell data\n",
+      "Select the directory with the test cell data\n",
       "Working directory to save model: /Users/mleming/Desktop/CellDataLoader_playground\n",
       "Location of train data: /Users/mleming/Desktop/CellDataLoader_playground/dropbox_downloads/train\n",
       "Location of test data: /Users/mleming/Desktop/CellDataLoader_playground/dropbox_downloads/test\n"
@@ -77,8 +64,9 @@
     "if use_tkinter:\n",
     "    print(\"Select your working directory to save the model\")\n",
     "    wd = tk.filedialog.askdirectory(parent=root,title=\"Please select the directory to save the model\")\n",
-    "    print(\"Select the directory with the cell data\")\n",
+    "    print(\"Select the directory with the train cell data\")\n",
     "    train_folder_name = tk.filedialog.askdirectory(title=\"Please select the directory with training cell data\")\n",
+    "    print(\"Select the directory with the test cell data\")\n",
     "    test_folder_name = tk.filedialog.askdirectory(title=\"Please select the directory with test cell data\")\n",
     "print(f\"Working directory to save model: {wd}\")\n",
     "print(f\"Location of train data: {train_folder_name}\")\n",
@@ -94,7 +82,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 35,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -108,7 +96,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 39,
+   "execution_count": 4,
    "metadata": {
     "id": "LgDYXyklF9BX"
    },
@@ -170,7 +158,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 40,
+   "execution_count": 8,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/",
@@ -179,7 +167,23 @@
     "id": "YxPqQuvcZMYJ",
     "outputId": "cb3a8158-b807-48cb-a68f-bed0fb022ebc"
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loading /Users/mleming/Desktop/CellDataLoader_playground/checkpoints/torch_model.pt\n",
+      "Preparing Train Data\n",
+      "Detected label format: Regex\n",
+      "427 image paths read\n",
+      "3 Channels Detected\n",
+      "Preparing Test Data\n",
+      "Detected label format: Regex\n",
+      "50 image paths read\n",
+      "3 Channels Detected\n"
+     ]
+    }
+   ],
    "source": [
     "# Get a pretrained model from torchvision\n",
     "if os.path.isfile(model_file):\n",
@@ -193,16 +197,18 @@
     "    #    torch.nn.Linear(2048, 1000)\n",
     "    #)\n",
     "\n",
+    "print(\"Preparing Train Data\")\n",
     "dataloader_train = CellDataloader(train_folder_name,label_regex=label_regex_strings,\n",
     "    dtype = \"torch\",\n",
-    "    verbose = False,\n",
+    "    verbose = True,\n",
     "    batch_size = 64,\n",
     "    gpu_ids = None,\n",
     "    n_channels=3)\n",
     "\n",
+    "print(\"Preparing Test Data\")\n",
     "dataloader_test = CellDataloader(test_folder_name,label_regex=label_regex_strings,\n",
     "    dtype = \"torch\",\n",
-    "    verbose = False,\n",
+    "    verbose = True,\n",
     "    batch_size = 64,\n",
     "    gpu_ids = None,\n",
     "    n_channels=3)"
@@ -212,77 +218,19 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Beginning training\n",
-      "Epoch 0/50: loss: 1.03203\n",
-      "Epoch 1/50: loss: 0.88285\n",
-      "Epoch 2/50: loss: 0.80278\n",
-      "Epoch 3/50: loss: 0.67617\n",
-      "Epoch 4/50: loss: 0.70672\n",
-      "Epoch 5/50: loss: 0.50680\n",
-      "Epoch 6/50: loss: 0.44564\n",
-      "Epoch 7/50: loss: 0.28783\n",
-      "Epoch 8/50: loss: 0.39078\n",
-      "Epoch 9/50: loss: 0.16406\n",
-      "Epoch 10/50: loss: 0.19954\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "train_torch(model,dataloader_train,model_file)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 32,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "tensor([[ 2.0775, -0.3837, -2.3550,  ..., -0.5430,  1.8986, -0.2228],\n",
-      "        [ 2.0775, -0.3837, -2.3550,  ..., -0.5430,  1.8986, -0.2228],\n",
-      "        [ 2.0775, -0.3837, -2.3550,  ..., -0.5430,  1.8986, -0.2228],\n",
-      "        ...,\n",
-      "        [ 2.0775, -0.3837, -2.3550,  ..., -0.5430,  1.8986, -0.2228],\n",
-      "        [ 2.0775, -0.3837, -2.3550,  ..., -0.5430,  1.8986, -0.2228],\n",
-      "        [ 2.0775, -0.3837, -2.3550,  ..., -0.5430,  1.8986, -0.2228]],\n",
-      "       grad_fn=<AddmmBackward0>)\n",
-      "tensor([[ 2.3903, -1.0563, -2.1164,  ..., -0.8507,  1.2689, -0.6499],\n",
-      "        [ 2.3903, -1.0563, -2.1164,  ..., -0.8507,  1.2689, -0.6499],\n",
-      "        [ 2.3903, -1.0563, -2.1164,  ..., -0.8507,  1.2689, -0.6499],\n",
-      "        ...,\n",
-      "        [ 2.3903, -1.0563, -2.1164,  ..., -0.8507,  1.2689, -0.6499],\n",
-      "        [ 2.3903, -1.0563, -2.1164,  ..., -0.8507,  1.2689, -0.6499],\n",
-      "        [ 2.3903, -1.0563, -2.1164,  ..., -0.8507,  1.2689, -0.6499]],\n",
-      "       grad_fn=<AddmmBackward0>)\n",
-      "tensor([[ 0.0843, -0.1752, -5.3073,  ..., -2.1159, -1.4730, -0.1893],\n",
-      "        [ 0.0843, -0.1752, -5.3073,  ..., -2.1159, -1.4730, -0.1893],\n",
-      "        [ 0.0843, -0.1752, -5.3073,  ..., -2.1159, -1.4730, -0.1893],\n",
-      "        ...,\n",
-      "        [ 0.0843, -0.1752, -5.3073,  ..., -2.1159, -1.4730, -0.1893],\n",
-      "        [ 0.0843, -0.1752, -5.3073,  ..., -2.1159, -1.4730, -0.1893],\n",
-      "        [ 0.0843, -0.1752, -5.3073,  ..., -2.1159, -1.4730, -0.1893]],\n",
-      "       grad_fn=<AddmmBackward0>)\n",
-      "Final accuracy: 0.9375\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "test_torch(model,dataloader_test)"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {

{cell_data_loader-0.0.23 → cell_data_loader-0.0.25}/pyproject.toml

@@ -2,7 +2,7 @@
 
 name = "cell_data_loader"
 
-version = "0.0.23"
+version = "0.0.25"
 
 dependencies = [
   "numpy==1.22.0",

{cell_data_loader-0.0.23 → cell_data_loader-0.0.25}/src/cell_data_loader/__init__.py

@@ -1,3 +1,3 @@
-__version__ = "0.0.20"
+__version__ = "0.0.25"
 
 from .cell_data_loader import *

{cell_data_loader-0.0.23 → cell_data_loader-0.0.25}/src/cell_data_loader/cell_data_loader.py

@@ -277,7 +277,16 @@ class CellDataloader():#BaseDataset):
 			assert(isinstance(label_regex,list) or isinstance(label_regex,str))
 			if isinstance(label_regex,list):
 				assert(len(label_regex) > 0)
-				self.label_regex = [re.compile(_) for _ in label_regex]
+				self.label_regex = []
+				for l in label_regex:
+					if isinstance(l,str):
+						self.label_regex.append(re.compile(l))
+					elif isinstance(l,list):
+						self.label_regex.append([re.compile(_) for _ in l])
+					else:
+						raise Exception(
+							f"Label regex {l} must be string or list"
+						)
 				if len(self.label_regex) == 1:
 					self.label_regex = self.label_regex[0]
 				self.n_labels = len(self.label_regex) + 1
@@ -329,6 +338,7 @@ class CellDataloader():#BaseDataset):
 		for j,all_filename_list in enumerate(all_filename_lists):
 			for i,filename in enumerate(all_filename_list):
 					if is_image_file(filename):
+						skip = False
 						imlabel = ImageLabelObject(filename,
 									gpu_ids=self.gpu_ids,
 									dtype=self.dtype,
@@ -338,6 +348,7 @@ class CellDataloader():#BaseDataset):
 							imlabel.label = j
 						elif self.label_input_format == "Regex":
 							imlabel.label = self.__matchitem__(filename)
+							if imlabel.label < 0: skip = True
 						elif self.label_input_format == "List":
 							imlabel.label = label_file_dict[filename]
 						elif self.label_input_format == "Cell_Box_Filelist":
@@ -345,6 +356,7 @@ class CellDataloader():#BaseDataset):
 						elif self.label_input_format == "Cell_Box_Regex":
 							raise Exception("Unimplemented")
 							imlabel.boxlabel = self.cell_im_regex(filename)
+						if skip: continue
 						self.image_objects.append(imlabel)
 		random.shuffle(self.image_objects)
 		if self.match_labels:
@@ -426,10 +438,15 @@ class CellDataloader():#BaseDataset):
 				Label input format must be regex, is currently %s
 				""" % self.label_input_format)
 		if isinstance(self.label_regex,list):
-			m = 0
+			m = -1
 			for i,reg in enumerate(self.label_regex):
-				if bool(reg.search(image_file)):
-					if m > 0:
+				if isinstance(reg,list):
+					expr = np.any(
+						[ bool(reg_.search(image_file)) for reg_ in reg ]
+					)
+				else: expr = bool(reg.search(image_file))
+				if expr:
+					if m > -1:
 						warnings.warn(
 							("Image file %s matches at"+\
 							" least two regular expressions") % image_file)