cell-data-loader 0.0.17__tar.gz → 0.0.18__tar.gz

{cell_data_loader-0.0.17 → cell_data_loader-0.0.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: cell_data_loader
-Version: 0.0.17
+Version: 0.0.18
 Summary: Converts general images of cells into formats and labels for deep learning pipelines
 Project-URL: Homepage, https://github.com/mleming/CellDataLoader
 Author-email: Matt Leming <mleming@mgh.harvard.edu>

cell_data_loader-0.0.18/example/CellDataLoaderPlayground.ipynb

@@ -0,0 +1,213 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": 2,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "RcU4j0yvYQo5",
+        "outputId": "bf38507e-b5ca-4b15-f07f-c399ae53a388"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Requirement already satisfied: cell-data-loader==0.0.17 in /usr/local/lib/python3.10/dist-packages (0.0.17)\n"
+          ]
+        }
+      ],
+      "source": [
+        "!pip3 install --no-deps cell-data-loader"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from cell_data_loader import CellDataloader\n",
+        "\n",
+        "imfolder = '../data'\n",
+        "\n",
+        "dataloader = CellDataloader(imfolder)\n"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 385
+        },
+        "id": "mqEfUykaam6g",
+        "outputId": "d4a384a1-27e3-43a9-a7fb-3f1956a0c799"
+      },
+      "execution_count": 3,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "/usr/local/lib/python3.10/dist-packages/cell_data_loader/util.py:19: UserWarning: Working Cellpose not installed -- cell segmentation not possible\n",
+            "  warnings.warn(\"Working Cellpose not installed -- cell segmentation not\"+\\\n"
+          ]
+        },
+        {
+          "output_type": "error",
+          "ename": "Exception",
+          "evalue": "Invalid string input: /path/to/my/images",
+          "traceback": [
+            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+            "\u001b[0;31mException\u001b[0m                                 Traceback (most recent call last)",
+            "\u001b[0;32m<ipython-input-3-0452ef4833b3>\u001b[0m in \u001b[0;36m<cell line: 5>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mimfolder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'/path/to/my/images'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mdataloader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCellDataloader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimfolder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/cell_data_loader/cell_data_loader.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, label_regex, label_file, segment_image, augment_image, dim, batch_size, verbose, dtype, gpu_ids, label_balance, cell_box_regex, cell_box_filelist, n_channels, channels_first, match_labels, normalize, *image_folders)\u001b[0m\n\u001b[1;32m    249\u001b[0m                 \u001b[0mduplicate_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    250\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mimg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mimage_folders\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 251\u001b[0;31m                         \u001b[0mflist\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_file_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    252\u001b[0m                         \u001b[0mflist_set\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mflist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    253\u001b[0m                         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mflist_set\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mintersection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mduplicate_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m>\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/cell_data_loader/util.py\u001b[0m in \u001b[0;36mget_file_list\u001b[0;34m(obj, allow_list_of_list)\u001b[0m\n\u001b[1;32m    856\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_file_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mallow_list_of_list\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    857\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 858\u001b[0;31m                 \u001b[0mobj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_file_list_from_str\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    859\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    860\u001b[0m                 \u001b[0mobj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_file_list_from_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/cell_data_loader/util.py\u001b[0m in \u001b[0;36mget_file_list_from_str\u001b[0;34m(obj)\u001b[0m\n\u001b[1;32m    833\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mall_filename_list\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    834\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 835\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Invalid string input: %s\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    836\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    837\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_file_list_from_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mallow_list_of_list\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+            "\u001b[0;31mException\u001b[0m: Invalid string input: /path/to/my/images"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from torchvision.models import resnet50 #, ResNet50_Weights\n",
+        "import torch\n",
+        "import os\n",
+        "import glob"
+      ],
+      "metadata": {
+        "id": "rIUibEMEF98J"
+      },
+      "execution_count": 4,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "def example_torch(gpu_ids = None,verbose=True):\n",
+        "\n",
+        "\t\"\"\"\n",
+        "\tReplace these folders with whatever folders of cells you may have. Note that\n",
+        "\tthe test/train folders in this script are the same -- you would need to\n",
+        "\thave a separate train/test set in your own implementation\n",
+        "\t\"\"\"\n",
+        "\n",
+        "\twd = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))\n",
+        "\tmodel_folder = os.path.join(wd,'checkpoints')\n",
+        "\tos.makedirs(model_folder,exist_ok=True)\n",
+        "\tmodel_file = os.path.join(model_folder,'torch_model.pt')\n",
+        "\n",
+        "\t# Training data folders\n",
+        "\timfolder1_train = os.path.join(wd,'data',\n",
+        "\t\t'3368914_4_non_tumor')\n",
+        "\timfolder2_train = os.path.join(wd,'data',\n",
+        "\t\t'4173633_5')\n",
+        "\n",
+        "\t# Test data folders\n",
+        "\timfolder1_test = os.path.join(wd,'data','3368914_4_non_tumor')\n",
+        "\timfolder2_test = os.path.join(wd,'data','4173633_5')\n",
+        "\n",
+        "\t# Get a pretrained model from torchvision\n",
+        "\tif os.path.isfile(model_file):\n",
+        "\t\tif verbose: print(\"Loading %s\" % model_file)\n",
+        "\t\tmodel = torch.load(model_file)\n",
+        "\telse:\n",
+        "\t\tmodel = resnet50(pretrained=True)\n",
+        "\n",
+        "\tif gpu_ids is not None:\n",
+        "\t\tmodel.to(gpu_ids)\n",
+        "\n",
+        "\t# Train\n",
+        "\n",
+        "\tmodel.train()\n",
+        "\tloss_fn = torch.nn.MSELoss(reduction='sum')\n",
+        "\toptimizer = torch.optim.Adam(model.parameters(), lr=1e-5)\n",
+        "\tdataloader_train = CellDataloader(imfolder1_train,imfolder2_train,\n",
+        "\t\tdtype=\"torch\",\n",
+        "\t\tverbose=False, gpu_ids=gpu_ids)\n",
+        "\tif verbose: print(\"Beginning training\")\n",
+        "\tn_epochs = 1\n",
+        "\tfor epoch in range(n_epochs):\n",
+        "\t\tl = 0\n",
+        "\t\tc = 0\n",
+        "\t\tfor image,y in dataloader_train:\n",
+        "\t\t\ty_pred = model(image)\n",
+        "\t\t\ty_pred = y_pred[:,:y.size()[1]]\n",
+        "\t\t\tloss = loss_fn(y_pred, y)\n",
+        "\t\t\tl += float(loss)\n",
+        "\t\t\tc += 1\n",
+        "\t\t\toptimizer.zero_grad()\n",
+        "\t\t\tloss.backward()\n",
+        "\t\t\toptimizer.step()\n",
+        "\t\ttorch.save(model,model_file)\n",
+        "\t\tif verbose:\n",
+        "\t\t\tprint(\n",
+        "\t\t\t\t\"Epoch {epoch:d}/{n_epochs:d}: loss: {loss:.5f}\".format(\n",
+        "\t\t\t\t\tepoch=epoch,n_epochs=n_epochs,loss=l/c)\n",
+        "\t\t\t)\n",
+        "\t# Test\n",
+        "\n",
+        "\tmodel.eval()\n",
+        "\n",
+        "\tdataloader_test = CellDataloader(imfolder1_test,imfolder2_test,\n",
+        "\t\tdtype=\"torch\",verbose=False, gpu_ids = gpu_ids)\n",
+        "\ttotal_images = 0\n",
+        "\tsum_accuracy = 0\n",
+        "\tfor image,y in dataloader_test:\n",
+        "\t\ttotal_images += image.size()[0]\n",
+        "\t\ty_pred = model(image)\n",
+        "\t\ty_pred = y_pred[:,:y.size()[1]]\n",
+        "\t\tsum_accuracy += torch.sum(torch.argmax(y_pred,axis=1) == \\\n",
+        "\t\t\ttorch.argmax(y,axis=1))\n",
+        "\taccuracy = sum_accuracy / total_images\n",
+        "\tif verbose: print(\"Final accuracy: %.4f\" % accuracy)\n"
+      ],
+      "metadata": {
+        "id": "LgDYXyklF9BX"
+      },
+      "execution_count": 5,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 35
+        },
+        "id": "YxPqQuvcZMYJ",
+        "outputId": "cb3a8158-b807-48cb-a68f-bed0fb022ebc"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "'0.14.1+cu117'"
+            ],
+            "application/vnd.google.colaboratory.intrinsic+json": {
+              "type": "string"
+            }
+          },
+          "metadata": {},
+          "execution_count": 6
+        }
+      ]
+    }
+  ]
+}

{cell_data_loader-0.0.17 → cell_data_loader-0.0.18}/pyproject.toml

@@ -2,7 +2,7 @@
 
 name = "cell_data_loader"
 
-version = "0.0.17"
+version = "0.0.18"
 
 dependencies = [
   "numpy==1.22.0",

{cell_data_loader-0.0.17 → cell_data_loader-0.0.18}/src/cell_data_loader/__init__.py

@@ -1,3 +1,3 @@
-__version__ = "0.0.17"
+__version__ = "0.0.19"
 
 from .cell_data_loader import *

{cell_data_loader-0.0.17 → cell_data_loader-0.0.18}/src/cell_data_loader/cell_data_loader.py

@@ -281,7 +281,7 @@ class CellDataloader():#BaseDataset):
 				self.label_regex = [re.compile(_) for _ in label_regex]
 				if len(self.label_regex) == 1:
 					self.label_regex = self.label_regex[0]
-				self.n_labels = len(self.label_regex)
+				self.n_labels = len(self.label_regex) + 1
 			else:
 				self.label_regex = re.compile(label_regex)
 				self.n_labels = 1
@@ -434,7 +434,7 @@ class CellDataloader():#BaseDataset):
 						warnings.warn(
 							("Image file %s matches at"+\
 							" least two regular expressions") % image_file)
-					m = i + 1
+					m = i
 			return m
 		else:
 			if bool(self.label_regex.search(image_file)):