cell-data-loader 0.0.20__tar.gz → 0.0.21__tar.gz

{cell_data_loader-0.0.20 → cell_data_loader-0.0.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: cell_data_loader
-Version: 0.0.20
+Version: 0.0.21
 Summary: Converts general images of cells into formats and labels for deep learning pipelines
 Project-URL: Homepage, https://github.com/mleming/CellDataLoader
 Author-email: Matt Leming <mleming@mgh.harvard.edu>
@@ -26,10 +26,21 @@ Cell Data Loader is a simple AI support tool in Python that can take in images o
 
 It can also be used for normal computer vision research, which is why CellPose is not a strict dependency.
 
+Quick Start
+-----------
+
 To install Cell Data Loader, simply type into a standard UNIX terminal
 
     pip install cell-data-loader
 
+To use a quick Jupyter Notebook example, navigate to [example/CellDataLoaderPlayground.ipynb](https://github.com/mleming/CellDataLoader/blob/main/example/CellDataLoaderPlayground.ipynb), download it, and run it in [Jupyter Notebook](https://jupyterlab.readthedocs.io/en/stable/getting_started/installation.html). On a Mac, that may be run with the following command:
+
+	python3 -m jupyterlab ~/Downloads/CellDataLoaderPlayground.ipynb
+
+
+Python
+------
+
 The simplest way to use Cell Data Loader is to instantiate a dataloader as such:
 
 ~~~python

{cell_data_loader-0.0.20 → cell_data_loader-0.0.21}/README.md

@@ -5,10 +5,21 @@ Cell Data Loader is a simple AI support tool in Python that can take in images o
 
 It can also be used for normal computer vision research, which is why CellPose is not a strict dependency.
 
+Quick Start
+-----------
+
 To install Cell Data Loader, simply type into a standard UNIX terminal
 
     pip install cell-data-loader
 
+To use a quick Jupyter Notebook example, navigate to [example/CellDataLoaderPlayground.ipynb](https://github.com/mleming/CellDataLoader/blob/main/example/CellDataLoaderPlayground.ipynb), download it, and run it in [Jupyter Notebook](https://jupyterlab.readthedocs.io/en/stable/getting_started/installation.html). On a Mac, that may be run with the following command:
+
+	python3 -m jupyterlab ~/Downloads/CellDataLoaderPlayground.ipynb
+
+
+Python
+------
+
 The simplest way to use Cell Data Loader is to instantiate a dataloader as such:
 
 ~~~python

cell_data_loader-0.0.21/example/CellDataLoaderPlayground.ipynb

@@ -0,0 +1,312 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "RcU4j0yvYQo5",
+    "outputId": "bf38507e-b5ca-4b15-f07f-c399ae53a388"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: cell-data-loader in /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages (0.0.18)\n",
+      "\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
+      "\u001b[31mERROR: Could not find a version that satisfies the requirement tkinter (from versions: none)\u001b[0m\u001b[31m\n",
+      "\u001b[0m\u001b[31mERROR: No matching distribution found for tkinter\u001b[0m\u001b[31m\n",
+      "\u001b[0m\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip3 install --no-deps cell-data-loader\n",
+    "!pip3 install tkinter"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 385
+    },
+    "id": "mqEfUykaam6g",
+    "outputId": "d4a384a1-27e3-43a9-a7fb-3f1956a0c799"
+   },
+   "outputs": [],
+   "source": [
+    "from cell_data_loader import CellDataloader\n",
+    "import torch,os,glob\n",
+    "from torchvision.models import resnet50 #, ResNet50_Weights\n",
+    "import tkinter as tk\n",
+    "import tkinter.filedialog"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Working directory to save model: /Users/mleming/Desktop/CellDataLoader_playground\n",
+      "Location of train data: /Users/mleming/Desktop/CellDataLoader_playground/dropbox_downloads/train\n",
+      "Location of test data: /Users/mleming/Desktop/CellDataLoader_playground/dropbox_downloads/test\n"
+     ]
+    }
+   ],
+   "source": [
+    "### CHANGE THESE LINES\n",
+    "wd = '/Users/mleming/Desktop/CellDataLoader_playground'\n",
+    "train_folder_name = os.path.join(wd,'dropbox_downloads','train')\n",
+    "test_folder_name = os.path.join(wd,'dropbox_downloads','test')\n",
+    "use_tkinter = True\n",
+    "root = tk.Tk()\n",
+    "if use_tkinter:\n",
+    "    print(\"Select your working directory to save the model\")\n",
+    "    wd = tk.filedialog.askdirectory(parent=root,title=\"Please select the directory to save the model\")\n",
+    "    print(\"Select the directory with the cell data\")\n",
+    "    train_folder_name = tk.filedialog.askdirectory(title=\"Please select the directory with training cell data\")\n",
+    "    test_folder_name = tk.filedialog.askdirectory(title=\"Please select the directory with test cell data\")\n",
+    "print(f\"Working directory to save model: {wd}\")\n",
+    "print(f\"Location of train data: {train_folder_name}\")\n",
+    "print(f\"Location of test data: {test_folder_name}\")\n",
+    "\n",
+    "## These strings are used to match between the files with one label and those with another\n",
+    "## In this case, if some folders have the word \"blurry\" and others have the word \"clear\",\n",
+    "## they will have labels 1 and 2. Note that CellDataLoader pushes a warning if any given\n",
+    "## path matches both strings (i.e., \"/this/path/is/Blurry/and/Clear/img.png\")\n",
+    "\n",
+    "label_regex_strings = [\"Blurry\",\"Clear\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "os.makedirs(wd, exist_ok = True)\n",
+    "\n",
+    "# Checkpoints\n",
+    "model_folder = os.path.join(wd,'checkpoints')\n",
+    "os.makedirs(model_folder,exist_ok=True)\n",
+    "model_file = os.path.join(model_folder,'torch_model.pt')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "metadata": {
+    "id": "LgDYXyklF9BX"
+   },
+   "outputs": [],
+   "source": [
+    "def train_torch(model,\n",
+    "                dataloader,\n",
+    "                model_file = None,\n",
+    "                epochs = 50,\n",
+    "                gpu_ids = None,\n",
+    "                verbose = True):\n",
+    "\n",
+    "    if gpu_ids is not None:\n",
+    "        model.to(gpu_ids)\n",
+    "\n",
+    "    # Train\n",
+    "\n",
+    "    model.train()\n",
+    "    #loss_fn = torch.nn.CrossEntropyLoss()\n",
+    "    loss_fn = torch.nn.NLLLoss()\n",
+    "    optimizer = torch.optim.Adam(model.parameters(), lr=1e-5)\n",
+    "    if verbose: print(\"Beginning training\")\n",
+    "    for epoch in range(epochs):\n",
+    "        loss_total,count = 0,0\n",
+    "        for image,y in dataloader_train:\n",
+    "            y_pred = model(image)\n",
+    "            y_pred = y_pred[:,:y.size()[1]]\n",
+    "            loss = loss_fn(y_pred, torch.argmax(y,1))\n",
+    "            loss_total += float(loss)\n",
+    "            count += 1\n",
+    "            optimizer.zero_grad()\n",
+    "            loss.backward()\n",
+    "            optimizer.step()\n",
+    "        if model_file is not None:\n",
+    "            torch.save(model,model_file)\n",
+    "        if verbose:\n",
+    "            print(\n",
+    "                \"Epoch {epoch:d}/{epochs:d}: loss: {loss:.5f}\".format(\n",
+    "                    epoch=epoch,epochs=epochs,loss=loss_total/count)\n",
+    "            )\n",
+    "\n",
+    "def test_torch(model,\n",
+    "               dataloader,\n",
+    "               gpu_ids = None,\n",
+    "               verbose=True):\n",
+    "    \n",
+    "    model.eval()\n",
+    "    total_images = 0\n",
+    "    sum_accuracy = 0\n",
+    "    for image,y in dataloader:\n",
+    "        total_images += image.size()[0]\n",
+    "        y_pred = model(image)\n",
+    "        y_pred = y_pred[:,:y.size()[1]]\n",
+    "        sum_accuracy += torch.sum(torch.argmax(y_pred,axis=1) == \\\n",
+    "            torch.argmax(y,axis=1))\n",
+    "    accuracy = sum_accuracy / total_images\n",
+    "    if verbose: print(\"Final accuracy: %.4f\" % accuracy)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 35
+    },
+    "id": "YxPqQuvcZMYJ",
+    "outputId": "cb3a8158-b807-48cb-a68f-bed0fb022ebc"
+   },
+   "outputs": [],
+   "source": [
+    "# Get a pretrained model from torchvision\n",
+    "if os.path.isfile(model_file):\n",
+    "    print(\"Loading %s\" % model_file)\n",
+    "    model = torch.load(model_file)\n",
+    "else:\n",
+    "    # Download a pretrained resnet and edit its outputs to be compatible with cross entropy loss\n",
+    "    model = resnet50(pretrained=True)\n",
+    "    #model.fc = torch.nn.Sequential(\n",
+    "    #    torch.nn.Dropout(0.5),\n",
+    "    #    torch.nn.Linear(2048, 1000)\n",
+    "    #)\n",
+    "\n",
+    "dataloader_train = CellDataloader(train_folder_name,label_regex=label_regex_strings,\n",
+    "    dtype = \"torch\",\n",
+    "    verbose = False,\n",
+    "    batch_size = 64,\n",
+    "    gpu_ids = None,\n",
+    "    n_channels=3)\n",
+    "\n",
+    "dataloader_test = CellDataloader(test_folder_name,label_regex=label_regex_strings,\n",
+    "    dtype = \"torch\",\n",
+    "    verbose = False,\n",
+    "    batch_size = 64,\n",
+    "    gpu_ids = None,\n",
+    "    n_channels=3)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Beginning training\n",
+      "Epoch 0/50: loss: 1.03203\n",
+      "Epoch 1/50: loss: 0.88285\n",
+      "Epoch 2/50: loss: 0.80278\n",
+      "Epoch 3/50: loss: 0.67617\n",
+      "Epoch 4/50: loss: 0.70672\n",
+      "Epoch 5/50: loss: 0.50680\n",
+      "Epoch 6/50: loss: 0.44564\n",
+      "Epoch 7/50: loss: 0.28783\n",
+      "Epoch 8/50: loss: 0.39078\n",
+      "Epoch 9/50: loss: 0.16406\n",
+      "Epoch 10/50: loss: 0.19954\n"
+     ]
+    }
+   ],
+   "source": [
+    "train_torch(model,dataloader_train,model_file)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "tensor([[ 2.0775, -0.3837, -2.3550,  ..., -0.5430,  1.8986, -0.2228],\n",
+      "        [ 2.0775, -0.3837, -2.3550,  ..., -0.5430,  1.8986, -0.2228],\n",
+      "        [ 2.0775, -0.3837, -2.3550,  ..., -0.5430,  1.8986, -0.2228],\n",
+      "        ...,\n",
+      "        [ 2.0775, -0.3837, -2.3550,  ..., -0.5430,  1.8986, -0.2228],\n",
+      "        [ 2.0775, -0.3837, -2.3550,  ..., -0.5430,  1.8986, -0.2228],\n",
+      "        [ 2.0775, -0.3837, -2.3550,  ..., -0.5430,  1.8986, -0.2228]],\n",
+      "       grad_fn=<AddmmBackward0>)\n",
+      "tensor([[ 2.3903, -1.0563, -2.1164,  ..., -0.8507,  1.2689, -0.6499],\n",
+      "        [ 2.3903, -1.0563, -2.1164,  ..., -0.8507,  1.2689, -0.6499],\n",
+      "        [ 2.3903, -1.0563, -2.1164,  ..., -0.8507,  1.2689, -0.6499],\n",
+      "        ...,\n",
+      "        [ 2.3903, -1.0563, -2.1164,  ..., -0.8507,  1.2689, -0.6499],\n",
+      "        [ 2.3903, -1.0563, -2.1164,  ..., -0.8507,  1.2689, -0.6499],\n",
+      "        [ 2.3903, -1.0563, -2.1164,  ..., -0.8507,  1.2689, -0.6499]],\n",
+      "       grad_fn=<AddmmBackward0>)\n",
+      "tensor([[ 0.0843, -0.1752, -5.3073,  ..., -2.1159, -1.4730, -0.1893],\n",
+      "        [ 0.0843, -0.1752, -5.3073,  ..., -2.1159, -1.4730, -0.1893],\n",
+      "        [ 0.0843, -0.1752, -5.3073,  ..., -2.1159, -1.4730, -0.1893],\n",
+      "        ...,\n",
+      "        [ 0.0843, -0.1752, -5.3073,  ..., -2.1159, -1.4730, -0.1893],\n",
+      "        [ 0.0843, -0.1752, -5.3073,  ..., -2.1159, -1.4730, -0.1893],\n",
+      "        [ 0.0843, -0.1752, -5.3073,  ..., -2.1159, -1.4730, -0.1893]],\n",
+      "       grad_fn=<AddmmBackward0>)\n",
+      "Final accuracy: 0.9375\n"
+     ]
+    }
+   ],
+   "source": [
+    "test_torch(model,dataloader_test)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "colab": {
+   "provenance": []
+  },
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

{cell_data_loader-0.0.20 → cell_data_loader-0.0.21}/pyproject.toml

@@ -2,7 +2,7 @@
 
 name = "cell_data_loader"
 
-version = "0.0.20"
+version = "0.0.21"
 
 dependencies = [
   "numpy==1.22.0",

{cell_data_loader-0.0.20 → cell_data_loader-0.0.21}/src/cell_data_loader/cell_data_loader.py

@@ -12,9 +12,8 @@ from scipy.ndimage.interpolation import map_coordinates
 from scipy.ndimage.filters import gaussian_filter
 from .base_dataset import BaseDataset
 import torch,torchvision
-torchvision.disable_beta_transforms_warning()
+#torchvision.disable_beta_transforms_warning()
 from .util import *
-import warnings
 
 
 #import openslide