cell-data-loader 0.0.17__tar.gz → 0.0.20__tar.gz

{cell_data_loader-0.0.17 → cell_data_loader-0.0.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: cell_data_loader
-Version: 0.0.17
+Version: 0.0.20
 Summary: Converts general images of cells into formats and labels for deep learning pipelines
 Project-URL: Homepage, https://github.com/mleming/CellDataLoader
 Author-email: Matt Leming <mleming@mgh.harvard.edu>
@@ -8,6 +8,7 @@ Maintainer-email: Matt Leming <mleming@mgh.harvard.edu>
 License: MIT License
 Keywords: biomedical,cell,cell image,cellpose,csz,dataloader
 Requires-Python: >=3.8
+Requires-Dist: cell-data-loader
 Requires-Dist: numpy==1.22.0
 Requires-Dist: opencv-python>=4.5.4
 Requires-Dist: pillow
@@ -15,7 +16,7 @@ Requires-Dist: scikit-image
 Requires-Dist: scipy
 Requires-Dist: slideio==2.4.1
 Requires-Dist: torch>=2.0.0
-Requires-Dist: torchvision>=0.15.0
+Requires-Dist: torchvision==0.14.0
 Description-Content-Type: text/markdown
 
 Cell Data Loader

cell_data_loader-0.0.20/example/CellDataLoaderPlayground.ipynb

@@ -0,0 +1,293 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "RcU4j0yvYQo5",
+    "outputId": "bf38507e-b5ca-4b15-f07f-c399ae53a388"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: cell-data-loader in /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages (0.0.18)\n",
+      "\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip3 install --no-deps cell-data-loader\n",
+    "!pip2 install tkinter"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 385
+    },
+    "id": "mqEfUykaam6g",
+    "outputId": "d4a384a1-27e3-43a9-a7fb-3f1956a0c799"
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/cell_data_loader/util.py:19: UserWarning: Working Cellpose not installed -- cell segmentation not possible\n",
+      "  warnings.warn(\"Working Cellpose not installed -- cell segmentation not\"+\\\n"
+     ]
+    }
+   ],
+   "source": [
+    "from cell_data_loader import CellDataloader\n",
+    "import torch,os,glob\n",
+    "from torchvision.models import resnet50 #, ResNet50_Weights\n",
+    "import tkinter as tk\n",
+    "import tkinter.filedialog\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Working directory to save model: /Users/mleming/Desktop/CellDataLoader_playground\n",
+      "Location of data: /Users/mleming/Desktop/CellDataLoader_playground/dropbox_downloads\n"
+     ]
+    }
+   ],
+   "source": [
+    "### CHANGE THESE LINES\n",
+    "wd = '/Users/mleming/Desktop/CellDataLoader_playground'\n",
+    "local_folder_name = os.path.join(wd,'dropbox_downloads')\n",
+    "use_tkinter = True\n",
+    "root = tk.Tk()\n",
+    "if use_tkinter:\n",
+    "    print(\"Select your working directory to save the model\")\n",
+    "    wd = tk.filedialog.askdirectory(parent=root,title=\"Please select the directory to save the model\")\n",
+    "    print(\"Select the directory with the cell data\")\n",
+    "    local_folder_name = tk.filedialog.askdirectory(title=\"Please select the directory with cell data\")\n",
+    "print(f\"Working directory to save model: {wd}\")\n",
+    "print(f\"Location of data: {local_folder_name}\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "os.makedirs(wd, exist_ok = True)\n",
+    "os.makedirs(local_folder_name, exist_ok = True)\n",
+    "\n",
+    "# Checkpoints\n",
+    "model_folder = os.path.join(wd,'checkpoints')\n",
+    "os.makedirs(model_folder,exist_ok=True)\n",
+    "model_file = os.path.join(model_folder,'torch_model.pt')\n",
+    "\n",
+    "# Training data folders\n",
+    "imfolder1_train = glob.glob(os.path.join(local_folder_name,'*','Blurry'))\n",
+    "imfolder2_train =  glob.glob(os.path.join(local_folder_name,'*','Clear'))\n",
+    "\n",
+    "imfolder1_test = os.path.join(local_folder_name,'3368914_4_non_tumor')\n",
+    "imfolder2_test = os.path.join(local_folder_name,'4173633_5')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "id": "LgDYXyklF9BX"
+   },
+   "outputs": [],
+   "source": [
+    "def train_torch(model,\n",
+    "                dataloader,\n",
+    "                model_file = None,\n",
+    "                epochs = 50,\n",
+    "                gpu_ids = None,\n",
+    "                verbose = True):\n",
+    "\n",
+    "    if gpu_ids is not None:\n",
+    "        model.to(gpu_ids)\n",
+    "\n",
+    "    # Train\n",
+    "\n",
+    "    model.train()\n",
+    "    loss_fn = torch.nn.CrossEntropyLoss()\n",
+    "    optimizer = torch.optim.Adam(model.parameters(), lr=1e-5)\n",
+    "    if verbose: print(\"Beginning training\")\n",
+    "    for epoch in range(epochs):\n",
+    "        loss_total,count = 0,0\n",
+    "        for image,y in dataloader_train:\n",
+    "            y_pred = model(image)\n",
+    "            y_pred = y_pred[:,:y.size()[1]]\n",
+    "            \n",
+    "            loss = loss_fn(y_pred, torch.argmax(y,1))\n",
+    "            loss_total += float(loss)\n",
+    "            count += 1\n",
+    "            optimizer.zero_grad()\n",
+    "            loss.backward()\n",
+    "            optimizer.step()\n",
+    "        if model_file is not None:\n",
+    "            torch.save(model,model_file)\n",
+    "        if verbose:\n",
+    "            print(\n",
+    "                \"Epoch {epoch:d}/{epochs:d}: loss: {loss:.5f}\".format(\n",
+    "                    epoch=epoch,epochs=epochs,loss=loss_total/count)\n",
+    "            )\n",
+    "\n",
+    "def test_torch(model,\n",
+    "               dataloader,\n",
+    "               gpu_ids = None,\n",
+    "               verbose=True):\n",
+    "    \n",
+    "    model.eval()\n",
+    "    total_images = 0\n",
+    "    sum_accuracy = 0\n",
+    "    for image,y in dataloader:\n",
+    "        total_images += image.size()[0]\n",
+    "        y_pred = model(image)\n",
+    "        y_pred = y_pred[:,:y.size()[1]]\n",
+    "        sum_accuracy += torch.sum(torch.argmax(y_pred,axis=1) == \\\n",
+    "            torch.argmax(y,axis=1))\n",
+    "    accuracy = sum_accuracy / total_images\n",
+    "    if verbose: print(\"Final accuracy: %.4f\" % accuracy)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 35
+    },
+    "id": "YxPqQuvcZMYJ",
+    "outputId": "cb3a8158-b807-48cb-a68f-bed0fb022ebc"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loading /Users/mleming/Desktop/CellDataLoader_playground/checkpoints/torch_model.pt\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Get a pretrained model from torchvision\n",
+    "if os.path.isfile(model_file):\n",
+    "    print(\"Loading %s\" % model_file)\n",
+    "    model = torch.load(model_file)\n",
+    "else:\n",
+    "    model = resnet50(pretrained=True)\n",
+    "\n",
+    "dataloader_train = CellDataloader(local_folder_name,label_regex=[\"Blurry\",\"Clear\"],\n",
+    "    dtype = \"torch\",\n",
+    "    verbose = False,\n",
+    "    batch_size = 64,\n",
+    "    gpu_ids = None)\n",
+    "\n",
+    "dataloader_test = "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Beginning training\n",
+      "torch.Size([64, 3, 64, 64])\n",
+      "tensor([0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638,\n",
+      "        0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638,\n",
+      "        0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638,\n",
+      "        0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638,\n",
+      "        0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638,\n",
+      "        0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638,\n",
+      "        0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638, 0.0638,\n",
+      "        0.0638])\n",
+      "torch.Size([64, 3, 64, 64])\n",
+      "tensor([0.8844, 0.8844, 0.8844, 0.8767, 0.8844, 0.8844, 0.8765, 0.8768, 0.8844,\n",
+      "        0.8874, 0.8844, 0.8844, 0.8844, 0.8844, 0.8844, 0.8868, 0.8844, 0.8844,\n",
+      "        0.8844, 0.8844, 0.8844, 0.8844, 0.8844, 0.8844, 0.8880, 0.8844, 0.8844,\n",
+      "        0.8869, 0.8844, 0.8844, 0.8844, 0.8844, 0.8844, 0.8844, 0.8873, 0.8844,\n",
+      "        0.8844, 0.8844, 0.8844, 0.8844, 0.8771, 0.8844, 0.8844, 0.8844, 0.8844,\n",
+      "        0.8844, 0.8844, 0.8844, 0.8844, 0.8844, 0.8844, 0.8844, 0.8870, 0.8844,\n",
+      "        0.8844, 0.8844, 0.8844, 0.8844, 0.8844, 0.8844, 0.8844, 0.8844, 0.8844,\n",
+      "        0.8765])\n"
+     ]
+    },
+    {
+     "ename": "KeyboardInterrupt",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[44], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mtrain_torch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43mdataloader_train\u001b[49m\u001b[43m)\u001b[49m\n",
+      "Cell \u001b[0;32mIn[42], line 19\u001b[0m, in \u001b[0;36mtrain_torch\u001b[0;34m(model, dataloader, model_file, epochs, gpu_ids, verbose)\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m epoch \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(epochs):\n\u001b[1;32m     18\u001b[0m     loss_total,count \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m,\u001b[38;5;241m0\u001b[39m\n\u001b[0;32m---> 19\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m image,y \u001b[38;5;129;01min\u001b[39;00m dataloader_train:\n\u001b[1;32m     20\u001b[0m         \u001b[38;5;28mprint\u001b[39m(image\u001b[38;5;241m.\u001b[39msize())\n\u001b[1;32m     21\u001b[0m         \u001b[38;5;28mprint\u001b[39m(image\u001b[38;5;241m.\u001b[39mmean((\u001b[38;5;241m1\u001b[39m,\u001b[38;5;241m2\u001b[39m,\u001b[38;5;241m3\u001b[39m)))\n",
+      "File \u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/cell_data_loader/cell_data_loader.py:494\u001b[0m, in \u001b[0;36mCellDataloader.__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    492\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbatch_size):\n\u001b[1;32m    493\u001b[0m \t\u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreturn_labels():\n\u001b[0;32m--> 494\u001b[0m \t\tim,y \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnext_im\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    495\u001b[0m \t\t\u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(im,\u001b[38;5;28mint\u001b[39m) \u001b[38;5;129;01mand\u001b[39;00m im \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m:\n\u001b[1;32m    496\u001b[0m \t\t\tim,y \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnext_im()\n",
+      "File \u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/cell_data_loader/cell_data_loader.py:468\u001b[0m, in \u001b[0;36mCellDataloader.next_im\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    466\u001b[0m \t\u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m\n\u001b[1;32m    467\u001b[0m \u001b[38;5;66;03m#assert(len(self.image_objects[self.index]) > 0)\u001b[39;00m\n\u001b[0;32m--> 468\u001b[0m im \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mimage_objects\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mindex\u001b[49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mim_index\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m    469\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreturn_labels(): label \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mimage_objects[\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mindex]\u001b[38;5;241m.\u001b[39mlabel\n\u001b[1;32m    470\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mim_index \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n",
+      "File \u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/cell_data_loader/cell_data_loader.py:168\u001b[0m, in \u001b[0;36mImageLabelObject.__getitem__\u001b[0;34m(self, index)\u001b[0m\n\u001b[1;32m    166\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__getitem__\u001b[39m(\u001b[38;5;28mself\u001b[39m,index):\n\u001b[1;32m    167\u001b[0m \t\u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmode \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mwhole\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m--> 168\u001b[0m \t\t\u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_image\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    169\u001b[0m \t\u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmode \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msliced\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m    170\u001b[0m \t\tx_dim,y_dim \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_scaled_dims()\n",
+      "File \u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/cell_data_loader/cell_data_loader.py:81\u001b[0m, in \u001b[0;36mImageLabelObject.get_image\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     79\u001b[0m n_scenes \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mimage\u001b[38;5;241m.\u001b[39mnum_scenes\n\u001b[1;32m     80\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mimage \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mimage\u001b[38;5;241m.\u001b[39mget_scene(n_scenes\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\n\u001b[0;32m---> 81\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mimage \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mimage\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_block\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     82\u001b[0m \u001b[43m\t\u001b[49m\u001b[43mslices\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m,\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mimage\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnum_z_slices\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     83\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     84\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mimage \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39msqueeze(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mimage)\n\u001b[1;32m     85\u001b[0m \u001b[38;5;66;03m#self.image = np.moveaxis(self.image,0,-1)\u001b[39;00m\n",
+      "File \u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/slideio/py_slideio.py:145\u001b[0m, in \u001b[0;36mScene.read_block\u001b[0;34m(self, rect, size, channel_indices, slices, frames)\u001b[0m\n\u001b[1;32m    132\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mread_block\u001b[39m(\u001b[38;5;28mself\u001b[39m, rect\u001b[38;5;241m=\u001b[39m(\u001b[38;5;241m0\u001b[39m,\u001b[38;5;241m0\u001b[39m,\u001b[38;5;241m0\u001b[39m,\u001b[38;5;241m0\u001b[39m), size\u001b[38;5;241m=\u001b[39m(\u001b[38;5;241m0\u001b[39m,\u001b[38;5;241m0\u001b[39m), channel_indices\u001b[38;5;241m=\u001b[39m[], slices\u001b[38;5;241m=\u001b[39m(\u001b[38;5;241m0\u001b[39m,\u001b[38;5;241m1\u001b[39m), frames\u001b[38;5;241m=\u001b[39m(\u001b[38;5;241m0\u001b[39m,\u001b[38;5;241m1\u001b[39m)):\n\u001b[1;32m    133\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m'''Reads rectangular block of the scene with optional rescaling.\u001b[39;00m\n\u001b[1;32m    134\u001b[0m \n\u001b[1;32m    135\u001b[0m \u001b[38;5;124;03m    Args:\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    143\u001b[0m \u001b[38;5;124;03m        numpy array with pixel values\u001b[39;00m\n\u001b[1;32m    144\u001b[0m \u001b[38;5;124;03m    '''\u001b[39;00m\n\u001b[0;32m--> 145\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mscene\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_block\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrect\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msize\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mchannel_indices\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mslices\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mframes\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
+     ]
+    }
+   ],
+   "source": [
+    "train_torch(model,dataloader_train)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "colab": {
+   "provenance": []
+  },
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

{cell_data_loader-0.0.17 → cell_data_loader-0.0.20}/pyproject.toml

@@ -2,17 +2,18 @@
 
 name = "cell_data_loader"
 
-version = "0.0.17"
+version = "0.0.20"
 
 dependencies = [
   "numpy==1.22.0",
   "scipy",
   "torch>=2.0.0",
-  "torchvision>=0.15.0",
+  "torchvision==0.14.0",
   "opencv-python>=4.5.4",
   "slideio==2.4.1",
   "scikit-image",
-  "pillow"
+  "pillow",
+  "cell_data_loader"
 ]
 requires-python = ">=3.8"
 

{cell_data_loader-0.0.17 → cell_data_loader-0.0.20}/src/cell_data_loader/__init__.py

@@ -1,3 +1,3 @@
-__version__ = "0.0.17"
+__version__ = "0.0.20"
 
 from .cell_data_loader import *

{cell_data_loader-0.0.17 → cell_data_loader-0.0.20}/src/cell_data_loader/cell_data_loader.py

@@ -281,7 +281,7 @@ class CellDataloader():#BaseDataset):
 				self.label_regex = [re.compile(_) for _ in label_regex]
 				if len(self.label_regex) == 1:
 					self.label_regex = self.label_regex[0]
-				self.n_labels = len(self.label_regex)
+				self.n_labels = len(self.label_regex) + 1
 			else:
 				self.label_regex = re.compile(label_regex)
 				self.n_labels = 1
@@ -434,7 +434,7 @@ class CellDataloader():#BaseDataset):
 						warnings.warn(
 							("Image file %s matches at"+\
 							" least two regular expressions") % image_file)
-					m = i + 1
+					m = i
 			return m
 		else:
 			if bool(self.label_regex.search(image_file)):

cell_data_loader-0.0.17/example/example_numpy.py

@@ -1,71 +0,0 @@
-#!/usr/bin/python3
-from cell_data_loader import CellDataloader
-
-try:
-	import tensorflow as tf
-except:
-	try:
-		from keras import tensorflow as tf
-	except:
-		raise Exception("Need valid tensorflow")
-
-import numpy as np
-import os
-
-def example_numpy(verbose=True):
-
-	"""
-	Replace these folders with whatever folders of cells you may have. Note that
-	the test/train folders in this script are the same -- you would need to 
-	have a separate train/test set in your own implementation
-	"""
-	wd = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-	imfolder1 = os.path.join(wd,'data',
-		'3368914_4_non_tumor')
-	imfolder2 = os.path.join(wd,'data',
-		'4173633_5')
-
-	#model = resnet50()
-	model = tf.keras.applications.resnet50.ResNet50(
-		include_top=True,
-		weights='imagenet',
-		input_tensor=None,
-		input_shape=None,
-		pooling=None,
-		classes=1000
-	)
-
-	# Train
-
-	dataloader_train = CellDataloader(imfolder1,imfolder2,
-		dtype="numpy",verbose=False)
-
-	z = None
-	for epoch in range(1):
-		for image,y in dataloader_train:
-			if z is None:
-				z_dim = list(y.shape)
-				z_dim[1] = 1000 - z_dim[1]
-				z = np.zeros(z_dim)
-			y = np.concatenate((y,z),axis=1)
-			model.fit(image,y)
-
-	# Test
-
-	model.eval()
-	dataloader_test = CellDataloader(imfolder1,imfolder2,dtype="torch",
-		verbose=False)
-	total_images = 0
-	sum_accuracy = 0
-	for image,y in dataloader_test:
-		total_images += image.shape[0]
-		y_pred = model(image)
-		y_pred = y_pred[:,:y.shape[1]]
-		sum_accuracy += torch.sum(torch.argmax(y_pred,axis=1) == \
-			torch.argmax(y,axis=1))
-
-	accuracy = sum_accuracy / total_images
-	if verbose: print("Final accuracy: %.4f" % sum_accuracy)
-
-if __name__ == "__main__":
-	example_numpy()

cell_data_loader-0.0.17/example/example_torch.py

@@ -1,88 +0,0 @@
-#!/usr/bin/python3
-
-from cell_data_loader import CellDataloader
-from torchvision.models import resnet50 #, ResNet50_Weights
-import torch
-import os
-import glob
-
-def example_torch(gpu_ids = None,verbose=True):
-
-	"""
-	Replace these folders with whatever folders of cells you may have. Note that
-	the test/train folders in this script are the same -- you would need to 
-	have a separate train/test set in your own implementation
-	"""
-
-	wd = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-	model_folder = os.path.join(wd,'checkpoints')
-	os.makedirs(model_folder,exist_ok=True)
-	model_file = os.path.join(model_folder,'torch_model.pt')
-	
-	# Training data folders
-	imfolder1_train = os.path.join(wd,'data',
-		'3368914_4_non_tumor')
-	imfolder2_train = os.path.join(wd,'data',
-		'4173633_5')
-	
-	# Test data folders
-	imfolder1_test = os.path.join(wd,'data','3368914_4_non_tumor')
-	imfolder2_test = os.path.join(wd,'data','4173633_5')
-	
-	# Get a pretrained model from torchvision
-	if os.path.isfile(model_file):
-		if verbose: print("Loading %s" % model_file)
-		model = torch.load(model_file)
-	else:
-		model = resnet50(pretrained=True)
-
-	if gpu_ids is not None:
-		model.to(gpu_ids)
-	
-	# Train
-	
-	model.train()
-	loss_fn = torch.nn.MSELoss(reduction='sum')
-	optimizer = torch.optim.Adam(model.parameters(), lr=1e-5)
-	dataloader_train = CellDataloader(imfolder1_train,imfolder2_train,
-		dtype="torch",
-		verbose=False, gpu_ids=gpu_ids)
-	if verbose: print("Beginning training")
-	n_epochs = 1
-	for epoch in range(n_epochs):
-		l = 0
-		c = 0
-		for image,y in dataloader_train:
-			y_pred = model(image)
-			y_pred = y_pred[:,:y.size()[1]]
-			loss = loss_fn(y_pred, y)
-			l += float(loss)
-			c += 1
-			optimizer.zero_grad()
-			loss.backward()
-			optimizer.step()
-		torch.save(model,model_file)
-		if verbose:
-			print(
-				"Epoch {epoch:d}/{n_epochs:d}: loss: {loss:.5f}".format(
-					epoch=epoch,n_epochs=n_epochs,loss=l/c)
-			)
-	# Test
-	
-	model.eval()
-
-	dataloader_test = CellDataloader(imfolder1_test,imfolder2_test,
-		dtype="torch",verbose=False, gpu_ids = gpu_ids)
-	total_images = 0
-	sum_accuracy = 0
-	for image,y in dataloader_test:
-		total_images += image.size()[0]
-		y_pred = model(image)
-		y_pred = y_pred[:,:y.size()[1]]
-		sum_accuracy += torch.sum(torch.argmax(y_pred,axis=1) == \
-			torch.argmax(y,axis=1))
-	accuracy = sum_accuracy / total_images
-	if verbose: print("Final accuracy: %.4f" % accuracy)
-
-if __name__ == "__main__":
-	example_torch()