brainscore-vision 2.1__py3-none-any.whl

brainscore_vision/data/david2004/data_packaging/2018-06-05_gallant_data.ipynb

@@ -0,0 +1,3628 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import re"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import zipfile"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import glob"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import itertools"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import xarray as xr"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import matplotlib.pyplot as plt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import mkgu"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from mkgu.knownfile import KnownFile as kf"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from mkgu.knownfile import FileRecord, Sighting"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from mkgu.lookup import pwdb"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from mkgu.assemblies import AssemblyModel, AssemblyStoreMap, AssemblyStoreModel"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from mkgu.stimuli import ImageModel, AttributeModel, ImageMetaModel, StimulusSetModel, ImageStoreModel, \\\n",
+    "    StimulusSetImageMap, ImageStoreMap"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Process .nc files"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "v2_base_path = \"/braintree/data2/active/users/jjpr/mkgu_packaging/crcns/v2-1\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['/braintree/data2/active/users/jjpr/mkgu_packaging/crcns/v2-1/V2Data1/DGrat/data.nc',\n",
+       " '/braintree/data2/active/users/jjpr/mkgu_packaging/crcns/v2-1/V2Data10/NatRev/data.nc',\n",
+       " '/braintree/data2/active/users/jjpr/mkgu_packaging/crcns/v2-1/V2Data11/NatRev/data.nc',\n",
+       " '/braintree/data2/active/users/jjpr/mkgu_packaging/crcns/v2-1/V2Data12/NatRev/data.nc',\n",
+       " '/braintree/data2/active/users/jjpr/mkgu_packaging/crcns/v2-1/V2Data13/NatRev/data.nc',\n",
+       " '/braintree/data2/active/users/jjpr/mkgu_packaging/crcns/v2-1/V2Data14/NatRev/data.nc',\n",
+       " '/braintree/data2/active/users/jjpr/mkgu_packaging/crcns/v2-1/V2Data15/NatRev/data.nc',\n",
+       " '/braintree/data2/active/users/jjpr/mkgu_packaging/crcns/v2-1/V2Data16/NatRev/data.nc',\n",
+       " '/braintree/data2/active/users/jjpr/mkgu_packaging/crcns/v2-1/V2Data17/NatRev/data.nc',\n",
+       " '/braintree/data2/active/users/jjpr/mkgu_packaging/crcns/v2-1/V2Data18/NatRev/data.nc',\n",
+       " '/braintree/data2/active/users/jjpr/mkgu_packaging/crcns/v2-1/V2Data19/NatRev/data.nc',\n",
+       " '/braintree/data2/active/users/jjpr/mkgu_packaging/crcns/v2-1/V2Data2/DGrat/data.nc',\n",
+       " '/braintree/data2/active/users/jjpr/mkgu_packaging/crcns/v2-1/V2Data20/NatRev/data.nc',\n",
+       " '/braintree/data2/active/users/jjpr/mkgu_packaging/crcns/v2-1/V2Data3/DGrat/data.nc',\n",
+       " '/braintree/data2/active/users/jjpr/mkgu_packaging/crcns/v2-1/V2Data4/DGrat/data.nc',\n",
+       " '/braintree/data2/active/users/jjpr/mkgu_packaging/crcns/v2-1/V2Data5/DGrat/data.nc',\n",
+       " '/braintree/data2/active/users/jjpr/mkgu_packaging/crcns/v2-1/V2Data6/DGrat/data.nc',\n",
+       " '/braintree/data2/active/users/jjpr/mkgu_packaging/crcns/v2-1/V2Data7/Nat/data.nc',\n",
+       " '/braintree/data2/active/users/jjpr/mkgu_packaging/crcns/v2-1/V2Data7/NatRev/data.nc',\n",
+       " '/braintree/data2/active/users/jjpr/mkgu_packaging/crcns/v2-1/V2Data8/NatRev/data.nc',\n",
+       " '/braintree/data2/active/users/jjpr/mkgu_packaging/crcns/v2-1/V2Data9/NatRev/data.nc']"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "nc_files = glob.glob(os.path.join(v2_base_path, \"*/*/*.nc\"), recursive=True)\n",
+    "sorted(nc_files)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gd_arrays = {}\n",
+    "for f in nc_files:\n",
+    "    gd_arrays[f] = xr.open_dataarray(f)\n",
+    "gd_arrays"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for gd_array_key in gd_arrays:\n",
+    "    gd_array = gd_arrays[gd_array_key]\n",
+    "    gd_array = gd_array.T.rename({\"image_file_name\": \"presentation\"})\n",
+    "    gd_array.coords[\"presentation_id\"] = (\"presentation\", range(gd_array.shape[1]))\n",
+    "    gd_array.coords[\"neuroid_id\"] = (\"neuroid\", gd_array[\"neuroid\"].values)\n",
+    "    gd_arrays[gd_array_key] = gd_array\n",
+    "gd_arrays"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def massage_file_name(file_name):\n",
+    "    split = re.split(\"\\\\\\\\|/\", file_name)\n",
+    "    split = [t for t in split if t]\n",
+    "    relative_path = os.path.join(*split[-5:])\n",
+    "    full_path = os.path.join(\"/\", *split)\n",
+    "    basename = split[-1]\n",
+    "    exists = os.path.exists(full_path)\n",
+    "    sha1 = kf(full_path).sha1\n",
+    "    result = {\n",
+    "        \"image_file_path_original\": relative_path,\n",
+    "        \"image_id\": sha1\n",
+    "    }\n",
+    "    return result"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for gd_array_key in gd_arrays:\n",
+    "    print(gd_array_key)\n",
+    "    gd_array = gd_arrays[gd_array_key]\n",
+    "    df_massage = pd.DataFrame(list(map(massage_file_name, gd_array[\"presentation\"].values)))\n",
+    "    for column in df_massage.columns:\n",
+    "        gd_array.coords[column] = (\"presentation\", df_massage[column])\n",
+    "    gd_array.reset_index([\"neuroid\", \"presentation\"], drop=True, inplace=True)\n",
+    "    mkgu.assemblies.gather_indexes(gd_array)\n",
+    "    gd_array.reset_index([\"category_name\"], drop=True, inplace=True)\n",
+    "gd_arrays"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Combine arrays"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "aligned = xr.align(*list(gd_arrays.values()), join=\"outer\")\n",
+    "aligned"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "aligned[0].shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "[(~np.isnan(da)).any() for da in aligned]"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "metadata": {},
+   "source": [
+    "if the length of the presentation axis is the sum of the presentation axes of all the data sets, there won't be collision if we use xr.combine_first()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "non_nan_indices = []\n",
+    "for da in aligned:\n",
+    "    non_nan_indices.append(np.flatnonzero(~np.isnan(da.values))) \n",
+    "non_nan_indices"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# should all be False\n",
+    "for a, b in itertools.combinations(non_nan_indices, 2):\n",
+    "    print(np.in1d(a, b).any())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "blank = np.full_like(aligned[0], np.nan)\n",
+    "blank"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "da_result = xr.DataArray(blank, coords=aligned[0].coords, dims=aligned[0].dims)\n",
+    "da_result"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for da in aligned:\n",
+    "    da_result = da_result.combine_first(da)\n",
+    "da_result"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def levels_for_index(xr_data, index):\n",
+    "    return xr_data.indexes[index].names\n",
+    "\n",
+    "def all_index_levels(xr_data):\n",
+    "    nested = [levels_for_index(xr_data, index) for index in xr_data.indexes]\n",
+    "    return [x for inner in nested for x in inner]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "da_result.reset_index(all_index_levels(da_result), inplace=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "target_netcdf_file = \"/braintree/data2/active/users/jjpr/mkgu_packaging/crcns/v2-1/crcns_v2-1_neuronal.nc\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "da_result.to_netcdf(target_netcdf_file)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "da_loaded = xr.open_dataarray(target_netcdf_file)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<xarray.DataArray (neuroid: 135, presentation: 3494103)>\n",
+       "[471703905 values with dtype=float64]\n",
+       "Coordinates:\n",
+       "    region                    (neuroid) object ...\n",
+       "    animal                    (neuroid) object ...\n",
+       "    neuroid_id                (neuroid) object ...\n",
+       "    stimulusRepeats           (presentation) int64 ...\n",
+       "    presentation_id           (presentation) int64 ...\n",
+       "    image_file_path_original  (presentation) object ...\n",
+       "    image_id                  (presentation) object ...\n",
+       "Dimensions without coordinates: neuroid, presentation"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "da_loaded"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<xarray.DataArray (neuroid: 2, presentation: 74740)>\n",
+       "array([[      0,       0,       0, ...,     134,     134,     134],\n",
+       "       [     16,      35,      54, ..., 3481545, 3481546, 3481547]])\n",
+       "Coordinates:\n",
+       "    region                    (neuroid) object ...\n",
+       "    animal                    (neuroid) object ...\n",
+       "    neuroid_id                (neuroid) object ...\n",
+       "    stimulusRepeats           (presentation) int64 ...\n",
+       "    presentation_id           (presentation) int64 ...\n",
+       "    image_file_path_original  (presentation) object ...\n",
+       "    image_id                  (presentation) object '437fcb15ba2caa46c4a2aea77796c8316b8a98cf' ...\n",
+       "Dimensions without coordinates: neuroid, presentation"
+      ]
+     },
+     "execution_count": 43,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "non_nan = np.nonzero(~np.isnan(da_loaded))\n",
+    "non_nan"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Make the image zip file"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_image_meta = pd.DataFrame({\"image_id\": np.unique(da_loaded[\"image_id\"].values)})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>image_id</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>00000a2518d148e9b455537488e325a7f42f39d8</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>00001776837935437e2784af953a4db24cc19536</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>00003bb7e9ca9265dc521763f73514f9c4d48f8a</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>00005ad0cbb2bafc56814919fda60268a1077ba3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>00005e46dace8673b631b75f1df3a6d1a8b2c1c2</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>00006b3962b738bb25ab44265092ac6ef71943ed</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>00007b43039dc9f26f12d6e222460a70a1400d1f</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>00008a40da04647d4350b9acb0b6657326c1ba54</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>00008be32ed56378291bc536589d97bc39e35557</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>00008eead95d140fb4591430b11193525dfc379a</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10</th>\n",
+       "      <td>00009e9fd517bd1aa8c72046dc51b23beb14bd6b</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11</th>\n",
+       "      <td>00009f4607ffef290fa043eaaad9a61792cdf52d</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>12</th>\n",
+       "      <td>0000b4496034903fcfa414566b171f4c59959aa1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13</th>\n",
+       "      <td>0000c1747157700a98bb17f0712f61a55bfc76be</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14</th>\n",
+       "      <td>0000c18032e90501faa945854e0a8279f35f113b</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>15</th>\n",
+       "      <td>0000c941ce6c69a723ca4842ec52f7c39c1fcf6e</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>16</th>\n",
+       "      <td>0000ec7b6ffd8e31f84d0b81b387154838ad9605</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>17</th>\n",
+       "      <td>00010a8473ad131508f1d21353221dd3038d7d00</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>18</th>\n",
+       "      <td>00010ac420ed0bd3f212fc4a0a1c51bda60ec929</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>19</th>\n",
+       "      <td>00011aedc68af6f9c23c35ba1bd512dbc2e8e8b9</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>20</th>\n",
+       "      <td>000128de8497372d32b8237b085b2ca2bf679ebf</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>21</th>\n",
+       "      <td>00013841c33fbd35df4e025b04db78cbc330a380</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>22</th>\n",
+       "      <td>00014c83ca4993261ebb88a75cd9207e569d0bba</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>23</th>\n",
+       "      <td>0001548f9772a3870f9f57d7ddd15372fdd498d4</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>24</th>\n",
+       "      <td>00019d74a4b7ca4e84a402e33f25028996345ce3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>25</th>\n",
+       "      <td>0001c0abf05394d6438f48e8376b66297938e888</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>26</th>\n",
+       "      <td>0001d851793da81785d86b4205b184a74b60ce23</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>27</th>\n",
+       "      <td>0001d99d7f29638060839746013c043f7857ae47</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>28</th>\n",
+       "      <td>000209ba9ff4f70dce2baf79e5dd0328c688bc11</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>29</th>\n",
+       "      <td>00023835d1cb1095d5c97526fa460438c25490a3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770987</th>\n",
+       "      <td>fffd5b92538382d74078624393b5ac341724a25d</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770988</th>\n",
+       "      <td>fffd70f1ebd24394b678671016d7da2b044302c7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770989</th>\n",
+       "      <td>fffd8533bbb0982e3d9927d4834d17c9068a1857</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770990</th>\n",
+       "      <td>fffd8bdbb214a1a9772ac138501cae9b990a8731</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770991</th>\n",
+       "      <td>fffd9e0889381e54b1bd12e0576b595d9976b9dd</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770992</th>\n",
+       "      <td>fffdad566df3a4bd3a660d193ada8eb8ffaa88c2</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770993</th>\n",
+       "      <td>fffdb264a410114f2c9bed3765786c474bdea1a9</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770994</th>\n",
+       "      <td>fffdb9a37c6a1f0cd100f9b340a2788600f591f7</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770995</th>\n",
+       "      <td>fffdd183dc71df6de59b077cec3f00e6bf04f5cd</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770996</th>\n",
+       "      <td>fffdd57f5450722f7c756b2ff3f0974ffa610124</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770997</th>\n",
+       "      <td>fffde0392a7a1b70ada752dbe6c34bbbd2aa7eb5</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770998</th>\n",
+       "      <td>fffde7360aaed78e7ae839d80c7474710394b16c</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770999</th>\n",
+       "      <td>fffdedf0b18b842a566e36eb1143be87fa89ec5d</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771000</th>\n",
+       "      <td>fffdefeee340b6c60d2f8c8a6f27d6943addefa9</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771001</th>\n",
+       "      <td>fffdf93ab97bce745ef9275492278030872f14ab</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771002</th>\n",
+       "      <td>fffe11a3ac0619b14fd577005ddb7d379ac5f9d0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771003</th>\n",
+       "      <td>fffe2f1973fe927a07b51982606717600caf2e0e</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771004</th>\n",
+       "      <td>fffe3525e7a24fdf9ab76601f220e29ea2baf327</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771005</th>\n",
+       "      <td>fffea406c12823e5864ea00ae98d8b99c9589145</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771006</th>\n",
+       "      <td>fffeb23f36412d1a940abdab9f03643b57fc4027</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771007</th>\n",
+       "      <td>fffebd9e7eabeda8839774bf15bdba921e19e693</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771008</th>\n",
+       "      <td>fffec462f80db47772e9f0afe431162d47350c82</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771009</th>\n",
+       "      <td>fffeeae0ab84c261d0a6b1e57e5a89c7d6217984</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771010</th>\n",
+       "      <td>ffff06ab149541f11cb8a2612b0da87eba2ed950</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771011</th>\n",
+       "      <td>ffff14ee8bef7387a05a504808bc77d9e38de3ab</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771012</th>\n",
+       "      <td>ffff333854bfd2aef1adb44eedaee86842d38c96</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771013</th>\n",
+       "      <td>ffff666f06cff0c5f167f96a37fffb2cb2867d2c</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771014</th>\n",
+       "      <td>ffff7fbfc6e2ee2af05646a03c3b1bfee05dd82f</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771015</th>\n",
+       "      <td>ffffe8068e37504568db803945cfdeee68e68859</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771016</th>\n",
+       "      <td>fffff7e11f1a8d38d9378103e0ca6b25ed219a5e</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>771017 rows × 1 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                        image_id\n",
+       "0       00000a2518d148e9b455537488e325a7f42f39d8\n",
+       "1       00001776837935437e2784af953a4db24cc19536\n",
+       "2       00003bb7e9ca9265dc521763f73514f9c4d48f8a\n",
+       "3       00005ad0cbb2bafc56814919fda60268a1077ba3\n",
+       "4       00005e46dace8673b631b75f1df3a6d1a8b2c1c2\n",
+       "5       00006b3962b738bb25ab44265092ac6ef71943ed\n",
+       "6       00007b43039dc9f26f12d6e222460a70a1400d1f\n",
+       "7       00008a40da04647d4350b9acb0b6657326c1ba54\n",
+       "8       00008be32ed56378291bc536589d97bc39e35557\n",
+       "9       00008eead95d140fb4591430b11193525dfc379a\n",
+       "10      00009e9fd517bd1aa8c72046dc51b23beb14bd6b\n",
+       "11      00009f4607ffef290fa043eaaad9a61792cdf52d\n",
+       "12      0000b4496034903fcfa414566b171f4c59959aa1\n",
+       "13      0000c1747157700a98bb17f0712f61a55bfc76be\n",
+       "14      0000c18032e90501faa945854e0a8279f35f113b\n",
+       "15      0000c941ce6c69a723ca4842ec52f7c39c1fcf6e\n",
+       "16      0000ec7b6ffd8e31f84d0b81b387154838ad9605\n",
+       "17      00010a8473ad131508f1d21353221dd3038d7d00\n",
+       "18      00010ac420ed0bd3f212fc4a0a1c51bda60ec929\n",
+       "19      00011aedc68af6f9c23c35ba1bd512dbc2e8e8b9\n",
+       "20      000128de8497372d32b8237b085b2ca2bf679ebf\n",
+       "21      00013841c33fbd35df4e025b04db78cbc330a380\n",
+       "22      00014c83ca4993261ebb88a75cd9207e569d0bba\n",
+       "23      0001548f9772a3870f9f57d7ddd15372fdd498d4\n",
+       "24      00019d74a4b7ca4e84a402e33f25028996345ce3\n",
+       "25      0001c0abf05394d6438f48e8376b66297938e888\n",
+       "26      0001d851793da81785d86b4205b184a74b60ce23\n",
+       "27      0001d99d7f29638060839746013c043f7857ae47\n",
+       "28      000209ba9ff4f70dce2baf79e5dd0328c688bc11\n",
+       "29      00023835d1cb1095d5c97526fa460438c25490a3\n",
+       "...                                          ...\n",
+       "770987  fffd5b92538382d74078624393b5ac341724a25d\n",
+       "770988  fffd70f1ebd24394b678671016d7da2b044302c7\n",
+       "770989  fffd8533bbb0982e3d9927d4834d17c9068a1857\n",
+       "770990  fffd8bdbb214a1a9772ac138501cae9b990a8731\n",
+       "770991  fffd9e0889381e54b1bd12e0576b595d9976b9dd\n",
+       "770992  fffdad566df3a4bd3a660d193ada8eb8ffaa88c2\n",
+       "770993  fffdb264a410114f2c9bed3765786c474bdea1a9\n",
+       "770994  fffdb9a37c6a1f0cd100f9b340a2788600f591f7\n",
+       "770995  fffdd183dc71df6de59b077cec3f00e6bf04f5cd\n",
+       "770996  fffdd57f5450722f7c756b2ff3f0974ffa610124\n",
+       "770997  fffde0392a7a1b70ada752dbe6c34bbbd2aa7eb5\n",
+       "770998  fffde7360aaed78e7ae839d80c7474710394b16c\n",
+       "770999  fffdedf0b18b842a566e36eb1143be87fa89ec5d\n",
+       "771000  fffdefeee340b6c60d2f8c8a6f27d6943addefa9\n",
+       "771001  fffdf93ab97bce745ef9275492278030872f14ab\n",
+       "771002  fffe11a3ac0619b14fd577005ddb7d379ac5f9d0\n",
+       "771003  fffe2f1973fe927a07b51982606717600caf2e0e\n",
+       "771004  fffe3525e7a24fdf9ab76601f220e29ea2baf327\n",
+       "771005  fffea406c12823e5864ea00ae98d8b99c9589145\n",
+       "771006  fffeb23f36412d1a940abdab9f03643b57fc4027\n",
+       "771007  fffebd9e7eabeda8839774bf15bdba921e19e693\n",
+       "771008  fffec462f80db47772e9f0afe431162d47350c82\n",
+       "771009  fffeeae0ab84c261d0a6b1e57e5a89c7d6217984\n",
+       "771010  ffff06ab149541f11cb8a2612b0da87eba2ed950\n",
+       "771011  ffff14ee8bef7387a05a504808bc77d9e38de3ab\n",
+       "771012  ffff333854bfd2aef1adb44eedaee86842d38c96\n",
+       "771013  ffff666f06cff0c5f167f96a37fffb2cb2867d2c\n",
+       "771014  ffff7fbfc6e2ee2af05646a03c3b1bfee05dd82f\n",
+       "771015  ffffe8068e37504568db803945cfdeee68e68859\n",
+       "771016  fffff7e11f1a8d38d9378103e0ca6b25ed219a5e\n",
+       "\n",
+       "[771017 rows x 1 columns]"
+      ]
+     },
+     "execution_count": 22,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_image_meta"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def first_dupe(sha1):\n",
+    "    fr = FileRecord.get(sha1=sha1)\n",
+    "    return fr.sightings[0].location\n",
+    "# order is not guaranteed, so on subsequent runs test that you got the same result, see below"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>image_id</th>\n",
+       "      <th>first_dupe</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>00000a2518d148e9b455537488e325a7f42f39d8</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>00001776837935437e2784af953a4db24cc19536</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>00003bb7e9ca9265dc521763f73514f9c4d48f8a</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>00005ad0cbb2bafc56814919fda60268a1077ba3</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>00005e46dace8673b631b75f1df3a6d1a8b2c1c2</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>00006b3962b738bb25ab44265092ac6ef71943ed</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>00007b43039dc9f26f12d6e222460a70a1400d1f</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>00008a40da04647d4350b9acb0b6657326c1ba54</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>00008be32ed56378291bc536589d97bc39e35557</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>00008eead95d140fb4591430b11193525dfc379a</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10</th>\n",
+       "      <td>00009e9fd517bd1aa8c72046dc51b23beb14bd6b</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11</th>\n",
+       "      <td>00009f4607ffef290fa043eaaad9a61792cdf52d</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>12</th>\n",
+       "      <td>0000b4496034903fcfa414566b171f4c59959aa1</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13</th>\n",
+       "      <td>0000c1747157700a98bb17f0712f61a55bfc76be</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14</th>\n",
+       "      <td>0000c18032e90501faa945854e0a8279f35f113b</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>15</th>\n",
+       "      <td>0000c941ce6c69a723ca4842ec52f7c39c1fcf6e</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>16</th>\n",
+       "      <td>0000ec7b6ffd8e31f84d0b81b387154838ad9605</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>17</th>\n",
+       "      <td>00010a8473ad131508f1d21353221dd3038d7d00</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>18</th>\n",
+       "      <td>00010ac420ed0bd3f212fc4a0a1c51bda60ec929</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>19</th>\n",
+       "      <td>00011aedc68af6f9c23c35ba1bd512dbc2e8e8b9</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>20</th>\n",
+       "      <td>000128de8497372d32b8237b085b2ca2bf679ebf</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>21</th>\n",
+       "      <td>00013841c33fbd35df4e025b04db78cbc330a380</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>22</th>\n",
+       "      <td>00014c83ca4993261ebb88a75cd9207e569d0bba</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>23</th>\n",
+       "      <td>0001548f9772a3870f9f57d7ddd15372fdd498d4</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>24</th>\n",
+       "      <td>00019d74a4b7ca4e84a402e33f25028996345ce3</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>25</th>\n",
+       "      <td>0001c0abf05394d6438f48e8376b66297938e888</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>26</th>\n",
+       "      <td>0001d851793da81785d86b4205b184a74b60ce23</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>27</th>\n",
+       "      <td>0001d99d7f29638060839746013c043f7857ae47</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>28</th>\n",
+       "      <td>000209ba9ff4f70dce2baf79e5dd0328c688bc11</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>29</th>\n",
+       "      <td>00023835d1cb1095d5c97526fa460438c25490a3</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770987</th>\n",
+       "      <td>fffd5b92538382d74078624393b5ac341724a25d</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770988</th>\n",
+       "      <td>fffd70f1ebd24394b678671016d7da2b044302c7</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770989</th>\n",
+       "      <td>fffd8533bbb0982e3d9927d4834d17c9068a1857</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770990</th>\n",
+       "      <td>fffd8bdbb214a1a9772ac138501cae9b990a8731</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770991</th>\n",
+       "      <td>fffd9e0889381e54b1bd12e0576b595d9976b9dd</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770992</th>\n",
+       "      <td>fffdad566df3a4bd3a660d193ada8eb8ffaa88c2</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770993</th>\n",
+       "      <td>fffdb264a410114f2c9bed3765786c474bdea1a9</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770994</th>\n",
+       "      <td>fffdb9a37c6a1f0cd100f9b340a2788600f591f7</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770995</th>\n",
+       "      <td>fffdd183dc71df6de59b077cec3f00e6bf04f5cd</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770996</th>\n",
+       "      <td>fffdd57f5450722f7c756b2ff3f0974ffa610124</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770997</th>\n",
+       "      <td>fffde0392a7a1b70ada752dbe6c34bbbd2aa7eb5</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770998</th>\n",
+       "      <td>fffde7360aaed78e7ae839d80c7474710394b16c</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770999</th>\n",
+       "      <td>fffdedf0b18b842a566e36eb1143be87fa89ec5d</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771000</th>\n",
+       "      <td>fffdefeee340b6c60d2f8c8a6f27d6943addefa9</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771001</th>\n",
+       "      <td>fffdf93ab97bce745ef9275492278030872f14ab</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771002</th>\n",
+       "      <td>fffe11a3ac0619b14fd577005ddb7d379ac5f9d0</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771003</th>\n",
+       "      <td>fffe2f1973fe927a07b51982606717600caf2e0e</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771004</th>\n",
+       "      <td>fffe3525e7a24fdf9ab76601f220e29ea2baf327</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771005</th>\n",
+       "      <td>fffea406c12823e5864ea00ae98d8b99c9589145</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771006</th>\n",
+       "      <td>fffeb23f36412d1a940abdab9f03643b57fc4027</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771007</th>\n",
+       "      <td>fffebd9e7eabeda8839774bf15bdba921e19e693</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771008</th>\n",
+       "      <td>fffec462f80db47772e9f0afe431162d47350c82</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771009</th>\n",
+       "      <td>fffeeae0ab84c261d0a6b1e57e5a89c7d6217984</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771010</th>\n",
+       "      <td>ffff06ab149541f11cb8a2612b0da87eba2ed950</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771011</th>\n",
+       "      <td>ffff14ee8bef7387a05a504808bc77d9e38de3ab</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771012</th>\n",
+       "      <td>ffff333854bfd2aef1adb44eedaee86842d38c96</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771013</th>\n",
+       "      <td>ffff666f06cff0c5f167f96a37fffb2cb2867d2c</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771014</th>\n",
+       "      <td>ffff7fbfc6e2ee2af05646a03c3b1bfee05dd82f</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771015</th>\n",
+       "      <td>ffffe8068e37504568db803945cfdeee68e68859</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771016</th>\n",
+       "      <td>fffff7e11f1a8d38d9378103e0ca6b25ed219a5e</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>771017 rows × 2 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                        image_id  \\\n",
+       "0       00000a2518d148e9b455537488e325a7f42f39d8   \n",
+       "1       00001776837935437e2784af953a4db24cc19536   \n",
+       "2       00003bb7e9ca9265dc521763f73514f9c4d48f8a   \n",
+       "3       00005ad0cbb2bafc56814919fda60268a1077ba3   \n",
+       "4       00005e46dace8673b631b75f1df3a6d1a8b2c1c2   \n",
+       "5       00006b3962b738bb25ab44265092ac6ef71943ed   \n",
+       "6       00007b43039dc9f26f12d6e222460a70a1400d1f   \n",
+       "7       00008a40da04647d4350b9acb0b6657326c1ba54   \n",
+       "8       00008be32ed56378291bc536589d97bc39e35557   \n",
+       "9       00008eead95d140fb4591430b11193525dfc379a   \n",
+       "10      00009e9fd517bd1aa8c72046dc51b23beb14bd6b   \n",
+       "11      00009f4607ffef290fa043eaaad9a61792cdf52d   \n",
+       "12      0000b4496034903fcfa414566b171f4c59959aa1   \n",
+       "13      0000c1747157700a98bb17f0712f61a55bfc76be   \n",
+       "14      0000c18032e90501faa945854e0a8279f35f113b   \n",
+       "15      0000c941ce6c69a723ca4842ec52f7c39c1fcf6e   \n",
+       "16      0000ec7b6ffd8e31f84d0b81b387154838ad9605   \n",
+       "17      00010a8473ad131508f1d21353221dd3038d7d00   \n",
+       "18      00010ac420ed0bd3f212fc4a0a1c51bda60ec929   \n",
+       "19      00011aedc68af6f9c23c35ba1bd512dbc2e8e8b9   \n",
+       "20      000128de8497372d32b8237b085b2ca2bf679ebf   \n",
+       "21      00013841c33fbd35df4e025b04db78cbc330a380   \n",
+       "22      00014c83ca4993261ebb88a75cd9207e569d0bba   \n",
+       "23      0001548f9772a3870f9f57d7ddd15372fdd498d4   \n",
+       "24      00019d74a4b7ca4e84a402e33f25028996345ce3   \n",
+       "25      0001c0abf05394d6438f48e8376b66297938e888   \n",
+       "26      0001d851793da81785d86b4205b184a74b60ce23   \n",
+       "27      0001d99d7f29638060839746013c043f7857ae47   \n",
+       "28      000209ba9ff4f70dce2baf79e5dd0328c688bc11   \n",
+       "29      00023835d1cb1095d5c97526fa460438c25490a3   \n",
+       "...                                          ...   \n",
+       "770987  fffd5b92538382d74078624393b5ac341724a25d   \n",
+       "770988  fffd70f1ebd24394b678671016d7da2b044302c7   \n",
+       "770989  fffd8533bbb0982e3d9927d4834d17c9068a1857   \n",
+       "770990  fffd8bdbb214a1a9772ac138501cae9b990a8731   \n",
+       "770991  fffd9e0889381e54b1bd12e0576b595d9976b9dd   \n",
+       "770992  fffdad566df3a4bd3a660d193ada8eb8ffaa88c2   \n",
+       "770993  fffdb264a410114f2c9bed3765786c474bdea1a9   \n",
+       "770994  fffdb9a37c6a1f0cd100f9b340a2788600f591f7   \n",
+       "770995  fffdd183dc71df6de59b077cec3f00e6bf04f5cd   \n",
+       "770996  fffdd57f5450722f7c756b2ff3f0974ffa610124   \n",
+       "770997  fffde0392a7a1b70ada752dbe6c34bbbd2aa7eb5   \n",
+       "770998  fffde7360aaed78e7ae839d80c7474710394b16c   \n",
+       "770999  fffdedf0b18b842a566e36eb1143be87fa89ec5d   \n",
+       "771000  fffdefeee340b6c60d2f8c8a6f27d6943addefa9   \n",
+       "771001  fffdf93ab97bce745ef9275492278030872f14ab   \n",
+       "771002  fffe11a3ac0619b14fd577005ddb7d379ac5f9d0   \n",
+       "771003  fffe2f1973fe927a07b51982606717600caf2e0e   \n",
+       "771004  fffe3525e7a24fdf9ab76601f220e29ea2baf327   \n",
+       "771005  fffea406c12823e5864ea00ae98d8b99c9589145   \n",
+       "771006  fffeb23f36412d1a940abdab9f03643b57fc4027   \n",
+       "771007  fffebd9e7eabeda8839774bf15bdba921e19e693   \n",
+       "771008  fffec462f80db47772e9f0afe431162d47350c82   \n",
+       "771009  fffeeae0ab84c261d0a6b1e57e5a89c7d6217984   \n",
+       "771010  ffff06ab149541f11cb8a2612b0da87eba2ed950   \n",
+       "771011  ffff14ee8bef7387a05a504808bc77d9e38de3ab   \n",
+       "771012  ffff333854bfd2aef1adb44eedaee86842d38c96   \n",
+       "771013  ffff666f06cff0c5f167f96a37fffb2cb2867d2c   \n",
+       "771014  ffff7fbfc6e2ee2af05646a03c3b1bfee05dd82f   \n",
+       "771015  ffffe8068e37504568db803945cfdeee68e68859   \n",
+       "771016  fffff7e11f1a8d38d9378103e0ca6b25ed219a5e   \n",
+       "\n",
+       "                                               first_dupe  \n",
+       "0       /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "1       /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "2       /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "3       /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "4       /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "5       /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "6       /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "7       /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "8       /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "9       /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "10      /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "11      /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "12      /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "13      /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "14      /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "15      /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "16      /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "17      /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "18      /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "19      /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "20      /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "21      /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "22      /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "23      /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "24      /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "25      /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "26      /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "27      /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "28      /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "29      /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "...                                                   ...  \n",
+       "770987  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "770988  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "770989  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "770990  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "770991  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "770992  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "770993  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "770994  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "770995  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "770996  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "770997  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "770998  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "770999  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "771000  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "771001  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "771002  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "771003  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "771004  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "771005  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "771006  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "771007  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "771008  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "771009  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "771010  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "771011  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "771012  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "771013  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "771014  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "771015  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "771016  /braintree/data2/active/users/jjpr/mkgu_packag...  \n",
+       "\n",
+       "[771017 rows x 2 columns]"
+      ]
+     },
+     "execution_count": 24,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_image_meta[\"first_dupe\"] = list(map(first_dupe, df_image_meta[\"image_id\"]))\n",
+    "df_image_meta"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_relative(path, base):\n",
+    "    split_path = path.split(\"/\")\n",
+    "    split_base = base.split(\"/\")\n",
+    "    target_path = \"/\".join(split_path[len(split_base):])\n",
+    "    return target_path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "file_base = \"/braintree/data2/active/users/jjpr/mkgu_packaging\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>image_id</th>\n",
+       "      <th>first_dupe</th>\n",
+       "      <th>relative_path</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>00000a2518d148e9b455537488e325a7f42f39d8</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data8/NatRev/stimuli/41590fd57c0b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>00001776837935437e2784af953a4db24cc19536</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data14/NatRev/stimuli/2f03d6fc365...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>00003bb7e9ca9265dc521763f73514f9c4d48f8a</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data4/DGrat/stimuli/6954025a16641...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>00005ad0cbb2bafc56814919fda60268a1077ba3</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data14/NatRev/stimuli/beee725f0d2...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>00005e46dace8673b631b75f1df3a6d1a8b2c1c2</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data1/DGrat/stimuli/616577d68f558...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>00006b3962b738bb25ab44265092ac6ef71943ed</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/b1178068ddf...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>00007b43039dc9f26f12d6e222460a70a1400d1f</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/f803727e2e6...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>00008a40da04647d4350b9acb0b6657326c1ba54</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data15/NatRev/stimuli/aa2ec48ff96...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>00008be32ed56378291bc536589d97bc39e35557</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/1f67ce6b1df...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>00008eead95d140fb4591430b11193525dfc379a</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/727a677da46...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10</th>\n",
+       "      <td>00009e9fd517bd1aa8c72046dc51b23beb14bd6b</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data9/NatRev/stimuli/b2b4abe27f40...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11</th>\n",
+       "      <td>00009f4607ffef290fa043eaaad9a61792cdf52d</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data16/NatRev/stimuli/e1845ec2d6a...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>12</th>\n",
+       "      <td>0000b4496034903fcfa414566b171f4c59959aa1</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data1/DGrat/stimuli/8aeac1a5fd78f...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13</th>\n",
+       "      <td>0000c1747157700a98bb17f0712f61a55bfc76be</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/0e886b29ee9...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14</th>\n",
+       "      <td>0000c18032e90501faa945854e0a8279f35f113b</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data3/DGrat/stimuli/6a048770f7e0d...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>15</th>\n",
+       "      <td>0000c941ce6c69a723ca4842ec52f7c39c1fcf6e</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data1/DGrat/stimuli/a67aec394883e...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>16</th>\n",
+       "      <td>0000ec7b6ffd8e31f84d0b81b387154838ad9605</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/dcb7785ebe6...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>17</th>\n",
+       "      <td>00010a8473ad131508f1d21353221dd3038d7d00</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/5d478210b5c...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>18</th>\n",
+       "      <td>00010ac420ed0bd3f212fc4a0a1c51bda60ec929</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data1/DGrat/stimuli/2408273f04739...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>19</th>\n",
+       "      <td>00011aedc68af6f9c23c35ba1bd512dbc2e8e8b9</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/4732de4356e...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>20</th>\n",
+       "      <td>000128de8497372d32b8237b085b2ca2bf679ebf</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/18160fa581d...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>21</th>\n",
+       "      <td>00013841c33fbd35df4e025b04db78cbc330a380</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data5/DGrat/stimuli/0938334bd9d5f...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>22</th>\n",
+       "      <td>00014c83ca4993261ebb88a75cd9207e569d0bba</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data4/DGrat/stimuli/ee44f2758175b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>23</th>\n",
+       "      <td>0001548f9772a3870f9f57d7ddd15372fdd498d4</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/05c361feae1...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>24</th>\n",
+       "      <td>00019d74a4b7ca4e84a402e33f25028996345ce3</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/73f2da8ebc7...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>25</th>\n",
+       "      <td>0001c0abf05394d6438f48e8376b66297938e888</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/9dd99a69b02...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>26</th>\n",
+       "      <td>0001d851793da81785d86b4205b184a74b60ce23</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data19/NatRev/stimuli/fe5d9d38f0e...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>27</th>\n",
+       "      <td>0001d99d7f29638060839746013c043f7857ae47</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/02991dc3ec4...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>28</th>\n",
+       "      <td>000209ba9ff4f70dce2baf79e5dd0328c688bc11</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/f9ca4c3ff84...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>29</th>\n",
+       "      <td>00023835d1cb1095d5c97526fa460438c25490a3</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/38e13dd6b14...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770987</th>\n",
+       "      <td>fffd5b92538382d74078624393b5ac341724a25d</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data11/NatRev/stimuli/3e3248e7fdb...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770988</th>\n",
+       "      <td>fffd70f1ebd24394b678671016d7da2b044302c7</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data15/NatRev/stimuli/3c551cba386...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770989</th>\n",
+       "      <td>fffd8533bbb0982e3d9927d4834d17c9068a1857</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data4/DGrat/stimuli/ae77e07ad79cd...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770990</th>\n",
+       "      <td>fffd8bdbb214a1a9772ac138501cae9b990a8731</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data8/NatRev/stimuli/73a16cc24970...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770991</th>\n",
+       "      <td>fffd9e0889381e54b1bd12e0576b595d9976b9dd</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data9/NatRev/stimuli/b409fa07bb5f...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770992</th>\n",
+       "      <td>fffdad566df3a4bd3a660d193ada8eb8ffaa88c2</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data13/NatRev/stimuli/38ce2a5517b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770993</th>\n",
+       "      <td>fffdb264a410114f2c9bed3765786c474bdea1a9</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/5d931ba3722...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770994</th>\n",
+       "      <td>fffdb9a37c6a1f0cd100f9b340a2788600f591f7</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data16/NatRev/stimuli/21ca02a9cf9...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770995</th>\n",
+       "      <td>fffdd183dc71df6de59b077cec3f00e6bf04f5cd</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data1/DGrat/stimuli/7832e1b72ceac...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770996</th>\n",
+       "      <td>fffdd57f5450722f7c756b2ff3f0974ffa610124</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/2cad235ec23...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770997</th>\n",
+       "      <td>fffde0392a7a1b70ada752dbe6c34bbbd2aa7eb5</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/75b4af00ed0...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770998</th>\n",
+       "      <td>fffde7360aaed78e7ae839d80c7474710394b16c</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/93a7daa37b6...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770999</th>\n",
+       "      <td>fffdedf0b18b842a566e36eb1143be87fa89ec5d</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data1/DGrat/stimuli/242989d62300a...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771000</th>\n",
+       "      <td>fffdefeee340b6c60d2f8c8a6f27d6943addefa9</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data14/NatRev/stimuli/8eb55148a7d...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771001</th>\n",
+       "      <td>fffdf93ab97bce745ef9275492278030872f14ab</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data7/NatRev/stimuli/57de6957435a...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771002</th>\n",
+       "      <td>fffe11a3ac0619b14fd577005ddb7d379ac5f9d0</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/4c9ea7a1ccf...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771003</th>\n",
+       "      <td>fffe2f1973fe927a07b51982606717600caf2e0e</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data4/DGrat/stimuli/44a1c41b7c7e1...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771004</th>\n",
+       "      <td>fffe3525e7a24fdf9ab76601f220e29ea2baf327</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data1/DGrat/stimuli/7613fc37f14de...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771005</th>\n",
+       "      <td>fffea406c12823e5864ea00ae98d8b99c9589145</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/598765f127f...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771006</th>\n",
+       "      <td>fffeb23f36412d1a940abdab9f03643b57fc4027</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data1/DGrat/stimuli/b9c11cf935638...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771007</th>\n",
+       "      <td>fffebd9e7eabeda8839774bf15bdba921e19e693</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/df6b12ffde3...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771008</th>\n",
+       "      <td>fffec462f80db47772e9f0afe431162d47350c82</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data13/NatRev/stimuli/d093a74b17a...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771009</th>\n",
+       "      <td>fffeeae0ab84c261d0a6b1e57e5a89c7d6217984</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data8/NatRev/stimuli/1c76bf23e30d...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771010</th>\n",
+       "      <td>ffff06ab149541f11cb8a2612b0da87eba2ed950</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/95678ef6c6e...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771011</th>\n",
+       "      <td>ffff14ee8bef7387a05a504808bc77d9e38de3ab</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/5e8910ffcfd...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771012</th>\n",
+       "      <td>ffff333854bfd2aef1adb44eedaee86842d38c96</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data11/NatRev/stimuli/ab19f537bc6...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771013</th>\n",
+       "      <td>ffff666f06cff0c5f167f96a37fffb2cb2867d2c</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data3/DGrat/stimuli/4d35448df2c44...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771014</th>\n",
+       "      <td>ffff7fbfc6e2ee2af05646a03c3b1bfee05dd82f</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data9/NatRev/stimuli/1e63ca6a93bc...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771015</th>\n",
+       "      <td>ffffe8068e37504568db803945cfdeee68e68859</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/55e31de78f9...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771016</th>\n",
+       "      <td>fffff7e11f1a8d38d9378103e0ca6b25ed219a5e</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data11/NatRev/stimuli/17d9020839b...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>771017 rows × 3 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                        image_id  \\\n",
+       "0       00000a2518d148e9b455537488e325a7f42f39d8   \n",
+       "1       00001776837935437e2784af953a4db24cc19536   \n",
+       "2       00003bb7e9ca9265dc521763f73514f9c4d48f8a   \n",
+       "3       00005ad0cbb2bafc56814919fda60268a1077ba3   \n",
+       "4       00005e46dace8673b631b75f1df3a6d1a8b2c1c2   \n",
+       "5       00006b3962b738bb25ab44265092ac6ef71943ed   \n",
+       "6       00007b43039dc9f26f12d6e222460a70a1400d1f   \n",
+       "7       00008a40da04647d4350b9acb0b6657326c1ba54   \n",
+       "8       00008be32ed56378291bc536589d97bc39e35557   \n",
+       "9       00008eead95d140fb4591430b11193525dfc379a   \n",
+       "10      00009e9fd517bd1aa8c72046dc51b23beb14bd6b   \n",
+       "11      00009f4607ffef290fa043eaaad9a61792cdf52d   \n",
+       "12      0000b4496034903fcfa414566b171f4c59959aa1   \n",
+       "13      0000c1747157700a98bb17f0712f61a55bfc76be   \n",
+       "14      0000c18032e90501faa945854e0a8279f35f113b   \n",
+       "15      0000c941ce6c69a723ca4842ec52f7c39c1fcf6e   \n",
+       "16      0000ec7b6ffd8e31f84d0b81b387154838ad9605   \n",
+       "17      00010a8473ad131508f1d21353221dd3038d7d00   \n",
+       "18      00010ac420ed0bd3f212fc4a0a1c51bda60ec929   \n",
+       "19      00011aedc68af6f9c23c35ba1bd512dbc2e8e8b9   \n",
+       "20      000128de8497372d32b8237b085b2ca2bf679ebf   \n",
+       "21      00013841c33fbd35df4e025b04db78cbc330a380   \n",
+       "22      00014c83ca4993261ebb88a75cd9207e569d0bba   \n",
+       "23      0001548f9772a3870f9f57d7ddd15372fdd498d4   \n",
+       "24      00019d74a4b7ca4e84a402e33f25028996345ce3   \n",
+       "25      0001c0abf05394d6438f48e8376b66297938e888   \n",
+       "26      0001d851793da81785d86b4205b184a74b60ce23   \n",
+       "27      0001d99d7f29638060839746013c043f7857ae47   \n",
+       "28      000209ba9ff4f70dce2baf79e5dd0328c688bc11   \n",
+       "29      00023835d1cb1095d5c97526fa460438c25490a3   \n",
+       "...                                          ...   \n",
+       "770987  fffd5b92538382d74078624393b5ac341724a25d   \n",
+       "770988  fffd70f1ebd24394b678671016d7da2b044302c7   \n",
+       "770989  fffd8533bbb0982e3d9927d4834d17c9068a1857   \n",
+       "770990  fffd8bdbb214a1a9772ac138501cae9b990a8731   \n",
+       "770991  fffd9e0889381e54b1bd12e0576b595d9976b9dd   \n",
+       "770992  fffdad566df3a4bd3a660d193ada8eb8ffaa88c2   \n",
+       "770993  fffdb264a410114f2c9bed3765786c474bdea1a9   \n",
+       "770994  fffdb9a37c6a1f0cd100f9b340a2788600f591f7   \n",
+       "770995  fffdd183dc71df6de59b077cec3f00e6bf04f5cd   \n",
+       "770996  fffdd57f5450722f7c756b2ff3f0974ffa610124   \n",
+       "770997  fffde0392a7a1b70ada752dbe6c34bbbd2aa7eb5   \n",
+       "770998  fffde7360aaed78e7ae839d80c7474710394b16c   \n",
+       "770999  fffdedf0b18b842a566e36eb1143be87fa89ec5d   \n",
+       "771000  fffdefeee340b6c60d2f8c8a6f27d6943addefa9   \n",
+       "771001  fffdf93ab97bce745ef9275492278030872f14ab   \n",
+       "771002  fffe11a3ac0619b14fd577005ddb7d379ac5f9d0   \n",
+       "771003  fffe2f1973fe927a07b51982606717600caf2e0e   \n",
+       "771004  fffe3525e7a24fdf9ab76601f220e29ea2baf327   \n",
+       "771005  fffea406c12823e5864ea00ae98d8b99c9589145   \n",
+       "771006  fffeb23f36412d1a940abdab9f03643b57fc4027   \n",
+       "771007  fffebd9e7eabeda8839774bf15bdba921e19e693   \n",
+       "771008  fffec462f80db47772e9f0afe431162d47350c82   \n",
+       "771009  fffeeae0ab84c261d0a6b1e57e5a89c7d6217984   \n",
+       "771010  ffff06ab149541f11cb8a2612b0da87eba2ed950   \n",
+       "771011  ffff14ee8bef7387a05a504808bc77d9e38de3ab   \n",
+       "771012  ffff333854bfd2aef1adb44eedaee86842d38c96   \n",
+       "771013  ffff666f06cff0c5f167f96a37fffb2cb2867d2c   \n",
+       "771014  ffff7fbfc6e2ee2af05646a03c3b1bfee05dd82f   \n",
+       "771015  ffffe8068e37504568db803945cfdeee68e68859   \n",
+       "771016  fffff7e11f1a8d38d9378103e0ca6b25ed219a5e   \n",
+       "\n",
+       "                                               first_dupe  \\\n",
+       "0       /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "1       /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "2       /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "3       /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "4       /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "5       /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "6       /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "7       /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "8       /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "9       /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "10      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "11      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "12      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "13      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "14      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "15      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "16      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "17      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "18      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "19      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "20      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "21      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "22      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "23      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "24      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "25      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "26      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "27      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "28      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "29      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "...                                                   ...   \n",
+       "770987  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "770988  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "770989  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "770990  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "770991  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "770992  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "770993  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "770994  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "770995  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "770996  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "770997  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "770998  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "770999  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771000  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771001  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771002  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771003  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771004  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771005  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771006  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771007  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771008  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771009  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771010  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771011  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771012  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771013  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771014  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771015  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771016  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "\n",
+       "                                            relative_path  \n",
+       "0       crcns/v2-1/V2Data8/NatRev/stimuli/41590fd57c0b...  \n",
+       "1       crcns/v2-1/V2Data14/NatRev/stimuli/2f03d6fc365...  \n",
+       "2       crcns/v2-1/V2Data4/DGrat/stimuli/6954025a16641...  \n",
+       "3       crcns/v2-1/V2Data14/NatRev/stimuli/beee725f0d2...  \n",
+       "4       crcns/v2-1/V2Data1/DGrat/stimuli/616577d68f558...  \n",
+       "5       crcns/v2-1/V2Data18/NatRev/stimuli/b1178068ddf...  \n",
+       "6       crcns/v2-1/V2Data18/NatRev/stimuli/f803727e2e6...  \n",
+       "7       crcns/v2-1/V2Data15/NatRev/stimuli/aa2ec48ff96...  \n",
+       "8       crcns/v2-1/V2Data17/NatRev/stimuli/1f67ce6b1df...  \n",
+       "9       crcns/v2-1/V2Data17/NatRev/stimuli/727a677da46...  \n",
+       "10      crcns/v2-1/V2Data9/NatRev/stimuli/b2b4abe27f40...  \n",
+       "11      crcns/v2-1/V2Data16/NatRev/stimuli/e1845ec2d6a...  \n",
+       "12      crcns/v2-1/V2Data1/DGrat/stimuli/8aeac1a5fd78f...  \n",
+       "13      crcns/v2-1/V2Data18/NatRev/stimuli/0e886b29ee9...  \n",
+       "14      crcns/v2-1/V2Data3/DGrat/stimuli/6a048770f7e0d...  \n",
+       "15      crcns/v2-1/V2Data1/DGrat/stimuli/a67aec394883e...  \n",
+       "16      crcns/v2-1/V2Data17/NatRev/stimuli/dcb7785ebe6...  \n",
+       "17      crcns/v2-1/V2Data18/NatRev/stimuli/5d478210b5c...  \n",
+       "18      crcns/v2-1/V2Data1/DGrat/stimuli/2408273f04739...  \n",
+       "19      crcns/v2-1/V2Data18/NatRev/stimuli/4732de4356e...  \n",
+       "20      crcns/v2-1/V2Data17/NatRev/stimuli/18160fa581d...  \n",
+       "21      crcns/v2-1/V2Data5/DGrat/stimuli/0938334bd9d5f...  \n",
+       "22      crcns/v2-1/V2Data4/DGrat/stimuli/ee44f2758175b...  \n",
+       "23      crcns/v2-1/V2Data18/NatRev/stimuli/05c361feae1...  \n",
+       "24      crcns/v2-1/V2Data18/NatRev/stimuli/73f2da8ebc7...  \n",
+       "25      crcns/v2-1/V2Data18/NatRev/stimuli/9dd99a69b02...  \n",
+       "26      crcns/v2-1/V2Data19/NatRev/stimuli/fe5d9d38f0e...  \n",
+       "27      crcns/v2-1/V2Data17/NatRev/stimuli/02991dc3ec4...  \n",
+       "28      crcns/v2-1/V2Data18/NatRev/stimuli/f9ca4c3ff84...  \n",
+       "29      crcns/v2-1/V2Data18/NatRev/stimuli/38e13dd6b14...  \n",
+       "...                                                   ...  \n",
+       "770987  crcns/v2-1/V2Data11/NatRev/stimuli/3e3248e7fdb...  \n",
+       "770988  crcns/v2-1/V2Data15/NatRev/stimuli/3c551cba386...  \n",
+       "770989  crcns/v2-1/V2Data4/DGrat/stimuli/ae77e07ad79cd...  \n",
+       "770990  crcns/v2-1/V2Data8/NatRev/stimuli/73a16cc24970...  \n",
+       "770991  crcns/v2-1/V2Data9/NatRev/stimuli/b409fa07bb5f...  \n",
+       "770992  crcns/v2-1/V2Data13/NatRev/stimuli/38ce2a5517b...  \n",
+       "770993  crcns/v2-1/V2Data17/NatRev/stimuli/5d931ba3722...  \n",
+       "770994  crcns/v2-1/V2Data16/NatRev/stimuli/21ca02a9cf9...  \n",
+       "770995  crcns/v2-1/V2Data1/DGrat/stimuli/7832e1b72ceac...  \n",
+       "770996  crcns/v2-1/V2Data18/NatRev/stimuli/2cad235ec23...  \n",
+       "770997  crcns/v2-1/V2Data17/NatRev/stimuli/75b4af00ed0...  \n",
+       "770998  crcns/v2-1/V2Data18/NatRev/stimuli/93a7daa37b6...  \n",
+       "770999  crcns/v2-1/V2Data1/DGrat/stimuli/242989d62300a...  \n",
+       "771000  crcns/v2-1/V2Data14/NatRev/stimuli/8eb55148a7d...  \n",
+       "771001  crcns/v2-1/V2Data7/NatRev/stimuli/57de6957435a...  \n",
+       "771002  crcns/v2-1/V2Data17/NatRev/stimuli/4c9ea7a1ccf...  \n",
+       "771003  crcns/v2-1/V2Data4/DGrat/stimuli/44a1c41b7c7e1...  \n",
+       "771004  crcns/v2-1/V2Data1/DGrat/stimuli/7613fc37f14de...  \n",
+       "771005  crcns/v2-1/V2Data18/NatRev/stimuli/598765f127f...  \n",
+       "771006  crcns/v2-1/V2Data1/DGrat/stimuli/b9c11cf935638...  \n",
+       "771007  crcns/v2-1/V2Data18/NatRev/stimuli/df6b12ffde3...  \n",
+       "771008  crcns/v2-1/V2Data13/NatRev/stimuli/d093a74b17a...  \n",
+       "771009  crcns/v2-1/V2Data8/NatRev/stimuli/1c76bf23e30d...  \n",
+       "771010  crcns/v2-1/V2Data18/NatRev/stimuli/95678ef6c6e...  \n",
+       "771011  crcns/v2-1/V2Data17/NatRev/stimuli/5e8910ffcfd...  \n",
+       "771012  crcns/v2-1/V2Data11/NatRev/stimuli/ab19f537bc6...  \n",
+       "771013  crcns/v2-1/V2Data3/DGrat/stimuli/4d35448df2c44...  \n",
+       "771014  crcns/v2-1/V2Data9/NatRev/stimuli/1e63ca6a93bc...  \n",
+       "771015  crcns/v2-1/V2Data18/NatRev/stimuli/55e31de78f9...  \n",
+       "771016  crcns/v2-1/V2Data11/NatRev/stimuli/17d9020839b...  \n",
+       "\n",
+       "[771017 rows x 3 columns]"
+      ]
+     },
+     "execution_count": 29,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_image_meta[\"relative_path\"] = list(map(lambda x: get_relative(x, file_base), df_image_meta[\"first_dupe\"]))\n",
+    "df_image_meta"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "target_zip_path = \"/braintree/home/jjpr/.mkgu/data/gallant.Willmore2010/gallant_crcns_v2-1_stimuli.zip\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!mkdir -p /braintree/home/jjpr/.mkgu/data/gallant.Willmore2010"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with zipfile.ZipFile(target_zip_path, 'w') as target_zip:\n",
+    "    for image in df_image_meta.itertuples():\n",
+    "        target_zip.write(image.first_dupe, arcname=image.relative_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "containing_dir = os.path.dirname(target_zip_path)\n",
+    "with zipfile.ZipFile(target_zip_path, 'r') as new_zip:\n",
+    "    new_zip.extractall(containing_dir)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def copied(source):\n",
+    "    relative = get_relative(source, file_base)\n",
+    "    target = os.path.join(containing_dir, relative)\n",
+    "    return os.path.exists(target)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>image_id</th>\n",
+       "      <th>first_dupe</th>\n",
+       "      <th>relative_path</th>\n",
+       "      <th>copied</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>00000a2518d148e9b455537488e325a7f42f39d8</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data8/NatRev/stimuli/41590fd57c0b...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>00001776837935437e2784af953a4db24cc19536</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data14/NatRev/stimuli/2f03d6fc365...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>00003bb7e9ca9265dc521763f73514f9c4d48f8a</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data4/DGrat/stimuli/6954025a16641...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>00005ad0cbb2bafc56814919fda60268a1077ba3</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data14/NatRev/stimuli/beee725f0d2...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>00005e46dace8673b631b75f1df3a6d1a8b2c1c2</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data1/DGrat/stimuli/616577d68f558...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>00006b3962b738bb25ab44265092ac6ef71943ed</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/b1178068ddf...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>00007b43039dc9f26f12d6e222460a70a1400d1f</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/f803727e2e6...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>00008a40da04647d4350b9acb0b6657326c1ba54</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data15/NatRev/stimuli/aa2ec48ff96...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>00008be32ed56378291bc536589d97bc39e35557</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/1f67ce6b1df...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>00008eead95d140fb4591430b11193525dfc379a</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/727a677da46...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10</th>\n",
+       "      <td>00009e9fd517bd1aa8c72046dc51b23beb14bd6b</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data9/NatRev/stimuli/b2b4abe27f40...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11</th>\n",
+       "      <td>00009f4607ffef290fa043eaaad9a61792cdf52d</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data16/NatRev/stimuli/e1845ec2d6a...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>12</th>\n",
+       "      <td>0000b4496034903fcfa414566b171f4c59959aa1</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data1/DGrat/stimuli/8aeac1a5fd78f...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13</th>\n",
+       "      <td>0000c1747157700a98bb17f0712f61a55bfc76be</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/0e886b29ee9...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14</th>\n",
+       "      <td>0000c18032e90501faa945854e0a8279f35f113b</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data3/DGrat/stimuli/6a048770f7e0d...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>15</th>\n",
+       "      <td>0000c941ce6c69a723ca4842ec52f7c39c1fcf6e</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data1/DGrat/stimuli/a67aec394883e...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>16</th>\n",
+       "      <td>0000ec7b6ffd8e31f84d0b81b387154838ad9605</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/dcb7785ebe6...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>17</th>\n",
+       "      <td>00010a8473ad131508f1d21353221dd3038d7d00</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/5d478210b5c...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>18</th>\n",
+       "      <td>00010ac420ed0bd3f212fc4a0a1c51bda60ec929</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data1/DGrat/stimuli/2408273f04739...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>19</th>\n",
+       "      <td>00011aedc68af6f9c23c35ba1bd512dbc2e8e8b9</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/4732de4356e...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>20</th>\n",
+       "      <td>000128de8497372d32b8237b085b2ca2bf679ebf</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/18160fa581d...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>21</th>\n",
+       "      <td>00013841c33fbd35df4e025b04db78cbc330a380</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data5/DGrat/stimuli/0938334bd9d5f...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>22</th>\n",
+       "      <td>00014c83ca4993261ebb88a75cd9207e569d0bba</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data4/DGrat/stimuli/ee44f2758175b...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>23</th>\n",
+       "      <td>0001548f9772a3870f9f57d7ddd15372fdd498d4</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/05c361feae1...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>24</th>\n",
+       "      <td>00019d74a4b7ca4e84a402e33f25028996345ce3</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/73f2da8ebc7...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>25</th>\n",
+       "      <td>0001c0abf05394d6438f48e8376b66297938e888</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/9dd99a69b02...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>26</th>\n",
+       "      <td>0001d851793da81785d86b4205b184a74b60ce23</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data19/NatRev/stimuli/fe5d9d38f0e...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>27</th>\n",
+       "      <td>0001d99d7f29638060839746013c043f7857ae47</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/02991dc3ec4...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>28</th>\n",
+       "      <td>000209ba9ff4f70dce2baf79e5dd0328c688bc11</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/f9ca4c3ff84...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>29</th>\n",
+       "      <td>00023835d1cb1095d5c97526fa460438c25490a3</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/38e13dd6b14...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770987</th>\n",
+       "      <td>fffd5b92538382d74078624393b5ac341724a25d</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data11/NatRev/stimuli/3e3248e7fdb...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770988</th>\n",
+       "      <td>fffd70f1ebd24394b678671016d7da2b044302c7</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data15/NatRev/stimuli/3c551cba386...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770989</th>\n",
+       "      <td>fffd8533bbb0982e3d9927d4834d17c9068a1857</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data4/DGrat/stimuli/ae77e07ad79cd...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770990</th>\n",
+       "      <td>fffd8bdbb214a1a9772ac138501cae9b990a8731</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data8/NatRev/stimuli/73a16cc24970...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770991</th>\n",
+       "      <td>fffd9e0889381e54b1bd12e0576b595d9976b9dd</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data9/NatRev/stimuli/b409fa07bb5f...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770992</th>\n",
+       "      <td>fffdad566df3a4bd3a660d193ada8eb8ffaa88c2</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data13/NatRev/stimuli/38ce2a5517b...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770993</th>\n",
+       "      <td>fffdb264a410114f2c9bed3765786c474bdea1a9</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/5d931ba3722...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770994</th>\n",
+       "      <td>fffdb9a37c6a1f0cd100f9b340a2788600f591f7</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data16/NatRev/stimuli/21ca02a9cf9...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770995</th>\n",
+       "      <td>fffdd183dc71df6de59b077cec3f00e6bf04f5cd</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data1/DGrat/stimuli/7832e1b72ceac...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770996</th>\n",
+       "      <td>fffdd57f5450722f7c756b2ff3f0974ffa610124</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/2cad235ec23...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770997</th>\n",
+       "      <td>fffde0392a7a1b70ada752dbe6c34bbbd2aa7eb5</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/75b4af00ed0...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770998</th>\n",
+       "      <td>fffde7360aaed78e7ae839d80c7474710394b16c</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/93a7daa37b6...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770999</th>\n",
+       "      <td>fffdedf0b18b842a566e36eb1143be87fa89ec5d</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data1/DGrat/stimuli/242989d62300a...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771000</th>\n",
+       "      <td>fffdefeee340b6c60d2f8c8a6f27d6943addefa9</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data14/NatRev/stimuli/8eb55148a7d...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771001</th>\n",
+       "      <td>fffdf93ab97bce745ef9275492278030872f14ab</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data7/NatRev/stimuli/57de6957435a...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771002</th>\n",
+       "      <td>fffe11a3ac0619b14fd577005ddb7d379ac5f9d0</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/4c9ea7a1ccf...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771003</th>\n",
+       "      <td>fffe2f1973fe927a07b51982606717600caf2e0e</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data4/DGrat/stimuli/44a1c41b7c7e1...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771004</th>\n",
+       "      <td>fffe3525e7a24fdf9ab76601f220e29ea2baf327</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data1/DGrat/stimuli/7613fc37f14de...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771005</th>\n",
+       "      <td>fffea406c12823e5864ea00ae98d8b99c9589145</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/598765f127f...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771006</th>\n",
+       "      <td>fffeb23f36412d1a940abdab9f03643b57fc4027</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data1/DGrat/stimuli/b9c11cf935638...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771007</th>\n",
+       "      <td>fffebd9e7eabeda8839774bf15bdba921e19e693</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/df6b12ffde3...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771008</th>\n",
+       "      <td>fffec462f80db47772e9f0afe431162d47350c82</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data13/NatRev/stimuli/d093a74b17a...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771009</th>\n",
+       "      <td>fffeeae0ab84c261d0a6b1e57e5a89c7d6217984</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data8/NatRev/stimuli/1c76bf23e30d...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771010</th>\n",
+       "      <td>ffff06ab149541f11cb8a2612b0da87eba2ed950</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/95678ef6c6e...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771011</th>\n",
+       "      <td>ffff14ee8bef7387a05a504808bc77d9e38de3ab</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/5e8910ffcfd...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771012</th>\n",
+       "      <td>ffff333854bfd2aef1adb44eedaee86842d38c96</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data11/NatRev/stimuli/ab19f537bc6...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771013</th>\n",
+       "      <td>ffff666f06cff0c5f167f96a37fffb2cb2867d2c</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data3/DGrat/stimuli/4d35448df2c44...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771014</th>\n",
+       "      <td>ffff7fbfc6e2ee2af05646a03c3b1bfee05dd82f</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data9/NatRev/stimuli/1e63ca6a93bc...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771015</th>\n",
+       "      <td>ffffe8068e37504568db803945cfdeee68e68859</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/55e31de78f9...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771016</th>\n",
+       "      <td>fffff7e11f1a8d38d9378103e0ca6b25ed219a5e</td>\n",
+       "      <td>/braintree/data2/active/users/jjpr/mkgu_packag...</td>\n",
+       "      <td>crcns/v2-1/V2Data11/NatRev/stimuli/17d9020839b...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>771017 rows × 4 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                        image_id  \\\n",
+       "0       00000a2518d148e9b455537488e325a7f42f39d8   \n",
+       "1       00001776837935437e2784af953a4db24cc19536   \n",
+       "2       00003bb7e9ca9265dc521763f73514f9c4d48f8a   \n",
+       "3       00005ad0cbb2bafc56814919fda60268a1077ba3   \n",
+       "4       00005e46dace8673b631b75f1df3a6d1a8b2c1c2   \n",
+       "5       00006b3962b738bb25ab44265092ac6ef71943ed   \n",
+       "6       00007b43039dc9f26f12d6e222460a70a1400d1f   \n",
+       "7       00008a40da04647d4350b9acb0b6657326c1ba54   \n",
+       "8       00008be32ed56378291bc536589d97bc39e35557   \n",
+       "9       00008eead95d140fb4591430b11193525dfc379a   \n",
+       "10      00009e9fd517bd1aa8c72046dc51b23beb14bd6b   \n",
+       "11      00009f4607ffef290fa043eaaad9a61792cdf52d   \n",
+       "12      0000b4496034903fcfa414566b171f4c59959aa1   \n",
+       "13      0000c1747157700a98bb17f0712f61a55bfc76be   \n",
+       "14      0000c18032e90501faa945854e0a8279f35f113b   \n",
+       "15      0000c941ce6c69a723ca4842ec52f7c39c1fcf6e   \n",
+       "16      0000ec7b6ffd8e31f84d0b81b387154838ad9605   \n",
+       "17      00010a8473ad131508f1d21353221dd3038d7d00   \n",
+       "18      00010ac420ed0bd3f212fc4a0a1c51bda60ec929   \n",
+       "19      00011aedc68af6f9c23c35ba1bd512dbc2e8e8b9   \n",
+       "20      000128de8497372d32b8237b085b2ca2bf679ebf   \n",
+       "21      00013841c33fbd35df4e025b04db78cbc330a380   \n",
+       "22      00014c83ca4993261ebb88a75cd9207e569d0bba   \n",
+       "23      0001548f9772a3870f9f57d7ddd15372fdd498d4   \n",
+       "24      00019d74a4b7ca4e84a402e33f25028996345ce3   \n",
+       "25      0001c0abf05394d6438f48e8376b66297938e888   \n",
+       "26      0001d851793da81785d86b4205b184a74b60ce23   \n",
+       "27      0001d99d7f29638060839746013c043f7857ae47   \n",
+       "28      000209ba9ff4f70dce2baf79e5dd0328c688bc11   \n",
+       "29      00023835d1cb1095d5c97526fa460438c25490a3   \n",
+       "...                                          ...   \n",
+       "770987  fffd5b92538382d74078624393b5ac341724a25d   \n",
+       "770988  fffd70f1ebd24394b678671016d7da2b044302c7   \n",
+       "770989  fffd8533bbb0982e3d9927d4834d17c9068a1857   \n",
+       "770990  fffd8bdbb214a1a9772ac138501cae9b990a8731   \n",
+       "770991  fffd9e0889381e54b1bd12e0576b595d9976b9dd   \n",
+       "770992  fffdad566df3a4bd3a660d193ada8eb8ffaa88c2   \n",
+       "770993  fffdb264a410114f2c9bed3765786c474bdea1a9   \n",
+       "770994  fffdb9a37c6a1f0cd100f9b340a2788600f591f7   \n",
+       "770995  fffdd183dc71df6de59b077cec3f00e6bf04f5cd   \n",
+       "770996  fffdd57f5450722f7c756b2ff3f0974ffa610124   \n",
+       "770997  fffde0392a7a1b70ada752dbe6c34bbbd2aa7eb5   \n",
+       "770998  fffde7360aaed78e7ae839d80c7474710394b16c   \n",
+       "770999  fffdedf0b18b842a566e36eb1143be87fa89ec5d   \n",
+       "771000  fffdefeee340b6c60d2f8c8a6f27d6943addefa9   \n",
+       "771001  fffdf93ab97bce745ef9275492278030872f14ab   \n",
+       "771002  fffe11a3ac0619b14fd577005ddb7d379ac5f9d0   \n",
+       "771003  fffe2f1973fe927a07b51982606717600caf2e0e   \n",
+       "771004  fffe3525e7a24fdf9ab76601f220e29ea2baf327   \n",
+       "771005  fffea406c12823e5864ea00ae98d8b99c9589145   \n",
+       "771006  fffeb23f36412d1a940abdab9f03643b57fc4027   \n",
+       "771007  fffebd9e7eabeda8839774bf15bdba921e19e693   \n",
+       "771008  fffec462f80db47772e9f0afe431162d47350c82   \n",
+       "771009  fffeeae0ab84c261d0a6b1e57e5a89c7d6217984   \n",
+       "771010  ffff06ab149541f11cb8a2612b0da87eba2ed950   \n",
+       "771011  ffff14ee8bef7387a05a504808bc77d9e38de3ab   \n",
+       "771012  ffff333854bfd2aef1adb44eedaee86842d38c96   \n",
+       "771013  ffff666f06cff0c5f167f96a37fffb2cb2867d2c   \n",
+       "771014  ffff7fbfc6e2ee2af05646a03c3b1bfee05dd82f   \n",
+       "771015  ffffe8068e37504568db803945cfdeee68e68859   \n",
+       "771016  fffff7e11f1a8d38d9378103e0ca6b25ed219a5e   \n",
+       "\n",
+       "                                               first_dupe  \\\n",
+       "0       /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "1       /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "2       /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "3       /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "4       /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "5       /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "6       /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "7       /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "8       /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "9       /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "10      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "11      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "12      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "13      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "14      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "15      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "16      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "17      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "18      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "19      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "20      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "21      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "22      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "23      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "24      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "25      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "26      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "27      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "28      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "29      /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "...                                                   ...   \n",
+       "770987  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "770988  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "770989  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "770990  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "770991  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "770992  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "770993  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "770994  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "770995  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "770996  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "770997  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "770998  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "770999  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771000  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771001  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771002  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771003  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771004  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771005  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771006  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771007  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771008  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771009  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771010  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771011  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771012  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771013  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771014  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771015  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "771016  /braintree/data2/active/users/jjpr/mkgu_packag...   \n",
+       "\n",
+       "                                            relative_path  copied  \n",
+       "0       crcns/v2-1/V2Data8/NatRev/stimuli/41590fd57c0b...    True  \n",
+       "1       crcns/v2-1/V2Data14/NatRev/stimuli/2f03d6fc365...    True  \n",
+       "2       crcns/v2-1/V2Data4/DGrat/stimuli/6954025a16641...    True  \n",
+       "3       crcns/v2-1/V2Data14/NatRev/stimuli/beee725f0d2...    True  \n",
+       "4       crcns/v2-1/V2Data1/DGrat/stimuli/616577d68f558...    True  \n",
+       "5       crcns/v2-1/V2Data18/NatRev/stimuli/b1178068ddf...    True  \n",
+       "6       crcns/v2-1/V2Data18/NatRev/stimuli/f803727e2e6...    True  \n",
+       "7       crcns/v2-1/V2Data15/NatRev/stimuli/aa2ec48ff96...    True  \n",
+       "8       crcns/v2-1/V2Data17/NatRev/stimuli/1f67ce6b1df...    True  \n",
+       "9       crcns/v2-1/V2Data17/NatRev/stimuli/727a677da46...    True  \n",
+       "10      crcns/v2-1/V2Data9/NatRev/stimuli/b2b4abe27f40...    True  \n",
+       "11      crcns/v2-1/V2Data16/NatRev/stimuli/e1845ec2d6a...    True  \n",
+       "12      crcns/v2-1/V2Data1/DGrat/stimuli/8aeac1a5fd78f...    True  \n",
+       "13      crcns/v2-1/V2Data18/NatRev/stimuli/0e886b29ee9...    True  \n",
+       "14      crcns/v2-1/V2Data3/DGrat/stimuli/6a048770f7e0d...    True  \n",
+       "15      crcns/v2-1/V2Data1/DGrat/stimuli/a67aec394883e...    True  \n",
+       "16      crcns/v2-1/V2Data17/NatRev/stimuli/dcb7785ebe6...    True  \n",
+       "17      crcns/v2-1/V2Data18/NatRev/stimuli/5d478210b5c...    True  \n",
+       "18      crcns/v2-1/V2Data1/DGrat/stimuli/2408273f04739...    True  \n",
+       "19      crcns/v2-1/V2Data18/NatRev/stimuli/4732de4356e...    True  \n",
+       "20      crcns/v2-1/V2Data17/NatRev/stimuli/18160fa581d...    True  \n",
+       "21      crcns/v2-1/V2Data5/DGrat/stimuli/0938334bd9d5f...    True  \n",
+       "22      crcns/v2-1/V2Data4/DGrat/stimuli/ee44f2758175b...    True  \n",
+       "23      crcns/v2-1/V2Data18/NatRev/stimuli/05c361feae1...    True  \n",
+       "24      crcns/v2-1/V2Data18/NatRev/stimuli/73f2da8ebc7...    True  \n",
+       "25      crcns/v2-1/V2Data18/NatRev/stimuli/9dd99a69b02...    True  \n",
+       "26      crcns/v2-1/V2Data19/NatRev/stimuli/fe5d9d38f0e...    True  \n",
+       "27      crcns/v2-1/V2Data17/NatRev/stimuli/02991dc3ec4...    True  \n",
+       "28      crcns/v2-1/V2Data18/NatRev/stimuli/f9ca4c3ff84...    True  \n",
+       "29      crcns/v2-1/V2Data18/NatRev/stimuli/38e13dd6b14...    True  \n",
+       "...                                                   ...     ...  \n",
+       "770987  crcns/v2-1/V2Data11/NatRev/stimuli/3e3248e7fdb...    True  \n",
+       "770988  crcns/v2-1/V2Data15/NatRev/stimuli/3c551cba386...    True  \n",
+       "770989  crcns/v2-1/V2Data4/DGrat/stimuli/ae77e07ad79cd...    True  \n",
+       "770990  crcns/v2-1/V2Data8/NatRev/stimuli/73a16cc24970...    True  \n",
+       "770991  crcns/v2-1/V2Data9/NatRev/stimuli/b409fa07bb5f...    True  \n",
+       "770992  crcns/v2-1/V2Data13/NatRev/stimuli/38ce2a5517b...    True  \n",
+       "770993  crcns/v2-1/V2Data17/NatRev/stimuli/5d931ba3722...    True  \n",
+       "770994  crcns/v2-1/V2Data16/NatRev/stimuli/21ca02a9cf9...    True  \n",
+       "770995  crcns/v2-1/V2Data1/DGrat/stimuli/7832e1b72ceac...    True  \n",
+       "770996  crcns/v2-1/V2Data18/NatRev/stimuli/2cad235ec23...    True  \n",
+       "770997  crcns/v2-1/V2Data17/NatRev/stimuli/75b4af00ed0...    True  \n",
+       "770998  crcns/v2-1/V2Data18/NatRev/stimuli/93a7daa37b6...    True  \n",
+       "770999  crcns/v2-1/V2Data1/DGrat/stimuli/242989d62300a...    True  \n",
+       "771000  crcns/v2-1/V2Data14/NatRev/stimuli/8eb55148a7d...    True  \n",
+       "771001  crcns/v2-1/V2Data7/NatRev/stimuli/57de6957435a...    True  \n",
+       "771002  crcns/v2-1/V2Data17/NatRev/stimuli/4c9ea7a1ccf...    True  \n",
+       "771003  crcns/v2-1/V2Data4/DGrat/stimuli/44a1c41b7c7e1...    True  \n",
+       "771004  crcns/v2-1/V2Data1/DGrat/stimuli/7613fc37f14de...    True  \n",
+       "771005  crcns/v2-1/V2Data18/NatRev/stimuli/598765f127f...    True  \n",
+       "771006  crcns/v2-1/V2Data1/DGrat/stimuli/b9c11cf935638...    True  \n",
+       "771007  crcns/v2-1/V2Data18/NatRev/stimuli/df6b12ffde3...    True  \n",
+       "771008  crcns/v2-1/V2Data13/NatRev/stimuli/d093a74b17a...    True  \n",
+       "771009  crcns/v2-1/V2Data8/NatRev/stimuli/1c76bf23e30d...    True  \n",
+       "771010  crcns/v2-1/V2Data18/NatRev/stimuli/95678ef6c6e...    True  \n",
+       "771011  crcns/v2-1/V2Data17/NatRev/stimuli/5e8910ffcfd...    True  \n",
+       "771012  crcns/v2-1/V2Data11/NatRev/stimuli/ab19f537bc6...    True  \n",
+       "771013  crcns/v2-1/V2Data3/DGrat/stimuli/4d35448df2c44...    True  \n",
+       "771014  crcns/v2-1/V2Data9/NatRev/stimuli/1e63ca6a93bc...    True  \n",
+       "771015  crcns/v2-1/V2Data18/NatRev/stimuli/55e31de78f9...    True  \n",
+       "771016  crcns/v2-1/V2Data11/NatRev/stimuli/17d9020839b...    True  \n",
+       "\n",
+       "[771017 rows x 4 columns]"
+      ]
+     },
+     "execution_count": 41,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_image_meta[\"copied\"] = list(map(copied, df_image_meta[\"first_dupe\"]))\n",
+    "df_image_meta"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 42,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 42,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "all(df_image_meta[\"copied\"])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Make the StimulusSet lookup meta"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 44,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "pwdb.connect(reuse_if_open=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pwdb.create_tables(models=[ImageModel, AttributeModel, ImageMetaModel, StimulusSetModel, ImageStoreModel, StimulusSetImageMap, ImageStoreMap])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 46,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gallant_v2_images, created = StimulusSetModel.get_or_create(name=\"gallant.Willmore2010\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 49,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gallant_v2_image_store, created = ImageStoreModel.get_or_create(location_type=\"S3\", store_type=\"zip\",\n",
+    "                                  location=\"https://mkgu-gallant-crcns.s3.amazonaws.com/gallant_crcns_v2-1_stimuli.zip\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 50,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "eav_image_file_sha1, created = AttributeModel.get_or_create(name=\"image_file_sha1\", type=\"str\")\n",
+    "eav_image_file_path_unique, created = AttributeModel.get_or_create(name=\"image_file_path_unique\", type=\"str\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 52,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for image in df_image_meta.itertuples():\n",
+    "    pw_image = ImageModel(image_id=image.image_id)\n",
+    "    pw_stimulus_set_image_map = StimulusSetImageMap(stimulus_set=gallant_v2_images, image=pw_image)\n",
+    "    pw_image_image_store_map = ImageStoreMap(image=pw_image, image_store=gallant_v2_image_store,\n",
+    "                                             path=image.relative_path)\n",
+    "    pw_image.save()\n",
+    "    pw_stimulus_set_image_map.save()\n",
+    "    pw_image_image_store_map.save()\n",
+    "    ImageMetaModel(image=pw_image, attribute=eav_image_file_sha1, value=str(image.image_id)).save()\n",
+    "    ImageMetaModel(image=pw_image, attribute=eav_image_file_path_unique, value=str(image.relative_path)).save()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 53,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>id</th>\n",
+       "      <th>image_id</th>\n",
+       "      <th>image_file_sha1</th>\n",
+       "      <th>image_file_path_unique</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>125778</td>\n",
+       "      <td>00000a2518d148e9b455537488e325a7f42f39d8</td>\n",
+       "      <td>00000a2518d148e9b455537488e325a7f42f39d8</td>\n",
+       "      <td>crcns/v2-1/V2Data8/NatRev/stimuli/41590fd57c0b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>125779</td>\n",
+       "      <td>00001776837935437e2784af953a4db24cc19536</td>\n",
+       "      <td>00001776837935437e2784af953a4db24cc19536</td>\n",
+       "      <td>crcns/v2-1/V2Data14/NatRev/stimuli/2f03d6fc365...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>125780</td>\n",
+       "      <td>00003bb7e9ca9265dc521763f73514f9c4d48f8a</td>\n",
+       "      <td>00003bb7e9ca9265dc521763f73514f9c4d48f8a</td>\n",
+       "      <td>crcns/v2-1/V2Data4/DGrat/stimuli/6954025a16641...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>125781</td>\n",
+       "      <td>00005ad0cbb2bafc56814919fda60268a1077ba3</td>\n",
+       "      <td>00005ad0cbb2bafc56814919fda60268a1077ba3</td>\n",
+       "      <td>crcns/v2-1/V2Data14/NatRev/stimuli/beee725f0d2...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>125782</td>\n",
+       "      <td>00005e46dace8673b631b75f1df3a6d1a8b2c1c2</td>\n",
+       "      <td>00005e46dace8673b631b75f1df3a6d1a8b2c1c2</td>\n",
+       "      <td>crcns/v2-1/V2Data1/DGrat/stimuli/616577d68f558...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>125783</td>\n",
+       "      <td>00006b3962b738bb25ab44265092ac6ef71943ed</td>\n",
+       "      <td>00006b3962b738bb25ab44265092ac6ef71943ed</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/b1178068ddf...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>125784</td>\n",
+       "      <td>00007b43039dc9f26f12d6e222460a70a1400d1f</td>\n",
+       "      <td>00007b43039dc9f26f12d6e222460a70a1400d1f</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/f803727e2e6...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>125785</td>\n",
+       "      <td>00008a40da04647d4350b9acb0b6657326c1ba54</td>\n",
+       "      <td>00008a40da04647d4350b9acb0b6657326c1ba54</td>\n",
+       "      <td>crcns/v2-1/V2Data15/NatRev/stimuli/aa2ec48ff96...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>125786</td>\n",
+       "      <td>00008be32ed56378291bc536589d97bc39e35557</td>\n",
+       "      <td>00008be32ed56378291bc536589d97bc39e35557</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/1f67ce6b1df...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>125787</td>\n",
+       "      <td>00008eead95d140fb4591430b11193525dfc379a</td>\n",
+       "      <td>00008eead95d140fb4591430b11193525dfc379a</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/727a677da46...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10</th>\n",
+       "      <td>125788</td>\n",
+       "      <td>00009e9fd517bd1aa8c72046dc51b23beb14bd6b</td>\n",
+       "      <td>00009e9fd517bd1aa8c72046dc51b23beb14bd6b</td>\n",
+       "      <td>crcns/v2-1/V2Data9/NatRev/stimuli/b2b4abe27f40...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11</th>\n",
+       "      <td>125789</td>\n",
+       "      <td>00009f4607ffef290fa043eaaad9a61792cdf52d</td>\n",
+       "      <td>00009f4607ffef290fa043eaaad9a61792cdf52d</td>\n",
+       "      <td>crcns/v2-1/V2Data16/NatRev/stimuli/e1845ec2d6a...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>12</th>\n",
+       "      <td>125790</td>\n",
+       "      <td>0000b4496034903fcfa414566b171f4c59959aa1</td>\n",
+       "      <td>0000b4496034903fcfa414566b171f4c59959aa1</td>\n",
+       "      <td>crcns/v2-1/V2Data1/DGrat/stimuli/8aeac1a5fd78f...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13</th>\n",
+       "      <td>125791</td>\n",
+       "      <td>0000c1747157700a98bb17f0712f61a55bfc76be</td>\n",
+       "      <td>0000c1747157700a98bb17f0712f61a55bfc76be</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/0e886b29ee9...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14</th>\n",
+       "      <td>125792</td>\n",
+       "      <td>0000c18032e90501faa945854e0a8279f35f113b</td>\n",
+       "      <td>0000c18032e90501faa945854e0a8279f35f113b</td>\n",
+       "      <td>crcns/v2-1/V2Data3/DGrat/stimuli/6a048770f7e0d...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>15</th>\n",
+       "      <td>125793</td>\n",
+       "      <td>0000c941ce6c69a723ca4842ec52f7c39c1fcf6e</td>\n",
+       "      <td>0000c941ce6c69a723ca4842ec52f7c39c1fcf6e</td>\n",
+       "      <td>crcns/v2-1/V2Data1/DGrat/stimuli/a67aec394883e...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>16</th>\n",
+       "      <td>125794</td>\n",
+       "      <td>0000ec7b6ffd8e31f84d0b81b387154838ad9605</td>\n",
+       "      <td>0000ec7b6ffd8e31f84d0b81b387154838ad9605</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/dcb7785ebe6...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>17</th>\n",
+       "      <td>125795</td>\n",
+       "      <td>00010a8473ad131508f1d21353221dd3038d7d00</td>\n",
+       "      <td>00010a8473ad131508f1d21353221dd3038d7d00</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/5d478210b5c...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>18</th>\n",
+       "      <td>125796</td>\n",
+       "      <td>00010ac420ed0bd3f212fc4a0a1c51bda60ec929</td>\n",
+       "      <td>00010ac420ed0bd3f212fc4a0a1c51bda60ec929</td>\n",
+       "      <td>crcns/v2-1/V2Data1/DGrat/stimuli/2408273f04739...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>19</th>\n",
+       "      <td>125797</td>\n",
+       "      <td>00011aedc68af6f9c23c35ba1bd512dbc2e8e8b9</td>\n",
+       "      <td>00011aedc68af6f9c23c35ba1bd512dbc2e8e8b9</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/4732de4356e...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>20</th>\n",
+       "      <td>125798</td>\n",
+       "      <td>000128de8497372d32b8237b085b2ca2bf679ebf</td>\n",
+       "      <td>000128de8497372d32b8237b085b2ca2bf679ebf</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/18160fa581d...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>21</th>\n",
+       "      <td>125799</td>\n",
+       "      <td>00013841c33fbd35df4e025b04db78cbc330a380</td>\n",
+       "      <td>00013841c33fbd35df4e025b04db78cbc330a380</td>\n",
+       "      <td>crcns/v2-1/V2Data5/DGrat/stimuli/0938334bd9d5f...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>22</th>\n",
+       "      <td>125800</td>\n",
+       "      <td>00014c83ca4993261ebb88a75cd9207e569d0bba</td>\n",
+       "      <td>00014c83ca4993261ebb88a75cd9207e569d0bba</td>\n",
+       "      <td>crcns/v2-1/V2Data4/DGrat/stimuli/ee44f2758175b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>23</th>\n",
+       "      <td>125801</td>\n",
+       "      <td>0001548f9772a3870f9f57d7ddd15372fdd498d4</td>\n",
+       "      <td>0001548f9772a3870f9f57d7ddd15372fdd498d4</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/05c361feae1...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>24</th>\n",
+       "      <td>125802</td>\n",
+       "      <td>00019d74a4b7ca4e84a402e33f25028996345ce3</td>\n",
+       "      <td>00019d74a4b7ca4e84a402e33f25028996345ce3</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/73f2da8ebc7...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>25</th>\n",
+       "      <td>125803</td>\n",
+       "      <td>0001c0abf05394d6438f48e8376b66297938e888</td>\n",
+       "      <td>0001c0abf05394d6438f48e8376b66297938e888</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/9dd99a69b02...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>26</th>\n",
+       "      <td>125804</td>\n",
+       "      <td>0001d851793da81785d86b4205b184a74b60ce23</td>\n",
+       "      <td>0001d851793da81785d86b4205b184a74b60ce23</td>\n",
+       "      <td>crcns/v2-1/V2Data19/NatRev/stimuli/fe5d9d38f0e...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>27</th>\n",
+       "      <td>125805</td>\n",
+       "      <td>0001d99d7f29638060839746013c043f7857ae47</td>\n",
+       "      <td>0001d99d7f29638060839746013c043f7857ae47</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/02991dc3ec4...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>28</th>\n",
+       "      <td>125806</td>\n",
+       "      <td>000209ba9ff4f70dce2baf79e5dd0328c688bc11</td>\n",
+       "      <td>000209ba9ff4f70dce2baf79e5dd0328c688bc11</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/f9ca4c3ff84...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>29</th>\n",
+       "      <td>125807</td>\n",
+       "      <td>00023835d1cb1095d5c97526fa460438c25490a3</td>\n",
+       "      <td>00023835d1cb1095d5c97526fa460438c25490a3</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/38e13dd6b14...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770987</th>\n",
+       "      <td>896765</td>\n",
+       "      <td>fffd5b92538382d74078624393b5ac341724a25d</td>\n",
+       "      <td>fffd5b92538382d74078624393b5ac341724a25d</td>\n",
+       "      <td>crcns/v2-1/V2Data11/NatRev/stimuli/3e3248e7fdb...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770988</th>\n",
+       "      <td>896766</td>\n",
+       "      <td>fffd70f1ebd24394b678671016d7da2b044302c7</td>\n",
+       "      <td>fffd70f1ebd24394b678671016d7da2b044302c7</td>\n",
+       "      <td>crcns/v2-1/V2Data15/NatRev/stimuli/3c551cba386...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770989</th>\n",
+       "      <td>896767</td>\n",
+       "      <td>fffd8533bbb0982e3d9927d4834d17c9068a1857</td>\n",
+       "      <td>fffd8533bbb0982e3d9927d4834d17c9068a1857</td>\n",
+       "      <td>crcns/v2-1/V2Data4/DGrat/stimuli/ae77e07ad79cd...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770990</th>\n",
+       "      <td>896768</td>\n",
+       "      <td>fffd8bdbb214a1a9772ac138501cae9b990a8731</td>\n",
+       "      <td>fffd8bdbb214a1a9772ac138501cae9b990a8731</td>\n",
+       "      <td>crcns/v2-1/V2Data8/NatRev/stimuli/73a16cc24970...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770991</th>\n",
+       "      <td>896769</td>\n",
+       "      <td>fffd9e0889381e54b1bd12e0576b595d9976b9dd</td>\n",
+       "      <td>fffd9e0889381e54b1bd12e0576b595d9976b9dd</td>\n",
+       "      <td>crcns/v2-1/V2Data9/NatRev/stimuli/b409fa07bb5f...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770992</th>\n",
+       "      <td>896770</td>\n",
+       "      <td>fffdad566df3a4bd3a660d193ada8eb8ffaa88c2</td>\n",
+       "      <td>fffdad566df3a4bd3a660d193ada8eb8ffaa88c2</td>\n",
+       "      <td>crcns/v2-1/V2Data13/NatRev/stimuli/38ce2a5517b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770993</th>\n",
+       "      <td>896771</td>\n",
+       "      <td>fffdb264a410114f2c9bed3765786c474bdea1a9</td>\n",
+       "      <td>fffdb264a410114f2c9bed3765786c474bdea1a9</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/5d931ba3722...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770994</th>\n",
+       "      <td>896772</td>\n",
+       "      <td>fffdb9a37c6a1f0cd100f9b340a2788600f591f7</td>\n",
+       "      <td>fffdb9a37c6a1f0cd100f9b340a2788600f591f7</td>\n",
+       "      <td>crcns/v2-1/V2Data16/NatRev/stimuli/21ca02a9cf9...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770995</th>\n",
+       "      <td>896773</td>\n",
+       "      <td>fffdd183dc71df6de59b077cec3f00e6bf04f5cd</td>\n",
+       "      <td>fffdd183dc71df6de59b077cec3f00e6bf04f5cd</td>\n",
+       "      <td>crcns/v2-1/V2Data1/DGrat/stimuli/7832e1b72ceac...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770996</th>\n",
+       "      <td>896774</td>\n",
+       "      <td>fffdd57f5450722f7c756b2ff3f0974ffa610124</td>\n",
+       "      <td>fffdd57f5450722f7c756b2ff3f0974ffa610124</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/2cad235ec23...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770997</th>\n",
+       "      <td>896775</td>\n",
+       "      <td>fffde0392a7a1b70ada752dbe6c34bbbd2aa7eb5</td>\n",
+       "      <td>fffde0392a7a1b70ada752dbe6c34bbbd2aa7eb5</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/75b4af00ed0...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770998</th>\n",
+       "      <td>896776</td>\n",
+       "      <td>fffde7360aaed78e7ae839d80c7474710394b16c</td>\n",
+       "      <td>fffde7360aaed78e7ae839d80c7474710394b16c</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/93a7daa37b6...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>770999</th>\n",
+       "      <td>896777</td>\n",
+       "      <td>fffdedf0b18b842a566e36eb1143be87fa89ec5d</td>\n",
+       "      <td>fffdedf0b18b842a566e36eb1143be87fa89ec5d</td>\n",
+       "      <td>crcns/v2-1/V2Data1/DGrat/stimuli/242989d62300a...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771000</th>\n",
+       "      <td>896778</td>\n",
+       "      <td>fffdefeee340b6c60d2f8c8a6f27d6943addefa9</td>\n",
+       "      <td>fffdefeee340b6c60d2f8c8a6f27d6943addefa9</td>\n",
+       "      <td>crcns/v2-1/V2Data14/NatRev/stimuli/8eb55148a7d...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771001</th>\n",
+       "      <td>896779</td>\n",
+       "      <td>fffdf93ab97bce745ef9275492278030872f14ab</td>\n",
+       "      <td>fffdf93ab97bce745ef9275492278030872f14ab</td>\n",
+       "      <td>crcns/v2-1/V2Data7/NatRev/stimuli/57de6957435a...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771002</th>\n",
+       "      <td>896780</td>\n",
+       "      <td>fffe11a3ac0619b14fd577005ddb7d379ac5f9d0</td>\n",
+       "      <td>fffe11a3ac0619b14fd577005ddb7d379ac5f9d0</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/4c9ea7a1ccf...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771003</th>\n",
+       "      <td>896781</td>\n",
+       "      <td>fffe2f1973fe927a07b51982606717600caf2e0e</td>\n",
+       "      <td>fffe2f1973fe927a07b51982606717600caf2e0e</td>\n",
+       "      <td>crcns/v2-1/V2Data4/DGrat/stimuli/44a1c41b7c7e1...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771004</th>\n",
+       "      <td>896782</td>\n",
+       "      <td>fffe3525e7a24fdf9ab76601f220e29ea2baf327</td>\n",
+       "      <td>fffe3525e7a24fdf9ab76601f220e29ea2baf327</td>\n",
+       "      <td>crcns/v2-1/V2Data1/DGrat/stimuli/7613fc37f14de...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771005</th>\n",
+       "      <td>896783</td>\n",
+       "      <td>fffea406c12823e5864ea00ae98d8b99c9589145</td>\n",
+       "      <td>fffea406c12823e5864ea00ae98d8b99c9589145</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/598765f127f...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771006</th>\n",
+       "      <td>896784</td>\n",
+       "      <td>fffeb23f36412d1a940abdab9f03643b57fc4027</td>\n",
+       "      <td>fffeb23f36412d1a940abdab9f03643b57fc4027</td>\n",
+       "      <td>crcns/v2-1/V2Data1/DGrat/stimuli/b9c11cf935638...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771007</th>\n",
+       "      <td>896785</td>\n",
+       "      <td>fffebd9e7eabeda8839774bf15bdba921e19e693</td>\n",
+       "      <td>fffebd9e7eabeda8839774bf15bdba921e19e693</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/df6b12ffde3...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771008</th>\n",
+       "      <td>896786</td>\n",
+       "      <td>fffec462f80db47772e9f0afe431162d47350c82</td>\n",
+       "      <td>fffec462f80db47772e9f0afe431162d47350c82</td>\n",
+       "      <td>crcns/v2-1/V2Data13/NatRev/stimuli/d093a74b17a...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771009</th>\n",
+       "      <td>896787</td>\n",
+       "      <td>fffeeae0ab84c261d0a6b1e57e5a89c7d6217984</td>\n",
+       "      <td>fffeeae0ab84c261d0a6b1e57e5a89c7d6217984</td>\n",
+       "      <td>crcns/v2-1/V2Data8/NatRev/stimuli/1c76bf23e30d...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771010</th>\n",
+       "      <td>896788</td>\n",
+       "      <td>ffff06ab149541f11cb8a2612b0da87eba2ed950</td>\n",
+       "      <td>ffff06ab149541f11cb8a2612b0da87eba2ed950</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/95678ef6c6e...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771011</th>\n",
+       "      <td>896789</td>\n",
+       "      <td>ffff14ee8bef7387a05a504808bc77d9e38de3ab</td>\n",
+       "      <td>ffff14ee8bef7387a05a504808bc77d9e38de3ab</td>\n",
+       "      <td>crcns/v2-1/V2Data17/NatRev/stimuli/5e8910ffcfd...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771012</th>\n",
+       "      <td>896790</td>\n",
+       "      <td>ffff333854bfd2aef1adb44eedaee86842d38c96</td>\n",
+       "      <td>ffff333854bfd2aef1adb44eedaee86842d38c96</td>\n",
+       "      <td>crcns/v2-1/V2Data11/NatRev/stimuli/ab19f537bc6...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771013</th>\n",
+       "      <td>896791</td>\n",
+       "      <td>ffff666f06cff0c5f167f96a37fffb2cb2867d2c</td>\n",
+       "      <td>ffff666f06cff0c5f167f96a37fffb2cb2867d2c</td>\n",
+       "      <td>crcns/v2-1/V2Data3/DGrat/stimuli/4d35448df2c44...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771014</th>\n",
+       "      <td>896792</td>\n",
+       "      <td>ffff7fbfc6e2ee2af05646a03c3b1bfee05dd82f</td>\n",
+       "      <td>ffff7fbfc6e2ee2af05646a03c3b1bfee05dd82f</td>\n",
+       "      <td>crcns/v2-1/V2Data9/NatRev/stimuli/1e63ca6a93bc...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771015</th>\n",
+       "      <td>896793</td>\n",
+       "      <td>ffffe8068e37504568db803945cfdeee68e68859</td>\n",
+       "      <td>ffffe8068e37504568db803945cfdeee68e68859</td>\n",
+       "      <td>crcns/v2-1/V2Data18/NatRev/stimuli/55e31de78f9...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>771016</th>\n",
+       "      <td>896794</td>\n",
+       "      <td>fffff7e11f1a8d38d9378103e0ca6b25ed219a5e</td>\n",
+       "      <td>fffff7e11f1a8d38d9378103e0ca6b25ed219a5e</td>\n",
+       "      <td>crcns/v2-1/V2Data11/NatRev/stimuli/17d9020839b...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>771017 rows × 4 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "            id                                  image_id  \\\n",
+       "0       125778  00000a2518d148e9b455537488e325a7f42f39d8   \n",
+       "1       125779  00001776837935437e2784af953a4db24cc19536   \n",
+       "2       125780  00003bb7e9ca9265dc521763f73514f9c4d48f8a   \n",
+       "3       125781  00005ad0cbb2bafc56814919fda60268a1077ba3   \n",
+       "4       125782  00005e46dace8673b631b75f1df3a6d1a8b2c1c2   \n",
+       "5       125783  00006b3962b738bb25ab44265092ac6ef71943ed   \n",
+       "6       125784  00007b43039dc9f26f12d6e222460a70a1400d1f   \n",
+       "7       125785  00008a40da04647d4350b9acb0b6657326c1ba54   \n",
+       "8       125786  00008be32ed56378291bc536589d97bc39e35557   \n",
+       "9       125787  00008eead95d140fb4591430b11193525dfc379a   \n",
+       "10      125788  00009e9fd517bd1aa8c72046dc51b23beb14bd6b   \n",
+       "11      125789  00009f4607ffef290fa043eaaad9a61792cdf52d   \n",
+       "12      125790  0000b4496034903fcfa414566b171f4c59959aa1   \n",
+       "13      125791  0000c1747157700a98bb17f0712f61a55bfc76be   \n",
+       "14      125792  0000c18032e90501faa945854e0a8279f35f113b   \n",
+       "15      125793  0000c941ce6c69a723ca4842ec52f7c39c1fcf6e   \n",
+       "16      125794  0000ec7b6ffd8e31f84d0b81b387154838ad9605   \n",
+       "17      125795  00010a8473ad131508f1d21353221dd3038d7d00   \n",
+       "18      125796  00010ac420ed0bd3f212fc4a0a1c51bda60ec929   \n",
+       "19      125797  00011aedc68af6f9c23c35ba1bd512dbc2e8e8b9   \n",
+       "20      125798  000128de8497372d32b8237b085b2ca2bf679ebf   \n",
+       "21      125799  00013841c33fbd35df4e025b04db78cbc330a380   \n",
+       "22      125800  00014c83ca4993261ebb88a75cd9207e569d0bba   \n",
+       "23      125801  0001548f9772a3870f9f57d7ddd15372fdd498d4   \n",
+       "24      125802  00019d74a4b7ca4e84a402e33f25028996345ce3   \n",
+       "25      125803  0001c0abf05394d6438f48e8376b66297938e888   \n",
+       "26      125804  0001d851793da81785d86b4205b184a74b60ce23   \n",
+       "27      125805  0001d99d7f29638060839746013c043f7857ae47   \n",
+       "28      125806  000209ba9ff4f70dce2baf79e5dd0328c688bc11   \n",
+       "29      125807  00023835d1cb1095d5c97526fa460438c25490a3   \n",
+       "...        ...                                       ...   \n",
+       "770987  896765  fffd5b92538382d74078624393b5ac341724a25d   \n",
+       "770988  896766  fffd70f1ebd24394b678671016d7da2b044302c7   \n",
+       "770989  896767  fffd8533bbb0982e3d9927d4834d17c9068a1857   \n",
+       "770990  896768  fffd8bdbb214a1a9772ac138501cae9b990a8731   \n",
+       "770991  896769  fffd9e0889381e54b1bd12e0576b595d9976b9dd   \n",
+       "770992  896770  fffdad566df3a4bd3a660d193ada8eb8ffaa88c2   \n",
+       "770993  896771  fffdb264a410114f2c9bed3765786c474bdea1a9   \n",
+       "770994  896772  fffdb9a37c6a1f0cd100f9b340a2788600f591f7   \n",
+       "770995  896773  fffdd183dc71df6de59b077cec3f00e6bf04f5cd   \n",
+       "770996  896774  fffdd57f5450722f7c756b2ff3f0974ffa610124   \n",
+       "770997  896775  fffde0392a7a1b70ada752dbe6c34bbbd2aa7eb5   \n",
+       "770998  896776  fffde7360aaed78e7ae839d80c7474710394b16c   \n",
+       "770999  896777  fffdedf0b18b842a566e36eb1143be87fa89ec5d   \n",
+       "771000  896778  fffdefeee340b6c60d2f8c8a6f27d6943addefa9   \n",
+       "771001  896779  fffdf93ab97bce745ef9275492278030872f14ab   \n",
+       "771002  896780  fffe11a3ac0619b14fd577005ddb7d379ac5f9d0   \n",
+       "771003  896781  fffe2f1973fe927a07b51982606717600caf2e0e   \n",
+       "771004  896782  fffe3525e7a24fdf9ab76601f220e29ea2baf327   \n",
+       "771005  896783  fffea406c12823e5864ea00ae98d8b99c9589145   \n",
+       "771006  896784  fffeb23f36412d1a940abdab9f03643b57fc4027   \n",
+       "771007  896785  fffebd9e7eabeda8839774bf15bdba921e19e693   \n",
+       "771008  896786  fffec462f80db47772e9f0afe431162d47350c82   \n",
+       "771009  896787  fffeeae0ab84c261d0a6b1e57e5a89c7d6217984   \n",
+       "771010  896788  ffff06ab149541f11cb8a2612b0da87eba2ed950   \n",
+       "771011  896789  ffff14ee8bef7387a05a504808bc77d9e38de3ab   \n",
+       "771012  896790  ffff333854bfd2aef1adb44eedaee86842d38c96   \n",
+       "771013  896791  ffff666f06cff0c5f167f96a37fffb2cb2867d2c   \n",
+       "771014  896792  ffff7fbfc6e2ee2af05646a03c3b1bfee05dd82f   \n",
+       "771015  896793  ffffe8068e37504568db803945cfdeee68e68859   \n",
+       "771016  896794  fffff7e11f1a8d38d9378103e0ca6b25ed219a5e   \n",
+       "\n",
+       "                                 image_file_sha1  \\\n",
+       "0       00000a2518d148e9b455537488e325a7f42f39d8   \n",
+       "1       00001776837935437e2784af953a4db24cc19536   \n",
+       "2       00003bb7e9ca9265dc521763f73514f9c4d48f8a   \n",
+       "3       00005ad0cbb2bafc56814919fda60268a1077ba3   \n",
+       "4       00005e46dace8673b631b75f1df3a6d1a8b2c1c2   \n",
+       "5       00006b3962b738bb25ab44265092ac6ef71943ed   \n",
+       "6       00007b43039dc9f26f12d6e222460a70a1400d1f   \n",
+       "7       00008a40da04647d4350b9acb0b6657326c1ba54   \n",
+       "8       00008be32ed56378291bc536589d97bc39e35557   \n",
+       "9       00008eead95d140fb4591430b11193525dfc379a   \n",
+       "10      00009e9fd517bd1aa8c72046dc51b23beb14bd6b   \n",
+       "11      00009f4607ffef290fa043eaaad9a61792cdf52d   \n",
+       "12      0000b4496034903fcfa414566b171f4c59959aa1   \n",
+       "13      0000c1747157700a98bb17f0712f61a55bfc76be   \n",
+       "14      0000c18032e90501faa945854e0a8279f35f113b   \n",
+       "15      0000c941ce6c69a723ca4842ec52f7c39c1fcf6e   \n",
+       "16      0000ec7b6ffd8e31f84d0b81b387154838ad9605   \n",
+       "17      00010a8473ad131508f1d21353221dd3038d7d00   \n",
+       "18      00010ac420ed0bd3f212fc4a0a1c51bda60ec929   \n",
+       "19      00011aedc68af6f9c23c35ba1bd512dbc2e8e8b9   \n",
+       "20      000128de8497372d32b8237b085b2ca2bf679ebf   \n",
+       "21      00013841c33fbd35df4e025b04db78cbc330a380   \n",
+       "22      00014c83ca4993261ebb88a75cd9207e569d0bba   \n",
+       "23      0001548f9772a3870f9f57d7ddd15372fdd498d4   \n",
+       "24      00019d74a4b7ca4e84a402e33f25028996345ce3   \n",
+       "25      0001c0abf05394d6438f48e8376b66297938e888   \n",
+       "26      0001d851793da81785d86b4205b184a74b60ce23   \n",
+       "27      0001d99d7f29638060839746013c043f7857ae47   \n",
+       "28      000209ba9ff4f70dce2baf79e5dd0328c688bc11   \n",
+       "29      00023835d1cb1095d5c97526fa460438c25490a3   \n",
+       "...                                          ...   \n",
+       "770987  fffd5b92538382d74078624393b5ac341724a25d   \n",
+       "770988  fffd70f1ebd24394b678671016d7da2b044302c7   \n",
+       "770989  fffd8533bbb0982e3d9927d4834d17c9068a1857   \n",
+       "770990  fffd8bdbb214a1a9772ac138501cae9b990a8731   \n",
+       "770991  fffd9e0889381e54b1bd12e0576b595d9976b9dd   \n",
+       "770992  fffdad566df3a4bd3a660d193ada8eb8ffaa88c2   \n",
+       "770993  fffdb264a410114f2c9bed3765786c474bdea1a9   \n",
+       "770994  fffdb9a37c6a1f0cd100f9b340a2788600f591f7   \n",
+       "770995  fffdd183dc71df6de59b077cec3f00e6bf04f5cd   \n",
+       "770996  fffdd57f5450722f7c756b2ff3f0974ffa610124   \n",
+       "770997  fffde0392a7a1b70ada752dbe6c34bbbd2aa7eb5   \n",
+       "770998  fffde7360aaed78e7ae839d80c7474710394b16c   \n",
+       "770999  fffdedf0b18b842a566e36eb1143be87fa89ec5d   \n",
+       "771000  fffdefeee340b6c60d2f8c8a6f27d6943addefa9   \n",
+       "771001  fffdf93ab97bce745ef9275492278030872f14ab   \n",
+       "771002  fffe11a3ac0619b14fd577005ddb7d379ac5f9d0   \n",
+       "771003  fffe2f1973fe927a07b51982606717600caf2e0e   \n",
+       "771004  fffe3525e7a24fdf9ab76601f220e29ea2baf327   \n",
+       "771005  fffea406c12823e5864ea00ae98d8b99c9589145   \n",
+       "771006  fffeb23f36412d1a940abdab9f03643b57fc4027   \n",
+       "771007  fffebd9e7eabeda8839774bf15bdba921e19e693   \n",
+       "771008  fffec462f80db47772e9f0afe431162d47350c82   \n",
+       "771009  fffeeae0ab84c261d0a6b1e57e5a89c7d6217984   \n",
+       "771010  ffff06ab149541f11cb8a2612b0da87eba2ed950   \n",
+       "771011  ffff14ee8bef7387a05a504808bc77d9e38de3ab   \n",
+       "771012  ffff333854bfd2aef1adb44eedaee86842d38c96   \n",
+       "771013  ffff666f06cff0c5f167f96a37fffb2cb2867d2c   \n",
+       "771014  ffff7fbfc6e2ee2af05646a03c3b1bfee05dd82f   \n",
+       "771015  ffffe8068e37504568db803945cfdeee68e68859   \n",
+       "771016  fffff7e11f1a8d38d9378103e0ca6b25ed219a5e   \n",
+       "\n",
+       "                                   image_file_path_unique  \n",
+       "0       crcns/v2-1/V2Data8/NatRev/stimuli/41590fd57c0b...  \n",
+       "1       crcns/v2-1/V2Data14/NatRev/stimuli/2f03d6fc365...  \n",
+       "2       crcns/v2-1/V2Data4/DGrat/stimuli/6954025a16641...  \n",
+       "3       crcns/v2-1/V2Data14/NatRev/stimuli/beee725f0d2...  \n",
+       "4       crcns/v2-1/V2Data1/DGrat/stimuli/616577d68f558...  \n",
+       "5       crcns/v2-1/V2Data18/NatRev/stimuli/b1178068ddf...  \n",
+       "6       crcns/v2-1/V2Data18/NatRev/stimuli/f803727e2e6...  \n",
+       "7       crcns/v2-1/V2Data15/NatRev/stimuli/aa2ec48ff96...  \n",
+       "8       crcns/v2-1/V2Data17/NatRev/stimuli/1f67ce6b1df...  \n",
+       "9       crcns/v2-1/V2Data17/NatRev/stimuli/727a677da46...  \n",
+       "10      crcns/v2-1/V2Data9/NatRev/stimuli/b2b4abe27f40...  \n",
+       "11      crcns/v2-1/V2Data16/NatRev/stimuli/e1845ec2d6a...  \n",
+       "12      crcns/v2-1/V2Data1/DGrat/stimuli/8aeac1a5fd78f...  \n",
+       "13      crcns/v2-1/V2Data18/NatRev/stimuli/0e886b29ee9...  \n",
+       "14      crcns/v2-1/V2Data3/DGrat/stimuli/6a048770f7e0d...  \n",
+       "15      crcns/v2-1/V2Data1/DGrat/stimuli/a67aec394883e...  \n",
+       "16      crcns/v2-1/V2Data17/NatRev/stimuli/dcb7785ebe6...  \n",
+       "17      crcns/v2-1/V2Data18/NatRev/stimuli/5d478210b5c...  \n",
+       "18      crcns/v2-1/V2Data1/DGrat/stimuli/2408273f04739...  \n",
+       "19      crcns/v2-1/V2Data18/NatRev/stimuli/4732de4356e...  \n",
+       "20      crcns/v2-1/V2Data17/NatRev/stimuli/18160fa581d...  \n",
+       "21      crcns/v2-1/V2Data5/DGrat/stimuli/0938334bd9d5f...  \n",
+       "22      crcns/v2-1/V2Data4/DGrat/stimuli/ee44f2758175b...  \n",
+       "23      crcns/v2-1/V2Data18/NatRev/stimuli/05c361feae1...  \n",
+       "24      crcns/v2-1/V2Data18/NatRev/stimuli/73f2da8ebc7...  \n",
+       "25      crcns/v2-1/V2Data18/NatRev/stimuli/9dd99a69b02...  \n",
+       "26      crcns/v2-1/V2Data19/NatRev/stimuli/fe5d9d38f0e...  \n",
+       "27      crcns/v2-1/V2Data17/NatRev/stimuli/02991dc3ec4...  \n",
+       "28      crcns/v2-1/V2Data18/NatRev/stimuli/f9ca4c3ff84...  \n",
+       "29      crcns/v2-1/V2Data18/NatRev/stimuli/38e13dd6b14...  \n",
+       "...                                                   ...  \n",
+       "770987  crcns/v2-1/V2Data11/NatRev/stimuli/3e3248e7fdb...  \n",
+       "770988  crcns/v2-1/V2Data15/NatRev/stimuli/3c551cba386...  \n",
+       "770989  crcns/v2-1/V2Data4/DGrat/stimuli/ae77e07ad79cd...  \n",
+       "770990  crcns/v2-1/V2Data8/NatRev/stimuli/73a16cc24970...  \n",
+       "770991  crcns/v2-1/V2Data9/NatRev/stimuli/b409fa07bb5f...  \n",
+       "770992  crcns/v2-1/V2Data13/NatRev/stimuli/38ce2a5517b...  \n",
+       "770993  crcns/v2-1/V2Data17/NatRev/stimuli/5d931ba3722...  \n",
+       "770994  crcns/v2-1/V2Data16/NatRev/stimuli/21ca02a9cf9...  \n",
+       "770995  crcns/v2-1/V2Data1/DGrat/stimuli/7832e1b72ceac...  \n",
+       "770996  crcns/v2-1/V2Data18/NatRev/stimuli/2cad235ec23...  \n",
+       "770997  crcns/v2-1/V2Data17/NatRev/stimuli/75b4af00ed0...  \n",
+       "770998  crcns/v2-1/V2Data18/NatRev/stimuli/93a7daa37b6...  \n",
+       "770999  crcns/v2-1/V2Data1/DGrat/stimuli/242989d62300a...  \n",
+       "771000  crcns/v2-1/V2Data14/NatRev/stimuli/8eb55148a7d...  \n",
+       "771001  crcns/v2-1/V2Data7/NatRev/stimuli/57de6957435a...  \n",
+       "771002  crcns/v2-1/V2Data17/NatRev/stimuli/4c9ea7a1ccf...  \n",
+       "771003  crcns/v2-1/V2Data4/DGrat/stimuli/44a1c41b7c7e1...  \n",
+       "771004  crcns/v2-1/V2Data1/DGrat/stimuli/7613fc37f14de...  \n",
+       "771005  crcns/v2-1/V2Data18/NatRev/stimuli/598765f127f...  \n",
+       "771006  crcns/v2-1/V2Data1/DGrat/stimuli/b9c11cf935638...  \n",
+       "771007  crcns/v2-1/V2Data18/NatRev/stimuli/df6b12ffde3...  \n",
+       "771008  crcns/v2-1/V2Data13/NatRev/stimuli/d093a74b17a...  \n",
+       "771009  crcns/v2-1/V2Data8/NatRev/stimuli/1c76bf23e30d...  \n",
+       "771010  crcns/v2-1/V2Data18/NatRev/stimuli/95678ef6c6e...  \n",
+       "771011  crcns/v2-1/V2Data17/NatRev/stimuli/5e8910ffcfd...  \n",
+       "771012  crcns/v2-1/V2Data11/NatRev/stimuli/ab19f537bc6...  \n",
+       "771013  crcns/v2-1/V2Data3/DGrat/stimuli/4d35448df2c44...  \n",
+       "771014  crcns/v2-1/V2Data9/NatRev/stimuli/1e63ca6a93bc...  \n",
+       "771015  crcns/v2-1/V2Data18/NatRev/stimuli/55e31de78f9...  \n",
+       "771016  crcns/v2-1/V2Data11/NatRev/stimuli/17d9020839b...  \n",
+       "\n",
+       "[771017 rows x 4 columns]"
+      ]
+     },
+     "execution_count": 53,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "gallant_v2_stimulus_set = mkgu.get_stimulus_set(\"gallant.Willmore2010\")\n",
+    "gallant_v2_stimulus_set"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Make the DataAssembly lookup meta"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 54,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pwdb.create_tables(models=[AssemblyModel, AssemblyStoreMap, AssemblyStoreModel])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 55,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "1"
+      ]
+     },
+     "execution_count": 55,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "assy = AssemblyModel(name=\"gallant.Willmore2010\", assembly_class=\"NeuronRecordingAssembly\",\n",
+    "                     stimulus_set=gallant_v2_images)\n",
+    "assy.save()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 56,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "1"
+      ]
+     },
+     "execution_count": 56,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "store = AssemblyStoreModel(assembly_type=\"netCDF\",\n",
+    "                           location_type=\"S3\",\n",
+    "                           location=\"https://mkgu-gallant-crcns.s3.amazonaws.com/crcns_v2-1_neuronal.nc\")\n",
+    "store.save()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 57,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "1"
+      ]
+     },
+     "execution_count": 57,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "assy_store_map = AssemblyStoreMap(assembly_model=assy, assembly_store_model=store, role=\"gallant.Willmore2010\")\n",
+    "assy_store_map.save()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 59,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<xarray.NeuronRecordingAssembly (neuroid: 135, presentation: 3494103)>\n",
+       "array([[nan, nan, nan, ..., nan, nan, nan],\n",
+       "       [nan, nan, nan, ..., nan, nan, nan],\n",
+       "       [nan, nan, nan, ..., nan, nan, nan],\n",
+       "       ...,\n",
+       "       [nan, nan, nan, ..., nan, nan, nan],\n",
+       "       [nan, nan, nan, ..., nan, nan, nan],\n",
+       "       [nan, nan, nan, ..., nan, nan, nan]])\n",
+       "Coordinates:\n",
+       "  * neuroid                   (neuroid) MultiIndex\n",
+       "  - region                    (neuroid) object 'V2' 'V2' 'V2' 'V2' 'V2' 'V2' ...\n",
+       "  - animal                    (neuroid) object 'e' 'e' 'e' 'e' 'e' 'e' 'e' ...\n",
+       "  - neuroid_id                (neuroid) object 'e0017' 'e0018' 'e0021' ...\n",
+       "  * presentation              (presentation) MultiIndex\n",
+       "  - stimulusRepeats           (presentation) int64 1 1 1 1 1 1 1 1 1 1 1 1 1 ...\n",
+       "  - presentation_id           (presentation) int64 0 0 0 0 0 0 0 0 0 0 0 0 0 ...\n",
+       "  - image_file_path_original  (presentation) object 'v2-1/V2Data1/DGrat/stimuli/1b2a86f3f332f098b9eb2567fe8f04cd.jpg' ...\n",
+       "  - image_id                  (presentation) object '437fcb15ba2caa46c4a2aea77796c8316b8a98cf' ...\n",
+       "  - id                        (presentation) int64 329152 251876 691891 ...\n",
+       "  - image_file_sha1           (presentation) object '437fcb15ba2caa46c4a2aea77796c8316b8a98cf' ...\n",
+       "  - image_file_path_unique    (presentation) object 'crcns/v2-1/V2Data1/DGrat/stimuli/1b2a86f3f332f098b9eb2567fe8f04cd.jpg' ...\n",
+       "Attributes:\n",
+       "    stimulus_set_name:  gallant.Willmore2010\n",
+       "    stimulus_set:                   id                                  image..."
+      ]
+     },
+     "execution_count": 59,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "gallant_v2 = mkgu.get_assembly(\"gallant.Willmore2010\")\n",
+    "gallant_v2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 60,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "771017"
+      ]
+     },
+     "execution_count": 60,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(np.unique(gallant_v2[\"image_file_path_unique\"].values))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "mkgu_packaging",
+   "language": "python",
+   "name": "mkgu_packaging"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}