ethoscopy 2.2.1__tar.gz → 2.4.0__tar.gz

ethoscopy-2.4.0/.claude/settings.local.json

@@ -0,0 +1,48 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(wc:*)",
+      "Bash(paplay:*)",
+      "Bash(ETHOSCOPE_DATA_PATH=/mnt/ethoscope_data docker compose:*)",
+      "Bash(docker ps:*)",
+      "Bash(docker logs:*)",
+      "Bash(docker compose:*)",
+      "Bash(curl:*)",
+      "Bash(docker exec:*)",
+      "Bash(mkdir -p /tmp/ethoscopy_wheel_inspect)",
+      "Read(//tmp/**)",
+      "Bash(pip download *)",
+      "Bash(unzip -l ethoscopy-2.0.4-py3-none-any.whl)",
+      "Bash(git check-ignore *)",
+      "Bash(/home/gg/Code/ethoscope_project/ethoscopy/.venv/bin/pip install *)",
+      "Bash(.venv/bin/python -m pytest tests/test_get_tutorials.py -v)",
+      "Bash(.venv/bin/python *)",
+      "Bash(.venv/bin/pip install *)",
+      "Bash(gh auth *)",
+      "Bash(gh repo *)",
+      "Bash(docker info *)",
+      "mcp__bookstack__bookstack_search",
+      "Bash(.venv/bin/ruff check *)",
+      "Bash(.venv/bin/black --check src/ tests/)",
+      "Bash(.venv/bin/black src/ethoscopy/misc/get_tutorials.py tests/test_get_tutorials.py)",
+      "mcp__bookstack__bookstack_pages_read",
+      "mcp__bookstack__bookstack_pages_update",
+      "Bash(gh issue create --repo gilestrolab/ethoscopy --title 'Tutorial data pickles missing from PyPI wheel \\(2.0.0 – 2.0.4\\)' --body ' *)",
+      "Bash(git add *)",
+      "Bash(git commit *)",
+      "Bash(git push *)",
+      "Bash(git tag *)",
+      "Bash(gh release create v2.0.5 --repo gilestrolab/ethoscopy --title 'v2.0.5 — package tutorial data separately' --notes ' *)",
+      "Bash(gh run *)",
+      "Bash(gh issue *)",
+      "Bash(.venv/bin/twine check *)",
+      "Bash(.venv/bin/twine upload *)",
+      "Bash(.venv/bin/pip index *)",
+      "Bash(ETHOSCOPE_LAB_TAG=1.2 docker compose build)",
+      "Bash(gh release *)",
+      "Bash(ETHOSCOPE_LAB_TAG=1.2 docker compose build --no-cache --progress=plain)",
+      "Bash(grep -E --line-buffered '^#[0-9]+ \\(ERROR|CANCELED\\)|^#[0-9]+ DONE [0-9.]+s$|^ERROR|^failed to solve|Execution halted|^Error: Required R packages|non-zero code|^naming to|^ *=> .* done *$')",
+      "Bash(grep -E --line-buffered '^#[0-9]+ \\(ERROR|CANCELED\\)$|^#[0-9]+ DONE [0-9.]+s$|^failed to solve|Execution halted|^Error: Required R packages|Successfully built|naming to docker\\\\.io')"
+    ]
+  }
+}

ethoscopy-2.4.0/.gitignore

@@ -0,0 +1,24 @@
+*.txt
+*.pyc
+*venv
+dist/
+*vscode
+__pycache__/
+ethoscopy.egg-info/
+.gitignore
+*.pkl
+.python-version
+tmp
+
+# JupyterHub runtime state, generated when the Docker stack first runs.
+# jupyterhub_cookie_secret signs session cookies and jupyterhub.sqlite holds
+# user records and API tokens; neither belongs in the repo. Committing the
+# secret would be the same failure as the tracked .env.keycloak removed in
+# 6cbcbbe. jupyterhub_config.py stays tracked - it is configuration, not state.
+Docker/jupyterhub_data/jupyterhub_cookie_secret
+Docker/jupyterhub_data/jupyterhub.sqlite
+
+# Coverage output
+.coverage
+htmlcov/
+coverage.xml

{ethoscopy-2.2.1 → ethoscopy-2.4.0}/Docker/Dockerfile

@@ -105,19 +105,13 @@ RUN pip3 install \
         openpyxl \
         pyarrow \
         tqdm \
-        jupyterlab-git \
-        ethoscopy==2.2.0
+        jupyterlab-git
        # pycatch22
 
-# Pre-populate tutorial datasets inside the installed package so non-root
-# JupyterHub users don't have to download them on first notebook run. The
-# pickles (~36 MB total) live in the ethoscopy repo, not the PyPI wheel.
-RUN python3 -c "from ethoscopy.misc.get_tutorials import download_tutorial_data, package_tutorial_data_dir; download_tutorial_data(dest_dir=package_tutorial_data_dir(), verbose=True)"
-
 # Install Ethoscope from source
 RUN git clone https://github.com/gilestrolab/ethoscope.git /opt/ethoscope && \
     cd /opt/ethoscope/ && git checkout dev && \
-    cd /opt/ethoscope/src/ethoscope && pip install . 
+    cd /opt/ethoscope/src/ethoscope && pip install .
 
 # ============================================================================
 # USER SETUP
@@ -127,6 +121,38 @@ RUN git clone https://github.com/gilestrolab/ethoscope.git /opt/ethoscope && \
 RUN useradd -m ethoscopelab && \
     echo 'ethoscopelab:ethoscope' | chpasswd
 
+# ============================================================================
+# ETHOSCOPY (kept last on purpose)
+# ============================================================================
+# Reason: ethoscopy is the fastest-moving pin in this image and the one most
+# often bumped for a bugfix. Everything above — the ~1 h R-package compile, the
+# scientific pip stack, ethoscope-from-source — is stable, so installing
+# ethoscopy in its own late layer means a version bump rebuilds only these two
+# steps (seconds) instead of invalidating the whole pip block. Keep this the
+# last software install; put nothing slow after it. Bumping the pin below is the
+# only change needed to ship a new ethoscopy into the image.
+RUN pip3 install ethoscopy==2.3.0
+
+# Pre-populate tutorial datasets inside the installed package so non-root
+# JupyterHub users don't have to download them on first notebook run. The
+# pickles (~36 MB total) live in the ethoscopy repo, not the PyPI wheel.
+#
+# This is best-effort. urlretrieve() has no timeout, so a stalled connection to
+# GitHub would hang the build forever (observed: a 36 MB / ~8 s download stuck
+# >25 min); cap each connection with socket.setdefaulttimeout and retry, using
+# overwrite so a socket killed mid-write cannot leave a partial file the next
+# attempt skips as "already present". If it still cannot reach GitHub (e.g. a
+# build host whose Docker bridge network is firewalled off from GitHub while the
+# container's runtime network is not), warn and continue rather than failing the
+# whole image: ethoscopy fetches the tutorial data on demand at first use, so a
+# missing prefetch only costs a one-time download inside the user's session.
+RUN for attempt in 1 2 3; do \
+        python3 -c "import socket; socket.setdefaulttimeout(60); from ethoscopy.misc.get_tutorials import download_tutorial_data, package_tutorial_data_dir; download_tutorial_data(dest_dir=package_tutorial_data_dir(), overwrite=True, verbose=True)" && exit 0; \
+        echo "tutorial download attempt $attempt failed; retrying in 5s..."; \
+        sleep 5; \
+    done; \
+    echo "WARNING: could not pre-populate tutorial data at build time; it will download on first use at runtime." >&2
+
 # ============================================================================
 # FINAL CONFIGURATION
 # ============================================================================

{ethoscopy-2.2.1 → ethoscopy-2.4.0}/Docker/README.md

@@ -12,6 +12,41 @@ The command to use to recreate that image is `JUPYTER_HUB_TAG=5.3.0 ETHOSCOPE_LA
 
 After creation, the image can be run using the enclosed `docker-compose.yml` file, replacing values as fit.
 
+### The GPU does not speed up the build (it is a *runtime* resource)
+
+A recurring confusion: **`docker compose build` cannot use the GPU, no matter what.**
+The host does have an NVIDIA GPU (e.g. an RTX A4000) and the *running* container is
+given access to it via the `deploy.resources.reservations.devices: capabilities: ["gpu"]`
+block in `docker-compose.yml` — that is what GPU-aware analysis inside the notebook uses.
+But building the image is an entirely separate phase that runs on CPU: BuildKit has no
+GPU path, and every slow step here is CPU/IO-bound anyway.
+
+What actually makes the build slow (~1–1.5 h from cold) is, in order:
+
+1. **Compiling the 8 R packages** from source (`Rscript install_r_packages.r`) — by far
+   the biggest cost, and pure CPU.
+2. `pip install` of the JupyterLab / notebook-intelligence / scientific stack.
+3. `apt-get` and the `git clone` of ethoscope.
+
+None of these is GPU-accelerable. If a rebuild is taking the full hour+ it is almost
+always because the **build cache was lost**, forcing the R compile to re-run. To avoid
+re-paying it:
+
+- **Do not kill a build mid-layer.** BuildKit only commits a layer's cache when that
+  layer *finishes*; a build killed during the R step discards it and the next build
+  recompiles from scratch.
+- **Bumping `ethoscopy==X.Y.Z` only invalidates the pip layer and below.** The R compile
+  sits earlier in the `Dockerfile`, so a version-pin bump should reuse the cached R layer
+  and finish in minutes — *provided the cache still exists*. Old cache is eventually
+  evicted (`docker buildx du` shows what is reclaimable), which is the usual reason an
+  "only changed the pin" rebuild still takes an hour.
+- **Prefer pulling over building.** Regular deployments should `docker pull
+  ggilestro/ethoscope-lab:<tag>` rather than rebuild; only rebuild when the recipe itself
+  changes, then push the result so nobody else has to.
+
+Rule of thumb: reach for the GPU to make *analysis* faster, never to make the *image build*
+faster — there is nothing to accelerate there.
+
 ## Authentication and User Management
 
 This JupyterHub instance supports multiple authentication methods including:

{ethoscopy-2.2.1 → ethoscopy-2.4.0}/Docker/docker-compose.yml

@@ -6,7 +6,7 @@ services:
       dockerfile: Dockerfile
       args:
         JUPYTER_HUB_TAG: ${JUPYTER_HUB_TAG:-5.4.0}
-    image: ggilestro/ethoscope-lab:${ETHOSCOPE_LAB_TAG:-1.3}
+    image: ggilestro/ethoscope-lab:${ETHOSCOPE_LAB_TAG:-1.4}
     container_name: ethoscope-lab
     ports:
       - 8082:8000

{ethoscopy-2.2.1 → ethoscopy-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: ethoscopy
-Version: 2.2.1
+Version: 2.4.0
 Summary: "A python based toolkit to download and anlyse data from the Ethoscope hardware system."
 Author-email: Lblackhurst29 <lblackhurst29@gmail.com>
 License-File: LICENSE
@@ -19,6 +19,7 @@ Requires-Dist: hmmlearn<0.4.0,>=0.3.2
 Requires-Dist: nbformat<6.0.0,>=5.10.4
 Requires-Dist: numpy<3.0.0,>=2.0.0
 Requires-Dist: pandas<3.0.0,>=2.2.2
+Requires-Dist: pillow<13.0,>=10.0
 Requires-Dist: plotly<6.0.0,>=5.22.0
 Requires-Dist: pywavelets<2.0.0,>=1.6.0
 Requires-Dist: seaborn<0.14.0,>=0.13.2
@@ -55,6 +56,8 @@ At its core ethoscopy is a subclass of the data manipulation tool Pandas. The da
 
 Ethoscopy contains methods to perform common analytical techniques per specimen in the data table, such as removing dead specimens, interpolating missing values, or calculating sleep from movement. Addtionally, specialist anlysing tools have been implemented for analysing circadian rhythm, such as periodograms, and for generating hidden Markov models (HMM) to understand latent behavioural states. HMMs are trained utilising hmmlearn in the background and come accompanied with a range of visualisation tools to understand the generated model.
 
+For lifespan experiments, `survival_table()` and `km_survival_plot()` estimate Kaplan-Meier survival curves with Greenwood confidence bands. Death is detected from sustained immobility using the same rule as `curate_dead_animals()`, and specimens still moving when their recording ends are censored rather than counted as deaths. Pass `subject_cols` when one animal was recorded across several sessions so its recordings are treated as one individual.
+
 ### -- Update to 2.0 --
 
 This new update sees a whole refactoring of the code base to make everything more streamline and keep the package up to date with the new versions of pandas and numpy. Gone are seperate classes for periodograms and HMM based analysis, all are under one class behavpy(). Addtioanlly, now the user can choose between plotter packages, Seaborn and Plotly, and choose a desired colour pallete. The previous used package Plotly can balloon the size of jupyter notebooks, putting a strain on storage, despite being great for data exploration. If you just want static plots, use Seaborn. But be wary of comparison, the backend for Plotly plots is all calculated in ethoscopy applying z-score and bootstrapping to quantification plots, whereas Seaborn based plots will use the Seaborn internal tools for errors and averaging.

{ethoscopy-2.2.1 → ethoscopy-2.4.0}/README.md

@@ -17,6 +17,8 @@ At its core ethoscopy is a subclass of the data manipulation tool Pandas. The da
 
 Ethoscopy contains methods to perform common analytical techniques per specimen in the data table, such as removing dead specimens, interpolating missing values, or calculating sleep from movement. Addtionally, specialist anlysing tools have been implemented for analysing circadian rhythm, such as periodograms, and for generating hidden Markov models (HMM) to understand latent behavioural states. HMMs are trained utilising hmmlearn in the background and come accompanied with a range of visualisation tools to understand the generated model.
 
+For lifespan experiments, `survival_table()` and `km_survival_plot()` estimate Kaplan-Meier survival curves with Greenwood confidence bands. Death is detected from sustained immobility using the same rule as `curate_dead_animals()`, and specimens still moving when their recording ends are censored rather than counted as deaths. Pass `subject_cols` when one animal was recorded across several sessions so its recordings are treated as one individual.
+
 ### -- Update to 2.0 --
 
 This new update sees a whole refactoring of the code base to make everything more streamline and keep the package up to date with the new versions of pandas and numpy. Gone are seperate classes for periodograms and HMM based analysis, all are under one class behavpy(). Addtioanlly, now the user can choose between plotter packages, Seaborn and Plotly, and choose a desired colour pallete. The previous used package Plotly can balloon the size of jupyter notebooks, putting a strain on storage, despite being great for data exploration. If you just want static plots, use Seaborn. But be wary of comparison, the backend for Plotly plots is all calculated in ethoscopy applying z-score and bootstrapping to quantification plots, whereas Seaborn based plots will use the Seaborn internal tools for errors and averaging.

{ethoscopy-2.2.1 → ethoscopy-2.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ethoscopy"
-version = "2.2.1"
+version = "2.4.0"
 description = "\"A python based toolkit to download and anlyse data from the Ethoscope hardware system.\""
 authors = [{name = "Lblackhurst29",email = "lblackhurst29@gmail.com"}]
 readme = "README.md"
@@ -16,6 +16,10 @@ dependencies = [
     "astropy>=7.0,<8.0",
     "nbformat>=5.10.4,<6.0.0",
     "tqdm>=4.66,<5.0",
+    # estimate_light_cycle() decodes the stored IMG_SNAPSHOTS JPEGs. Pillow
+    # already arrived transitively through seaborn -> matplotlib; this makes the
+    # direct dependency explicit.
+    "pillow>=10.0,<13.0",
 ]
 requires-python = ">=3.12,<4.0"
 classifiers = [

{ethoscopy-2.2.1 → ethoscopy-2.4.0}/src/ethoscopy/__init__.py

@@ -15,8 +15,11 @@ from ethoscopy.analyse import (
 from ethoscopy.behavpy import behavpy
 from ethoscopy.load import (
     download_from_remote_dir,
+    estimate_light_cycle,
     link_meta_index,
     load_ethoscope,
+    load_ethoscope_diagnostics,
+    load_ethoscope_light_schedule,
     load_ethoscope_metadata,
 )
 from ethoscopy.misc.general_functions import concat
@@ -28,9 +31,12 @@ __all__ = [
     "behavpy",
     "download_from_remote_dir",
     "download_tutorial_data",
+    "estimate_light_cycle",
     "get_tutorial",
     "link_meta_index",
     "load_ethoscope",
+    "load_ethoscope_diagnostics",
+    "load_ethoscope_light_schedule",
     "load_ethoscope_metadata",
     "max_velocity_detector",
     "sleep_annotation",