ethoscopy 2.0__tar.gz → 2.0.3__tar.gz

ethoscopy-2.0.3/.claude/settings.local.json

@@ -0,0 +1,25 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(git add:*)",
+      "Bash(git commit:*)",
+      "Bash(python:*)",
+      "Bash(ls:*)",
+      "Bash(find:*)",
+      "Bash(docker exec ethoscope-lab ls -la \"/mnt/ethoscope_results/26634d78becf48628fe5d9ffc67fae04/ETHOSCOPE_266/2025-07-29_19-12-16/2025-07-29_19-12-16_26634d78becf48628fe5d9ffc67fae04.db\")",
+      "Bash(docker exec:*)",
+      "Bash(source:*)",
+      "Bash(git push:*)",
+      "Bash(mkdir:*)",
+      "Bash(cp:*)",
+      "Bash(sqlite3:*)",
+      "Bash(black:*)",
+      "Bash(grep:*)",
+      "Bash(pip install:*)",
+      "Bash(git stash pop:*)",
+      "Bash(git checkout:*)",
+      "Bash(git tag:*)"
+    ],
+    "deny": []
+  }
+}

ethoscopy-2.0.3/.codecov.yml

@@ -0,0 +1,26 @@
+coverage:
+  range: 50..90
+  round: down
+  precision: 2
+
+  status:
+    project:
+      default:
+        target: auto
+        threshold: 1%
+        base: auto
+    patch:
+      default:
+        target: auto
+        threshold: 1%
+        base: auto
+
+comment:
+  layout: "reach, diff, flags, files"
+  behavior: default
+
+ignore:
+  - "tests/"
+  - "tutorial_notebook/"
+  - "setup.py"
+  - "run_tests.py"

ethoscopy-2.0.3/.github/workflows/ci.yml

@@ -0,0 +1,151 @@
+name: CI/CD Pipeline
+
+on:
+  push:
+    branches: [ main, develop ]
+  pull_request:
+    branches: [ main ]
+  schedule:
+    # Run tests daily at 6 AM UTC
+    - cron: '0 6 * * *'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.11', '3.12']
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Cache pip dependencies
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e ".[dev]"
+        pip install pytest-cov pytest-xdist pytest-mock
+
+    - name: Lint with ruff
+      run: |
+        pip install ruff
+        ruff check src/
+
+    - name: Format check with black
+      run: |
+        pip install black
+        black --check src/ tests/
+
+    - name: Run unit tests
+      run: |
+        python -m pytest tests/ -v --cov=ethoscopy --cov-report=xml --cov-report=term-missing -m "unit"
+
+    - name: Run integration tests
+      run: |
+        python -m pytest tests/ -v -m "integration"
+
+    - name: Upload coverage reports to Codecov
+      uses: codecov/codecov-action@v5
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        slug: gilestrolab/ethoscopy
+
+  test-notebooks:
+    runs-on: ubuntu-latest
+    needs: test
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e ".[dev]"
+        pip install nbval jupyter
+
+    - name: Test notebooks
+      run: |
+        pytest --nbval-lax tutorial_notebook/*.ipynb -v
+      continue-on-error: true  # Notebooks may fail due to data dependencies
+
+  build:
+    runs-on: ubuntu-latest
+    needs: [test, test-notebooks]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+
+    - name: Install build dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build twine
+
+    - name: Build package
+      run: |
+        python -m build
+
+    - name: Check package
+      run: |
+        twine check dist/*
+
+    - name: Upload build artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: dist-files
+        path: dist/
+
+  security-scan:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install bandit safety
+
+    - name: Run security scan with bandit
+      run: |
+        bandit -r src/ -f json -o bandit-report.json
+      continue-on-error: true
+
+    - name: Run safety check
+      run: |
+        safety check --json --output safety-report.json
+      continue-on-error: true
+
+    - name: Upload security reports
+      uses: actions/upload-artifact@v4
+      with:
+        name: security-reports
+        path: |
+          bandit-report.json
+          safety-report.json

ethoscopy-2.0.3/.github/workflows/release.yml

@@ -0,0 +1,74 @@
+name: Release to PyPI
+
+on:
+  release:
+    types: [published]
+  push:
+    tags:
+      - 'v*.*.*'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.11', '3.12']
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e ".[dev]"
+        pip install pytest-cov
+
+    - name: Run tests
+      run: |
+        python -m pytest tests/ -v --cov=ethoscopy -m "not slow"
+
+  build-and-publish:
+    needs: test
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+
+    - name: Install build dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build twine
+
+    - name: Build package
+      run: |
+        python -m build
+
+    - name: Check package
+      run: |
+        twine check dist/*
+
+    - name: Publish to Test PyPI
+      if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }}
+      run: |
+        twine upload --repository testpypi dist/*
+
+    - name: Publish to PyPI
+      if: github.event_name == 'release'
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+      run: |
+        twine upload dist/*

ethoscopy-2.0.3/.gitignore

@@ -0,0 +1,11 @@
+*.txt
+*.pyc
+*venv
+dist/
+*vscode
+__pycache__/
+ethoscopy.egg-info/
+.gitignore
+*.pkl
+.python-version
+tmp

ethoscopy-2.0.3/.pre-commit-config.yaml

@@ -0,0 +1,62 @@
+repos:
+  # General file checks
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+        files: \.(py|yml|yaml|md|txt)$
+      - id: end-of-file-fixer
+        files: \.(py|yml|yaml|md|txt)$
+      - id: check-yaml
+      - id: check-toml
+      - id: check-merge-conflict
+      - id: check-added-large-files
+        args: ['--maxkb=1000']
+      - id: debug-statements
+        files: ^(src/|tests/).*\.py$
+      - id: check-ast
+        files: ^(src/|tests/).*\.py$
+
+  # Black - Code formatting
+  - repo: https://github.com/psf/black
+    rev: 24.10.0
+    hooks:
+      - id: black
+        args: [--line-length=88]
+        files: ^(src/|tests/).*\.py$
+
+  # Ruff - Linting and code analysis
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.8.4
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+        files: ^src/.*\.py$
+      - id: ruff-format
+        files: ^(src/|tests/).*\.py$
+
+  # Import sorting
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+        args: [--profile, black, --line-length=88]
+        files: ^(src/|tests/).*\.py$
+
+  - repo: local
+    hooks:
+      - id: pytest-unit
+        name: pytest-unit
+        entry: .venv/bin/python -m pytest tests/ -m "unit" -v
+        language: system
+        pass_filenames: false
+        always_run: true
+        stages: [commit]
+
+      - id: pytest-fast
+        name: pytest-fast
+        entry: .venv/bin/python -m pytest tests/ -m "not slow" -v --tb=short
+        language: system
+        pass_filenames: false
+        always_run: true
+        stages: [push]

ethoscopy-2.0.3/CLAUDE.md

@@ -0,0 +1,111 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+Ethoscopy is a Python data analysis toolbox for behavioral time series data from Ethoscope (Drosophila monitoring system). It's built as a specialized pandas DataFrame subclass that maintains linked metadata and provides analysis/visualization tools for circadian rhythms, sleep patterns, and behavioral states using HMM.
+
+## Development Environment
+
+### Python Environment
+- **Use Python 3.10+** (project requires >=3.10,<4.0)
+- **Always use virtual environment** (`.venv` as mentioned in user's global instructions)
+- **Install in development mode**: `pip install -e .`
+
+### Package Management
+- Uses **pyproject.toml** for modern Python packaging (not setup.py for production)
+- **Hatchling** as build backend
+- **Rye** for dependency management (managed = true in pyproject.toml)
+
+### Key Dependencies
+- Core: pandas >=2.2.2, numpy >=2.0.0
+- Visualization: plotly >=5.22.0, seaborn >=0.13.2
+- Analysis: hmmlearn >=0.3.2, astropy >=6.1, pywavelets >=1.6.0
+- Dev: ipykernel for Jupyter notebook support
+
+## Build and Development Commands
+
+### Installation
+```bash
+# Install in development mode
+pip install -e .
+
+# For Jupyter notebook development
+pip install -e ".[dev]"
+```
+
+### Testing
+- **No formal test suite found** - project relies on Jupyter notebooks for validation
+- Use the tutorial notebooks in `tutorial_notebook/` for testing functionality
+- Run notebooks: `jupyter notebook tutorial_notebook/`
+
+### Docker Environment
+- **Docker available** for JupyterHub deployment
+- Build: `JUPYTER_HUB_TAG=5.3.0 ETHOSCOPE_LAB_TAG=1.0 docker compose build`
+- Run: `docker compose up -d` (from Docker/ directory)
+
+## Architecture and Code Structure
+
+### Core Architecture
+- **behavpy_core**: Base DataFrame subclass with core functionality (xmv, curate, summary)
+- **behavpy**: Main user-facing class for backward compatibility
+- **Canvas System**: Supports both 'plotly' and 'seaborn' for visualization backends
+
+### Key Components
+1. **Data Loading** (`load.py`): FTP download, ethoscope database loading, metadata linking
+2. **Analysis** (`analyse.py`): Sleep annotation, velocity detection, stimulus response
+3. **Visualization**: Split between plotly (`behavpy_plotly.py`) and seaborn (`behavpy_seaborn.py`)
+4. **Specialized Classes**:
+   - `behavpy_HMM_class.py`: Hidden Markov Model analysis
+   - `behavpy_periodogram_class.py`: Circadian rhythm analysis
+5. **Utilities** (`misc/`): General functions, validation, tutorial data
+
+### Data Structure
+- **Dual DataFrame design**: Main data + linked metadata via shared 'id' column
+- **Metadata cleaning**: Automatically removes columns like 'path', 'file_name', 'file_size', 'machine_id'
+- **Index requirement**: Data must have 'id' as index name
+
+### Import Pattern
+```python
+import ethoscopy as etho
+df = etho.behavpy(data, metadata, check=True, canvas='plotly', palette='Set2')
+```
+
+## Key Design Patterns
+
+### Version 2.0 Migration
+- **Backward compatible** with pre-2.0 pickled data
+- **Unified class structure**: All analysis under single `behavpy()` class
+- **Canvas selection**: Choose between plotly/seaborn at initialization
+- **New concat function**: Use `etho.concat()` instead of `behavpy_object.concat()`
+
+### Plotting Architecture
+- **Dual backend support**: Plotly (interactive) vs Seaborn (static)
+- **Built-in statistical processing**: Z-score normalization, bootstrapping for plotly
+- **Seaborn backend**: Uses seaborn's internal statistics
+- **Performance consideration**: Plotly can create large notebook files
+
+### Analysis Capabilities
+- **Sleep detection**: Movement-based sleep annotation
+- **Circadian analysis**: Periodograms (Lomb-Scargle, Fourier, Wavelet)
+- **HMM behavioral states**: Using hmmlearn with visualization tools
+- **Data curation**: Dead specimen removal, interpolation, filtering
+
+## Development Guidelines
+
+### Code Organization
+- **src/ethoscopy/**: Main package code
+- **tutorial_notebook/**: Jupyter notebooks for testing and examples
+- **Docker/**: JupyterHub deployment configuration
+- **Keep files under 500 lines** (per user's global instructions)
+
+### Testing Approach
+- **Use Jupyter notebooks** for validation and testing
+- **Tutorial notebooks serve as integration tests**
+- **Always test with both canvas options** (plotly/seaborn)
+
+### Documentation
+- **Docstrings**: Google style required for all functions
+- **Tutorial notebooks**: Primary documentation method
+- **README.md**: Keep updated with installation and basic usage

ethoscopy-2.0.3/Docker/.claude/settings.local.json

@@ -0,0 +1,19 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(ls:*)",
+      "Bash(git checkout:*)",
+      "Bash(docker images:*)",
+      "Read(//tmp/**)",
+      "Bash(JUPYTER_HUB_TAG=5.3.0 ETHOSCOPE_LAB_TAG=1.1 docker compose build:*)",
+      "Bash(tee:*)",
+      "Bash(docker compose:*)",
+      "Bash(find:*)",
+      "Bash(sqlite3:*)",
+      "Bash(python3:*)",
+      "Bash(docker exec:*)",
+      "Bash(docker cp:*)"
+    ],
+    "deny": []
+  }
+}

ethoscopy-2.0.3/Docker/.env

@@ -0,0 +1,39 @@
+# Keycloak OAuth Configuration for JupyterHub
+# To use: cp .env.keycloak .env && docker compose restart
+
+# Image versions
+JUPYTER_HUB_TAG=5.3.0
+ETHOSCOPE_LAB_TAG=1.1
+
+# Authentication
+AUTHENTICATOR_CLASS=generic
+DUMMY_PASSWORD=ethoscope
+
+# User configuration
+ALLOWED_USERS=amadabhushi,ggilestro,mjoyce,lguo,labguest1,labguest2,labguest3,labguest4,labguest5,labguest6,labguest7,ethoscopelab
+ADMIN_USERS=ggilestro
+
+# User/Group IDs (must match your host system)
+UID=1000
+GID=1000
+
+# Keycloak OAuth Settings
+OAUTH_CLIENT_ID=jupyter-lab
+OAUTH_CLIENT_SECRET=pBUaqP6b8bO5IamJ7jw4lD3l5EyKJaKq
+OAUTH_CALLBACK_URL=https://jupyter.lab.gilest.ro/hub/oauth_callback
+
+# Keycloak endpoints (realm: gilestro-lab)
+OAUTH_AUTHORIZE_URL=https://auth.gilest.ro/realms/gilestro-lab/protocol/openid-connect/auth
+OAUTH_TOKEN_URL=https://auth.gilest.ro/realms/gilestro-lab/protocol/openid-connect/token
+OAUTH_USERDATA_URL=https://auth.gilest.ro/realms/gilestro-lab/protocol/openid-connect/userinfo
+
+# Username claim (Keycloak typically uses 'preferred_username')
+OAUTH_USERNAME_CLAIM=preferred_username
+
+# OAuth scopes
+OAUTH_SCOPE=openid profile email
+
+# Provider-specific (not used for Keycloak, but needed for env validation)
+GITHUB_ORG=
+GOOGLE_HOSTED_DOMAIN=
+GITLAB_URL=

ethoscopy-2.0.3/Docker/.env.keycloak

@@ -0,0 +1,39 @@
+# Keycloak OAuth Configuration for JupyterHub
+# To use: cp .env.keycloak .env && docker compose restart
+
+# Image versions
+JUPYTER_HUB_TAG=5.3.0
+ETHOSCOPE_LAB_TAG=1.1
+
+# Authentication
+AUTHENTICATOR_CLASS=generic
+DUMMY_PASSWORD=ethoscope
+
+# User configuration
+ALLOWED_USERS=amadabhushi,ggilestro,mjoyce,lguo,labguest1,labguest2,labguest3,labguest4,labguest5,labguest6,labguest7,ethoscopelab
+ADMIN_USERS=ggilestro
+
+# User/Group IDs (must match your host system)
+UID=1000
+GID=1000
+
+# Keycloak OAuth Settings
+OAUTH_CLIENT_ID=jupyter-lab
+OAUTH_CLIENT_SECRET=pBUaqP6b8bO5IamJ7jw4lD3l5EyKJaKq
+OAUTH_CALLBACK_URL=https://jupyter.lab.gilest.ro/hub/oauth_callback
+
+# Keycloak endpoints (realm: gilestro-lab)
+OAUTH_AUTHORIZE_URL=https://auth.gilest.ro/realms/gilestro-lab/protocol/openid-connect/auth
+OAUTH_TOKEN_URL=https://auth.gilest.ro/realms/gilestro-lab/protocol/openid-connect/token
+OAUTH_USERDATA_URL=https://auth.gilest.ro/realms/gilestro-lab/protocol/openid-connect/userinfo
+
+# Username claim (Keycloak typically uses 'preferred_username')
+OAUTH_USERNAME_CLAIM=preferred_username
+
+# OAuth scopes
+OAUTH_SCOPE=openid profile email
+
+# Provider-specific (not used for Keycloak, but needed for env validation)
+GITHUB_ORG=
+GOOGLE_HOSTED_DOMAIN=
+GITLAB_URL=

ethoscopy-2.0.3/Docker/Dockerfile

@@ -0,0 +1,108 @@
+# Dockerfile for jupyter-rethomics
+ARG JUPYTER_HUB_TAG=5.3.0
+FROM jupyterhub/jupyterhub:${JUPYTER_HUB_TAG}
+
+# ============================================================================
+# SYSTEM DEPENDENCIES AND TOOLS
+# ============================================================================
+
+# Install system prerequisites and build tools
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        fonts-dejavu \
+        apt-utils \
+        nano \
+        jq \
+        git \
+        gfortran \
+        file \
+        gcc \
+        cmake \
+        libcurl4-openssl-dev \
+        libfontconfig1-dev \
+        libharfbuzz-dev \
+        libfribidi-dev \
+        libtiff-dev \
+        libxml2-dev \
+        libpam0g-dev \
+        wget \
+        lsb-release \
+        software-properties-common \
+        dirmngr \
+        jupyter && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# ============================================================================
+# R INSTALLATION AND SETUP
+# ============================================================================
+
+# Add CRAN repository for latest R version
+RUN wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc && \
+    add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/" -y
+
+# Install R from CRAN repository
+RUN apt-get update && \
+    apt-get install -y r-base r-base-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install R packages and rethomics
+COPY install_r_packages.r /root/install_r_packages.r
+RUN ln -s /bin/tar /bin/gtar && \
+    Rscript /root/install_r_packages.r
+
+# ============================================================================
+# JUPYTER INSTALLATION AND CONFIGURATION
+# ============================================================================
+
+# Install JupyterLab (version 3.6.4)
+RUN pip3 install jupyterlab --ignore-installed
+
+# To upgrade to version 4.0.2 (commented out)
+# RUN pip3 install jupyterlab --upgrade
+
+# Install Jupyter extensions (compatible with 3.x only)
+RUN pip3 install jupyterlab-spreadsheet-editor
+
+# ============================================================================
+# PYTHON PACKAGES AND ETHOSCOPE SETUP
+# ============================================================================
+
+# Install Python data science packages
+RUN pip3 install \
+        pandas \
+        seaborn \
+        bokeh \
+        lifelines \
+        opencv-python \
+        mysql-connector \
+        ethoscopy 
+       # pycatch22
+
+# Install Ethoscope from source
+RUN git clone https://github.com/gilestrolab/ethoscope.git /opt/ethoscope && \
+    cd /opt/ethoscope/ && git checkout dev && \
+    cd /opt/ethoscope/src/ethoscope && pip install . 
+
+# ============================================================================
+# USER SETUP
+# ============================================================================
+
+# Add default user ethoscopelab with password ethoscopelab
+RUN useradd -m ethoscopelab && \
+    echo 'ethoscopelab:ethoscope' | chpasswd
+
+# ============================================================================
+# FINAL CONFIGURATION
+# ============================================================================
+
+WORKDIR /srv/jupyterhub/
+# JupyterHub config will be mounted via docker-compose
+
+EXPOSE 8000
+
+LABEL maintainer="Giorgio Gilestro <giorgio@gilest.ro>" \
+      org.jupyter.service="ethoscope-lab"
+
+CMD ["jupyterhub"]