ethoscopy 2.0.1__tar.gz → 2.0.4__tar.gz

ethoscopy-2.0.4/.claude/settings.local.json

@@ -0,0 +1,25 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(git add:*)",
+      "Bash(git commit:*)",
+      "Bash(python:*)",
+      "Bash(ls:*)",
+      "Bash(find:*)",
+      "Bash(docker exec ethoscope-lab ls -la \"/mnt/ethoscope_results/26634d78becf48628fe5d9ffc67fae04/ETHOSCOPE_266/2025-07-29_19-12-16/2025-07-29_19-12-16_26634d78becf48628fe5d9ffc67fae04.db\")",
+      "Bash(docker exec:*)",
+      "Bash(source:*)",
+      "Bash(git push:*)",
+      "Bash(mkdir:*)",
+      "Bash(cp:*)",
+      "Bash(sqlite3:*)",
+      "Bash(black:*)",
+      "Bash(grep:*)",
+      "Bash(pip install:*)",
+      "Bash(git stash pop:*)",
+      "Bash(git checkout:*)",
+      "Bash(git tag:*)"
+    ],
+    "deny": []
+  }
+}

ethoscopy-2.0.4/.codecov.yml

@@ -0,0 +1,26 @@
+coverage:
+  range: 50..90
+  round: down
+  precision: 2
+
+  status:
+    project:
+      default:
+        target: auto
+        threshold: 1%
+        base: auto
+    patch:
+      default:
+        target: auto
+        threshold: 1%
+        base: auto
+
+comment:
+  layout: "reach, diff, flags, files"
+  behavior: default
+
+ignore:
+  - "tests/"
+  - "tutorial_notebook/"
+  - "setup.py"
+  - "run_tests.py"

ethoscopy-2.0.4/.github/workflows/ci.yml

@@ -0,0 +1,151 @@
+name: CI/CD Pipeline
+
+on:
+  push:
+    branches: [ main, develop ]
+  pull_request:
+    branches: [ main ]
+  schedule:
+    # Run tests daily at 6 AM UTC
+    - cron: '0 6 * * *'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.11', '3.12']
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Cache pip dependencies
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e ".[dev]"
+        pip install pytest-cov pytest-xdist pytest-mock
+
+    - name: Lint with ruff
+      run: |
+        pip install ruff
+        ruff check src/
+
+    - name: Format check with black
+      run: |
+        pip install black
+        black --check src/ tests/
+
+    - name: Run unit tests
+      run: |
+        python -m pytest tests/ -v --cov=ethoscopy --cov-report=xml --cov-report=term-missing -m "unit"
+
+    - name: Run integration tests
+      run: |
+        python -m pytest tests/ -v -m "integration"
+
+    - name: Upload coverage reports to Codecov
+      uses: codecov/codecov-action@v5
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        slug: gilestrolab/ethoscopy
+
+  test-notebooks:
+    runs-on: ubuntu-latest
+    needs: test
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e ".[dev]"
+        pip install nbval jupyter
+
+    - name: Test notebooks
+      run: |
+        pytest --nbval-lax tutorial_notebook/*.ipynb -v
+      continue-on-error: true  # Notebooks may fail due to data dependencies
+
+  build:
+    runs-on: ubuntu-latest
+    needs: [test, test-notebooks]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+
+    - name: Install build dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build twine
+
+    - name: Build package
+      run: |
+        python -m build
+
+    - name: Check package
+      run: |
+        twine check dist/*
+
+    - name: Upload build artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: dist-files
+        path: dist/
+
+  security-scan:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install bandit safety
+
+    - name: Run security scan with bandit
+      run: |
+        bandit -r src/ -f json -o bandit-report.json
+      continue-on-error: true
+
+    - name: Run safety check
+      run: |
+        safety check --json --output safety-report.json
+      continue-on-error: true
+
+    - name: Upload security reports
+      uses: actions/upload-artifact@v4
+      with:
+        name: security-reports
+        path: |
+          bandit-report.json
+          safety-report.json

ethoscopy-2.0.4/.github/workflows/release.yml

@@ -0,0 +1,74 @@
+name: Release to PyPI
+
+on:
+  release:
+    types: [published]
+  push:
+    tags:
+      - 'v*.*.*'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.11', '3.12']
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e ".[dev]"
+        pip install pytest-cov
+
+    - name: Run tests
+      run: |
+        python -m pytest tests/ -v --cov=ethoscopy -m "not slow"
+
+  build-and-publish:
+    needs: test
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+
+    - name: Install build dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build twine
+
+    - name: Build package
+      run: |
+        python -m build
+
+    - name: Check package
+      run: |
+        twine check dist/*
+
+    - name: Publish to Test PyPI
+      if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }}
+      run: |
+        twine upload --repository testpypi dist/*
+
+    - name: Publish to PyPI
+      if: github.event_name == 'release'
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+      run: |
+        twine upload dist/*

{ethoscopy-2.0.1 → ethoscopy-2.0.4}/.gitignore

@@ -7,4 +7,5 @@ __pycache__/
 ethoscopy.egg-info/
 .gitignore
 *.pkl
-.python-version
+.python-version
+tmp

ethoscopy-2.0.4/.pre-commit-config.yaml

@@ -0,0 +1,62 @@
+repos:
+  # General file checks
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+        files: \.(py|yml|yaml|md|txt)$
+      - id: end-of-file-fixer
+        files: \.(py|yml|yaml|md|txt)$
+      - id: check-yaml
+      - id: check-toml
+      - id: check-merge-conflict
+      - id: check-added-large-files
+        args: ['--maxkb=1000']
+      - id: debug-statements
+        files: ^(src/|tests/).*\.py$
+      - id: check-ast
+        files: ^(src/|tests/).*\.py$
+
+  # Black - Code formatting
+  - repo: https://github.com/psf/black
+    rev: 24.10.0
+    hooks:
+      - id: black
+        args: [--line-length=88]
+        files: ^(src/|tests/).*\.py$
+
+  # Ruff - Linting and code analysis
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.8.4
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+        files: ^src/.*\.py$
+      - id: ruff-format
+        files: ^(src/|tests/).*\.py$
+
+  # Import sorting
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+        args: [--profile, black, --line-length=88]
+        files: ^(src/|tests/).*\.py$
+
+  - repo: local
+    hooks:
+      - id: pytest-unit
+        name: pytest-unit
+        entry: .venv/bin/python -m pytest tests/ -m "unit" -v
+        language: system
+        pass_filenames: false
+        always_run: true
+        stages: [commit]
+
+      - id: pytest-fast
+        name: pytest-fast
+        entry: .venv/bin/python -m pytest tests/ -m "not slow" -v --tb=short
+        language: system
+        pass_filenames: false
+        always_run: true
+        stages: [push]

{ethoscopy-2.0.1 → ethoscopy-2.0.4}/CLAUDE.md

@@ -10,7 +10,7 @@ Ethoscopy is a Python data analysis toolbox for behavioral time series data from
 
 ### Python Environment
 - **Use Python 3.10+** (project requires >=3.10,<4.0)
-- **Always use virtual environment** (`venv_linux` as mentioned in user's global instructions)
+- **Always use virtual environment** (`.venv` as mentioned in user's global instructions)
 - **Install in development mode**: `pip install -e .`
 
 ### Package Management
@@ -45,6 +45,41 @@ pip install -e ".[dev]"
 - Build: `JUPYTER_HUB_TAG=5.3.0 ETHOSCOPE_LAB_TAG=1.0 docker compose build`
 - Run: `docker compose up -d` (from Docker/ directory)
 
+### Troubleshooting: Database "Malformed" Errors
+
+**Problem**: Intermittent "database disk image is malformed" errors when loading ethoscope data in Docker
+
+**Root Cause**: SQLite databases in WAL (Write-Ahead Logging) mode on read-only Docker mounts
+
+**Solution Options**:
+1. **Convert databases to DELETE mode** (recommended for immediate fix)
+   ```bash
+   # Using the conversion script (from project root)
+   python3 scripts/convert_wal_to_delete.py /mnt/ethoscope_data/results --verbose
+
+   # Or using bash wrapper
+   ./scripts/convert_databases.sh /mnt/ethoscope_data/results
+   ```
+
+2. **Improved connection handling** (v2.0.4+)
+   - The `_connect_db()` function in `load.py` now detects WAL mode automatically
+   - Uses `mode=ro&nolock=1` URI parameters for WAL databases on read-only mounts
+   - Includes retry logic in `read_single_roi_optimized()` for resilience
+
+**Key Implementation Details** (load.py):
+- Lines 18-70: `_connect_db()` - Smart connection with WAL detection and appropriate parameters
+- Lines 943-979: Retry logic in `read_single_roi_optimized()` - Handles transient errors with fresh connections
+
+**Testing After Changes**:
+```python
+import ethoscopy as etho
+metadata = etho.link_meta_index('metadata.csv', '/mnt/ethoscope_results')
+data = etho.load_ethoscope(metadata, reference_hour=9.0, FUN=etho.sleep_annotation)
+# Should load all ROIs without "malformed" errors
+```
+
+**See Also**: `Docker/README.md` for detailed database preparation instructions
+
 ## Architecture and Code Structure
 
 ### Core Architecture
@@ -108,4 +143,4 @@ df = etho.behavpy(data, metadata, check=True, canvas='plotly', palette='Set2')
 ### Documentation
 - **Docstrings**: Google style required for all functions
 - **Tutorial notebooks**: Primary documentation method
-- **README.md**: Keep updated with installation and basic usage
+- **README.md**: Keep updated with installation and basic usage

ethoscopy-2.0.4/Docker/.claude/settings.local.json

@@ -0,0 +1,41 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(ls:*)",
+      "Bash(git checkout:*)",
+      "Bash(docker images:*)",
+      "Read(//tmp/**)",
+      "Bash(JUPYTER_HUB_TAG=5.3.0 ETHOSCOPE_LAB_TAG=1.1 docker compose build:*)",
+      "Bash(tee:*)",
+      "Bash(docker compose:*)",
+      "Bash(find:*)",
+      "Bash(sqlite3:*)",
+      "Bash(python3:*)",
+      "Bash(docker exec:*)",
+      "Bash(docker cp:*)",
+      "Bash(ip addr:*)",
+      "Bash(systemctl status:*)",
+      "Bash(docker ps:*)",
+      "Bash(wg show:*)",
+      "Bash(ip route:*)",
+      "Bash(ip rule:*)",
+      "Bash(docker inspect:*)",
+      "Bash(docker logs:*)",
+      "Bash(nc:*)",
+      "Bash(timeout 3 bash -c 'cat < /dev/tcp/192.168.254.10/8082')",
+      "Bash(docker network inspect:*)",
+      "Bash(ss:*)",
+      "Bash(host:*)",
+      "Bash(chmod:*)",
+      "Bash(tree:*)",
+      "Bash(source:*)",
+      "Bash(pip install:*)",
+      "Bash(git add:*)",
+      "Bash(git commit -m \"$\\(cat <<''EOF''\nfix: resolve \"database disk image is malformed\" errors for WAL-mode databases\n\nAdd comprehensive solution for SQLite WAL-mode database errors on read-only Docker mounts:\n\n- Improve connection handling in load.py:\n  - Detect WAL mode databases and use appropriate connection parameters\n  - Add retry logic with fresh connections for transient errors\n  - Use mode=ro&nolock=1 for WAL databases on read-only filesystems\n\n- Add database conversion scripts:\n  - convert_wal_to_delete.py: Python script to convert databases from WAL to DELETE mode\n  - convert_databases.sh: Bash wrapper for easy bulk conversion\n  - scripts/README.md: Comprehensive documentation for database utilities\n\n- Update documentation:\n  - Docker/README.md: Add \"Database Preparation for Docker\" section\n  - CLAUDE.md: Add troubleshooting section for database errors\n\n- Bump version to 2.0.4\n\nThis resolves intermittent loading failures for ROIs when ethoscope databases\nin WAL mode are mounted read-only in Docker containers.\n\nCo-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>\nEOF\n\\)\")",
+      "Bash(git push:*)",
+      "Bash(git tag:*)",
+      "Bash(python -m build:*)"
+    ],
+    "deny": []
+  }
+}

ethoscopy-2.0.4/Docker/.env

@@ -0,0 +1,39 @@
+# Keycloak OAuth Configuration for JupyterHub
+# To use: cp .env.keycloak .env && docker compose restart
+
+# Image versions
+JUPYTER_HUB_TAG=5.3.0
+ETHOSCOPE_LAB_TAG=1.1
+
+# Authentication
+AUTHENTICATOR_CLASS=generic
+DUMMY_PASSWORD=ethoscope
+
+# User configuration
+ALLOWED_USERS=amadabhushi,ggilestro,mjoyce,lguo,labguest1,labguest2,labguest3,labguest4,labguest5,labguest6,labguest7,ethoscopelab
+ADMIN_USERS=ggilestro
+
+# User/Group IDs (must match your host system)
+UID=1000
+GID=1000
+
+# Keycloak OAuth Settings
+OAUTH_CLIENT_ID=jupyter-lab
+OAUTH_CLIENT_SECRET=pBUaqP6b8bO5IamJ7jw4lD3l5EyKJaKq
+OAUTH_CALLBACK_URL=https://jupyter.lab.gilest.ro/hub/oauth_callback
+
+# Keycloak endpoints (realm: gilestro-lab)
+OAUTH_AUTHORIZE_URL=https://auth.gilest.ro/realms/gilestro-lab/protocol/openid-connect/auth
+OAUTH_TOKEN_URL=https://auth.gilest.ro/realms/gilestro-lab/protocol/openid-connect/token
+OAUTH_USERDATA_URL=https://auth.gilest.ro/realms/gilestro-lab/protocol/openid-connect/userinfo
+
+# Username claim (Keycloak typically uses 'preferred_username')
+OAUTH_USERNAME_CLAIM=preferred_username
+
+# OAuth scopes
+OAUTH_SCOPE=openid profile email
+
+# Provider-specific (not used for Keycloak, but needed for env validation)
+GITHUB_ORG=
+GOOGLE_HOSTED_DOMAIN=
+GITLAB_URL=

ethoscopy-2.0.4/Docker/.env.keycloak

@@ -0,0 +1,39 @@
+# Keycloak OAuth Configuration for JupyterHub
+# To use: cp .env.keycloak .env && docker compose restart
+
+# Image versions
+JUPYTER_HUB_TAG=5.3.0
+ETHOSCOPE_LAB_TAG=1.1
+
+# Authentication
+AUTHENTICATOR_CLASS=generic
+DUMMY_PASSWORD=ethoscope
+
+# User configuration
+ALLOWED_USERS=amadabhushi,ggilestro,mjoyce,lguo,labguest1,labguest2,labguest3,labguest4,labguest5,labguest6,labguest7,ethoscopelab
+ADMIN_USERS=ggilestro
+
+# User/Group IDs (must match your host system)
+UID=1000
+GID=1000
+
+# Keycloak OAuth Settings
+OAUTH_CLIENT_ID=jupyter-lab
+OAUTH_CLIENT_SECRET=pBUaqP6b8bO5IamJ7jw4lD3l5EyKJaKq
+OAUTH_CALLBACK_URL=https://jupyter.lab.gilest.ro/hub/oauth_callback
+
+# Keycloak endpoints (realm: gilestro-lab)
+OAUTH_AUTHORIZE_URL=https://auth.gilest.ro/realms/gilestro-lab/protocol/openid-connect/auth
+OAUTH_TOKEN_URL=https://auth.gilest.ro/realms/gilestro-lab/protocol/openid-connect/token
+OAUTH_USERDATA_URL=https://auth.gilest.ro/realms/gilestro-lab/protocol/openid-connect/userinfo
+
+# Username claim (Keycloak typically uses 'preferred_username')
+OAUTH_USERNAME_CLAIM=preferred_username
+
+# OAuth scopes
+OAUTH_SCOPE=openid profile email
+
+# Provider-specific (not used for Keycloak, but needed for env validation)
+GITHUB_ORG=
+GOOGLE_HOSTED_DOMAIN=
+GITLAB_URL=