arraybridge 0.2.0__tar.gz

arraybridge-0.2.0/.github/workflows/ci.yml

@@ -0,0 +1,153 @@
+name: CI
+
+on:
+  push:
+    branches: [main, master]
+  pull_request:
+    branches: [main, master]
+  workflow_dispatch:
+
+jobs:
+  # Test across Python versions with CPU-compatible frameworks
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        framework: [none, torch]
+        exclude:
+          # Reduce matrix size - test torch mainly on ubuntu
+          - os: windows-latest
+            framework: torch
+          - os: macos-latest
+            framework: torch
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install base dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Install PyTorch (CPU-only)
+        if: matrix.framework == 'torch'
+        run: |
+          pip install torch --index-url https://download.pytorch.org/whl/cpu
+
+      - name: Run tests with coverage
+        run: |
+          pytest --cov=arraybridge --cov-report=xml --cov-report=html --cov-report=term-missing -v
+
+      - name: Upload coverage to Codecov
+        if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12' && matrix.framework == 'torch'
+        uses: codecov/codecov-action@v3
+        with:
+          file: ./coverage.xml
+          fail_ci_if_error: false
+
+  # GPU tests with GitHub Actions GPU runners (optional, non-blocking)
+  # Note: GPU runners may have long queue times, so this job is allowed to fail
+  gpu-test:
+    runs-on: ubuntu-latest-gpu-t4
+    continue-on-error: true  # Don't block PR merges if GPU tests fail or timeout
+    
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Check CUDA availability
+        run: |
+          nvidia-smi
+          nvcc --version || echo "NVCC not available"
+
+      - name: Install base dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Install GPU frameworks
+        run: |
+          # Install PyTorch with CUDA support
+          pip install torch torchvision --index-url https://download.pytorch.org/whl/cu121
+          
+          # Install CuPy with CUDA 12.x support
+          pip install cupy-cuda12x
+          
+          # Verify installations
+          python -c "import torch; print(f'PyTorch version: {torch.__version__}'); print(f'CUDA available: {torch.cuda.is_available()}'); print(f'CUDA device: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"N/A\"}')"
+          python -c "import cupy as cp; print(f'CuPy version: {cp.__version__}'); print(f'CUDA device: {cp.cuda.Device()}')"
+
+      - name: Run GPU tests
+        run: |
+          # Run all tests - GPU frameworks will be used when available
+          pytest -v --tb=short
+          
+      - name: Test GPU memory conversions
+        run: |
+          # Quick sanity check for GPU conversions
+          python -c "
+          import numpy as np
+          import torch
+          import cupy as cp
+          from arraybridge import convert_memory, detect_memory_type
+          
+          # Test NumPy -> CuPy
+          np_arr = np.array([1, 2, 3], dtype=np.float32)
+          cp_arr = convert_memory(np_arr, 'numpy', 'cupy', gpu_id=0)
+          print(f'NumPy -> CuPy: {type(cp_arr)}, device: {cp_arr.device}')
+          
+          # Test NumPy -> PyTorch GPU
+          torch_arr = convert_memory(np_arr, 'numpy', 'torch', gpu_id=0)
+          print(f'NumPy -> PyTorch: {type(torch_arr)}, device: {torch_arr.device}')
+          
+          # Test CuPy -> PyTorch
+          torch_from_cp = convert_memory(cp_arr, 'cupy', 'torch', gpu_id=0)
+          print(f'CuPy -> PyTorch: {type(torch_from_cp)}, device: {torch_from_cp.device}')
+          
+          print('✓ All GPU conversions successful!')
+          "
+
+  # Code quality checks
+  code-quality:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install ruff black mypy
+
+      - name: Run ruff (linting)
+        run: |
+          ruff check src/ --output-format=github
+
+      - name: Run black (formatting check)
+        run: |
+          black --check src/
+
+      - name: Run mypy (type checking)
+        continue-on-error: true  # Scientific code often has complex types
+        run: |
+          mypy src/ --ignore-missing-imports

arraybridge-0.2.0/.github/workflows/gpu-tests.yml

@@ -0,0 +1,69 @@
+name: GPU Tests (Optional)
+
+on:
+  workflow_dispatch:  # Manual trigger only
+  schedule:
+    # Run weekly on Sunday at 2am UTC (optional, can be removed)
+    - cron: '0 2 * * 0'
+
+jobs:
+  gpu-test:
+    runs-on: [self-hosted, gpu]  # Requires GPU runner
+    # Alternative: use GitHub's beta GPU runners when available
+    # runs-on: ubuntu-latest-gpu
+    
+    strategy:
+      fail-fast: false
+      matrix:
+        framework: [cupy, torch-gpu]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Check CUDA availability
+        run: |
+          nvidia-smi || echo "No NVIDIA GPU detected"
+          nvcc --version || echo "No CUDA compiler detected"
+
+      - name: Install base dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Install CuPy
+        if: matrix.framework == 'cupy'
+        run: |
+          pip install cupy-cuda12x  # Adjust CUDA version as needed
+
+      - name: Install PyTorch (GPU)
+        if: matrix.framework == 'torch-gpu'
+        run: |
+          pip install torch --index-url https://download.pytorch.org/whl/cu121
+
+      - name: Run GPU-specific tests
+        run: |
+          # Run only tests marked with @pytest.mark.gpu
+          pytest -v -m "gpu" --cov=arraybridge --cov-report=term-missing
+        continue-on-error: true  # Don't fail the workflow if GPU tests fail
+
+      - name: Run framework-specific tests
+        run: |
+          # Run tests for the specific framework
+          pytest -v -m "${{ matrix.framework }}" --cov=arraybridge --cov-report=term-missing
+        continue-on-error: true
+
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: gpu-test-results-${{ matrix.framework }}
+          path: |
+            htmlcov/
+            .coverage
+

arraybridge-0.2.0/.github/workflows/publish.yml

@@ -0,0 +1,51 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+permissions:
+  contents: write
+  id-token: write  # Required for trusted publishing
+
+jobs:
+  build-and-publish:
+    name: Build and publish to PyPI
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/arraybridge
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install build dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install build
+
+      - name: Build package
+        run: python -m build
+
+      - name: Check distribution
+        run: |
+          pip install twine
+          twine check dist/*
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          files: dist/*
+          generate_release_notes: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

arraybridge-0.2.0/.gitignore

@@ -0,0 +1,48 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual environments
+venv/
+ENV/
+env/
+.venv
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+
+# IDEs
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# Documentation
+docs/_build/
+site/
+
+# OS
+.DS_Store
+Thumbs.db

arraybridge-0.2.0/.readthedocs.yml

@@ -0,0 +1,33 @@
+# Read the Docs configuration file for arraybridge
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+  jobs:
+    post_install:
+      # Install the package in editable mode
+      - pip install -e .
+
+# Build documentation in the docs/source directory with Sphinx
+sphinx:
+  configuration: docs/source/conf.py
+  fail_on_warning: false
+
+# Optionally build your docs in additional formats such as PDF and ePub
+formats:
+  - pdf
+  - epub
+
+# Python configuration
+python:
+  install:
+    - method: pip
+      path: .
+      extra_requirements:
+        - docs

arraybridge-0.2.0/CI_IMPROVEMENTS.md

@@ -0,0 +1,193 @@
+# CI Testing Improvements
+
+## Summary
+
+This update restructures the CI testing strategy to eliminate GPU runner queue issues while maintaining comprehensive test coverage.
+
+## Problems Solved
+
+1. **CuPy Installation Failures**: CuPy requires CUDA drivers and cannot be installed on standard CI runners
+2. **GPU Runner Queue Times**: Beta GPU runners have long queue times, blocking PR feedback
+3. **Inefficient Test Matrix**: Too many redundant test combinations
+4. **Unclear Test Organization**: No markers to distinguish CPU vs GPU tests
+
+## Changes Made
+
+### 1. Updated CI Workflow (`.github/workflows/ci.yml`)
+
+**Before:**
+- Matrix included `cupy` framework option
+- Tried to install CuPy on standard runners (always failed)
+- Tested on Python 3.9-3.12 across all OS combinations
+- 36+ job combinations
+
+**After:**
+- Removed `cupy` from standard CI matrix
+- Reduced Python versions to 3.10-3.12 (dropped 3.9)
+- Simplified matrix: torch tests only on Ubuntu
+- GPU tests marked as `continue-on-error: true` (non-blocking)
+- **Result**: ~12 fast jobs on standard runners + 1 optional GPU job
+
+### 2. Added Pytest Markers (`pyproject.toml`)
+
+New markers for test organization:
+- `@pytest.mark.gpu` - Requires actual GPU hardware
+- `@pytest.mark.cupy` - Requires CuPy framework
+- `@pytest.mark.torch` - Requires PyTorch framework
+- `@pytest.mark.tensorflow` - Requires TensorFlow
+- `@pytest.mark.jax` - Requires JAX
+- `@pytest.mark.pyclesperanto` - Requires pyclesperanto
+- `@pytest.mark.slow` - Long-running tests
+
+### 3. Enhanced Test Fixtures (`tests/conftest.py`)
+
+- Added `pytest_configure()` to register markers
+- Improved `cupy_available` fixture to verify GPU access
+- Added `gpu_available` fixture for general GPU detection
+- Better error handling for framework imports
+
+### 4. Marked Framework-Specific Tests
+
+Added `@pytest.mark.torch` to PyTorch tests in `test_converters.py`:
+- `test_detect_torch_tensor`
+- `test_convert_numpy_to_torch`
+- `test_convert_torch_to_numpy`
+- `test_round_trip_conversion_numpy_torch`
+
+### 5. Created Optional GPU Workflow (`.github/workflows/gpu-tests.yml`)
+
+Separate workflow for GPU testing:
+- Manual trigger only (`workflow_dispatch`)
+- Optional weekly schedule
+- Runs on self-hosted or beta GPU runners
+- Tests CuPy and PyTorch GPU functionality
+- Non-blocking (failures don't block PRs)
+
+### 6. Documentation (`TESTING.md`)
+
+Comprehensive testing guide covering:
+- Test categories (CPU vs GPU)
+- Running tests locally
+- Using pytest markers
+- CI workflow explanation
+- Adding new tests
+- Coverage goals
+
+## Test Results
+
+### Without PyTorch (NumPy only)
+```
+61 passed, 4 deselected
+Coverage: 33%
+```
+
+### With PyTorch CPU
+```
+65 passed
+Coverage: ~35%
+```
+
+### Test Breakdown
+- **CPU-compatible tests**: 61 tests (run on every PR)
+- **PyTorch tests**: 4 tests (run on Ubuntu with torch-cpu)
+- **GPU tests**: 0 currently (would run on GPU runners when available)
+
+## Benefits
+
+### ✅ Fast CI Feedback
+- Standard tests complete in ~2-3 minutes
+- No waiting in GPU runner queues
+- Immediate feedback on PRs
+
+### ✅ Cost Efficiency
+- Uses free standard runners for 99% of tests
+- GPU runners only used when needed
+- Reduced total CI minutes
+
+### ✅ Better Test Organization
+- Clear markers for test categories
+- Easy to run specific test subsets
+- Better documentation
+
+### ✅ Comprehensive Coverage
+- Core conversion logic tested on CPU
+- PyTorch CPU tests validate API
+- Optional GPU tests for real hardware validation
+
+### ✅ Non-Blocking GPU Tests
+- GPU test failures don't block PRs
+- Can investigate GPU issues separately
+- Flexibility for GPU runner availability
+
+## Migration Guide
+
+### For Developers
+
+**Running tests locally:**
+```bash
+# CPU tests only (fast)
+pytest -m "not gpu"
+
+# With PyTorch CPU
+pip install torch --index-url https://download.pytorch.org/whl/cpu
+pytest
+
+# With GPU frameworks (requires CUDA)
+pip install cupy-cuda12x
+pytest -m gpu
+```
+
+**Adding new tests:**
+```python
+# CPU-compatible test (no marker needed)
+def test_my_feature():
+    pass
+
+# Framework-specific test
+@pytest.mark.torch
+def test_torch_feature(torch_available):
+    if not torch_available:
+        pytest.skip("PyTorch not available")
+    # test code
+
+# GPU-only test
+@pytest.mark.gpu
+@pytest.mark.cupy
+def test_gpu_feature(cupy_available, gpu_available):
+    if not cupy_available or not gpu_available:
+        pytest.skip("GPU not available")
+    # test code
+```
+
+### For CI/CD
+
+**Standard CI (automatic):**
+- Runs on every PR and push to main
+- Tests NumPy + PyTorch CPU
+- Fast feedback (~2-3 minutes)
+- Must pass for PR merge
+
+**GPU CI (optional):**
+- Manual trigger via Actions tab
+- Or weekly schedule
+- Non-blocking (can fail without blocking PRs)
+- Use for validating GPU-specific features
+
+## Future Improvements
+
+1. **Mock GPU Frameworks**: Add mocking for GPU frameworks to test API without hardware
+2. **Performance Benchmarks**: Add benchmarks for conversion operations
+3. **Integration Tests**: Real-world scientific workflow tests
+4. **Coverage Goals**: Increase coverage to 80%+ for core modules
+
+## Rollback Plan
+
+If issues arise, revert to previous CI by:
+1. Restore original `.github/workflows/ci.yml`
+2. Remove pytest markers (optional, won't break anything)
+3. Tests will still work, just with less organization
+
+## Questions?
+
+See `TESTING.md` for detailed testing documentation.
+