echr-extractor 0.0.1.dev1__tar.gz

echr_extractor-0.0.1.dev1/.flake8

@@ -0,0 +1,10 @@
+[flake8]
+max-line-length = 88
+extend-ignore = E203, W503, E501
+exclude = 
+    .git,
+    __pycache__,
+    .venv,
+    build,
+    dist,
+    *.egg-info

echr_extractor-0.0.1.dev1/.github/workflows/ci.yml

@@ -0,0 +1,163 @@
+name: Build, Test, Lint & Upload to TestPyPI and PyPI for ECHR-extractor
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+
+jobs:
+  lint:
+    if: github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/main')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+      
+      - name: Lint with super linter
+        uses: github/super-linter@v4
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          DEFAULT_BRANCH: 'main'
+          VALIDATE_PYTHON_BLACK: true
+          IGNORE_GITIGNORED_FILES: true
+          VALIDATE_ALL_CODEBASE: false
+      
+  test:
+    needs: lint
+    if: github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/main')
+    name: Test on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ['3.9', '3.10', '3.11', '3.12']
+
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel pytest pytest-cov
+          pip install -r requirements.txt
+          
+      
+      - name: Install package for testing
+        run: |
+          pip install -e .
+
+      - name: Run tests with pytest
+        run: |
+          mkdir -p junit
+          pytest tests/ --doctest-modules --junitxml=junit/test-results.xml --cov=src/echr_extractor --cov-report=xml --cov-report=html
+
+  build:
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
+    name: Build Python package
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+
+      - name: Install build dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel build setuptools_scm
+
+      - name: Show version that will be built
+        run: |
+          python -c "import setuptools_scm; print('Version:', setuptools_scm.get_version())"
+
+      - name: Build package
+        run: |
+          python -m build
+
+      - name: Upload build artifacts
+        if: (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+  pypi-publish:
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/echr-extractor/
+    permissions:
+      id-token: write
+    steps:
+      - name: Download all the artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+      - name: Publish distribution to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
+    name: Sign the Python distribution with Sigstore and upload them to GitHub Releases
+    needs:
+      - pypi-publish
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: write
+
+    steps:
+      - name: Download all the artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+      - name: Sign the Python distribution with Sigstore
+        uses: sigstore/gh-action-sigstore-python@v3.0.0
+        with:
+          inputs: >-
+            ./dist/*.tar.gz
+            ./dist/*.whl
+
+      - name: Get version from setuptools_scm
+        id: version
+        run: |
+          python -c "import setuptools_scm; print('version=' + setuptools_scm.get_version())" >> $GITHUB_OUTPUT
+
+      - name: Create Github release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh release create \
+          'echr-extractor-${{ steps.version.outputs.version }}' \
+          --repo '${{ github.repository }}' \
+          --target 'main' \
+          --notes ""
+
+      - name: Upload artifact signatures to Github release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh release upload \
+          'echr-extractor-${{ env.RELEASE_VERSION }}' ./dist/*.tar.gz ./dist/*.whl \
+          --repo '${{ github.repository }}'

echr_extractor-0.0.1.dev1/.gitignore

@@ -0,0 +1,139 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Data files
+data/
+*.csv
+*.json
+
+# Temporary files
+temp_repo/
+
+.vscode

echr_extractor-0.0.1.dev1/CHANGELOG.md

@@ -0,0 +1,71 @@
+# Changelog
+
+All notable changes to the ECHR Extractor project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [1.1.0] - 2025-09-12
+
+### Added
+
+- Date range batching for large datasets to prevent API timeouts
+- Enhanced error handling with exponential backoff retry logic
+- Progress tracking with tqdm progress bars for long-running operations
+- Memory management with chunked processing for large datasets
+- Configurable batch sizes, timeouts, and retry parameters
+- Comprehensive docstrings and usage examples for all functions
+- New parameters: `batch_size`, `timeout`, `retry_attempts`, `max_attempts`, `days_per_batch`, `progress_bar`, `memory_efficient`
+
+### Changed
+
+- Enhanced `get_r()` function with better error handling and exponential backoff
+- Completely rewritten `get_echr_metadata()` with batching and memory management
+- Updated `get_echr()` and `get_echr_extra()` functions with new configuration parameters
+- Improved logging and status reporting throughout the extraction process
+- Better handling of large date ranges by splitting them into manageable chunks
+
+### Fixed
+
+- Memory issues when processing very large datasets
+- API timeout issues for multi-year date range extractions
+- Better error recovery and retry mechanisms
+- Improved handling of network failures and connection errors
+
+### Performance
+
+- Significantly improved reliability for large-scale data extraction
+- Reduced memory usage through intelligent chunked processing
+- Better progress visibility for long-running operations
+- More robust handling of network interruptions
+
+## [1.0.44] - 2025-07-02
+
+### Added
+
+- Initial release as standalone package
+- Migrated from maastrichtlawtech/extraction_libraries echr branch
+- Modern Python package structure with pyproject.toml
+- Command-line interface (CLI) support
+- Comprehensive documentation and examples
+- CI/CD pipeline with GitHub Actions
+- Development tools setup (black, isort, flake8, pytest)
+
+### Changed
+
+- Restructured as proper Python package with src/ layout
+- Updated import statements for relative imports
+- Improved package metadata and dependencies
+- Enhanced README with comprehensive usage examples
+
+### Fixed
+
+- Import paths for proper package distribution
+- Package structure for PyPI publishing
+- Dependencies and requirements specifications
+
+## Previous Versions
+
+This package was previously part of the extraction_libraries repository
+under the 'echr' branch. For historical changes, please refer to:
+https://github.com/maastrichtlawtech/extraction_libraries/tree/echr

echr_extractor-0.0.1.dev1/IMPROVEMENTS.md

@@ -0,0 +1,216 @@
+# ECHR Extractor Improvements - Technical Documentation
+
+## Overview
+
+Improved error handling, memory management, and processing capabilities. These changes address reliability issues in large-scale data extraction while maintaining full backward compatibility.
+
+## Problem Statement
+
+### Issues Addressed
+
+- Large data extractions frequently failed due to API timeouts
+- Memory consumption issues when processing large datasets
+- Lack of progress visibility during long-running extractions
+- Limited error recovery mechanisms for network failures
+
+### Impact
+
+- Failed extractions required manual intervention and restart
+- Large datasets could not be processed due to memory constraints
+- Users had no visibility into extraction progress
+- Network interruptions caused complete extraction failures
+
+## Technical Improvements
+
+### 1. Date Range Batching System
+
+**Implementation:**
+
+- Automatically splits large date ranges into manageable chunks (default: 365 days)
+- Prevents API timeouts that previously caused extraction failures
+- Processes each chunk independently with proper error handling
+
+**Technical Details:**
+
+- Chunk size is configurable via `days_per_batch` parameter
+- Each chunk is processed as a separate API request
+- Failed chunks can be retried independently
+
+**Code Example:**
+
+```python
+# Automatic batching for large date ranges
+df = get_echr(
+    start_date='2010-01-01',
+    end_date='2020-12-31',
+    days_per_batch=365  # Processes in 1-year chunks
+)
+```
+
+### 2. Enhanced Error Handling and Retry Logic
+
+**Implementation:**
+
+- Implements exponential backoff retry strategy (2^count seconds, max 30s)
+- Better error detection with HTTP status code checking
+- Detailed error logging for troubleshooting
+
+**Technical Details:**
+
+- Retry strategy: 3 attempts with exponential backoff
+- Error types handled: timeouts, connection errors, HTTP errors
+- Comprehensive logging with error context
+
+### 3. Progress Tracking
+
+**Implementation:**
+
+- Real-time progress bars using tqdm library
+- Batch-level progress reporting
+- Clear status messages and completion estimates
+
+**Technical Details:**
+
+- Progress bars show current batch and overall progress
+- Configurable via `progress_bar` parameter
+- Displays processing speed and estimated completion time
+
+**Example Output:**
+
+```
+Batch 1/5: 2020-01-01 to 2020-12-31: 100%|██████████| 1500/1500 [05:30<00:00, 4.54it/s]
+Batch 2/5: 2021-01-01 to 2021-12-31: 100%|██████████| 1200/1200 [04:20<00:00, 4.61it/s]
+```
+
+### 4. Memory Management
+
+**Implementation:**
+
+- Chunked processing for large datasets (10,000 records per chunk)
+- Automatic garbage collection between chunks
+- Configurable memory efficiency settings
+
+**Technical Details:**
+
+- Memory-efficient processing controlled by `memory_efficient` parameter
+- Garbage collection forced between chunks to free memory
+- Prevents memory accumulation during large extractions
+
+### 5. Configuration Parameters
+
+**New Parameters:**
+
+- `batch_size`: Records per API request (default: 500, max: 500)
+- `timeout`: Request timeout in seconds (default: 60)
+- `retry_attempts`: Number of retry attempts (default: 3)
+- `max_attempts`: Maximum total attempts (default: 20)
+- `days_per_batch`: Days per date batch (default: 365)
+- `progress_bar`: Show progress bars (default: True)
+- `memory_efficient`: Use chunked processing (default: True)
+
+## Performance Improvements
+
+### Memory Usage
+
+- Memory consumption is now controlled through chunked processing
+- Large datasets are processed in 10,000-record chunks
+- Garbage collection is forced between chunks to prevent memory accumulation
+
+### Error Recovery
+
+- Automatic retry mechanism with exponential backoff
+- Network failures no longer cause complete extraction failures
+- Individual chunks can be retried independently
+
+### Progress Visibility
+
+- Real-time progress bars show extraction status
+- Users can monitor progress and estimate completion time
+- Batch-level reporting provides detailed progress information
+
+## Backward Compatibility
+
+### Compatibility Guarantee
+
+- All existing code continues to work unchanged
+- New parameters are optional with sensible defaults
+- Original function signatures preserved
+- Default behavior identical to previous version
+
+### Migration Path
+
+```python
+# Existing code - works exactly as before
+df = get_echr(start_id=0, end_id=1000, verbose=True)
+
+# Enhanced code - same result, additional features available
+df = get_echr(
+    start_id=0,
+    end_id=1000,
+    verbose=True,
+    batch_size=250,        # Optional: smaller batches
+    progress_bar=True,     # Optional: show progress
+    memory_efficient=True  # Optional: use chunked processing
+)
+```
+
+## User Benefits
+
+### For Researchers
+
+- Improved reliability for large dataset extractions
+- Progress visibility during long-running operations
+- Configurable parameters for different research needs
+- Better memory management for large extractions
+
+### For IT Support
+
+- Reduced extraction failure reports due to better error handling
+- Detailed error logs and status messages for troubleshooting
+- Clear progress indicators and error reporting
+
+### For Project Managers
+
+- More reliable extractions enable better project planning
+- Reduced support overhead due to improved error handling
+- Better scalability for large research projects
+
+## Implementation Details
+
+### Files Modified
+
+1. **`ECHR_metadata_harvester.py`**: Core extraction logic with batching and error handling
+2. **`echr.py`**: Main API functions with new configuration parameters
+3. **`CHANGELOG.md`**: Documentation of changes
+
+### Dependencies
+
+- **tqdm**: Progress bar library (already in requirements.txt)
+- **timedelta**: Date manipulation (part of standard library)
+
+### Testing
+
+- Backward compatibility verified with existing code
+- New features tested with real ECHR data
+- Memory usage improvements validated
+- Error handling tested with various failure scenarios
+
+## Technical Benefits
+
+### Scalability
+
+- Improved handling of large datasets through chunked processing
+- Memory usage controlled through configurable chunk sizes
+- Better suited for enterprise-level research projects
+
+### Maintainability
+
+- Enhanced error handling reduces debugging time
+- Comprehensive logging improves troubleshooting
+- Modular design facilitates future enhancements
+
+### Usability
+
+- Progress tracking improves user experience
+- Configurable parameters support diverse research needs
+- Better error messages aid in troubleshooting