exness-data-preprocess 0.1.0__tar.gz

exness_data_preprocess-0.1.0/.cz.toml

@@ -0,0 +1,12 @@
+[tool.commitizen]
+name = "cz_conventional_commits"
+version = "0.1.0"
+tag_format = "v$major.$minor.$patch"
+update_changelog_on_bump = true
+version_files = [
+    "pyproject.toml:version",
+    "src/exness_data_preprocess/__init__.py:__version__"
+]
+
+[tool.commitizen.settings]
+major_version_zero = true

exness_data_preprocess-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,50 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  pre-commit:
+    name: Pre-commit hooks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Run pre-commit hooks
+        uses: pre-commit/action@v3.0.1
+
+  test:
+    name: Tests (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+
+      - name: Install package with test dependencies
+        run: pip install -e ".[test]"
+
+      - name: Run tests with coverage
+        run: pytest --cov=exness_data_preprocess --cov-report=xml --cov-report=term
+
+      - name: Upload coverage to GitHub
+        if: matrix.python-version == '3.12'
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-report
+          path: coverage.xml

exness_data_preprocess-0.1.0/.github/workflows/pre-commit-autoupdate.yml

@@ -0,0 +1,38 @@
+name: Pre-commit Auto-update
+
+on:
+  schedule:
+    # Run weekly on Sunday at midnight UTC
+    - cron: '0 0 * * 0'
+  workflow_dispatch:  # Allow manual triggering
+
+jobs:
+  autoupdate:
+    name: Auto-update pre-commit hooks
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install pre-commit
+        run: pip install pre-commit
+
+      - name: Run pre-commit autoupdate
+        run: pre-commit autoupdate
+
+      - name: Create pull request
+        uses: peter-evans/create-pull-request@v6
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          branch: pre-commit-autoupdate
+          title: 'chore: update pre-commit hooks'
+          commit-message: 'chore: update pre-commit hooks'
+          body: |
+            Auto-update pre-commit hooks to latest versions.
+
+            - Updated by [pre-commit-autoupdate workflow](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
+          delete-branch: true

exness_data_preprocess-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,52 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build:
+    name: Build distribution
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install build dependencies
+        run: pip install build
+
+      - name: Build package
+        run: python -m build
+
+      - name: Upload distribution artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write  # REQUIRED for Trusted Publishing
+      contents: read
+    environment:
+      name: release
+      url: https://pypi.org/p/exness-data-preprocess
+
+    steps:
+      - name: Download distribution artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish to PyPI via Trusted Publishing
+        uses: pypa/gh-action-pypi-publish@release/v1
+        # No token needed! Uses GitHub OIDC via Trusted Publishing
+        # Digital attestations are generated automatically

exness_data_preprocess-0.1.0/.gitignore

@@ -0,0 +1,100 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Virtual environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# PyCharm
+.idea/
+
+# VSCode
+.vscode/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# Environments
+.env.local
+.env.*.local
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+.nox/
+.hypothesis/
+coverage.xml
+*.cover
+*.log
+
+# Type checking
+.mypy_cache/
+.dmypy.json
+dmypy.json
+.pytype/
+.pyre/
+
+# Data directories (for local development)
+data/
+*.parquet
+*.duckdb
+*.zip
+*.csv
+temp/
+
+# macOS
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Linux
+*~
+
+# Backup files
+*.bak
+*.swp
+*.swo
+*~
+
+# Distribution / packaging
+.Python
+pip-log.txt
+pip-delete-this-directory.txt
+
+# UV
+.uv/
+uv.lock
+
+# Pre-commit
+# .pre-commit-config.yaml should be tracked, not ignored

exness_data_preprocess-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,24 @@
+repos:
+- repo: https://github.com/commitizen-tools/commitizen
+  rev: v4.9.1
+  hooks:
+  - id: commitizen
+    stages: [commit-msg]
+  - id: commitizen-branch
+    stages: [pre-push]
+    args: [--rev-range, origin/main..HEAD]
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.4.0
+  hooks:
+  - id: trailing-whitespace
+  - id: end-of-file-fixer
+  - id: check-yaml
+  - id: check-added-large-files
+  - id: check-merge-conflict
+  - id: debug-statements
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.13.0
+  hooks:
+  - id: ruff
+    args: [--fix, --unsafe-fixes]
+  - id: ruff-format

exness_data_preprocess-0.1.0/AUTHORS.md

@@ -0,0 +1,24 @@
+# Authors
+
+## Project Lead
+
+- **Terry Li** <terry@eonlabs.com>
+  - Original author and maintainer
+  - Architecture and core implementation
+
+## Organization
+
+**Eon Labs**
+- Website: https://eonlabs.com
+- GitHub: https://github.com/Eon-Labs
+
+## Contributors
+
+We welcome contributions! See CONTRIBUTING.md for guidelines.
+
+## Acknowledgments
+
+- Exness for providing high-quality public tick data
+- DuckDB team for embedded OLAP database
+- Apache Arrow and Parquet communities
+- Facebook's Zstd compression library

exness_data_preprocess-0.1.0/CHANGELOG.md

@@ -0,0 +1,35 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- Initial release of exness-data-preprocess
+- Core `ExnessDataProcessor` class for tick data preprocessing
+- Simple Python API with wrapper functions
+- Command-line interface with `process`, `query`, `analyze`, and `stats` commands
+- Parquet compression with Zstd-22 (9% smaller than ZIP, lossless)
+- DuckDB OHLC generation with embedded metadata
+- Automatic resampling to higher timeframes (5m, 15m, 30m, 1h, 4h, 1d)
+- On-demand tick data analysis capabilities
+- Atomic file operations for data integrity
+- Comprehensive test suite with >90% coverage
+- Full documentation with usage examples and benchmarks
+
+### Technical Details
+- Optimal compression: Parquet + Zstd-22 (4.77 MB per month EURUSD)
+- DuckDB storage: 0.78 MB per month OHLC data
+- Zero precision loss (lossless compression)
+- Direct queryability via DuckDB SQL
+- Support for Python 3.9+
+
+## [0.1.0] - 2025-10-12
+
+Initial PyPI release.
+
+[Unreleased]: https://github.com/terrylica/exness-data-preprocess/compare/v0.1.0...HEAD
+[0.1.0]: https://github.com/terrylica/exness-data-preprocess/releases/tag/v0.1.0

exness_data_preprocess-0.1.0/CLAUDE.md

@@ -0,0 +1,274 @@
+# exness-data-preprocess - Project Memory
+
+**Architecture**: Professional forex tick data preprocessing with unified single-file DuckDB storage
+
+**Full Documentation**: [`README.md`](README.md) - Installation, usage, API reference
+
+---
+
+## Quick Links
+
+### Documentation
+- **[README.md](README.md)** - Full API reference, installation, usage examples
+- **[docs/README.md](docs/README.md)** - Documentation hub with research findings
+- **[docs/DATABASE_SCHEMA.md](docs/DATABASE_SCHEMA.md)** - Complete database schema with self-documentation
+- **[docs/UNIFIED_DUCKDB_PLAN_v2.md](docs/UNIFIED_DUCKDB_PLAN_v2.md)** - v2.0.0 architecture specification
+- **[docs/EXNESS_DATA_SOURCES.md](docs/EXNESS_DATA_SOURCES.md)** - Data source variants and URLs
+
+### Code Examples
+- **[examples/basic_usage.py](examples/basic_usage.py)** - Download, query, coverage operations
+- **[examples/batch_processing.py](examples/batch_processing.py)** - Multi-instrument parallel processing
+- **[examples/add_schema_comments.py](examples/add_schema_comments.py)** - Retrofit self-documentation
+
+### Development
+- **[CONTRIBUTING.md](CONTRIBUTING.md)** - Contribution guidelines
+- **[tests/README.md](tests/README.md)** - Test suite documentation
+
+---
+
+## Essential Architecture Decisions
+
+### Unified Single-File DuckDB Storage v2.0.0 (✅ Implemented & Validated 2025-10-12)
+
+**Decision**: Store all years of data in single DuckDB file per instrument (eurusd.duckdb, not monthly files)
+
+**Implementation**: v2.0.0 refactoring completed
+- ✅ **Single file per instrument**: eurusd.duckdb contains all years
+- ✅ **Dual-variant storage**: Raw_Spread + Standard in same database
+- ✅ **PRIMARY KEY constraints**: Prevents duplicates during incremental updates
+- ✅ **Automatic gap detection**: Downloads only missing months
+- ✅ **Phase7 9-column OHLC**: Dual spreads + dual tick counts
+- ✅ **Date range queries**: Sub-15ms query performance
+- ✅ **On-demand resampling**: Any timeframe (5m, 1h, 1d) in <15ms
+
+**Validation Results** (13 months, Oct 2024 - Oct 2025):
+- ✅ **Raw_Spread ticks**: 18.6M ticks
+- ✅ **Standard ticks**: 19.6M ticks
+- ✅ **OHLC bars**: 413K bars (1-minute)
+- ✅ **Database size**: 2.08 GB
+- ✅ **Query performance**: <15ms for all operations
+- ✅ **Incremental updates**: Working correctly (0 months added when up to date)
+
+**Architecture Benefits**:
+- **No file fragmentation**: All years in one database
+- **No duplicates**: PRIMARY KEY constraints prevent duplicate data
+- **Fast queries**: Date range filtering without loading entire dataset
+- **Scalability**: 2.08 GB for 13 months (3 years ~4.8 GB)
+
+**Comprehensive Plan**: [`docs/UNIFIED_DUCKDB_PLAN_v2.md`](docs/UNIFIED_DUCKDB_PLAN_v2.md) - Complete v2.0.0 specification
+
+**Legacy Plan**: [`docs/archive/UNIFIED_DUCKDB_PLAN_v1.0.0_LEGACY.md`](docs/archive/UNIFIED_DUCKDB_PLAN_v1.0.0_LEGACY.md) - Monthly-file architecture (archived)
+
+**Test Artifacts**:
+- `/tmp/exness-duckdb-test/refactored/eurusd.duckdb` - 2.08 GB unified database
+- `/tmp/exness-duckdb-test/test_refactored_processor.py` - Validation test
+- `/tmp/exness-duckdb-test/test_queries_only.py` - Query validation test
+
+### Phase7 9-Column OHLC Schema (✅ Implemented)
+
+**Decision**: Dual-variant BID-only OHLC with 9 columns capturing both Raw_Spread and Standard characteristics
+
+**Schema**: `Timestamp | Open | High | Low | Close | raw_spread_avg | standard_spread_avg | tick_count_raw_spread | tick_count_standard`
+
+**Key Features**:
+- ✅ **BID-only OHLC**: Uses Raw_Spread Bid prices (execution prices)
+- ✅ **Dual spreads**: Tracks both Raw_Spread (zero-spreads) and Standard (market spreads)
+- ✅ **Dual tick counts**: Records tick counts from both variants
+- ✅ **LEFT JOIN methodology**: Raw_Spread primary, Standard reference
+
+**Implementation**: [`src/exness_data_preprocess/processor.py`](src/exness_data_preprocess/processor.py) (lines 453-485)
+
+**Specification**: [`docs/research/eurusd-zero-spread-deviations/data/plan/phase7_bid_ohlc_construction_v1.1.0.md`](docs/research/eurusd-zero-spread-deviations/data/plan/phase7_bid_ohlc_construction_v1.1.0.md)
+
+**Complete Schema Details**: [`docs/DATABASE_SCHEMA.md`](docs/DATABASE_SCHEMA.md)
+
+### DuckDB Self-Documentation (✅ Implemented)
+
+**Decision**: Use DuckDB's `COMMENT ON` statements to store metadata inside the database
+
+**Implementation**: All tables and columns have embedded documentation
+- ✅ **Table comments**: Purpose, data source URLs, characteristics
+- ✅ **Column comments**: Type, constraints, nullability explanations
+- ✅ **Machine-readable**: Query via `duckdb_tables()`, `duckdb_columns()`
+- ✅ **Version-controlled**: Comments stored in database schema
+
+**Benefits**:
+- **Self-documenting**: Anyone connecting to database can query metadata
+- **Single source of truth**: Documentation lives with the data
+- **Tool integration**: BI tools and IDEs can display inline help
+- **No external docs needed**: Schema is fully self-explanatory
+
+**Query Examples**:
+```sql
+-- Get all table comments
+SELECT table_name, comment FROM duckdb_tables();
+
+-- Get all column comments with types
+SELECT table_name, column_name, data_type, comment FROM duckdb_columns();
+```
+
+**Retrofit Methods**:
+- `add_schema_comments(pair)` - Add comments to single database
+- `add_schema_comments_all()` - Batch-update all databases
+
+**Implementation**: [`src/exness_data_preprocess/processor.py`](src/exness_data_preprocess/processor.py) (lines 138-215, 698-797)
+
+**Usage Example**: [`examples/add_schema_comments.py`](examples/add_schema_comments.py)
+
+**Complete Schema Reference**: [`docs/DATABASE_SCHEMA.md`](docs/DATABASE_SCHEMA.md) - Human-readable documentation with introspection queries
+
+---
+
+## Exness Data Sources
+
+**Source**: https://ticks.ex2archive.com/ - Public tick data repository with 4 variants per instrument
+
+**Phase7 Uses**:
+- **Primary**: Raw_Spread variant (97.81% zero-spreads, execution prices)
+- **Reference**: Standard variant (0% zero-spreads, traditional quotes)
+
+**Key Characteristics**:
+- Monthly ZIP files with microsecond-precision CSV tick data
+- Institutional ECN/STP quality
+- Fixed URL pattern: `https://ticks.ex2archive.com/ticks/{VARIANT}/{YEAR}/{MONTH}/`
+
+**Complete Guide**: [`docs/EXNESS_DATA_SOURCES.md`](docs/EXNESS_DATA_SOURCES.md) - All 4 variants, URL patterns, download examples
+
+---
+
+## Research Areas
+
+### Zero-Spread Deviation Analysis
+
+**Research Period**: Sep 2024 baseline + 16-month validation (Jan-Aug 2024+2025)
+
+**Key Findings**:
+- ✅ **Mean Reversion**: 87.3% ± 1.9% stable across 16 months
+- ⚠️ **Volatility Prediction**: Regime shift between 2024 (R²=0.371) and 2025 (R²=0.209)
+- ✅ **Phase7 Methodology**: Dual-variant BID-only OHLC construction validated
+
+**Documentation**: [`docs/research/eurusd-zero-spread-deviations/README.md`](docs/research/eurusd-zero-spread-deviations/README.md)
+
+**Methodology**: [`docs/research/eurusd-zero-spread-deviations/01-methodology.md`](docs/research/eurusd-zero-spread-deviations/01-methodology.md)
+
+### Compression Benchmarks
+
+**Decision**: DuckDB native storage (no Parquet files in v2.0.0)
+
+**Legacy Benchmarks** (v1.0.0): Parquet Zstd-22 over Brotli-11 (too slow) and Delta Encoding (lossy)
+
+**Documentation**: [`docs/research/compression-benchmarks/README.md`](docs/research/compression-benchmarks/README.md)
+
+---
+
+## Development Setup
+
+**Quick Start**:
+```bash
+git clone https://github.com/Eon-Labs/exness-data-preprocess.git
+cd exness-data-preprocess
+uv sync --dev
+```
+
+**Complete Setup Guide**: [`README.md`](README.md) - Installation, testing, code quality, building
+
+**Contributing**: [`CONTRIBUTING.md`](CONTRIBUTING.md) - Contribution guidelines and workflow
+
+---
+
+## Current Implementation Status
+
+### v2.0.0 Architecture (✅ Completed 2025-10-12)
+
+- ✅ **Unified single-file DuckDB** - One file per instrument (eurusd.duckdb)
+- ✅ **Dual-variant storage** - Raw_Spread + Standard in same database
+- ✅ **PRIMARY KEY constraints** - Prevents duplicates during incremental updates
+- ✅ **Automatic gap detection** - Downloads only missing months
+- ✅ **Phase7 9-column OHLC** - Dual spreads + dual tick counts
+- ✅ **Date range queries** - Sub-15ms query performance
+- ✅ **On-demand resampling** - Any timeframe in <15ms
+- ✅ **SQL filter support** - Direct SQL WHERE clauses on ticks
+- ✅ **API refactoring** - Clean unified API
+- ✅ **Examples updated** - basic_usage.py, batch_processing.py
+- ✅ **Documentation updated** - README.md, CLAUDE.md, docs/README.md
+
+### Usage Examples
+
+**Basic Operations**: [`examples/basic_usage.py`](examples/basic_usage.py) - Download, query, coverage
+
+**Batch Processing**: [`examples/batch_processing.py`](examples/batch_processing.py) - Multi-instrument, parallel processing
+
+**Schema Comments**: [`examples/add_schema_comments.py`](examples/add_schema_comments.py) - Retrofit self-documentation to existing databases
+
+**Complete API Reference**: [`README.md`](README.md) - All methods, parameters, and usage patterns
+
+### Pending Tasks
+
+- ⏳ **CLI enhancements** - Add variant selection, OHLC resampling commands
+- ⏳ **Test suite** - Update test_processor.py, test_api.py, test_cli.py for v2.0.0
+- ⏳ **API expansion** - Add streaming query methods, batch operations
+
+---
+
+## File Locations
+
+**Project Root**: `/Users/terryli/eon/exness-data-preprocess/ `
+
+**Data Storage** (default): `~/eon/exness-data/`
+```
+~/eon/exness-data/
+├── eurusd.duckdb      # Single file for all EURUSD data
+├── gbpusd.duckdb      # Single file for all GBPUSD data
+├── xauusd.duckdb      # Single file for all XAUUSD data
+└── temp/
+    └── (temporary ZIP files)
+```
+
+**Database Schema** (per instrument):
+```
+eurusd.duckdb:
+├── raw_spread_ticks   # Timestamp (PK), Bid, Ask
+├── standard_ticks     # Timestamp (PK), Bid, Ask
+├── ohlc_1m            # Phase7 9-column schema
+└── metadata           # Coverage tracking
+```
+
+**Test Artifacts**: `/tmp/exness-duckdb-test/`
+- `refactored/eurusd.duckdb` - 2.08 GB unified database (13 months)
+- `test_refactored_processor.py` - Validation test
+- `test_queries_only.py` - Query validation test
+
+---
+
+## Migration from v1.0.0
+
+**v1.0.0 (Legacy)**:
+- Monthly DuckDB files: `eurusd_ohlc_2024_08.duckdb`
+- Parquet tick storage: `eurusd_ticks_2024_08.parquet`
+- Functions: `process_month()`, `process_date_range()`, `analyze_ticks()`
+
+**v2.0.0 (Current)**:
+- Single DuckDB file: `eurusd.duckdb`
+- No Parquet files (everything in DuckDB)
+- Unified API: `processor.update_data()`, `processor.query_ohlc()`, `processor.query_ticks()`
+
+**Migration Steps**:
+1. Run `processor.update_data(pair, start_date)` to create new unified database
+2. Delete old monthly files: `rm eurusd_ohlc_2024_*.duckdb eurusd_ticks_2024_*.parquet`
+3. Update code to use new API methods
+
+---
+
+## References
+
+- **Exness Data**: https://ticks.ex2archive.com/
+- **Official Exness**: https://www.exness.com/tick-history/ (Cloudflare protected)
+- **DuckDB**: https://duckdb.org/
+- **Apache Parquet**: https://parquet.apache.org/
+- **Zstd Compression**: https://facebook.github.io/zstd/
+
+---
+
+**Version**: 2.0.0
+**Last Updated**: 2025-10-12
+**Architecture**: Unified Single-File DuckDB Storage with Incremental Updates