exness-data-preprocess 1.2.1__tar.gz → 2.1.0__tar.gz

exness_data_preprocess-2.1.0/.github/workflows/ci.yml

@@ -0,0 +1,25 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  pre-commit:
+    name: Pre-commit hooks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Run pre-commit hooks
+        uses: pre-commit/action@v3.0.1
+
+  # NOTE: Tests removed per local-first testing standard
+  # See: ~/.claude/CLAUDE.md - "NO unit testing in GitHub Actions"
+  # Run tests locally with: mise run validate

{exness_data_preprocess-1.2.1 → exness_data_preprocess-2.1.0}/.gitignore

@@ -112,3 +112,7 @@ node_modules/
 # ADR: /docs/adr/2025-12-10-clickhouse-pydantic-config.md
 .dbeaver/
 .dbeaver-workspace/
+
+# Claude Code local files
+.claude/
+.lint-relative-paths-results.txt

{exness_data_preprocess-1.2.1 → exness_data_preprocess-2.1.0}/.mise.toml

@@ -35,8 +35,9 @@ python = "3.11"
 
 [tasks.validate]
 description = "Run full E2E validation pipeline"
-# ADR: /docs/adr/2025-12-10-clickhouse-e2e-validation-pipeline.md
+# ADR: /docs/adr/2025-12-11-duckdb-removal-clickhouse.md
 depends = [
+    "clickhouse:ensure",        # Fail fast if ClickHouse not running
     "validate:imports",
     "validate:lint",
     "validate:typecheck",
@@ -44,24 +45,36 @@ depends = [
     "validate:build",
     "validate:install",
     "validate:mixin",
+    "validate:clickhouse",      # ClickHouse E2E validation
 ]
 run = '''
 echo ""
 echo "=============================================="
 echo "✅ ALL VALIDATIONS PASSED"
 echo "=============================================="
-echo ""
-echo "💡 To run ClickHouse validation (requires ClickHouse running):"
-echo "   mise run clickhouse:start"
-echo "   mise run validate:clickhouse"
 '''
 
 # -----------------------------------------------------------------------------
 # Individual Validation Tasks
 # -----------------------------------------------------------------------------
 
+[tasks."clickhouse:ensure"]
+description = "Ensure ClickHouse is running (fail fast)"
+# ADR: /docs/adr/2025-12-11-duckdb-removal-clickhouse.md
+run = '''
+#!/usr/bin/env bash
+set -e
+if ! nc -z localhost 8123 2>/dev/null; then
+    echo "ERROR: ClickHouse not running on port 8123" >&2
+    echo "Start with: mise run clickhouse:start" >&2
+    exit 1
+fi
+echo "✅ ClickHouse connection verified"
+'''
+
 [tasks."validate:imports"]
 description = "Validate all module imports (catch circular imports, missing deps)"
+# ADR: /docs/adr/2025-12-11-duckdb-removal-clickhouse.md - DuckDB modules removed
 run = '''
 #!/usr/bin/env bash
 set -e
@@ -95,7 +108,7 @@ from exness_data_preprocess import (
 print('✅ All public exports importable')
 " || { echo "❌ Public exports import failed"; exit 1; }
 
-# Test all internal modules
+# Test all internal modules (ClickHouse-only, DuckDB modules removed)
 uv run python -c "
 modules = [
     'exness_data_preprocess.clickhouse_base',
@@ -105,15 +118,10 @@ modules = [
     'exness_data_preprocess.clickhouse_ohlc_generator',
     'exness_data_preprocess.clickhouse_query_engine',
     'exness_data_preprocess.config',
-    'exness_data_preprocess.database_manager',
     'exness_data_preprocess.downloader',
     'exness_data_preprocess.exchanges',
-    'exness_data_preprocess.gap_detector',
     'exness_data_preprocess.models',
-    'exness_data_preprocess.ohlc_generator',
     'exness_data_preprocess.processor',
-    'exness_data_preprocess.query_engine',
-    'exness_data_preprocess.schema',
     'exness_data_preprocess.session_detector',
     'exness_data_preprocess.tick_loader',
 ]
@@ -121,7 +129,7 @@ import importlib
 for mod in modules:
     importlib.import_module(mod)
     print(f'  ✓ {mod}')
-print('✅ All 18 modules importable')
+print('✅ All 13 modules importable')
 "
 echo "✅ Import validation passed"
 '''

{exness_data_preprocess-1.2.1 → exness_data_preprocess-2.1.0}/CHANGELOG.md

@@ -1,3 +1,46 @@
+# [2.1.0](https://github.com/terrylica/exness-data-preprocess/compare/v2.0.0...v2.1.0) (2025-12-28)
+
+
+### Bug Fixes
+
+* **clickhouse:** add parameters support to execute_command and fix OHLC generator ([391b939](https://github.com/terrylica/exness-data-preprocess/commit/391b939bd2f7e6baad8945d98fc63e8843799682))
+* **docs:** fix 22 broken links identified by lychee link checker ([a85862e](https://github.com/terrylica/exness-data-preprocess/commit/a85862e7af58bdd9e28eef59f364f56e0df37c24))
+
+
+### Features
+
+* **scripts:** add DuckDB to ClickHouse migration scripts ([1920903](https://github.com/terrylica/exness-data-preprocess/commit/1920903df113bea810cc3607273d6f2ececd3f4b))
+
+# [2.0.0](https://github.com/terrylica/exness-data-preprocess/compare/v1.2.1...v2.0.0) (2025-12-27)
+
+
+* feat!: migrate to ClickHouse-only backend, remove DuckDB ([7fd83bc](https://github.com/terrylica/exness-data-preprocess/commit/7fd83bc5720b2f5997a661ef8060757abd602a81))
+
+
+### BREAKING CHANGES
+
+* This release removes DuckDB and makes ClickHouse the sole storage backend.
+
+API Changes:
+- `duckdb_path` → `database` (str, ClickHouse database name)
+- `duckdb_size_mb` → `storage_bytes` (int, bytes not MB)
+- `database_exists` → removed
+- `base_dir` constructor parameter → removed
+
+Schema Changes:
+- OHLC schema reduced from 30 to 26 columns (removed 4 normalized metrics)
+
+Deleted Files (~1,219 lines):
+- database_manager.py (209 lines)
+- gap_detector.py (134 lines)
+- ohlc_generator.py (266 lines)
+- query_engine.py (291 lines)
+- schema.py (324 lines)
+
+Requires: ClickHouse running on localhost:8123
+
+ADR: docs/adr/2025-12-11-duckdb-removal-clickhouse.md
+
 ## [1.2.1](https://github.com/terrylica/exness-data-preprocess/compare/v1.2.0...v1.2.1) (2025-12-11)
 
 

exness_data_preprocess-2.1.0/CLAUDE.md

@@ -0,0 +1,152 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+**Package**: `exness-data-preprocess` — Forex tick data preprocessing with ClickHouse backend
+
+**Version**: See [`pyproject.toml`](/pyproject.toml) line 3 (SSoT via semantic-release)
+
+**Quick Start**: [`README.md`](/README.md) — Installation, API reference, usage examples
+
+---
+
+## Documentation Hub (Progressive Disclosure)
+
+### Essential Links
+
+| Topic            | SSoT Document                                                                                          | Purpose                      |
+| ---------------- | ------------------------------------------------------------------------------------------------------ | ---------------------------- |
+| **User Guide**   | [`README.md`](/README.md)                                                                               | Installation, API, examples  |
+| **Architecture** | [`docs/adr/2025-12-11-duckdb-removal-clickhouse.md`](/docs/adr/2025-12-11-duckdb-removal-clickhouse.md) | ClickHouse-only decision     |
+| **Schema**       | [`docs/DATABASE_SCHEMA.md`](/docs/DATABASE_SCHEMA.md)                                                   | 26-column OHLC specification |
+| **Modules**      | [`docs/MODULE_ARCHITECTURE.md`](/docs/MODULE_ARCHITECTURE.md)                                           | 13 modules with SLOs         |
+| **Data Sources** | [`docs/EXNESS_DATA_SOURCES.md`](/docs/EXNESS_DATA_SOURCES.md)                                           | Exness tick data variants    |
+| **Tasks**        | [`.mise.toml`](/.mise.toml)                                                                             | Validation, ClickHouse tasks |
+| **Contributing** | [`CONTRIBUTING.md`](/CONTRIBUTING.md)                                                                   | Development guidelines       |
+
+### Deep Dives (On-Demand)
+
+| Topic              | Location                               | When to Read              |
+| ------------------ | -------------------------------------- | ------------------------- |
+| ADRs               | [`docs/adr/`](/docs/adr/)               | Architecture decisions    |
+| Design Specs       | [`docs/design/`](/docs/design/)         | Implementation blueprints |
+| Validation Results | [`docs/validation/`](/docs/validation/) | Spike test proofs         |
+
+---
+
+## Architecture Overview (v2.0.0)
+
+### ClickHouse-Only Backend
+
+**ADR**: [`docs/adr/2025-12-11-duckdb-removal-clickhouse.md`](/docs/adr/2025-12-11-duckdb-removal-clickhouse.md)
+
+| Feature         | Description                                      |
+| --------------- | ------------------------------------------------ |
+| **Database**    | `exness` (single database for all instruments)   |
+| **Tables**      | `raw_spread_ticks`, `standard_ticks`, `ohlc_1m`  |
+| **Engine**      | ReplacingMergeTree (deduplication at merge time) |
+| **Connection**  | localhost:8123 (local) or cloud via env vars     |
+| **Entry Point** | `ExnessDataProcessor`                            |
+
+### Module Structure (13 Modules)
+
+**ClickHouse Backend (7 modules)**:
+
+- `processor.py` — Main orchestrator with ClickHouse backend
+- `clickhouse_manager.py` — Schema management, data loading
+- `clickhouse_gap_detector.py` — Gap detection for incremental updates
+- `clickhouse_query_engine.py` — Query with pagination
+- `clickhouse_ohlc_generator.py` — OHLC generation
+- `clickhouse_client.py` — Connection factory with error handling
+- `clickhouse_base.py` — Shared mixin for client lifecycle
+
+**Shared (6 modules)**:
+
+- `downloader.py` — HTTP download operations
+- `tick_loader.py` — CSV parsing
+- `session_detector.py` — Holiday/session detection (vectorized)
+- `config.py` — Configuration management
+- `models.py` — Pydantic models (UpdateResult, CoverageInfo, etc.)
+- `exchanges.py` — Exchange calendar integration
+
+**Deleted in v2.0.0**:
+
+- ~~`database_manager.py`~~ — DuckDB database operations
+- ~~`gap_detector.py`~~ — DuckDB gap detection
+- ~~`ohlc_generator.py`~~ — DuckDB OHLC generation
+- ~~`query_engine.py`~~ — DuckDB query operations
+- ~~`schema.py`~~ — DuckDB SQL schema definitions
+
+---
+
+## Development Commands
+
+### Quick Reference
+
+```bash
+# Setup
+uv sync --dev
+
+# Testing (45 tests)
+uv run pytest                          # All tests
+mise run validate                      # Full E2E validation (requires ClickHouse)
+
+# Linting
+uv run ruff check --fix src/ tests/
+
+# ClickHouse (required for full validation)
+mise run clickhouse:start              # Start local server
+mise run clickhouse:status             # Check connection
+mise run clickhouse:ensure             # Fail fast if not running
+
+# Build & Publish
+uv build
+```
+
+### mise Tasks
+
+All validation and tooling tasks are defined in [`.mise.toml`](/.mise.toml):
+
+| Task                           | Description                             |
+| ------------------------------ | --------------------------------------- |
+| `mise run validate`            | Full E2E pipeline (requires ClickHouse) |
+| `mise run clickhouse:ensure`   | Fail fast if ClickHouse not running     |
+| `mise run clickhouse:start`    | Start mise-installed ClickHouse         |
+| `mise run validate:clickhouse` | ClickHouse E2E tests                    |
+
+---
+
+## Key Changes in v2.0.0
+
+### Breaking Changes
+
+| v1.x (DuckDB)     | v2.0.0 (ClickHouse)             |
+| ----------------- | ------------------------------- |
+| `duckdb_path`     | `database` (str)                |
+| `duckdb_size_mb`  | `storage_bytes` (int)           |
+| `database_exists` | removed                         |
+| `base_dir` param  | removed (ClickHouse manages)    |
+| 30-column OHLC    | 26-column OHLC (4 cols removed) |
+
+### Removed Files
+
+```
+src/exness_data_preprocess/database_manager.py   (209 lines)
+src/exness_data_preprocess/gap_detector.py       (134 lines)
+src/exness_data_preprocess/ohlc_generator.py     (266 lines)
+src/exness_data_preprocess/query_engine.py       (291 lines)
+src/exness_data_preprocess/schema.py             (324 lines)
+```
+
+**Total: ~1,224 lines deleted**
+
+---
+
+## References
+
+- **Exness Data**: <https://ticks.ex2archive.com/>
+- **ClickHouse**: <https://clickhouse.com/>
+
+---
+
+**Last Updated**: 2025-12-11

{exness_data_preprocess-1.2.1 → exness_data_preprocess-2.1.0}/CONTRIBUTING.md

@@ -144,7 +144,7 @@ test: add integration tests for DuckDB operations
 
 ### Project Structure
 
-**See [`CLAUDE.md`](CLAUDE.md) for complete module structure (19 modules).**
+**See [`CLAUDE.md`](/CLAUDE.md) for complete module structure (19 modules).**
 
 Key directories:
 

exness_data_preprocess-2.1.0/PKG-INFO

@@ -0,0 +1,227 @@
+Metadata-Version: 2.4
+Name: exness-data-preprocess
+Version: 2.1.0
+Summary: Professional Exness forex tick data preprocessing with ClickHouse backend. Provides efficient storage with lossless precision and direct queryability.
+Project-URL: Homepage, https://github.com/terrylica/exness-data-preprocess
+Project-URL: Documentation, https://github.com/terrylica/exness-data-preprocess#readme
+Project-URL: Repository, https://github.com/terrylica/exness-data-preprocess.git
+Project-URL: Bug Tracker, https://github.com/terrylica/exness-data-preprocess/issues
+Project-URL: Source Code, https://github.com/terrylica/exness-data-preprocess
+Project-URL: Changelog, https://github.com/terrylica/exness-data-preprocess/blob/main/CHANGELOG.md
+Author-email: Eon Labs <terry@eonlabs.com>
+Maintainer-email: Terry Li <terry@eonlabs.com>
+License: MIT
+License-File: LICENSE
+Keywords: backtesting,clickhouse,data-engineering,efficient-storage,eurusd,exness,financial-data,forex,forex-data,lossless,microstructure,ohlc,pandas,preprocessing,quantitative-finance,queryable,tick-data,time-series
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Financial and Insurance Industry
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Office/Business :: Financial :: Investment
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.11
+Requires-Dist: clickhouse-connect>=0.7.0
+Requires-Dist: exchange-calendars>=4.5.6
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: pyyaml>=6.0.0
+Provides-Extra: examples
+Requires-Dist: tqdm>=4.65.0; extra == 'examples'
+Description-Content-Type: text/markdown
+
+# exness-data-preprocess v2.0.0
+
+Forex tick data preprocessing with ClickHouse backend. Downloads, stores, and queries tick data from Exness public repository.
+
+## Data Inventory (Validated 2025-12-28)
+
+| Instrument | raw_spread_ticks | standard_ticks | ohlc_1m   | Date Range              |
+| ---------- | ---------------- | -------------- | --------- | ----------------------- |
+| EURUSD     | 56,687,313       | 62,203,144     | 1,466,724 | 2022-01-02 → 2025-12-26 |
+| XAUUSD     | 166,820,410      | 167,871,871    | 1,401,953 | 2022-01-02 → 2025-12-26 |
+
+**Total**: 453.6M ticks, 2.87M OHLC bars in 5.99 GiB
+
+## Requirements
+
+- Python 3.11+
+- ClickHouse running on `localhost:8123`
+
+## Verify ClickHouse
+
+```bash
+# Check ClickHouse is running
+clickhouse client --query "SELECT version()"
+
+# Check exness database exists
+clickhouse client --query "SELECT count() FROM exness.raw_spread_ticks"
+```
+
+## Installation
+
+```bash
+# Using uv
+uv pip install exness-data-preprocess
+
+# From source
+git clone https://github.com/Eon-Labs/exness-data-preprocess.git
+cd exness-data-preprocess
+uv sync
+```
+
+## Python API
+
+```python
+import exness_data_preprocess as edp
+
+# Context manager (recommended)
+with edp.ExnessDataProcessor() as processor:
+    # Query ticks
+    df = processor.query_ticks('EURUSD', 'raw_spread', '2024-01-01', '2024-01-31')
+
+    # Query OHLC
+    df = processor.query_ohlc('EURUSD', '1m', '2024-01-01', '2024-01-31')
+
+    # Get coverage info
+    cov = processor.get_data_coverage('EURUSD')
+
+    # Download new data (incremental)
+    result = processor.update_data('EURUSD', start_date='2022-01-01')
+```
+
+### API Methods
+
+| Method                                              | Parameters                                     | Returns      |
+| --------------------------------------------------- | ---------------------------------------------- | ------------ |
+| `query_ticks(pair, variant, start_date, end_date)`  | variant: `raw_spread` or `standard`            | DataFrame    |
+| `query_ohlc(pair, timeframe, start_date, end_date)` | timeframe: `1m`, `5m`, `15m`, `1h`, `4h`, `1d` | DataFrame    |
+| `get_data_coverage(pair)`                           |                                                | CoverageInfo |
+| `update_data(pair, start_date, delete_zip=True)`    |                                                | UpdateResult |
+
+### Return Models
+
+```python
+# UpdateResult
+result.months_added        # int: months downloaded
+result.raw_ticks_added     # int: raw spread ticks added
+result.standard_ticks_added # int: standard ticks added
+result.ohlc_bars           # int: OHLC bars generated
+result.storage_bytes       # int: storage used
+
+# CoverageInfo
+cov.raw_spread_ticks       # int: total raw spread ticks
+cov.standard_ticks         # int: total standard ticks
+cov.ohlc_bars              # int: total OHLC bars
+cov.earliest_date          # str: earliest timestamp
+cov.latest_date            # str: latest timestamp
+cov.storage_bytes          # int: storage used
+```
+
+## Direct SQL Access
+
+```bash
+# Count ticks
+clickhouse client --query "SELECT count() FROM exness.raw_spread_ticks WHERE instrument='EURUSD'"
+
+# Query tick data
+clickhouse client --query "
+SELECT timestamp, bid, ask
+FROM exness.raw_spread_ticks
+WHERE instrument='EURUSD'
+  AND timestamp >= '2024-01-01'
+  AND timestamp < '2024-01-02'
+LIMIT 10"
+
+# Query OHLC
+clickhouse client --query "
+SELECT timestamp, open, high, low, close
+FROM exness.ohlc_1m
+WHERE instrument='EURUSD'
+  AND timestamp >= '2024-01-01'
+LIMIT 10"
+
+# Resample to 1h
+clickhouse client --query "
+SELECT
+    toStartOfHour(timestamp) AS ts,
+    argMin(open, timestamp) AS open,
+    max(high) AS high,
+    min(low) AS low,
+    argMax(close, timestamp) AS close
+FROM exness.ohlc_1m
+WHERE instrument='EURUSD' AND timestamp >= '2024-01-01'
+GROUP BY ts
+ORDER BY ts"
+```
+
+## Schema
+
+### Tick Tables
+
+`exness.raw_spread_ticks`, `exness.standard_ticks`:
+
+| Column     | Type                   | Codec               |
+| ---------- | ---------------------- | ------------------- |
+| instrument | LowCardinality(String) | -                   |
+| timestamp  | DateTime64(6, 'UTC')   | DoubleDelta, LZ4    |
+| bid        | Float64                | Gorilla(8), ZSTD(1) |
+| ask        | Float64                | Gorilla(8), ZSTD(1) |
+
+### OHLC Table
+
+`exness.ohlc_1m` (27 columns):
+
+| Category    | Columns                                                                                                                                                                 |
+| ----------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Core        | instrument, timestamp, open, high, low, close                                                                                                                           |
+| Spreads     | raw_spread_avg, standard_spread_avg                                                                                                                                     |
+| Tick counts | tick_count_raw_spread, tick_count_standard                                                                                                                              |
+| Timezone    | ny_hour, london_hour, ny_session, london_session                                                                                                                        |
+| Holidays    | is_us_holiday, is_uk_holiday, is_major_holiday                                                                                                                          |
+| Sessions    | is_nyse_session, is_lse_session, is_xswx_session, is_xfra_session, is_xtse_session, is_xnze_session, is_xtks_session, is_xasx_session, is_xhkg_session, is_xses_session |
+
+Engine: `ReplacingMergeTree` (automatic deduplication)
+
+Partition: `toYYYYMM(timestamp)`
+
+Order: `(instrument, timestamp)`
+
+## Data Source
+
+- URL: <https://ticks.ex2archive.com/>
+- Format: Monthly ZIP files with CSV tick data
+- Variants: Raw_Spread (zero spreads) + Standard (market spreads)
+- Precision: Microsecond timestamps
+
+## Development
+
+```bash
+# Setup
+uv sync --dev
+
+# Test
+uv run pytest
+
+# Lint
+uv run ruff check --fix .
+
+# Type check
+uv run mypy src/
+```
+
+## Documentation
+
+- [ClickHouse User Guide](/docs/CLICKHOUSE_USER_GUIDE.md) - Detailed ClickHouse usage
+- [Database Schema](/docs/DATABASE_SCHEMA.md) - 27-column OHLC specification
+- [Module Architecture](/docs/MODULE_ARCHITECTURE.md) - 13 modules with SLOs
+- [ADR: ClickHouse Migration](/docs/adr/2025-12-11-duckdb-removal-clickhouse.md) - Architecture decision
+
+## License
+
+MIT