exness-data-preprocess 1.2.1__tar.gz → 2.0.0__tar.gz

exness_data_preprocess-2.0.0/.github/workflows/ci.yml

@@ -0,0 +1,25 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  pre-commit:
+    name: Pre-commit hooks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Run pre-commit hooks
+        uses: pre-commit/action@v3.0.1
+
+  # NOTE: Tests removed per local-first testing standard
+  # See: ~/.claude/CLAUDE.md - "NO unit testing in GitHub Actions"
+  # Run tests locally with: mise run validate

{exness_data_preprocess-1.2.1 → exness_data_preprocess-2.0.0}/.mise.toml

@@ -35,8 +35,9 @@ python = "3.11"
 
 [tasks.validate]
 description = "Run full E2E validation pipeline"
-# ADR: /docs/adr/2025-12-10-clickhouse-e2e-validation-pipeline.md
+# ADR: /docs/adr/2025-12-11-duckdb-removal-clickhouse.md
 depends = [
+    "clickhouse:ensure",        # Fail fast if ClickHouse not running
     "validate:imports",
     "validate:lint",
     "validate:typecheck",
@@ -44,24 +45,36 @@ depends = [
     "validate:build",
     "validate:install",
     "validate:mixin",
+    "validate:clickhouse",      # ClickHouse E2E validation
 ]
 run = '''
 echo ""
 echo "=============================================="
 echo "✅ ALL VALIDATIONS PASSED"
 echo "=============================================="
-echo ""
-echo "💡 To run ClickHouse validation (requires ClickHouse running):"
-echo "   mise run clickhouse:start"
-echo "   mise run validate:clickhouse"
 '''
 
 # -----------------------------------------------------------------------------
 # Individual Validation Tasks
 # -----------------------------------------------------------------------------
 
+[tasks."clickhouse:ensure"]
+description = "Ensure ClickHouse is running (fail fast)"
+# ADR: /docs/adr/2025-12-11-duckdb-removal-clickhouse.md
+run = '''
+#!/usr/bin/env bash
+set -e
+if ! nc -z localhost 8123 2>/dev/null; then
+    echo "ERROR: ClickHouse not running on port 8123" >&2
+    echo "Start with: mise run clickhouse:start" >&2
+    exit 1
+fi
+echo "✅ ClickHouse connection verified"
+'''
+
 [tasks."validate:imports"]
 description = "Validate all module imports (catch circular imports, missing deps)"
+# ADR: /docs/adr/2025-12-11-duckdb-removal-clickhouse.md - DuckDB modules removed
 run = '''
 #!/usr/bin/env bash
 set -e
@@ -95,7 +108,7 @@ from exness_data_preprocess import (
 print('✅ All public exports importable')
 " || { echo "❌ Public exports import failed"; exit 1; }
 
-# Test all internal modules
+# Test all internal modules (ClickHouse-only, DuckDB modules removed)
 uv run python -c "
 modules = [
     'exness_data_preprocess.clickhouse_base',
@@ -105,15 +118,10 @@ modules = [
     'exness_data_preprocess.clickhouse_ohlc_generator',
     'exness_data_preprocess.clickhouse_query_engine',
     'exness_data_preprocess.config',
-    'exness_data_preprocess.database_manager',
     'exness_data_preprocess.downloader',
     'exness_data_preprocess.exchanges',
-    'exness_data_preprocess.gap_detector',
     'exness_data_preprocess.models',
-    'exness_data_preprocess.ohlc_generator',
     'exness_data_preprocess.processor',
-    'exness_data_preprocess.query_engine',
-    'exness_data_preprocess.schema',
     'exness_data_preprocess.session_detector',
     'exness_data_preprocess.tick_loader',
 ]
@@ -121,7 +129,7 @@ import importlib
 for mod in modules:
     importlib.import_module(mod)
     print(f'  ✓ {mod}')
-print('✅ All 18 modules importable')
+print('✅ All 13 modules importable')
 "
 echo "✅ Import validation passed"
 '''

{exness_data_preprocess-1.2.1 → exness_data_preprocess-2.0.0}/CHANGELOG.md

@@ -1,3 +1,33 @@
+# [2.0.0](https://github.com/terrylica/exness-data-preprocess/compare/v1.2.1...v2.0.0) (2025-12-27)
+
+
+* feat!: migrate to ClickHouse-only backend, remove DuckDB ([7fd83bc](https://github.com/terrylica/exness-data-preprocess/commit/7fd83bc5720b2f5997a661ef8060757abd602a81))
+
+
+### BREAKING CHANGES
+
+* This release removes DuckDB and makes ClickHouse the sole storage backend.
+
+API Changes:
+- `duckdb_path` → `database` (str, ClickHouse database name)
+- `duckdb_size_mb` → `storage_bytes` (int, bytes not MB)
+- `database_exists` → removed
+- `base_dir` constructor parameter → removed
+
+Schema Changes:
+- OHLC schema reduced from 30 to 26 columns (removed 4 normalized metrics)
+
+Deleted Files (~1,219 lines):
+- database_manager.py (209 lines)
+- gap_detector.py (134 lines)
+- ohlc_generator.py (266 lines)
+- query_engine.py (291 lines)
+- schema.py (324 lines)
+
+Requires: ClickHouse running on localhost:8123
+
+ADR: docs/adr/2025-12-11-duckdb-removal-clickhouse.md
+
 ## [1.2.1](https://github.com/terrylica/exness-data-preprocess/compare/v1.2.0...v1.2.1) (2025-12-11)
 
 

exness_data_preprocess-2.0.0/CLAUDE.md

@@ -0,0 +1,152 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+**Package**: `exness-data-preprocess` — Forex tick data preprocessing with ClickHouse backend
+
+**Version**: See [`pyproject.toml`](pyproject.toml) line 3 (SSoT via semantic-release)
+
+**Quick Start**: [`README.md`](README.md) — Installation, API reference, usage examples
+
+---
+
+## Documentation Hub (Progressive Disclosure)
+
+### Essential Links
+
+| Topic            | SSoT Document                                                                                          | Purpose                      |
+| ---------------- | ------------------------------------------------------------------------------------------------------ | ---------------------------- |
+| **User Guide**   | [`README.md`](README.md)                                                                               | Installation, API, examples  |
+| **Architecture** | [`docs/adr/2025-12-11-duckdb-removal-clickhouse.md`](docs/adr/2025-12-11-duckdb-removal-clickhouse.md) | ClickHouse-only decision     |
+| **Schema**       | [`docs/DATABASE_SCHEMA.md`](docs/DATABASE_SCHEMA.md)                                                   | 26-column OHLC specification |
+| **Modules**      | [`docs/MODULE_ARCHITECTURE.md`](docs/MODULE_ARCHITECTURE.md)                                           | 13 modules with SLOs         |
+| **Data Sources** | [`docs/EXNESS_DATA_SOURCES.md`](docs/EXNESS_DATA_SOURCES.md)                                           | Exness tick data variants    |
+| **Tasks**        | [`.mise.toml`](.mise.toml)                                                                             | Validation, ClickHouse tasks |
+| **Contributing** | [`CONTRIBUTING.md`](CONTRIBUTING.md)                                                                   | Development guidelines       |
+
+### Deep Dives (On-Demand)
+
+| Topic              | Location                               | When to Read              |
+| ------------------ | -------------------------------------- | ------------------------- |
+| ADRs               | [`docs/adr/`](docs/adr/)               | Architecture decisions    |
+| Design Specs       | [`docs/design/`](docs/design/)         | Implementation blueprints |
+| Validation Results | [`docs/validation/`](docs/validation/) | Spike test proofs         |
+
+---
+
+## Architecture Overview (v2.0.0)
+
+### ClickHouse-Only Backend
+
+**ADR**: [`docs/adr/2025-12-11-duckdb-removal-clickhouse.md`](docs/adr/2025-12-11-duckdb-removal-clickhouse.md)
+
+| Feature         | Description                                      |
+| --------------- | ------------------------------------------------ |
+| **Database**    | `exness` (single database for all instruments)   |
+| **Tables**      | `raw_spread_ticks`, `standard_ticks`, `ohlc_1m`  |
+| **Engine**      | ReplacingMergeTree (deduplication at merge time) |
+| **Connection**  | localhost:8123 (local) or cloud via env vars     |
+| **Entry Point** | `ExnessDataProcessor`                            |
+
+### Module Structure (13 Modules)
+
+**ClickHouse Backend (7 modules)**:
+
+- `processor.py` — Main orchestrator with ClickHouse backend
+- `clickhouse_manager.py` — Schema management, data loading
+- `clickhouse_gap_detector.py` — Gap detection for incremental updates
+- `clickhouse_query_engine.py` — Query with pagination
+- `clickhouse_ohlc_generator.py` — OHLC generation
+- `clickhouse_client.py` — Connection factory with error handling
+- `clickhouse_base.py` — Shared mixin for client lifecycle
+
+**Shared (6 modules)**:
+
+- `downloader.py` — HTTP download operations
+- `tick_loader.py` — CSV parsing
+- `session_detector.py` — Holiday/session detection (vectorized)
+- `config.py` — Configuration management
+- `models.py` — Pydantic models (UpdateResult, CoverageInfo, etc.)
+- `exchanges.py` — Exchange calendar integration
+
+**Deleted in v2.0.0**:
+
+- ~~`database_manager.py`~~ — DuckDB database operations
+- ~~`gap_detector.py`~~ — DuckDB gap detection
+- ~~`ohlc_generator.py`~~ — DuckDB OHLC generation
+- ~~`query_engine.py`~~ — DuckDB query operations
+- ~~`schema.py`~~ — DuckDB SQL schema definitions
+
+---
+
+## Development Commands
+
+### Quick Reference
+
+```bash
+# Setup
+uv sync --dev
+
+# Testing (45 tests)
+uv run pytest                          # All tests
+mise run validate                      # Full E2E validation (requires ClickHouse)
+
+# Linting
+uv run ruff check --fix src/ tests/
+
+# ClickHouse (required for full validation)
+mise run clickhouse:start              # Start local server
+mise run clickhouse:status             # Check connection
+mise run clickhouse:ensure             # Fail fast if not running
+
+# Build & Publish
+uv build
+```
+
+### mise Tasks
+
+All validation and tooling tasks are defined in [`.mise.toml`](.mise.toml):
+
+| Task                           | Description                             |
+| ------------------------------ | --------------------------------------- |
+| `mise run validate`            | Full E2E pipeline (requires ClickHouse) |
+| `mise run clickhouse:ensure`   | Fail fast if ClickHouse not running     |
+| `mise run clickhouse:start`    | Start mise-installed ClickHouse         |
+| `mise run validate:clickhouse` | ClickHouse E2E tests                    |
+
+---
+
+## Key Changes in v2.0.0
+
+### Breaking Changes
+
+| v1.x (DuckDB)     | v2.0.0 (ClickHouse)             |
+| ----------------- | ------------------------------- |
+| `duckdb_path`     | `database` (str)                |
+| `duckdb_size_mb`  | `storage_bytes` (int)           |
+| `database_exists` | removed                         |
+| `base_dir` param  | removed (ClickHouse manages)    |
+| 30-column OHLC    | 26-column OHLC (4 cols removed) |
+
+### Removed Files
+
+```
+src/exness_data_preprocess/database_manager.py   (209 lines)
+src/exness_data_preprocess/gap_detector.py       (134 lines)
+src/exness_data_preprocess/ohlc_generator.py     (266 lines)
+src/exness_data_preprocess/query_engine.py       (291 lines)
+src/exness_data_preprocess/schema.py             (324 lines)
+```
+
+**Total: ~1,224 lines deleted**
+
+---
+
+## References
+
+- **Exness Data**: <https://ticks.ex2archive.com/>
+- **ClickHouse**: <https://clickhouse.com/>
+
+---
+
+**Last Updated**: 2025-12-11