exness-data-preprocess 1.0.0__tar.gz → 1.2.0__tar.gz

{exness_data_preprocess-1.0.0 → exness_data_preprocess-1.2.0}/.gitignore

@@ -107,3 +107,8 @@ node_modules/
 # Link checking (lychee)
 .lychee*
 .lycheecache
+
+# DBeaver (credentials in generated JSON)
+# ADR: /docs/adr/2025-12-10-clickhouse-pydantic-config.md
+.dbeaver/
+.dbeaver-workspace/

{exness_data_preprocess-1.0.0 → exness_data_preprocess-1.2.0}/.lint-relative-paths-results.txt

@@ -1,7 +1,7 @@
 🔍 Scanning for non-repo-relative paths in: /Users/terryli/eon/exness-data-preprocess
    Violations: paths not starting with / or http(s)
 
-❌ Found 141 violation(s):
+❌ Found 140 violation(s):
 
 /Users/terryli/eon/exness-data-preprocess/README.md:112:- **Column Definitions**: See [`schema.py`](src/exness_data_preprocess/schema.py) - Single source of truth
 /Users/terryli/eon/exness-data-preprocess/README.md:113:- **Comprehensive Reference**: See [`DATABASE_SCHEMA.md`](docs/DATABASE_SCHEMA.md) - Query examples and usage patterns
@@ -54,7 +54,7 @@
 /Users/terryli/eon/exness-data-preprocess/docs/DATABASE_SCHEMA.md:244:**Column Definitions**: See [`../src/exness_data_preprocess/schema.py`](../src/exness_data_preprocess/schema.py) for complete column definitions with types and descriptions.
 /Users/terryli/eon/exness-data-preprocess/docs/DATABASE_SCHEMA.md:258:**Architecture**: Exchange Registry Pattern (v1.6.0) - Session columns dynamically generated from centralized EXCHANGES dict in [`exchanges.py`](../src/exness_data_preprocess/exchanges.py)
 
-... and 91 more (showing first 50)
+... and 90 more (showing first 50)
 
 📝 Fix: Convert relative paths to repo-relative paths starting with /
    WRONG: [Link](../docs/foo.md)

exness_data_preprocess-1.2.0/.mise.toml

@@ -0,0 +1,509 @@
+# mise configuration for exness-data-preprocess
+# E2E validation workflow for codebase pruning verification
+
+[env]
+# Python version for validation
+PYTHON_VERSION = "3.14"
+# Package name
+PACKAGE_NAME = "exness_data_preprocess"
+# Validation temp directory
+VALIDATION_TMP = "./tmp/validation"
+
+# ADR: /docs/adr/2025-12-10-clickhouse-pydantic-config.md
+# ClickHouse SSoT Configuration
+# Connection identity
+CLICKHOUSE_NAME = "exness-clickhouse-local"
+CLICKHOUSE_MODE = "local"  # "local" or "cloud"
+CLICKHOUSE_TYPE = "dev"    # "dev", "test", or "prod"
+# Connection parameters (match existing clickhouse_client.py defaults)
+CLICKHOUSE_HOST = "localhost"
+CLICKHOUSE_PORT = "8123"
+CLICKHOUSE_DATABASE = "exness"
+# Output paths
+DBEAVER_CONFIG_PATH = ".dbeaver/data-sources.json"
+# Cloud credentials (read from .env for security)
+# CLICKHOUSE_USER_READONLY - set in .env
+# CLICKHOUSE_PASSWORD_READONLY - set in .env
+
+[tools]
+python = "3.14"
+
+# =============================================================================
+# Validation Tasks - E2E Testing Pipeline
+# =============================================================================
+
+[tasks.validate]
+description = "Run full E2E validation pipeline"
+# ADR: /docs/adr/2025-12-10-clickhouse-e2e-validation-pipeline.md
+depends = [
+    "validate:imports",
+    "validate:lint",
+    "validate:typecheck",
+    "validate:test",
+    "validate:build",
+    "validate:install",
+    "validate:mixin",
+]
+run = '''
+echo ""
+echo "=============================================="
+echo "✅ ALL VALIDATIONS PASSED"
+echo "=============================================="
+echo ""
+echo "💡 To run ClickHouse validation (requires ClickHouse running):"
+echo "   mise run clickhouse:start"
+echo "   mise run validate:clickhouse"
+'''
+
+# -----------------------------------------------------------------------------
+# Individual Validation Tasks
+# -----------------------------------------------------------------------------
+
+[tasks."validate:imports"]
+description = "Validate all module imports (catch circular imports, missing deps)"
+run = '''
+#!/usr/bin/env bash
+set -e
+echo "🔍 Validating module imports..."
+
+# Test package import
+uv run python -c "import exness_data_preprocess" || { echo "❌ Package import failed"; exit 1; }
+
+# Test all public exports
+uv run python -c "
+from exness_data_preprocess import (
+    ExnessDataProcessor,
+    UpdateResult,
+    CoverageInfo,
+    DryRunResult,
+    PairType,
+    TimeframeType,
+    VariantType,
+    supported_pairs,
+    supported_timeframes,
+    supported_variants,
+    # ClickHouse exports
+    ClickHouseManager,
+    ClickHouseGapDetector,
+    ClickHouseQueryEngine,
+    ClickHouseOHLCGenerator,
+    ClickHouseConnectionError,
+    ClickHouseQueryError,
+    get_clickhouse_client,
+)
+print('✅ All public exports importable')
+" || { echo "❌ Public exports import failed"; exit 1; }
+
+# Test all internal modules
+uv run python -c "
+modules = [
+    'exness_data_preprocess.clickhouse_base',
+    'exness_data_preprocess.clickhouse_client',
+    'exness_data_preprocess.clickhouse_gap_detector',
+    'exness_data_preprocess.clickhouse_manager',
+    'exness_data_preprocess.clickhouse_ohlc_generator',
+    'exness_data_preprocess.clickhouse_query_engine',
+    'exness_data_preprocess.config',
+    'exness_data_preprocess.database_manager',
+    'exness_data_preprocess.downloader',
+    'exness_data_preprocess.exchanges',
+    'exness_data_preprocess.gap_detector',
+    'exness_data_preprocess.models',
+    'exness_data_preprocess.ohlc_generator',
+    'exness_data_preprocess.processor',
+    'exness_data_preprocess.query_engine',
+    'exness_data_preprocess.schema',
+    'exness_data_preprocess.session_detector',
+    'exness_data_preprocess.tick_loader',
+]
+import importlib
+for mod in modules:
+    importlib.import_module(mod)
+    print(f'  ✓ {mod}')
+print('✅ All 18 modules importable')
+"
+echo "✅ Import validation passed"
+'''
+
+[tasks."validate:lint"]
+description = "Run ruff linting"
+run = '''
+#!/usr/bin/env bash
+set -e
+echo "🔍 Running ruff check..."
+uv run ruff check src/ tests/ --output-format=concise
+echo "✅ Lint validation passed"
+'''
+
+[tasks."validate:typecheck"]
+description = "Run mypy type checking"
+run = '''
+#!/usr/bin/env bash
+set -e
+echo "🔍 Running mypy..."
+uv run mypy src/exness_data_preprocess/ --ignore-missing-imports --no-error-summary 2>&1 | head -50
+echo "✅ Type check validation passed (warnings may exist)"
+'''
+
+[tasks."validate:test"]
+description = "Run pytest test suite"
+run = '''
+#!/usr/bin/env bash
+set -e
+echo "🔍 Running pytest..."
+uv run pytest tests/ -v --tb=short -q
+echo "✅ Test validation passed"
+'''
+
+[tasks."validate:test-coverage"]
+description = "Run pytest with coverage report"
+run = '''
+#!/usr/bin/env bash
+set -e
+echo "🔍 Running pytest with coverage..."
+uv run pytest tests/ --cov=exness_data_preprocess --cov-report=term-missing --cov-report=html:tmp/coverage
+echo ""
+echo "📊 Coverage report: tmp/coverage/index.html"
+'''
+
+[tasks."validate:build"]
+description = "Validate package builds correctly"
+run = '''
+#!/usr/bin/env bash
+set -e
+echo "🔍 Building package..."
+rm -rf dist/
+uv build 2>&1 | tail -5
+echo ""
+ls -la dist/
+echo "✅ Build validation passed"
+'''
+
+[tasks."validate:install"]
+description = "Test fresh install in isolated environment"
+depends = ["validate:build"]
+run = '''
+#!/usr/bin/env bash
+set -e
+echo "🔍 Testing fresh install..."
+
+# Create isolated temp venv
+TMPDIR=$(mktemp -d)
+trap "rm -rf $TMPDIR" EXIT
+
+# Create fresh venv and install wheel using uv
+uv venv "$TMPDIR/venv" --python 3.14 -q
+WHEEL=$(ls dist/*.whl | head -1)
+uv pip install "$WHEEL" --python "$TMPDIR/venv/bin/python" -q
+
+# Test import in fresh environment
+"$TMPDIR/venv/bin/python" -c "
+import exness_data_preprocess as edp
+print(f'Version: {edp.__version__}')
+print(f'Processor: {edp.ExnessDataProcessor}')
+print(f'ClickHouseManager: {edp.ClickHouseManager}')
+print('✅ Fresh install works')
+"
+echo "✅ Install validation passed"
+'''
+
+# -----------------------------------------------------------------------------
+# Mixin Validation
+# -----------------------------------------------------------------------------
+
+[tasks."validate:mixin"]
+description = "Validate ClickHouseClientMixin inheritance"
+run = '''
+#!/usr/bin/env bash
+set -e
+echo "🔍 Validating ClickHouseClientMixin..."
+
+uv run python -c "
+from exness_data_preprocess.clickhouse_base import ClickHouseClientMixin
+from exness_data_preprocess.clickhouse_manager import ClickHouseManager
+from exness_data_preprocess.clickhouse_gap_detector import ClickHouseGapDetector
+from exness_data_preprocess.clickhouse_query_engine import ClickHouseQueryEngine
+from exness_data_preprocess.clickhouse_ohlc_generator import ClickHouseOHLCGenerator
+
+# Verify inheritance
+classes = [
+    ('ClickHouseManager', ClickHouseManager),
+    ('ClickHouseGapDetector', ClickHouseGapDetector),
+    ('ClickHouseQueryEngine', ClickHouseQueryEngine),
+    ('ClickHouseOHLCGenerator', ClickHouseOHLCGenerator),
+]
+
+for name, cls in classes:
+    assert issubclass(cls, ClickHouseClientMixin), f'{name} should inherit from ClickHouseClientMixin'
+    print(f'  ✓ {name} inherits ClickHouseClientMixin')
+
+# Verify mixin methods exist at class level (properties/methods are on class, not instance)
+for name, cls in classes:
+    assert hasattr(cls, 'client'), f'{name} missing client property'
+    assert hasattr(cls, 'close'), f'{name} missing close method'
+    assert hasattr(cls, '_init_client'), f'{name} missing _init_client method'
+    print(f'  ✓ {name} has mixin methods')
+
+print('✅ Mixin validation passed')
+"
+'''
+
+# -----------------------------------------------------------------------------
+# Quick Tasks
+# -----------------------------------------------------------------------------
+
+[tasks.test]
+description = "Run tests (alias for validate:test)"
+alias = "t"
+run = "uv run pytest tests/ -v --tb=short"
+
+[tasks.lint]
+description = "Run linting (alias for validate:lint)"
+alias = "l"
+run = "uv run ruff check src/ tests/"
+
+[tasks.fix]
+description = "Auto-fix linting issues"
+run = "uv run ruff check --fix src/ tests/ && uv run ruff format src/ tests/"
+
+# -----------------------------------------------------------------------------
+# CI Simulation
+# -----------------------------------------------------------------------------
+
+[tasks.ci]
+description = "Simulate CI pipeline locally"
+depends = ["validate"]
+depends_post = ["validate:test-coverage"]
+run = '''
+echo ""
+echo "=============================================="
+echo "✅ CI SIMULATION COMPLETE"
+echo "=============================================="
+'''
+
+# =============================================================================
+# ClickHouse Server Management
+# ADR: /docs/adr/2025-12-10-clickhouse-e2e-validation-pipeline.md
+# =============================================================================
+
+[tasks."clickhouse:start"]
+description = "Start mise-installed ClickHouse server"
+run = '''
+#!/usr/bin/env bash
+set -euo pipefail
+
+SHIM="${HOME}/.local/share/mise/shims/clickhouse"
+
+# Check mise ClickHouse installation
+if [[ ! -x "${SHIM}" ]]; then
+    echo "ERROR: mise ClickHouse not found. Install: mise install clickhouse" >&2
+    exit 1
+fi
+
+# Check if already running
+if nc -z localhost 8123 2>/dev/null; then
+    echo "✅ ClickHouse already running on port 8123"
+    "${SHIM}" client --query "SELECT version()"
+    exit 0
+fi
+
+# Start server in daemon mode
+echo "Starting ClickHouse server..."
+"${SHIM}" server --daemon
+
+# Wait for server to be ready
+for i in $(seq 1 10); do
+    if nc -z localhost 8123 2>/dev/null; then
+        echo "✅ ClickHouse server started"
+        "${SHIM}" client --query "SELECT version()"
+        exit 0
+    fi
+    sleep 1
+done
+
+echo "ERROR: ClickHouse failed to start" >&2
+exit 1
+'''
+
+[tasks."clickhouse:stop"]
+description = "Stop mise-installed ClickHouse server"
+run = '''
+#!/usr/bin/env bash
+if pkill -f "clickhouse server" 2>/dev/null; then
+    echo "✅ ClickHouse server stopped"
+else
+    echo "ClickHouse was not running"
+fi
+'''
+
+[tasks."clickhouse:status"]
+description = "Check ClickHouse connection via Python client"
+run = '''
+#!/usr/bin/env bash
+set -e
+
+if ! nc -z localhost 8123 2>/dev/null; then
+    echo "❌ ClickHouse not running on port 8123"
+    exit 1
+fi
+
+uv run python -c "
+from exness_data_preprocess import get_clickhouse_client
+from exness_data_preprocess.clickhouse_client import check_connection
+client = get_clickhouse_client()
+check_connection(client)
+print('✅ ClickHouse connection OK')
+client.close()
+"
+'''
+
+# =============================================================================
+# ClickHouse E2E Validation Tasks
+# ADR: /docs/adr/2025-12-10-clickhouse-e2e-validation-pipeline.md
+# =============================================================================
+
+[tasks."validate:clickhouse"]
+description = "Run all ClickHouse E2E validations with real data"
+depends = ["clickhouse:status"]
+run = '''
+#!/usr/bin/env bash
+set -e
+echo "🔍 Running ClickHouse E2E validation..."
+mise run validate:clickhouse-schema
+mise run validate:clickhouse-data
+mise run validate:clickhouse-pagination
+mise run validate:clickhouse-ohlc
+echo "✅ All ClickHouse validations passed"
+'''
+
+[tasks."validate:clickhouse-schema"]
+description = "Validate ClickHouse schema creation"
+depends = ["clickhouse:status"]
+run = '''
+#!/usr/bin/env bash
+set -e
+echo "🔍 Validating ClickHouse schema..."
+uv run python -c "
+from exness_data_preprocess import ClickHouseManager
+
+manager = ClickHouseManager()
+manager.ensure_schema()
+print('  ✓ ticks table created')
+print('  ✓ ohlc table created')
+manager.close()
+print('✅ Schema validation passed')
+"
+'''
+
+[tasks."validate:clickhouse-data"]
+description = "Validate ClickHouse data queries with real Exness data"
+depends = ["validate:clickhouse-schema"]
+run = '''
+#!/usr/bin/env bash
+set -e
+echo "🔍 Validating ClickHouse data operations..."
+uv run python -c "
+from exness_data_preprocess import ClickHouseQueryEngine
+
+engine = ClickHouseQueryEngine()
+
+# Query with limit to verify functionality
+df = engine.query_ticks('EURUSD', limit=1000)
+print(f'  ✓ query_ticks returned {len(df)} rows')
+
+# Verify columns
+expected_cols = ['timestamp', 'bid', 'ask']
+for col in expected_cols:
+    assert col in df.columns, f'Missing column: {col}'
+print('  ✓ All expected columns present')
+
+engine.close()
+print('✅ Data validation passed')
+"
+'''
+
+[tasks."validate:clickhouse-pagination"]
+description = "Validate ClickHouse pagination (LIMIT/OFFSET/cursor/batch)"
+depends = ["validate:clickhouse-schema"]
+run = '''
+#!/usr/bin/env bash
+set -e
+echo "🔍 Validating ClickHouse pagination..."
+uv run python -c "
+from exness_data_preprocess import ClickHouseQueryEngine
+
+engine = ClickHouseQueryEngine()
+
+# Test LIMIT
+df1 = engine.query_ticks('EURUSD', limit=100)
+assert len(df1) <= 100, f'LIMIT failed: got {len(df1)} rows'
+print(f'  ✓ LIMIT=100 returned {len(df1)} rows')
+
+# Test OFFSET
+df2 = engine.query_ticks('EURUSD', limit=100, offset=100)
+assert len(df2) <= 100, f'OFFSET failed: got {len(df2)} rows'
+print(f'  ✓ LIMIT=100 OFFSET=100 returned {len(df2)} rows')
+
+# Test cursor-based pagination
+result = engine.query_ticks_paginated('EURUSD', page_size=1000)
+print(f'  ✓ Cursor pagination: {len(result.data)} rows, has_more={result.has_more}')
+
+# Test batch iterator
+batches = list(engine.query_ticks_batches('EURUSD', batch_size=1000, max_batches=3))
+print(f'  ✓ Batch iterator: {len(batches)} batches')
+
+engine.close()
+print('✅ Pagination validation passed')
+"
+'''
+
+[tasks."validate:clickhouse-ohlc"]
+description = "Validate ClickHouse OHLC queries"
+depends = ["validate:clickhouse-schema"]
+run = '''
+#!/usr/bin/env bash
+set -e
+echo "🔍 Validating ClickHouse OHLC queries..."
+uv run python -c "
+from exness_data_preprocess import ClickHouseQueryEngine
+
+engine = ClickHouseQueryEngine()
+
+# Query OHLC with limit
+df = engine.query_ohlc('EURUSD', timeframe='1h', limit=100)
+print(f'  ✓ query_ohlc(1h) returned {len(df)} bars')
+
+# Verify OHLC columns exist (if data present)
+if len(df) > 0:
+    ohlc_cols = ['open', 'high', 'low', 'close']
+    for col in ohlc_cols:
+        assert col in df.columns, f'Missing OHLC column: {col}'
+    print('  ✓ All OHLC columns present')
+else:
+    print('  ⚠ No OHLC data available (expected if no data loaded)')
+
+engine.close()
+print('✅ OHLC validation passed')
+"
+'''
+
+# =============================================================================
+# DBeaver Config Generation
+# ADR: /docs/adr/2025-12-10-clickhouse-pydantic-config.md
+# =============================================================================
+
+[tasks.db-client-generate]
+description = "Generate DBeaver config from Pydantic model"
+run = "uv run scripts/generate_dbeaver_config.py --output ${DBEAVER_CONFIG_PATH}"
+
+[tasks."db-client:cloud"]
+description = "Generate config for ClickHouse Cloud"
+run = "uv run scripts/generate_dbeaver_config.py --mode cloud --output ${DBEAVER_CONFIG_PATH}"
+
+[tasks."db-client:dry-run"]
+description = "Preview generated config without writing"
+run = "uv run scripts/generate_dbeaver_config.py --dry-run"
+
+[tasks.dbeaver]
+description = "Launch DBeaver with project workspace"
+run = '"/Applications/DBeaver.app/Contents/MacOS/dbeaver" -data .dbeaver-workspace &'

{exness_data_preprocess-1.0.0 → exness_data_preprocess-1.2.0}/CHANGELOG.md

@@ -1,3 +1,22 @@
+# [1.2.0](https://github.com/terrylica/exness-data-preprocess/compare/v1.1.0...v1.2.0) (2025-12-10)
+
+
+### Bug Fixes
+
+* **clickhouse:** resolve E2E validation bugs in schema and query engine ([5c618f7](https://github.com/terrylica/exness-data-preprocess/commit/5c618f717cadc23097e9fd11d2fed87947ecf25f))
+
+
+### Features
+
+* add DBeaver config generation with CLICKHOUSE_MODE support ([0e97618](https://github.com/terrylica/exness-data-preprocess/commit/0e9761818f137461623f744af92f0b1e98b3a797))
+
+# [1.1.0](https://github.com/terrylica/exness-data-preprocess/compare/v1.0.0...v1.1.0) (2025-12-10)
+
+
+### Features
+
+* add ClickHouse E2E validation pipeline with pagination API ([2ec7325](https://github.com/terrylica/exness-data-preprocess/commit/2ec7325905459f3073e591af4f800b942de9a099))
+
 # [1.0.0](https://github.com/terrylica/exness-data-preprocess/compare/v0.8.0...v1.0.0) (2025-12-10)
 
 

{exness_data_preprocess-1.0.0 → exness_data_preprocess-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: exness-data-preprocess
-Version: 1.0.0
+Version: 1.2.0
 Summary: Professional Exness forex tick data preprocessing with optimal compression (Parquet Zstd-22) and DuckDB OHLC generation. Provides efficient storage (9% smaller than ZIP) with lossless precision and direct queryability.
 Project-URL: Homepage, https://github.com/terrylica/exness-data-preprocess
 Project-URL: Documentation, https://github.com/terrylica/exness-data-preprocess#readme