exness-data-preprocess 1.1.0__tar.gz → 1.2.0__tar.gz

{exness_data_preprocess-1.1.0 → exness_data_preprocess-1.2.0}/.gitignore

@@ -107,3 +107,8 @@ node_modules/
 # Link checking (lychee)
 .lychee*
 .lycheecache
+
+# DBeaver (credentials in generated JSON)
+# ADR: /docs/adr/2025-12-10-clickhouse-pydantic-config.md
+.dbeaver/
+.dbeaver-workspace/

{exness_data_preprocess-1.1.0 → exness_data_preprocess-1.2.0}/.mise.toml

@@ -9,6 +9,22 @@ PACKAGE_NAME = "exness_data_preprocess"
 # Validation temp directory
 VALIDATION_TMP = "./tmp/validation"
 
+# ADR: /docs/adr/2025-12-10-clickhouse-pydantic-config.md
+# ClickHouse SSoT Configuration
+# Connection identity
+CLICKHOUSE_NAME = "exness-clickhouse-local"
+CLICKHOUSE_MODE = "local"  # "local" or "cloud"
+CLICKHOUSE_TYPE = "dev"    # "dev", "test", or "prod"
+# Connection parameters (match existing clickhouse_client.py defaults)
+CLICKHOUSE_HOST = "localhost"
+CLICKHOUSE_PORT = "8123"
+CLICKHOUSE_DATABASE = "exness"
+# Output paths
+DBEAVER_CONFIG_PATH = ".dbeaver/data-sources.json"
+# Cloud credentials (read from .env for security)
+# CLICKHOUSE_USER_READONLY - set in .env
+# CLICKHOUSE_PASSWORD_READONLY - set in .env
+
 [tools]
 python = "3.14"
 
@@ -470,3 +486,24 @@ engine.close()
 print('✅ OHLC validation passed')
 "
 '''
+
+# =============================================================================
+# DBeaver Config Generation
+# ADR: /docs/adr/2025-12-10-clickhouse-pydantic-config.md
+# =============================================================================
+
+[tasks.db-client-generate]
+description = "Generate DBeaver config from Pydantic model"
+run = "uv run scripts/generate_dbeaver_config.py --output ${DBEAVER_CONFIG_PATH}"
+
+[tasks."db-client:cloud"]
+description = "Generate config for ClickHouse Cloud"
+run = "uv run scripts/generate_dbeaver_config.py --mode cloud --output ${DBEAVER_CONFIG_PATH}"
+
+[tasks."db-client:dry-run"]
+description = "Preview generated config without writing"
+run = "uv run scripts/generate_dbeaver_config.py --dry-run"
+
+[tasks.dbeaver]
+description = "Launch DBeaver with project workspace"
+run = '"/Applications/DBeaver.app/Contents/MacOS/dbeaver" -data .dbeaver-workspace &'

{exness_data_preprocess-1.1.0 → exness_data_preprocess-1.2.0}/CHANGELOG.md

@@ -1,3 +1,15 @@
+# [1.2.0](https://github.com/terrylica/exness-data-preprocess/compare/v1.1.0...v1.2.0) (2025-12-10)
+
+
+### Bug Fixes
+
+* **clickhouse:** resolve E2E validation bugs in schema and query engine ([5c618f7](https://github.com/terrylica/exness-data-preprocess/commit/5c618f717cadc23097e9fd11d2fed87947ecf25f))
+
+
+### Features
+
+* add DBeaver config generation with CLICKHOUSE_MODE support ([0e97618](https://github.com/terrylica/exness-data-preprocess/commit/0e9761818f137461623f744af92f0b1e98b3a797))
+
 # [1.1.0](https://github.com/terrylica/exness-data-preprocess/compare/v1.0.0...v1.1.0) (2025-12-10)
 
 

{exness_data_preprocess-1.1.0 → exness_data_preprocess-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: exness-data-preprocess
-Version: 1.1.0
+Version: 1.2.0
 Summary: Professional Exness forex tick data preprocessing with optimal compression (Parquet Zstd-22) and DuckDB OHLC generation. Provides efficient storage (9% smaller than ZIP) with lossless precision and direct queryability.
 Project-URL: Homepage, https://github.com/terrylica/exness-data-preprocess
 Project-URL: Documentation, https://github.com/terrylica/exness-data-preprocess#readme

exness_data_preprocess-1.2.0/docs/adr/2025-12-10-clickhouse-pydantic-config.md

@@ -0,0 +1,237 @@
+---
+status: accepted
+date: 2025-12-10
+decision-maker: Terry Li
+consulted:
+  [
+    Config-Discovery-Agent,
+    ClickHouse-Connection-Agent,
+    Skill-Reference-Agent,
+    ClickHouse-Pattern-Agent,
+    Mise-Pattern-Agent,
+  ]
+research-method: 9-agent-parallel-dctl
+clarification-iterations: 2
+perspectives: [SSoT-Configuration, DeveloperExperience, BackwardCompatibility]
+---
+
+# ADR: ClickHouse Pydantic Config Integration
+
+**Design Spec**: [Implementation Spec](/docs/design/2025-12-10-clickhouse-pydantic-config/spec.md)
+
+## Context and Problem Statement
+
+The `exness-data-preprocess` package has ClickHouse integration for tick data storage, but lacks:
+
+1. **Database client configuration** - No DBeaver connection config generation
+2. **SSoT for connection parameters** - Environment variables are scattered, not centralized in mise `[env]`
+3. **Inconsistent SSL handling** - Uses `CLICKHOUSE_SECURE` (boolean) instead of canonical `CLICKHOUSE_MODE` (local/cloud) pattern
+
+The `clickhouse-pydantic-config` skill from cc-skills provides a canonical pattern for generating DBeaver configs from Pydantic v2 models using mise as SSoT.
+
+### Before/After
+
+**Before**: Scattered configuration, no DBeaver integration
+
+```
+┌────────────────────┐
+│   Code Defaults    │
+│    (hardcoded)     │ ─┐
+└────────────────────┘  │
+  │                     │
+  │ port=8123           │ db='exness'
+  ▼                     ▼
+┌────────────────────────────────────────┐
+│          clickhouse_client.py          │
+└────────────────────────────────────────┘
+  ▲                     ▲
+  │ CLICKHOUSE_SECURE   │ CLICKHOUSE_HOST
+  │                     │
+┌────────────────────┐  │
+│      Env Vars      │  │
+│    (scattered)     │ ─┘
+└────────────────────┘
+┌────────────────────┐
+│      DBeaver       │
+│   (manual setup)   │
+└────────────────────┘
+```
+
+<details>
+<summary>graph-easy source (before.dot)</summary>
+
+```dot
+digraph before {
+  rankdir=TB;
+  env_vars [label="Env Vars\n(scattered)"];
+  code_defaults [label="Code Defaults\n(hardcoded)"];
+  clickhouse_client [label="clickhouse_client.py"];
+  env_vars -> clickhouse_client [label="CLICKHOUSE_SECURE"];
+  env_vars -> clickhouse_client [label="CLICKHOUSE_HOST"];
+  code_defaults -> clickhouse_client [label="port=8123"];
+  code_defaults -> clickhouse_client [label="db='exness'"];
+  dbeaver [label="DBeaver\n(manual setup)"];
+}
+```
+
+</details>
+
+**After**: mise `[env]` as SSoT, DBeaver config generation
+
+```
+                                             ┌────────────────────────────┐
+                                             │      .mise.toml [env]      │
+                                             │           (SSoT)           │
+                                             └────────────────────────────┘
+                                               │
+                                               │ CLICKHOUSE_*
+                                               ▼
+┌──────────────────────┐                     ┌────────────────────────────┐
+│ clickhouse_client.py │  validated config   │    ClickHouseConnection    │
+│                      │ ◀────────────────── │       (Pydantic v2)        │
+└──────────────────────┘                     └────────────────────────────┘
+                                               │
+                                               │ model fields
+                                               ▼
+                                             ┌────────────────────────────┐
+                                             │ generate_dbeaver_config.py │
+                                             └────────────────────────────┘
+                                               │
+                                               │ generates
+                                               ▼
+                                             ┌────────────────────────────┐
+                                             │ .dbeaver/data-sources.json │
+                                             └────────────────────────────┘
+```
+
+<details>
+<summary>graph-easy source (after.dot)</summary>
+
+```dot
+digraph after {
+  rankdir=TB;
+  mise_env [label=".mise.toml [env]\n(SSoT)"];
+  pydantic [label="ClickHouseConnection\n(Pydantic v2)"];
+  clickhouse_client [label="clickhouse_client.py"];
+  dbeaver_script [label="generate_dbeaver_config.py"];
+  dbeaver_json [label=".dbeaver/data-sources.json"];
+  mise_env -> pydantic [label="CLICKHOUSE_*"];
+  pydantic -> clickhouse_client [label="validated config"];
+  pydantic -> dbeaver_script [label="model fields"];
+  dbeaver_script -> dbeaver_json [label="generates"];
+}
+```
+
+</details>
+
+## Research Summary
+
+| Agent Perspective     | Key Finding                                                                                            | Confidence |
+| --------------------- | ------------------------------------------------------------------------------------------------------ | ---------- |
+| Config-Discovery      | `.mise.toml` has `[env]` section but no CLICKHOUSE\_\* vars; existing `config.py` has Pydantic pattern | High       |
+| ClickHouse-Connection | Current code uses `CLICKHOUSE_SECURE` for SSL; port 8123 for HTTP; database default is `exness`        | High       |
+| Skill-Reference       | Canonical model uses `CLICKHOUSE_MODE` (local/cloud) with mode-aware validation; scripts at `scripts/` | High       |
+| ClickHouse-Pattern    | Three skills form a chain: cloud-management → pydantic-config → architect                              | High       |
+| Mise-Pattern          | SSoT pattern: `[env]` section → scripts read via `os.environ.get()` with defaults                      | High       |
+
+## Decision Log
+
+| Decision Area    | Options Evaluated                       | Chosen                               | Rationale                                     |
+| ---------------- | --------------------------------------- | ------------------------------------ | --------------------------------------------- |
+| SSL Config       | CLICKHOUSE_SECURE vs CLICKHOUSE_MODE    | Hybrid (MODE + SECURE fallback)      | Backward compatibility for existing scripts   |
+| Script Location  | inline mise task, src/ module, scripts/ | `scripts/generate_dbeaver_config.py` | Canonical pattern from cc-skills              |
+| Database Default | `default` vs `exness`                   | `exness`                             | Match existing codebase convention            |
+| Connection Scope | Single mode vs Both local+cloud         | Single per invocation                | Canonical pattern; `--mode` flag for override |
+
+### Trade-offs Accepted
+
+| Trade-off      | Choice         | Accepted Cost                          |
+| -------------- | -------------- | -------------------------------------- |
+| MODE vs SECURE | Hybrid support | Slightly more complex connection logic |
+| PEP 723 script | Standalone     | Script not part of installed package   |
+
+## Decision Drivers
+
+- **SSoT Principle**: All configurable values should live in `.mise.toml` `[env]` section
+- **Developer Experience**: Zero-friction local setup, DBeaver integration
+- **Backward Compatibility**: Existing `CLICKHOUSE_SECURE` users should not break
+- **Canonical Conformity**: Follow cc-skills patterns exactly
+
+## Considered Options
+
+- **Option A**: Full migration to CLICKHOUSE_MODE only
+  - Breaks existing scripts using CLICKHOUSE_SECURE
+
+- **Option B**: Keep CLICKHOUSE_SECURE only, skip MODE
+  - Deviates from canonical cc-skills pattern
+
+- **Option C**: Hybrid - CLICKHOUSE_MODE primary, CLICKHOUSE_SECURE fallback ← Selected
+  - Best of both worlds: canonical pattern + backward compatibility
+
+## Decision Outcome
+
+Chosen option: **Option C (Hybrid)**, because it adopts the canonical `CLICKHOUSE_MODE` pattern from cc-skills while maintaining backward compatibility for any existing automation using `CLICKHOUSE_SECURE`.
+
+## Synthesis
+
+**Convergent findings**: All agents agreed on using mise `[env]` as SSoT, scripts at `scripts/`, and Pydantic v2 models.
+
+**Divergent findings**: Existing code uses `CLICKHOUSE_SECURE`; canonical skill uses `CLICKHOUSE_MODE`.
+
+**Resolution**: Implement hybrid support - MODE takes precedence when set, SECURE as fallback for backward compatibility.
+
+## Consequences
+
+### Positive
+
+- DBeaver config generation with zero manual setup
+- mise `[env]` becomes SSoT for all ClickHouse connection parameters
+- Follows canonical cc-skills pattern for maintainability
+- Backward compatible with existing CLICKHOUSE_SECURE usage
+
+### Negative
+
+- Slightly more complex SSL detection logic in `clickhouse_client.py`
+- Additional script file to maintain (`scripts/generate_dbeaver_config.py`)
+
+## Architecture
+
+```
+┌──────────────────────┐             ┌────────────────────────────┐                ┌──────────────────────┐              ┌───────────────────┐
+│      .mise.toml      │  env vars   │          scripts/          │  instantiate   │ ClickHouseConnection │  serialize   │     .dbeaver/     │
+│   [env] + [tasks]    │ ──────────▶ │ generate_dbeaver_config.py │ ─────────────▶ │    Pydantic Model    │ ───────────▶ │ data-sources.json │
+└──────────────────────┘             └────────────────────────────┘                └──────────────────────┘              └───────────────────┘
+  │
+  │ env vars
+  ▼
+┌──────────────────────┐
+│ clickhouse_client.py │
+│   (MODE fallback)    │
+└──────────────────────┘
+```
+
+<details>
+<summary>graph-easy source</summary>
+
+```dot
+digraph arch {
+  rankdir=LR;
+  mise [label=".mise.toml\n[env] + [tasks]"];
+  script [label="scripts/\ngenerate_dbeaver_config.py"];
+  model [label="ClickHouseConnection\nPydantic Model"];
+  client [label="clickhouse_client.py\n(MODE fallback)"];
+  dbeaver [label=".dbeaver/\ndata-sources.json"];
+  mise -> script [label="env vars"];
+  mise -> client [label="env vars"];
+  script -> model [label="instantiate"];
+  model -> dbeaver [label="serialize"];
+}
+```
+
+</details>
+
+## References
+
+- [clickhouse-pydantic-config skill](/Users/terryli/eon/cc-skills/plugins/devops-tools/skills/clickhouse-pydantic-config/)
+- [mise-configuration skill](/Users/terryli/eon/cc-skills/plugins/itp/skills/mise-configuration/)
+- [ADR: mise env centralized config](/docs/adr/2025-12-08-mise-env-centralized-config.md) (if exists)

exness_data_preprocess-1.2.0/docs/adr/diagrams/2025-12-10-clickhouse-pydantic-config-after.dot

@@ -0,0 +1,21 @@
+// After: mise [env] as SSoT, DBeaver config generation
+digraph after {
+  rankdir=TB;
+
+  // SSoT
+  mise_env [label=".mise.toml [env]\n(SSoT)"];
+
+  // Pydantic Model
+  pydantic [label="ClickHouseConnection\n(Pydantic v2)"];
+
+  // Consumers
+  clickhouse_client [label="clickhouse_client.py"];
+  dbeaver_script [label="generate_dbeaver_config.py"];
+  dbeaver_json [label=".dbeaver/data-sources.json"];
+
+  // Relationships
+  mise_env -> pydantic [label="CLICKHOUSE_*"];
+  pydantic -> clickhouse_client [label="validated config"];
+  pydantic -> dbeaver_script [label="model fields"];
+  dbeaver_script -> dbeaver_json [label="generates"];
+}

exness_data_preprocess-1.2.0/docs/adr/diagrams/2025-12-10-clickhouse-pydantic-config-arch.dot

@@ -0,0 +1,17 @@
+// Architecture: Config flow from mise to DBeaver
+digraph arch {
+  rankdir=LR;
+
+  // Layers
+  mise [label=".mise.toml\n[env] + [tasks]"];
+  script [label="scripts/\ngenerate_dbeaver_config.py"];
+  model [label="ClickHouseConnection\nPydantic Model"];
+  client [label="clickhouse_client.py\n(MODE fallback)"];
+  dbeaver [label=".dbeaver/\ndata-sources.json"];
+
+  // Flow
+  mise -> script [label="env vars"];
+  mise -> client [label="env vars"];
+  script -> model [label="instantiate"];
+  model -> dbeaver [label="serialize"];
+}

exness_data_preprocess-1.2.0/docs/adr/diagrams/2025-12-10-clickhouse-pydantic-config-before.dot

@@ -0,0 +1,20 @@
+// Before: Scattered configuration, no DBeaver integration
+digraph before {
+  rankdir=TB;
+
+  // Sources
+  env_vars [label="Env Vars\n(scattered)"];
+  code_defaults [label="Code Defaults\n(hardcoded)"];
+
+  // Consumer
+  clickhouse_client [label="clickhouse_client.py"];
+
+  // Relationships
+  env_vars -> clickhouse_client [label="CLICKHOUSE_SECURE"];
+  env_vars -> clickhouse_client [label="CLICKHOUSE_HOST"];
+  code_defaults -> clickhouse_client [label="port=8123"];
+  code_defaults -> clickhouse_client [label="db='exness'"];
+
+  // Missing
+  dbeaver [label="DBeaver\n(manual setup)"];
+}

exness_data_preprocess-1.2.0/docs/design/2025-12-10-clickhouse-pydantic-config/spec.md

@@ -0,0 +1,153 @@
+---
+title: ClickHouse Pydantic Config Integration
+adr: /docs/adr/2025-12-10-clickhouse-pydantic-config.md
+status: accepted
+created: 2025-12-10
+---
+
+# Design Spec: ClickHouse Pydantic Config Integration
+
+**ADR**: [ClickHouse Pydantic Config Integration](/docs/adr/2025-12-10-clickhouse-pydantic-config.md)
+
+## Goal
+
+Generate DBeaver database client configurations from Pydantic v2 models using mise `[env]` as Single Source of Truth (SSoT).
+
+## Canonical Reference
+
+`/Users/terryli/eon/cc-skills/plugins/devops-tools/skills/clickhouse-pydantic-config/`
+
+---
+
+## Adaptation Requirements
+
+| Aspect             | Current State               | Target State                         |
+| ------------------ | --------------------------- | ------------------------------------ |
+| **SSL config**     | `CLICKHOUSE_SECURE` env var | `CLICKHOUSE_MODE` (local/cloud)      |
+| **Database**       | `exness` (hardcoded)        | `exness` via `CLICKHOUSE_DATABASE`   |
+| **mise [env]**     | No CLICKHOUSE\_\* vars      | Full SSoT configuration              |
+| **DBeaver config** | Does not exist              | `.dbeaver/data-sources.json`         |
+| **Scripts**        | None                        | `scripts/generate_dbeaver_config.py` |
+
+---
+
+## Implementation Tasks
+
+### Task 1: Update `.mise.toml` [env] Section
+
+Add ClickHouse SSoT configuration:
+
+```toml
+# ADR: 2025-12-10-clickhouse-pydantic-config
+# ClickHouse SSoT Configuration
+
+# Connection identity
+CLICKHOUSE_NAME = "exness-clickhouse-local"
+CLICKHOUSE_MODE = "local"  # "local" or "cloud"
+CLICKHOUSE_TYPE = "dev"    # "dev", "test", or "prod"
+
+# Connection parameters (match existing clickhouse_client.py defaults)
+CLICKHOUSE_HOST = "localhost"
+CLICKHOUSE_PORT = "8123"
+CLICKHOUSE_DATABASE = "exness"
+
+# Output paths
+DBEAVER_CONFIG_PATH = ".dbeaver/data-sources.json"
+
+# Cloud credentials (read from .env for security)
+# CLICKHOUSE_USER_READONLY - set in .env
+# CLICKHOUSE_PASSWORD_READONLY - set in .env
+```
+
+### Task 2: Create `scripts/generate_dbeaver_config.py`
+
+Copy and adapt from canonical implementation:
+
+- Source: `/Users/terryli/eon/cc-skills/plugins/devops-tools/skills/clickhouse-pydantic-config/scripts/generate_dbeaver_config.py`
+- Adaptations:
+  - Change default database from `"default"` to `"exness"`
+  - Change default name from `"clickhouse-local"` to `"exness-clickhouse-local"`
+
+Key components:
+
+- `ClickHouseConnection` Pydantic model with `@model_validator` for mode-aware SSL
+- `DBeaver JSON format` with folders and connections structure
+- PEP 723 inline dependencies (`# /// script`)
+
+### Task 3: Add mise Tasks
+
+```toml
+[tasks.db-client-generate]
+description = "Generate DBeaver config from Pydantic model"
+run = "uv run scripts/generate_dbeaver_config.py --output ${DBEAVER_CONFIG_PATH}"
+
+[tasks."db-client:cloud"]
+description = "Generate config for ClickHouse Cloud"
+run = "uv run scripts/generate_dbeaver_config.py --mode cloud --output ${DBEAVER_CONFIG_PATH}"
+
+[tasks."db-client:dry-run"]
+description = "Preview generated config without writing"
+run = "uv run scripts/generate_dbeaver_config.py --dry-run"
+
+[tasks.dbeaver]
+description = "Launch DBeaver with project workspace"
+run = '"/Applications/DBeaver.app/Contents/MacOS/dbeaver" -data .dbeaver-workspace &'
+```
+
+### Task 4: Update `.gitignore`
+
+Add DBeaver entries:
+
+```
+# DBeaver (credentials in generated JSON)
+.dbeaver/
+.dbeaver-workspace/
+```
+
+### Task 5: Update `clickhouse_client.py` (Backward Compatibility)
+
+Add `CLICKHOUSE_MODE` support while keeping `CLICKHOUSE_SECURE` as fallback:
+
+```python
+# Mode-based SSL detection (canonical pattern)
+mode = os.environ.get("CLICKHOUSE_MODE", "").lower()
+if mode == "cloud":
+    resolved_secure = True
+    resolved_port = port or int(os.environ.get("CLICKHOUSE_PORT", "8443"))
+elif mode == "local":
+    resolved_secure = False
+else:
+    # Fallback: existing CLICKHOUSE_SECURE logic
+    env_secure = os.environ.get("CLICKHOUSE_SECURE", "").lower()
+    # ... existing logic
+```
+
+---
+
+## Critical Files
+
+| File                                              | Action                      |
+| ------------------------------------------------- | --------------------------- |
+| `.mise.toml`                                      | Add `[env]` vars + tasks    |
+| `scripts/generate_dbeaver_config.py`              | **CREATE** - PEP 723 script |
+| `.gitignore`                                      | Add `.dbeaver/` entries     |
+| `src/exness_data_preprocess/clickhouse_client.py` | Add MODE support            |
+
+---
+
+## Validation Checklist
+
+- [ ] `mise run db-client:dry-run` - Preview config
+- [ ] `mise run db-client-generate` - Generate local config
+- [ ] `jq . .dbeaver/data-sources.json` - Validate JSON
+- [ ] `mise run db-client:cloud` - Generate cloud config (requires .env)
+- [ ] DBeaver can import the connection
+- [ ] Existing `clickhouse_client.py` tests still pass
+
+---
+
+## Dependencies
+
+- `pydantic>=2.0.0` (already in pyproject.toml)
+- `clickhouse-connect>=0.7.0` (already in pyproject.toml)
+- DBeaver Community Edition (manual install)

{exness_data_preprocess-1.1.0 → exness_data_preprocess-1.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "exness-data-preprocess"
-version = "1.1.0"
+version = "1.2.0"
 description = "Professional Exness forex tick data preprocessing with optimal compression (Parquet Zstd-22) and DuckDB OHLC generation. Provides efficient storage (9% smaller than ZIP) with lossless precision and direct queryability."
 readme = { file = "README.md", content-type = "text/markdown" }
 authors = [