exness-data-preprocess 1.1.0__tar.gz → 1.2.1__tar.gz

{exness_data_preprocess-1.1.0 → exness_data_preprocess-1.2.1}/.gitignore

@@ -107,3 +107,8 @@ node_modules/
 # Link checking (lychee)
 .lychee*
 .lycheecache
+
+# DBeaver (credentials in generated JSON)
+# ADR: /docs/adr/2025-12-10-clickhouse-pydantic-config.md
+.dbeaver/
+.dbeaver-workspace/

{exness_data_preprocess-1.1.0 → exness_data_preprocess-1.2.1}/.mise.toml

@@ -2,15 +2,32 @@
 # E2E validation workflow for codebase pruning verification
 
 [env]
-# Python version for validation
-PYTHON_VERSION = "3.14"
+# Python version for validation (pinned for alpha-forge compatibility)
+PYTHON_VERSION = "3.11"
 # Package name
 PACKAGE_NAME = "exness_data_preprocess"
 # Validation temp directory
 VALIDATION_TMP = "./tmp/validation"
 
+# ADR: /docs/adr/2025-12-10-clickhouse-pydantic-config.md
+# ClickHouse SSoT Configuration
+# Connection identity
+CLICKHOUSE_NAME = "exness-clickhouse-local"
+CLICKHOUSE_MODE = "local"  # "local" or "cloud"
+CLICKHOUSE_TYPE = "dev"    # "dev", "test", or "prod"
+# Connection parameters (match existing clickhouse_client.py defaults)
+CLICKHOUSE_HOST = "localhost"
+CLICKHOUSE_PORT = "8123"
+CLICKHOUSE_DATABASE = "exness"
+# Output paths
+DBEAVER_CONFIG_PATH = ".dbeaver/data-sources.json"
+# Cloud credentials (read from .env for security)
+# CLICKHOUSE_USER_READONLY - set in .env
+# CLICKHOUSE_PASSWORD_READONLY - set in .env
+
 [tools]
-python = "3.14"
+# Pin Python 3.11 for alpha-forge compatibility (numpy requirements)
+python = "3.11"
 
 # =============================================================================
 # Validation Tasks - E2E Testing Pipeline
@@ -470,3 +487,24 @@ engine.close()
 print('✅ OHLC validation passed')
 "
 '''
+
+# =============================================================================
+# DBeaver Config Generation
+# ADR: /docs/adr/2025-12-10-clickhouse-pydantic-config.md
+# =============================================================================
+
+[tasks.db-client-generate]
+description = "Generate DBeaver config from Pydantic model"
+run = "uv run scripts/generate_dbeaver_config.py --output ${DBEAVER_CONFIG_PATH}"
+
+[tasks."db-client:cloud"]
+description = "Generate config for ClickHouse Cloud"
+run = "uv run scripts/generate_dbeaver_config.py --mode cloud --output ${DBEAVER_CONFIG_PATH}"
+
+[tasks."db-client:dry-run"]
+description = "Preview generated config without writing"
+run = "uv run scripts/generate_dbeaver_config.py --dry-run"
+
+[tasks.dbeaver]
+description = "Launch DBeaver with project workspace"
+run = '"/Applications/DBeaver.app/Contents/MacOS/dbeaver" -data .dbeaver-workspace &'

{exness_data_preprocess-1.1.0 → exness_data_preprocess-1.2.1}/CHANGELOG.md

@@ -1,3 +1,22 @@
+## [1.2.1](https://github.com/terrylica/exness-data-preprocess/compare/v1.2.0...v1.2.1) (2025-12-11)
+
+
+### Bug Fixes
+
+* **clickhouse:** bootstrap database creation without existing database context ([0fdfef1](https://github.com/terrylica/exness-data-preprocess/commit/0fdfef17cf797d64a66b3b163037a01c1e8245a7))
+
+# [1.2.0](https://github.com/terrylica/exness-data-preprocess/compare/v1.1.0...v1.2.0) (2025-12-10)
+
+
+### Bug Fixes
+
+* **clickhouse:** resolve E2E validation bugs in schema and query engine ([5c618f7](https://github.com/terrylica/exness-data-preprocess/commit/5c618f717cadc23097e9fd11d2fed87947ecf25f))
+
+
+### Features
+
+* add DBeaver config generation with CLICKHOUSE_MODE support ([0e97618](https://github.com/terrylica/exness-data-preprocess/commit/0e9761818f137461623f744af92f0b1e98b3a797))
+
 # [1.1.0](https://github.com/terrylica/exness-data-preprocess/compare/v1.0.0...v1.1.0) (2025-12-10)
 
 

exness_data_preprocess-1.2.1/CLAUDE.md

@@ -0,0 +1,150 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+**Package**: `exness-data-preprocess` — Forex tick data preprocessing with DuckDB (local) and ClickHouse (cloud) backends
+
+**Version**: See [`pyproject.toml`](pyproject.toml) line 3 (SSoT via semantic-release)
+
+**Quick Start**: [`README.md`](README.md) — Installation, API reference, usage examples
+
+---
+
+## Documentation Hub (Progressive Disclosure)
+
+### Essential Links
+
+| Topic            | SSoT Document                                                      | Purpose                               |
+| ---------------- | ------------------------------------------------------------------ | ------------------------------------- |
+| **User Guide**   | [`README.md`](README.md)                                           | Installation, API, examples           |
+| **Architecture** | [`docs/UNIFIED_DUCKDB_PLAN_v2.md`](docs/UNIFIED_DUCKDB_PLAN_v2.md) | Unified single-file DuckDB design     |
+| **Schema**       | [`docs/DATABASE_SCHEMA.md`](docs/DATABASE_SCHEMA.md)               | Phase7 30-column OHLC specification   |
+| **Modules**      | [`docs/MODULE_ARCHITECTURE.md`](docs/MODULE_ARCHITECTURE.md)       | 19 modules with SLOs                  |
+| **Data Sources** | [`docs/EXNESS_DATA_SOURCES.md`](docs/EXNESS_DATA_SOURCES.md)       | Exness tick data variants             |
+| **Research**     | [`docs/RESEARCH_PATTERNS.md`](docs/RESEARCH_PATTERNS.md)           | Hybrid materialization lifecycle      |
+| **Tasks**        | [`.mise.toml`](.mise.toml)                                         | Validation, ClickHouse, DBeaver tasks |
+| **Contributing** | [`CONTRIBUTING.md`](CONTRIBUTING.md)                               | Development guidelines                |
+
+### Deep Dives (On-Demand)
+
+| Topic              | Location                               | When to Read                     |
+| ------------------ | -------------------------------------- | -------------------------------- |
+| ADRs               | [`docs/adr/`](docs/adr/)               | Architecture decisions           |
+| Design Specs       | [`docs/design/`](docs/design/)         | Implementation blueprints        |
+| Optimization Plans | [`docs/phases/`](docs/phases/)         | Phase 2/3 SSoT YAML plans        |
+| Validation Results | [`docs/validation/`](docs/validation/) | Spike test proofs                |
+| Research Analysis  | [`docs/research/`](docs/research/)     | Zero-spread, compression studies |
+
+---
+
+## Architecture Overview
+
+### Dual Backend Design
+
+| Backend        | Storage                                   | Use Case          | Entry Point           |
+| -------------- | ----------------------------------------- | ----------------- | --------------------- |
+| **DuckDB**     | One file per instrument (`eurusd.duckdb`) | Local development | `ExnessDataProcessor` |
+| **ClickHouse** | Single table with instrument column       | Cloud deployment  | `ClickHouseManager`   |
+
+### Module Structure (19 Modules)
+
+**DuckDB Backend (8 modules)**:
+
+- `processor.py` — Thin orchestrator facade
+- `downloader.py` — HTTP download operations
+- `tick_loader.py` — CSV parsing
+- `database_manager.py` — Database operations
+- `session_detector.py` — Holiday/session detection (vectorized)
+- `gap_detector.py` — Incremental update logic (SQL-based)
+- `ohlc_generator.py` — Phase7 OHLC generation
+- `query_engine.py` — Query operations
+
+**ClickHouse Backend (7 modules)**:
+
+- `clickhouse_manager.py` — Schema management, data loading
+- `clickhouse_gap_detector.py` — Gap detection for ClickHouse
+- `clickhouse_query_engine.py` — Query with pagination
+- `clickhouse_ohlc_generator.py` — OHLC generation
+- `clickhouse_client.py` — Connection factory with error handling
+- `clickhouse_base.py` — Shared mixin for client lifecycle
+- `clickhouse_cloud.py` — Cloud-specific configuration
+
+**Shared (4 modules)**:
+
+- `config.py` — Configuration management
+- `models.py` — Pydantic models (UpdateResult, CoverageInfo, etc.)
+- `schema.py` — SQL schema definitions
+- `exchanges.py` — Exchange calendar integration
+
+**Complete Details**: [`docs/MODULE_ARCHITECTURE.md`](docs/MODULE_ARCHITECTURE.md)
+
+---
+
+## Development Commands
+
+### Quick Reference
+
+```bash
+# Setup
+uv sync --dev
+
+# Testing (48 tests)
+uv run pytest                          # All tests
+mise run validate                      # Full E2E validation
+
+# Linting
+uv run ruff check --fix src/ tests/
+
+# ClickHouse
+mise run clickhouse:start              # Start local server
+mise run clickhouse:status             # Check connection
+mise run validate:clickhouse           # E2E ClickHouse tests
+
+# Build & Publish
+uv build
+mise run db-client-generate            # Generate DBeaver config
+```
+
+### mise Tasks
+
+All validation and tooling tasks are defined in [`.mise.toml`](.mise.toml):
+
+| Task                           | Description                                                               |
+| ------------------------------ | ------------------------------------------------------------------------- |
+| `mise run validate`            | Full E2E pipeline (imports, lint, typecheck, test, build, install, mixin) |
+| `mise run validate:clickhouse` | ClickHouse E2E tests (requires running server)                            |
+| `mise run clickhouse:start`    | Start mise-installed ClickHouse                                           |
+| `mise run db-client-generate`  | Generate DBeaver config from Pydantic model                               |
+| `mise run ci`                  | Simulate CI pipeline locally                                              |
+
+---
+
+## Key Patterns
+
+### Performance Optimizations (~16x total speedup)
+
+| Phase | Optimization          | Speedup           | SSoT                                                                                                                 |
+| ----- | --------------------- | ----------------- | -------------------------------------------------------------------------------------------------------------------- |
+| 1     | Incremental OHLC      | 7.3x              | [`docs/validation/SPIKE_TEST_RESULTS_PHASE1_2025-10-18.md`](docs/validation/SPIKE_TEST_RESULTS_PHASE1_2025-10-18.md) |
+| 2     | Session Vectorization | 2.2x              | [`docs/phases/PHASE2_SESSION_VECTORIZATION_PLAN.yaml`](docs/phases/PHASE2_SESSION_VECTORIZATION_PLAN.yaml)           |
+| 3     | SQL Gap Detection     | 46% LOC reduction | [`docs/phases/PHASE3_SQL_GAP_DETECTION_PLAN.yaml`](docs/phases/PHASE3_SQL_GAP_DETECTION_PLAN.yaml)                   |
+
+### Research Lifecycle
+
+```
+Explore (pandas/Polars) → Validate → Graduate (DuckDB) → Query (SQL)
+```
+
+**SSoT**: [`docs/RESEARCH_PATTERNS.md`](docs/RESEARCH_PATTERNS.md)
+
+---
+
+## References
+
+- **Exness Data**: <https://ticks.ex2archive.com/>
+- **DuckDB**: <https://duckdb.org/>
+- **ClickHouse**: <https://clickhouse.com/>
+
+---
+
+**Last Updated**: 2025-12-10

{exness_data_preprocess-1.1.0 → exness_data_preprocess-1.2.1}/CONTRIBUTING.md

@@ -6,7 +6,7 @@ Thank you for your interest in contributing to Exness Data Preprocess! This docu
 
 ### Prerequisites
 
-- Python 3.9 or higher
+- Python 3.11 or higher (see `pyproject.toml` requires-python)
 - uv package manager (recommended) or pip
 - Git
 
@@ -114,6 +114,7 @@ When reporting issues, please include:
    ```
 
 7. **Push to your fork** and submit a pull request:
+
    ```bash
    git push origin feature/your-feature-name
    ```
@@ -143,31 +144,24 @@ test: add integration tests for DuckDB operations
 
 ### Project Structure
 
-```
-exness-data-preprocess/
-├── src/exness_data_preprocess/
-│   ├── __init__.py          # Package entry point
-│   ├── processor.py         # Core processor class
-│   ├── api.py              # Simple API functions
-│   └── cli.py              # Command-line interface
-├── tests/
-│   ├── conftest.py         # Pytest fixtures
-│   ├── test_processor.py   # Processor tests
-│   ├── test_api.py         # API tests
-│   └── test_cli.py         # CLI tests
-├── docs/                   # Documentation
-├── pyproject.toml         # Package configuration
-└── README.md             # Main documentation
-```
+**See [`CLAUDE.md`](CLAUDE.md) for complete module structure (19 modules).**
+
+Key directories:
+
+- `src/exness_data_preprocess/` — Source modules (DuckDB + ClickHouse backends)
+- `tests/` — Test suite (48 tests)
+- `docs/` — Documentation hub
+- `.mise.toml` — Validation and tooling tasks
+- `pyproject.toml` — Package configuration (SSoT for version via semantic-release)
 
 ### Adding New Features
 
 When adding new features:
 
-1. **Add implementation** in appropriate module (processor.py, api.py, or cli.py)
-2. **Add tests** in corresponding test file
+1. **Add implementation** in appropriate module (see CLAUDE.md Module Structure)
+2. **Add tests** in corresponding test file under `tests/`
 3. **Update API exports** in `__init__.py` if needed
-4. **Update README.md** with usage examples
+4. **Update CLAUDE.md** with module links if adding new modules
 5. **Add docstrings** following Google style
 
 Example docstring:

{exness_data_preprocess-1.1.0 → exness_data_preprocess-1.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: exness-data-preprocess
-Version: 1.1.0
+Version: 1.2.1
 Summary: Professional Exness forex tick data preprocessing with optimal compression (Parquet Zstd-22) and DuckDB OHLC generation. Provides efficient storage (9% smaller than ZIP) with lossless precision and direct queryability.
 Project-URL: Homepage, https://github.com/terrylica/exness-data-preprocess
 Project-URL: Documentation, https://github.com/terrylica/exness-data-preprocess#readme

exness_data_preprocess-1.2.1/docs/adr/2025-12-10-clickhouse-pydantic-config.md

@@ -0,0 +1,237 @@
+---
+status: accepted
+date: 2025-12-10
+decision-maker: Terry Li
+consulted:
+  [
+    Config-Discovery-Agent,
+    ClickHouse-Connection-Agent,
+    Skill-Reference-Agent,
+    ClickHouse-Pattern-Agent,
+    Mise-Pattern-Agent,
+  ]
+research-method: 9-agent-parallel-dctl
+clarification-iterations: 2
+perspectives: [SSoT-Configuration, DeveloperExperience, BackwardCompatibility]
+---
+
+# ADR: ClickHouse Pydantic Config Integration
+
+**Design Spec**: [Implementation Spec](/docs/design/2025-12-10-clickhouse-pydantic-config/spec.md)
+
+## Context and Problem Statement
+
+The `exness-data-preprocess` package has ClickHouse integration for tick data storage, but lacks:
+
+1. **Database client configuration** - No DBeaver connection config generation
+2. **SSoT for connection parameters** - Environment variables are scattered, not centralized in mise `[env]`
+3. **Inconsistent SSL handling** - Uses `CLICKHOUSE_SECURE` (boolean) instead of canonical `CLICKHOUSE_MODE` (local/cloud) pattern
+
+The `clickhouse-pydantic-config` skill from cc-skills provides a canonical pattern for generating DBeaver configs from Pydantic v2 models using mise as SSoT.
+
+### Before/After
+
+**Before**: Scattered configuration, no DBeaver integration
+
+```
+┌────────────────────┐
+│   Code Defaults    │
+│    (hardcoded)     │ ─┐
+└────────────────────┘  │
+  │                     │
+  │ port=8123           │ db='exness'
+  ▼                     ▼
+┌────────────────────────────────────────┐
+│          clickhouse_client.py          │
+└────────────────────────────────────────┘
+  ▲                     ▲
+  │ CLICKHOUSE_SECURE   │ CLICKHOUSE_HOST
+  │                     │
+┌────────────────────┐  │
+│      Env Vars      │  │
+│    (scattered)     │ ─┘
+└────────────────────┘
+┌────────────────────┐
+│      DBeaver       │
+│   (manual setup)   │
+└────────────────────┘
+```
+
+<details>
+<summary>graph-easy source (before.dot)</summary>
+
+```dot
+digraph before {
+  rankdir=TB;
+  env_vars [label="Env Vars\n(scattered)"];
+  code_defaults [label="Code Defaults\n(hardcoded)"];
+  clickhouse_client [label="clickhouse_client.py"];
+  env_vars -> clickhouse_client [label="CLICKHOUSE_SECURE"];
+  env_vars -> clickhouse_client [label="CLICKHOUSE_HOST"];
+  code_defaults -> clickhouse_client [label="port=8123"];
+  code_defaults -> clickhouse_client [label="db='exness'"];
+  dbeaver [label="DBeaver\n(manual setup)"];
+}
+```
+
+</details>
+
+**After**: mise `[env]` as SSoT, DBeaver config generation
+
+```
+                                             ┌────────────────────────────┐
+                                             │      .mise.toml [env]      │
+                                             │           (SSoT)           │
+                                             └────────────────────────────┘
+                                               │
+                                               │ CLICKHOUSE_*
+                                               ▼
+┌──────────────────────┐                     ┌────────────────────────────┐
+│ clickhouse_client.py │  validated config   │    ClickHouseConnection    │
+│                      │ ◀────────────────── │       (Pydantic v2)        │
+└──────────────────────┘                     └────────────────────────────┘
+                                               │
+                                               │ model fields
+                                               ▼
+                                             ┌────────────────────────────┐
+                                             │ generate_dbeaver_config.py │
+                                             └────────────────────────────┘
+                                               │
+                                               │ generates
+                                               ▼
+                                             ┌────────────────────────────┐
+                                             │ .dbeaver/data-sources.json │
+                                             └────────────────────────────┘
+```
+
+<details>
+<summary>graph-easy source (after.dot)</summary>
+
+```dot
+digraph after {
+  rankdir=TB;
+  mise_env [label=".mise.toml [env]\n(SSoT)"];
+  pydantic [label="ClickHouseConnection\n(Pydantic v2)"];
+  clickhouse_client [label="clickhouse_client.py"];
+  dbeaver_script [label="generate_dbeaver_config.py"];
+  dbeaver_json [label=".dbeaver/data-sources.json"];
+  mise_env -> pydantic [label="CLICKHOUSE_*"];
+  pydantic -> clickhouse_client [label="validated config"];
+  pydantic -> dbeaver_script [label="model fields"];
+  dbeaver_script -> dbeaver_json [label="generates"];
+}
+```
+
+</details>
+
+## Research Summary
+
+| Agent Perspective     | Key Finding                                                                                            | Confidence |
+| --------------------- | ------------------------------------------------------------------------------------------------------ | ---------- |
+| Config-Discovery      | `.mise.toml` has `[env]` section but no CLICKHOUSE\_\* vars; existing `config.py` has Pydantic pattern | High       |
+| ClickHouse-Connection | Current code uses `CLICKHOUSE_SECURE` for SSL; port 8123 for HTTP; database default is `exness`        | High       |
+| Skill-Reference       | Canonical model uses `CLICKHOUSE_MODE` (local/cloud) with mode-aware validation; scripts at `scripts/` | High       |
+| ClickHouse-Pattern    | Three skills form a chain: cloud-management → pydantic-config → architect                              | High       |
+| Mise-Pattern          | SSoT pattern: `[env]` section → scripts read via `os.environ.get()` with defaults                      | High       |
+
+## Decision Log
+
+| Decision Area    | Options Evaluated                       | Chosen                               | Rationale                                     |
+| ---------------- | --------------------------------------- | ------------------------------------ | --------------------------------------------- |
+| SSL Config       | CLICKHOUSE_SECURE vs CLICKHOUSE_MODE    | Hybrid (MODE + SECURE fallback)      | Backward compatibility for existing scripts   |
+| Script Location  | inline mise task, src/ module, scripts/ | `scripts/generate_dbeaver_config.py` | Canonical pattern from cc-skills              |
+| Database Default | `default` vs `exness`                   | `exness`                             | Match existing codebase convention            |
+| Connection Scope | Single mode vs Both local+cloud         | Single per invocation                | Canonical pattern; `--mode` flag for override |
+
+### Trade-offs Accepted
+
+| Trade-off      | Choice         | Accepted Cost                          |
+| -------------- | -------------- | -------------------------------------- |
+| MODE vs SECURE | Hybrid support | Slightly more complex connection logic |
+| PEP 723 script | Standalone     | Script not part of installed package   |
+
+## Decision Drivers
+
+- **SSoT Principle**: All configurable values should live in `.mise.toml` `[env]` section
+- **Developer Experience**: Zero-friction local setup, DBeaver integration
+- **Backward Compatibility**: Existing `CLICKHOUSE_SECURE` users should not break
+- **Canonical Conformity**: Follow cc-skills patterns exactly
+
+## Considered Options
+
+- **Option A**: Full migration to CLICKHOUSE_MODE only
+  - Breaks existing scripts using CLICKHOUSE_SECURE
+
+- **Option B**: Keep CLICKHOUSE_SECURE only, skip MODE
+  - Deviates from canonical cc-skills pattern
+
+- **Option C**: Hybrid - CLICKHOUSE_MODE primary, CLICKHOUSE_SECURE fallback ← Selected
+  - Best of both worlds: canonical pattern + backward compatibility
+
+## Decision Outcome
+
+Chosen option: **Option C (Hybrid)**, because it adopts the canonical `CLICKHOUSE_MODE` pattern from cc-skills while maintaining backward compatibility for any existing automation using `CLICKHOUSE_SECURE`.
+
+## Synthesis
+
+**Convergent findings**: All agents agreed on using mise `[env]` as SSoT, scripts at `scripts/`, and Pydantic v2 models.
+
+**Divergent findings**: Existing code uses `CLICKHOUSE_SECURE`; canonical skill uses `CLICKHOUSE_MODE`.
+
+**Resolution**: Implement hybrid support - MODE takes precedence when set, SECURE as fallback for backward compatibility.
+
+## Consequences
+
+### Positive
+
+- DBeaver config generation with zero manual setup
+- mise `[env]` becomes SSoT for all ClickHouse connection parameters
+- Follows canonical cc-skills pattern for maintainability
+- Backward compatible with existing CLICKHOUSE_SECURE usage
+
+### Negative
+
+- Slightly more complex SSL detection logic in `clickhouse_client.py`
+- Additional script file to maintain (`scripts/generate_dbeaver_config.py`)
+
+## Architecture
+
+```
+┌──────────────────────┐             ┌────────────────────────────┐                ┌──────────────────────┐              ┌───────────────────┐
+│      .mise.toml      │  env vars   │          scripts/          │  instantiate   │ ClickHouseConnection │  serialize   │     .dbeaver/     │
+│   [env] + [tasks]    │ ──────────▶ │ generate_dbeaver_config.py │ ─────────────▶ │    Pydantic Model    │ ───────────▶ │ data-sources.json │
+└──────────────────────┘             └────────────────────────────┘                └──────────────────────┘              └───────────────────┘
+  │
+  │ env vars
+  ▼
+┌──────────────────────┐
+│ clickhouse_client.py │
+│   (MODE fallback)    │
+└──────────────────────┘
+```
+
+<details>
+<summary>graph-easy source</summary>
+
+```dot
+digraph arch {
+  rankdir=LR;
+  mise [label=".mise.toml\n[env] + [tasks]"];
+  script [label="scripts/\ngenerate_dbeaver_config.py"];
+  model [label="ClickHouseConnection\nPydantic Model"];
+  client [label="clickhouse_client.py\n(MODE fallback)"];
+  dbeaver [label=".dbeaver/\ndata-sources.json"];
+  mise -> script [label="env vars"];
+  mise -> client [label="env vars"];
+  script -> model [label="instantiate"];
+  model -> dbeaver [label="serialize"];
+}
+```
+
+</details>
+
+## References
+
+- [clickhouse-pydantic-config skill](/Users/terryli/eon/cc-skills/plugins/devops-tools/skills/clickhouse-pydantic-config/)
+- [mise-configuration skill](/Users/terryli/eon/cc-skills/plugins/itp/skills/mise-configuration/)
+- [ADR: mise env centralized config](/docs/adr/2025-12-08-mise-env-centralized-config.md) (if exists)

exness_data_preprocess-1.2.1/docs/adr/diagrams/2025-12-10-clickhouse-pydantic-config-after.dot

@@ -0,0 +1,21 @@
+// After: mise [env] as SSoT, DBeaver config generation
+digraph after {
+  rankdir=TB;
+
+  // SSoT
+  mise_env [label=".mise.toml [env]\n(SSoT)"];
+
+  // Pydantic Model
+  pydantic [label="ClickHouseConnection\n(Pydantic v2)"];
+
+  // Consumers
+  clickhouse_client [label="clickhouse_client.py"];
+  dbeaver_script [label="generate_dbeaver_config.py"];
+  dbeaver_json [label=".dbeaver/data-sources.json"];
+
+  // Relationships
+  mise_env -> pydantic [label="CLICKHOUSE_*"];
+  pydantic -> clickhouse_client [label="validated config"];
+  pydantic -> dbeaver_script [label="model fields"];
+  dbeaver_script -> dbeaver_json [label="generates"];
+}

exness_data_preprocess-1.2.1/docs/adr/diagrams/2025-12-10-clickhouse-pydantic-config-arch.dot

@@ -0,0 +1,17 @@
+// Architecture: Config flow from mise to DBeaver
+digraph arch {
+  rankdir=LR;
+
+  // Layers
+  mise [label=".mise.toml\n[env] + [tasks]"];
+  script [label="scripts/\ngenerate_dbeaver_config.py"];
+  model [label="ClickHouseConnection\nPydantic Model"];
+  client [label="clickhouse_client.py\n(MODE fallback)"];
+  dbeaver [label=".dbeaver/\ndata-sources.json"];
+
+  // Flow
+  mise -> script [label="env vars"];
+  mise -> client [label="env vars"];
+  script -> model [label="instantiate"];
+  model -> dbeaver [label="serialize"];
+}

exness_data_preprocess-1.2.1/docs/adr/diagrams/2025-12-10-clickhouse-pydantic-config-before.dot

@@ -0,0 +1,20 @@
+// Before: Scattered configuration, no DBeaver integration
+digraph before {
+  rankdir=TB;
+
+  // Sources
+  env_vars [label="Env Vars\n(scattered)"];
+  code_defaults [label="Code Defaults\n(hardcoded)"];
+
+  // Consumer
+  clickhouse_client [label="clickhouse_client.py"];
+
+  // Relationships
+  env_vars -> clickhouse_client [label="CLICKHOUSE_SECURE"];
+  env_vars -> clickhouse_client [label="CLICKHOUSE_HOST"];
+  code_defaults -> clickhouse_client [label="port=8123"];
+  code_defaults -> clickhouse_client [label="db='exness'"];
+
+  // Missing
+  dbeaver [label="DBeaver\n(manual setup)"];
+}