exness-data-preprocess 2.1.0__tar.gz → 3.0.0__tar.gz

exness_data_preprocess-3.0.0/.github/dependabot.yml

@@ -0,0 +1,26 @@
+# Dependency update PRs. Replaces the scheduled pre-commit-autoupdate workflow.
+# No `uv` entry: uv.lock is not tracked and this is a library, so the only possible updates
+# would be raising the `>=` floors in pyproject.toml, which narrows what downstream users can install.
+version: 2
+updates:
+  - package-ecosystem: "pre-commit"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    commit-message:
+      prefix: "chore(deps)"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    commit-message:
+      prefix: "ci(deps)"
+  - package-ecosystem: "npm"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+    commit-message:
+      prefix: "chore(deps)"
+    groups:
+      semantic-release:
+        patterns: ["*"]

{exness_data_preprocess-2.1.0 → exness_data_preprocess-3.0.0}/.github/workflows/publish.yml

@@ -10,9 +10,9 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
 
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@v7
         with:
           python-version: '3.12'
 
@@ -23,7 +23,7 @@ jobs:
         run: python -m build
 
       - name: Upload distribution artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: dist
           path: dist/
@@ -41,7 +41,7 @@ jobs:
 
     steps:
       - name: Download distribution artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v8
         with:
           name: dist
           path: dist/

{exness_data_preprocess-2.1.0 → exness_data_preprocess-3.0.0}/.gitignore

@@ -41,7 +41,6 @@ venv.bak/
 .ipynb_checkpoints
 
 # pyenv
-.python-version
 
 # Environments
 .env.local
@@ -106,7 +105,6 @@ node_modules/
 
 # Link checking (lychee)
 .lychee*
-.lycheecache
 
 # DBeaver (credentials in generated JSON)
 # ADR: /docs/adr/2025-12-10-clickhouse-pydantic-config.md
@@ -116,3 +114,7 @@ node_modules/
 # Claude Code local files
 .claude/
 .lint-relative-paths-results.txt
+
+# moon (task runner) local cache
+.moon/cache/
+.moon/docker/

exness_data_preprocess-3.0.0/.moon/toolchain.yml

@@ -0,0 +1,2 @@
+# yaml-language-server: $schema=https://moonrepo.dev/schemas/toolchain.json
+# No moon-managed toolchains: tasks run on the `system` platform (uv from proto on PATH).

exness_data_preprocess-3.0.0/.moon/workspace.yml

@@ -0,0 +1,8 @@
+# yaml-language-server: $schema=https://moonrepo.dev/schemas/workspace.json
+# moon runs this repo's tasks; proto (.prototools) pins the tools.
+projects:
+  repo: '.'
+
+vcs:
+  client: 'git'
+  defaultBranch: 'main'

{exness_data_preprocess-2.1.0 → exness_data_preprocess-3.0.0}/.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 - repo: https://github.com/commitizen-tools/commitizen
-  rev: v4.9.1
+  rev: v4.19.2
   hooks:
   - id: commitizen
     stages: [commit-msg]
@@ -8,7 +8,7 @@ repos:
     stages: [pre-push]
     args: [--rev-range, origin/main..HEAD]
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
+  rev: v6.0.0
   hooks:
   - id: trailing-whitespace
   - id: end-of-file-fixer
@@ -17,8 +17,8 @@ repos:
   - id: check-merge-conflict
   - id: debug-statements
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.13.0
+  rev: v0.17.1
   hooks:
-  - id: ruff
+  - id: ruff-check
     args: [--fix, --unsafe-fixes]
   - id: ruff-format

exness_data_preprocess-3.0.0/.prototools

@@ -0,0 +1,5 @@
+# proto — repo-local toolchain versions.
+# uv owns Python: the interpreter version is .python-version, and uv.toml makes uv use only its own
+# managed build. ClickHouse comes from the system PATH (e.g. Homebrew), not from proto.
+uv = "latest"
+moon = "2.6.1"

exness_data_preprocess-3.0.0/.python-version

@@ -0,0 +1 @@
+3.11

exness_data_preprocess-3.0.0/.releaserc.yml

@@ -0,0 +1,24 @@
+# semantic-release configuration for Python project
+# Uses @semantic-release/exec instead of @semantic-release/npm
+branches:
+  - main
+
+plugins:
+  - "@semantic-release/commit-analyzer"
+  - "@semantic-release/release-notes-generator"
+  - "@semantic-release/changelog"
+  - - "@semantic-release/exec"
+    # `uv version --frozen` rewrites [project].version in pyproject.toml only (uv.lock is not tracked).
+    # __version__ is read from the installed distribution metadata, so nothing else needs bumping.
+    # The release narrative is a tracked file written in the PR before the release; without it
+    # there is no release. After @semantic-release/github publishes, the body is rebuilt as
+    # narrative + generated section (idempotent; see scripts/publish-release-notes.sh).
+    - verifyReleaseCmd: "test -s docs/release-notes/${nextRelease.version}.md || { echo 'missing docs/release-notes/${nextRelease.version}.md' >&2; exit 1; }"
+      prepareCmd: "uv version ${nextRelease.version} --frozen"
+      successCmd: "bash scripts/publish-release-notes.sh ${nextRelease.version}"
+  - - "@semantic-release/git"
+    - assets:
+        - pyproject.toml
+        - CHANGELOG.md
+      message: "chore(release): bump version ${lastRelease.version} → ${nextRelease.version} [skip ci]"
+  - "@semantic-release/github"

{exness_data_preprocess-2.1.0 → exness_data_preprocess-3.0.0}/CHANGELOG.md

@@ -1,3 +1,34 @@
+# [3.0.0](https://github.com/terrylica/exness-data-preprocess/compare/v2.1.1...v3.0.0) (2026-10-19)
+
+
+* fix!: keep distinct ticks that share a timestamp; read with FINAL; make force_redownload work ([e7ac22b](https://github.com/terrylica/exness-data-preprocess/commit/e7ac22ba659c7eb1c03f3ad1f223d23bff515304)), closes [#23](https://github.com/terrylica/exness-data-preprocess/issues/23)
+* refactor!: remove the unused config module, ExnessDataProcessor(config=...) and pyyaml ([b43be4a](https://github.com/terrylica/exness-data-preprocess/commit/b43be4a09702c6135a41b27256345952c83e6708)), closes [#21](https://github.com/terrylica/exness-data-preprocess/issues/21)
+
+
+### Bug Fixes
+
+* apply review findings (inclusive end_date day, cursor pruning, scoped ASOF build side, forced re-download) ([dd3a0b3](https://github.com/terrylica/exness-data-preprocess/commit/dd3a0b373689116d600f43ff489e3076a66eb13e)), closes [#23](https://github.com/terrylica/exness-data-preprocess/issues/23) [#24](https://github.com/terrylica/exness-data-preprocess/issues/24)
+* **query:** bind datetime parameters as DateTime64(6, 'UTC') so filters are not shifted on non-UTC servers ([1f88c41](https://github.com/terrylica/exness-data-preprocess/commit/1f88c41b372306bb5e929d131e1338d71e9ac539)), closes [#24](https://github.com/terrylica/exness-data-preprocess/issues/24)
+* report real storage_bytes and OHLC bar counts instead of placeholders ([a970be5](https://github.com/terrylica/exness-data-preprocess/commit/a970be5f75b8dbddff890e92a628388261f10026)), closes [#19](https://github.com/terrylica/exness-data-preprocess/issues/19)
+
+
+### BREAKING CHANGES
+
+* raw_spread_ticks and standard_ticks gain a `seq` column and sort by (instrument, timestamp, seq); existing tables are migrated in place by ensure_schema() (re-download months with update_data(..., force_redownload=True) to restore ticks the old key collapsed). CursorResult.next_cursor is now an opaque "<timestamp>#<seq>" string rather than a plain timestamp. Tick counts, tick queries and OHLC generation read with FINAL.
+* ExnessDataProcessor() no longer accepts a `config` argument, the exness_data_preprocess.config module (ConfigModel, load_config, get_default_config_path) is removed, and PyYAML is no longer installed as a dependency. Configure through environment variables (CLICKHOUSE_*, EXNESS_DOWNLOAD_DIR).
+
+## [2.1.1](https://github.com/terrylica/exness-data-preprocess/compare/v2.1.0...v2.1.1) (2026-10-19)
+
+
+### Bug Fixes
+
+* **clickhouse:** resolve the binary from PATH instead of a dead mise shim ([2e5cbe0](https://github.com/terrylica/exness-data-preprocess/commit/2e5cbe0bd3c51f0d95d6c1baf583a98727a12d36))
+* **e2e:** apply review findings (memory cap, isolated downloads, marker selection, signals) ([ec2d8aa](https://github.com/terrylica/exness-data-preprocess/commit/ec2d8aaea4cc1f2ca7bb5527208345ad2d526a4b)), closes [#6](https://github.com/terrylica/exness-data-preprocess/issues/6)
+* **processor:** bootstrap exness database before first get_client() call ([0cc55a4](https://github.com/terrylica/exness-data-preprocess/commit/0cc55a43fa482e48870f4685d170f2b2ff1aa941))
+* **release:** bump the version with `uv version`, read __version__ from metadata ([aa86693](https://github.com/terrylica/exness-data-preprocess/commit/aa866932bb41517629a51186af740555cc959774)), closes [#4](https://github.com/terrylica/exness-data-preprocess/issues/4)
+* **scripts:** resolve ruff lint errors and formatting issues ([68d939f](https://github.com/terrylica/exness-data-preprocess/commit/68d939f5fafdd2692b5ef6f1ad4e5811e3364a87))
+* **types:** make `moon run repo:check` green (mypy 15 errors to 0) ([79b2259](https://github.com/terrylica/exness-data-preprocess/commit/79b2259af62e4d5086016669d8dba825fd69dbd9)), closes [#7](https://github.com/terrylica/exness-data-preprocess/issues/7)
+
 # [2.1.0](https://github.com/terrylica/exness-data-preprocess/compare/v2.0.0...v2.1.0) (2025-12-28)
 
 

exness_data_preprocess-3.0.0/CLAUDE.md

@@ -0,0 +1,31 @@
+# CLAUDE.md
+
+`exness-data-preprocess`: downloads Exness forex ticks from ticks.ex2archive.com into ClickHouse (database `exness`: `raw_spread_ticks`, `standard_ticks`, `ohlc_1m`, plus `exchange_sessions` and `holidays`) and serves tick and OHLC queries. Entry point: `ExnessDataProcessor`. User docs: [README.md](/README.md); index of the rest: [docs/README.md](/docs/README.md).
+
+## Commands
+
+```bash
+uv sync --dev
+moon run repo:check      # offline gate: ruff check + ruff format --check + mypy + pytest
+moon run repo:e2e        # hermetic: throwaway ClickHouse on free ports, pytest with ClickHouse required,
+                         # validate_clickhouse.py all (incl. a real EURUSD download), examples/basic_usage.py
+moon run repo:build      # sdist + wheel into dist/
+```
+
+`repo:validate`, `repo:clickhouse-start` / `repo:clickhouse-stop` and `repo:validate-clickhouse-*` target a persistent local server on :8123 (see [moon.yml](/moon.yml)).
+
+## Toolchain
+
+- proto pins uv (`.prototools`); uv owns Python. `.python-version` pins 3.11 and wins. If `UV_PYTHON` is exported in the shell, it overrides the pin, so run `env -u UV_PYTHON uv ...` or the venv is built on the wrong interpreter.
+- Releases: write `docs/release-notes/<next>.md` (narrative paragraph + Keep a Changelog bullets, one line each) in the PR before releasing; semantic-release (`npm run release`) refuses to release without it, runs `uv version <next> --frozen`, and `scripts/publish-release-notes.sh` rebuilds the GitHub Release body as narrative + generated section. pyproject.toml is the only place the version lives; `__version__` reads installed metadata. Publishing is `.github/workflows/publish.yml` (PyPI Trusted Publishing on a GitHub Release), the only publish path.
+- Dependency update PRs come from `.github/dependabot.yml`. No other CI runs in Actions; gates are local.
+
+## Invariants
+
+- `ensure_database_bootstrap()` must run before the first `get_client()` that binds the `exness` database (`tests/test_database_bootstrap_order.py`).
+- Insert DataFrames with `client.insert_df(...)`, never `client.insert(data=df)` (that takes row sequences).
+- Tables are ReplacingMergeTree. Tick tables dedupe on `(instrument, timestamp, seq)`: `seq` (`assign_tick_seq`) numbers same-timestamp ticks in source order, so distinct ticks sharing a timestamp are kept and re-inserting a month is idempotent. Duplicates only collapse at merge time, so every read whose answer depends on duplicates (counts, tick/OHLC rows, the OHLC build) uses `FINAL`; keep it that way.
+- Datetime query parameters are typed `DateTime64(6, 'UTC')`, never bare `DateTime64`: a bare one parses naive text in the server's time zone and shifts every filter on a non-UTC server. `repo:e2e` runs its server in America/New_York to catch this.
+- ClickHouse-backed tests are marked `clickhouse` or use `processor_with_clickhouse` (fixture users get the marker at collection, so `-m clickhouse` selects them); `conftest.py` skips them with the probe's reason unless `EXNESS_REQUIRE_CLICKHOUSE=1`. A server that holds :8123 but rejects the credentials (el02's shared system ClickHouse) counts as unavailable. Use `repo:e2e`, not that server.
+- `gapless-crypto-clickhouse` also uses localhost:8123. `clickhouse-stop` only stops the PID in this repo's pidfile.
+- ADRs and design specs under `docs/adr/` and `docs/design/` are historical records: add new ones, don't rewrite old ones.

exness_data_preprocess-3.0.0/CONTRIBUTING.md

@@ -0,0 +1,32 @@
+# Contributing
+
+## Setup
+
+```bash
+git clone https://github.com/terrylica/exness-data-preprocess.git
+cd exness-data-preprocess
+uv sync --dev                 # Python version comes from .python-version
+```
+
+You also need [moon](https://moonrepo.dev/) (pinned through proto) for the task runner and a `clickhouse` binary on `PATH` for the end-to-end run.
+
+## Before you open a pull request
+
+```bash
+moon run repo:check   # ruff check, ruff format --check, mypy, pytest
+moon run repo:e2e     # throwaway ClickHouse + full pytest + validate checks + examples
+```
+
+Both gates run locally; GitHub Actions only publishes releases and runs Dependabot. See [tests/README.md](/tests/README.md) for how ClickHouse-backed tests are selected and skipped.
+
+## Commits and releases
+
+Commit messages follow [Conventional Commits](https://www.conventionalcommits.org/) (`feat:`, `fix:`, `docs:`, `test:`, `refactor:`, `perf:`, `chore:`, `ci:`); the commitizen pre-commit hook checks them (`uv run pre-commit install --hook-type commit-msg --hook-type pre-push`). semantic-release derives the next version and the changelog from them, so never edit the version or `CHANGELOG.md` by hand.
+
+## Reporting issues
+
+Include the package version (`uv pip show exness-data-preprocess`), Python version, OS, a minimal reproduction, and the full traceback.
+
+## License
+
+Contributions are licensed under the MIT License.

{exness_data_preprocess-2.1.0 → exness_data_preprocess-3.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: exness-data-preprocess
-Version: 2.1.0
+Version: 3.0.0
 Summary: Professional Exness forex tick data preprocessing with ClickHouse backend. Provides efficient storage with lossless precision and direct queryability.
 Project-URL: Homepage, https://github.com/terrylica/exness-data-preprocess
 Project-URL: Documentation, https://github.com/terrylica/exness-data-preprocess#readme
@@ -26,16 +26,15 @@ Classifier: Topic :: Office/Business :: Financial :: Investment
 Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.11
-Requires-Dist: clickhouse-connect>=0.7.0
+Requires-Dist: clickhouse-connect>=1.11.0
 Requires-Dist: exchange-calendars>=4.5.6
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: pydantic>=2.0.0
-Requires-Dist: pyyaml>=6.0.0
 Provides-Extra: examples
 Requires-Dist: tqdm>=4.65.0; extra == 'examples'
 Description-Content-Type: text/markdown
 
-# exness-data-preprocess v2.0.0
+# exness-data-preprocess
 
 Forex tick data preprocessing with ClickHouse backend. Downloads, stores, and queries tick data from Exness public repository.
 
@@ -83,16 +82,16 @@ import exness_data_preprocess as edp
 # Context manager (recommended)
 with edp.ExnessDataProcessor() as processor:
     # Query ticks
-    df = processor.query_ticks('EURUSD', 'raw_spread', '2024-01-01', '2024-01-31')
+    df = processor.query_ticks("EURUSD", "raw_spread", "2024-01-01", "2024-01-31")
 
     # Query OHLC
-    df = processor.query_ohlc('EURUSD', '1m', '2024-01-01', '2024-01-31')
+    df = processor.query_ohlc("EURUSD", "1m", "2024-01-01", "2024-01-31")
 
     # Get coverage info
-    cov = processor.get_data_coverage('EURUSD')
+    cov = processor.get_data_coverage("EURUSD")
 
     # Download new data (incremental)
-    result = processor.update_data('EURUSD', start_date='2022-01-01')
+    result = processor.update_data("EURUSD", start_date="2022-01-01")
 ```
 
 ### API Methods
@@ -108,19 +107,19 @@ with edp.ExnessDataProcessor() as processor:
 
 ```python
 # UpdateResult
-result.months_added        # int: months downloaded
-result.raw_ticks_added     # int: raw spread ticks added
-result.standard_ticks_added # int: standard ticks added
-result.ohlc_bars           # int: OHLC bars generated
-result.storage_bytes       # int: storage used
+result.months_added  # int: months downloaded
+result.raw_ticks_added  # int: raw spread ticks added
+result.standard_ticks_added  # int: standard ticks added
+result.ohlc_bars  # int: OHLC bars generated
+result.storage_bytes  # int: storage used
 
 # CoverageInfo
-cov.raw_spread_ticks       # int: total raw spread ticks
-cov.standard_ticks         # int: total standard ticks
-cov.ohlc_bars              # int: total OHLC bars
-cov.earliest_date          # str: earliest timestamp
-cov.latest_date            # str: latest timestamp
-cov.storage_bytes          # int: storage used
+cov.raw_spread_ticks  # int: total raw spread ticks
+cov.standard_ticks  # int: total standard ticks
+cov.ohlc_bars  # int: total OHLC bars
+cov.earliest_date  # str: earliest timestamp
+cov.latest_date  # str: latest timestamp
+cov.storage_bytes  # int: storage used
 ```
 
 ## Direct SQL Access
@@ -170,8 +169,8 @@ ORDER BY ts"
 | ---------- | ---------------------- | ------------------- |
 | instrument | LowCardinality(String) | -                   |
 | timestamp  | DateTime64(6, 'UTC')   | DoubleDelta, LZ4    |
-| bid        | Float64                | Gorilla(8), ZSTD(1) |
-| ask        | Float64                | Gorilla(8), ZSTD(1) |
+| bid        | Float64                | Gorilla, ZSTD       |
+| ask        | Float64                | Gorilla, ZSTD       |
 
 ### OHLC Table
 
@@ -218,8 +217,8 @@ uv run mypy src/
 ## Documentation
 
 - [ClickHouse User Guide](/docs/CLICKHOUSE_USER_GUIDE.md) - Detailed ClickHouse usage
-- [Database Schema](/docs/DATABASE_SCHEMA.md) - 27-column OHLC specification
-- [Module Architecture](/docs/MODULE_ARCHITECTURE.md) - 13 modules with SLOs
+- [Schema](#schema) - summary above; the DDL in [`clickhouse_manager.py`](/src/exness_data_preprocess/clickhouse_manager.py) is authoritative
+- [Exness Data Sources](/docs/EXNESS_DATA_SOURCES.md) - archive layout and variants
 - [ADR: ClickHouse Migration](/docs/adr/2025-12-11-duckdb-removal-clickhouse.md) - Architecture decision
 
 ## License

{exness_data_preprocess-2.1.0 → exness_data_preprocess-3.0.0}/README.md

@@ -1,4 +1,4 @@
-# exness-data-preprocess v2.0.0
+# exness-data-preprocess
 
 Forex tick data preprocessing with ClickHouse backend. Downloads, stores, and queries tick data from Exness public repository.
 
@@ -46,16 +46,16 @@ import exness_data_preprocess as edp
 # Context manager (recommended)
 with edp.ExnessDataProcessor() as processor:
     # Query ticks
-    df = processor.query_ticks('EURUSD', 'raw_spread', '2024-01-01', '2024-01-31')
+    df = processor.query_ticks("EURUSD", "raw_spread", "2024-01-01", "2024-01-31")
 
     # Query OHLC
-    df = processor.query_ohlc('EURUSD', '1m', '2024-01-01', '2024-01-31')
+    df = processor.query_ohlc("EURUSD", "1m", "2024-01-01", "2024-01-31")
 
     # Get coverage info
-    cov = processor.get_data_coverage('EURUSD')
+    cov = processor.get_data_coverage("EURUSD")
 
     # Download new data (incremental)
-    result = processor.update_data('EURUSD', start_date='2022-01-01')
+    result = processor.update_data("EURUSD", start_date="2022-01-01")
 ```
 
 ### API Methods
@@ -71,19 +71,19 @@ with edp.ExnessDataProcessor() as processor:
 
 ```python
 # UpdateResult
-result.months_added        # int: months downloaded
-result.raw_ticks_added     # int: raw spread ticks added
-result.standard_ticks_added # int: standard ticks added
-result.ohlc_bars           # int: OHLC bars generated
-result.storage_bytes       # int: storage used
+result.months_added  # int: months downloaded
+result.raw_ticks_added  # int: raw spread ticks added
+result.standard_ticks_added  # int: standard ticks added
+result.ohlc_bars  # int: OHLC bars generated
+result.storage_bytes  # int: storage used
 
 # CoverageInfo
-cov.raw_spread_ticks       # int: total raw spread ticks
-cov.standard_ticks         # int: total standard ticks
-cov.ohlc_bars              # int: total OHLC bars
-cov.earliest_date          # str: earliest timestamp
-cov.latest_date            # str: latest timestamp
-cov.storage_bytes          # int: storage used
+cov.raw_spread_ticks  # int: total raw spread ticks
+cov.standard_ticks  # int: total standard ticks
+cov.ohlc_bars  # int: total OHLC bars
+cov.earliest_date  # str: earliest timestamp
+cov.latest_date  # str: latest timestamp
+cov.storage_bytes  # int: storage used
 ```
 
 ## Direct SQL Access
@@ -133,8 +133,8 @@ ORDER BY ts"
 | ---------- | ---------------------- | ------------------- |
 | instrument | LowCardinality(String) | -                   |
 | timestamp  | DateTime64(6, 'UTC')   | DoubleDelta, LZ4    |
-| bid        | Float64                | Gorilla(8), ZSTD(1) |
-| ask        | Float64                | Gorilla(8), ZSTD(1) |
+| bid        | Float64                | Gorilla, ZSTD       |
+| ask        | Float64                | Gorilla, ZSTD       |
 
 ### OHLC Table
 
@@ -181,8 +181,8 @@ uv run mypy src/
 ## Documentation
 
 - [ClickHouse User Guide](/docs/CLICKHOUSE_USER_GUIDE.md) - Detailed ClickHouse usage
-- [Database Schema](/docs/DATABASE_SCHEMA.md) - 27-column OHLC specification
-- [Module Architecture](/docs/MODULE_ARCHITECTURE.md) - 13 modules with SLOs
+- [Schema](#schema) - summary above; the DDL in [`clickhouse_manager.py`](/src/exness_data_preprocess/clickhouse_manager.py) is authoritative
+- [Exness Data Sources](/docs/EXNESS_DATA_SOURCES.md) - archive layout and variants
 - [ADR: ClickHouse Migration](/docs/adr/2025-12-11-duckdb-removal-clickhouse.md) - Architecture decision
 
 ## License

{exness_data_preprocess-2.1.0 → exness_data_preprocess-3.0.0}/docs/CLICKHOUSE_USER_GUIDE.md

@@ -1,4 +1,4 @@
-# ClickHouse User Guide - exness-data-preprocess v2.0.0
+# ClickHouse User Guide - exness-data-preprocess
 
 **Package**: `exness-data-preprocess` - Professional forex tick data preprocessing with ClickHouse backend
 
@@ -17,9 +17,9 @@
 5. [Querying OHLC Data](#querying-ohlc-data)
 6. [Data Coverage](#data-coverage)
 7. [Updating Data](#updating-data)
-8. [26-Column OHLC Schema](#26-column-ohlc-schema)
+8. [27-Column OHLC Schema](#27-column-ohlc-schema)
 9. [Direct ClickHouse SQL Patterns](#direct-clickhouse-sql-patterns)
-10. [mise Tasks for ClickHouse](#mise-tasks-for-clickhouse)
+10. [moon Tasks for ClickHouse](#moon-tasks-for-clickhouse)
 11. [Environment Configuration](#environment-configuration)
 12. [Pagination Patterns](#pagination-patterns)
 13. [Troubleshooting](#troubleshooting)
@@ -66,10 +66,10 @@ processor.close()
 
 ```bash
 # Start ClickHouse server
-mise run clickhouse:start
+moon run repo:clickhouse-start
 
 # Verify connection
-mise run clickhouse:status
+uv run python scripts/validate_clickhouse.py status
 ```
 
 Default connection: `localhost:8123` (HTTP interface)
@@ -183,7 +183,7 @@ df_std = processor.query_ticks(
 )
 
 # Calculate spread statistics
-df_std['spread'] = df_std['ask'] - df_std['bid']
+df_std["spread"] = df_std["ask"] - df_std["bid"]
 print(f"Mean spread: {df_std['spread'].mean() * 10000:.4f} pips")
 ```
 
@@ -356,9 +356,9 @@ print(f"Gap months: {', '.join(result.gap_months)}")
 
 ---
 
-## 26-Column OHLC Schema
+## 27-Column OHLC Schema
 
-The `ohlc_1m` table contains 26 columns organized into categories:
+The `ohlc_1m` table contains 27 columns organized into categories:
 
 ### Core OHLC (5 columns)
 
@@ -542,53 +542,51 @@ ORDER BY instrument;
 
 ---
 
-## mise Tasks for ClickHouse
+## moon Tasks for ClickHouse
 
 ### Server Management
 
 ```bash
-# Start ClickHouse server (mise-installed)
-mise run clickhouse:start
+# Start ClickHouse server (clickhouse binary on PATH)
+moon run repo:clickhouse-start
 
-# Stop ClickHouse server
-mise run clickhouse:stop
+# Stop the server this repo started (by pidfile)
+moon run repo:clickhouse-stop
 
 # Check connection status
-mise run clickhouse:status
-
-# Fail fast if ClickHouse not running
-mise run clickhouse:ensure
+uv run python scripts/validate_clickhouse.py status
 ```
 
+The server listens on `localhost:8123`. `gapless-crypto-clickhouse` runs its own local server on the same port with a different data directory, so only one of the two can be up at a time.
+
 ### Validation Tasks
 
 ```bash
-# Run full E2E validation (requires ClickHouse)
-mise run validate
+# Run full local validation: lint + typecheck + tests + build + ClickHouse E2E (requires ClickHouse)
+moon run repo:validate
 
 # Validate ClickHouse schema
-mise run validate:clickhouse-schema
+moon run repo:validate-clickhouse-schema
 
 # Validate data queries
-mise run validate:clickhouse-data
+moon run repo:validate-clickhouse-data
 
 # Validate pagination
-mise run validate:clickhouse-pagination
+moon run repo:validate-clickhouse-pagination
 
 # Validate OHLC queries
-mise run validate:clickhouse-ohlc
+moon run repo:validate-clickhouse-ohlc
 ```
 
 ### Quick Reference
 
-| Task                           | Description                     |
-| ------------------------------ | ------------------------------- |
-| `mise run clickhouse:start`    | Start mise-installed ClickHouse |
-| `mise run clickhouse:stop`     | Stop ClickHouse server          |
-| `mise run clickhouse:status`   | Check Python client connection  |
-| `mise run clickhouse:ensure`   | Fail fast if not running        |
-| `mise run validate`            | Full E2E validation pipeline    |
-| `mise run validate:clickhouse` | All ClickHouse validations      |
+| Task                                | Description                                 |
+| ----------------------------------- | ------------------------------------------- |
+| `moon run repo:clickhouse-start`    | Start the ClickHouse server found on PATH   |
+| `moon run repo:clickhouse-stop`     | Stop the server this repo started           |
+| `moon run repo:check`               | Offline gate: lint + typecheck + tests      |
+| `moon run repo:validate`            | Full E2E validation pipeline                |
+| `moon run repo:validate-clickhouse` | All ClickHouse validations                  |
 
 ---
 
@@ -605,18 +603,7 @@ mise run validate:clickhouse-ohlc
 | `CLICKHOUSE_PASSWORD` | (empty)                        | Password           |
 | `CLICKHOUSE_DATABASE` | `exness`                       | Default database   |
 
-### mise Configuration
-
-Configuration is defined in `.mise.toml`:
-
-```toml
-[env]
-CLICKHOUSE_NAME = "exness-clickhouse-local"
-CLICKHOUSE_MODE = "local"
-CLICKHOUSE_HOST = "localhost"
-CLICKHOUSE_PORT = "8123"
-CLICKHOUSE_DATABASE = "exness"
-```
+The variables above are read with these defaults, so local mode needs no configuration.
 
 ---
 
@@ -690,7 +677,7 @@ ERROR: ClickHouse not running on port 8123
 **Solution**:
 
 ```bash
-mise run clickhouse:start
+moon run repo:clickhouse-start
 ```
 
 ### Connection Refused
@@ -715,7 +702,7 @@ If queries return empty DataFrames:
 
 ### Schema Mismatch
 
-If OHLC columns don't match expected 26-column schema:
+If OHLC columns don't match expected 27-column schema:
 
 ```bash
 # Regenerate OHLC for instrument

{exness_data_preprocess-2.1.0 → exness_data_preprocess-3.0.0}/docs/EXNESS_DATA_SOURCES.md

@@ -203,6 +203,7 @@ done
 import requests
 from pathlib import Path
 
+
 def download_exness_variant(pair: str, year: int, month: int, variant: str = "") -> Path:
     """
     Download Exness tick data for specific variant.
@@ -224,13 +225,14 @@ def download_exness_variant(pair: str, year: int, month: int, variant: str = "")
     response = requests.get(url, stream=True)
     response.raise_for_status()
 
-    with open(output, 'wb') as f:
+    with open(output, "wb") as f:
         for chunk in response.iter_content(chunk_size=8192):
             f.write(chunk)
 
     print(f"✅ Downloaded: {output} ({output.stat().st_size / 1024 / 1024:.2f} MB)")
     return output
 
+
 # Example: Phase7 dual-variant download
 raw_spread = download_exness_variant("EURUSD", 2024, 9, "Raw_Spread")
 standard = download_exness_variant("EURUSD", 2024, 9)