exness-data-preprocess 3.0.0__tar.gz → 3.0.1__tar.gz

{exness_data_preprocess-3.0.0 → exness_data_preprocess-3.0.1}/.prototools

@@ -2,4 +2,4 @@
 # uv owns Python: the interpreter version is .python-version, and uv.toml makes uv use only its own
 # managed build. ClickHouse comes from the system PATH (e.g. Homebrew), not from proto.
 uv = "latest"
-moon = "2.6.1"
+moon = "2.6.2"

exness_data_preprocess-3.0.1/.python-version

@@ -0,0 +1 @@
+3.15

{exness_data_preprocess-3.0.0 → exness_data_preprocess-3.0.1}/CHANGELOG.md

@@ -1,3 +1,10 @@
+## [3.0.1](https://github.com/terrylica/exness-data-preprocess/compare/v3.0.0...v3.0.1) (2026-10-19)
+
+
+### Bug Fixes
+
+* **ohlc:** compute tick_count_standard and standard_spread_avg per minute from standard_ticks ([a26ec2c](https://github.com/terrylica/exness-data-preprocess/commit/a26ec2cca0e8e2b326c8b5c6f93f2c9f2a658986)), closes [#25](https://github.com/terrylica/exness-data-preprocess/issues/25)
+
 # [3.0.0](https://github.com/terrylica/exness-data-preprocess/compare/v2.1.1...v3.0.0) (2026-10-19)
 
 

{exness_data_preprocess-3.0.0 → exness_data_preprocess-3.0.1}/CLAUDE.md

@@ -16,7 +16,7 @@ moon run repo:build      # sdist + wheel into dist/
 
 ## Toolchain
 
-- proto pins uv (`.prototools`); uv owns Python. `.python-version` pins 3.11 and wins. If `UV_PYTHON` is exported in the shell, it overrides the pin, so run `env -u UV_PYTHON uv ...` or the venv is built on the wrong interpreter.
+- proto pins uv (`latest`) and moon (`.prototools`); uv owns Python. Development runs on the latest CPython (`.python-version`, currently 3.15); the published floor stays `requires-python >=3.11`, and ruff/mypy target 3.11 so code keeps working there. Never export `UV_PYTHON`: an env var outranks `.python-version`.
 - Releases: write `docs/release-notes/<next>.md` (narrative paragraph + Keep a Changelog bullets, one line each) in the PR before releasing; semantic-release (`npm run release`) refuses to release without it, runs `uv version <next> --frozen`, and `scripts/publish-release-notes.sh` rebuilds the GitHub Release body as narrative + generated section. pyproject.toml is the only place the version lives; `__version__` reads installed metadata. Publishing is `.github/workflows/publish.yml` (PyPI Trusted Publishing on a GitHub Release), the only publish path.
 - Dependency update PRs come from `.github/dependabot.yml`. No other CI runs in Actions; gates are local.
 

{exness_data_preprocess-3.0.0 → exness_data_preprocess-3.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: exness-data-preprocess
-Version: 3.0.0
+Version: 3.0.1
 Summary: Professional Exness forex tick data preprocessing with ClickHouse backend. Provides efficient storage with lossless precision and direct queryability.
 Project-URL: Homepage, https://github.com/terrylica/exness-data-preprocess
 Project-URL: Documentation, https://github.com/terrylica/exness-data-preprocess#readme
@@ -22,6 +22,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.15
 Classifier: Topic :: Office/Business :: Financial :: Investment
 Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Classifier: Topic :: Software Development :: Libraries :: Python Modules

exness_data_preprocess-3.0.1/docs/release-notes/3.0.1.md

@@ -0,0 +1,18 @@
+## 3.0.1 (2026-10-19)
+
+Two `ohlc_1m` columns did not measure what their documentation says. `tick_count_standard` is documented as the number of Standard ticks in the bar, but it always equalled the Raw_Spread count: it was computed as `countIf(... IS NOT NULL)` over an ASOF join that never produces NULLs. `standard_spread_avg` is documented as the average Standard spread in the bar, but it averaged whichever Standard quote preceded each Raw_Spread tick, and the choice among tied quotes could change between rebuilds. Each variant is now aggregated per minute on its own and the two are joined on the minute, so both columns match their definitions, rebuilds are deterministic, and the build no longer runs a per-tick join. On the E2E months the counts now differ on 64,889 of 78,658 bars, where before they were identical on every bar.
+
+### Upgrading
+
+- Bars already stored keep the old values until they are rebuilt. Rebuild an instrument's bars with `update_data(pair, start_date=..., force_redownload=True)`, which re-downloads and regenerates from `start_date`, or call `ExnessDataProcessor().ch_ohlc_generator.regenerate_ohlc(pair)` to rebuild every bar from the stored ticks.
+
+### Fixed
+
+- `tick_count_standard` counts the Standard ticks in the bar's minute (0 when there are none) (#25).
+- `standard_spread_avg` is the mean Standard `ask - bid` in the bar's minute (NULL when there are none), deterministic across rebuilds (#25).
+- `repo:e2e` checks that `sum(tick_count_raw_spread)` equals the stored Raw_Spread ticks and `sum(tick_count_standard)` equals the Standard ticks in bar minutes, that the two counts differ, and that a sampled `standard_spread_avg` equals a direct per-minute average (#25).
+- Stale docstrings describing the v1 30-column schema, normalized metrics and the ASOF join are corrected (#25).
+
+### Changed
+
+- Development runs on Python 3.15.0 and moon 2.6.2; the supported range is unchanged (`requires-python >=3.11`) (#27, #28).

{exness_data_preprocess-3.0.0 → exness_data_preprocess-3.0.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "exness-data-preprocess"
-version = "3.0.0"
+version = "3.0.1"
 description = "Professional Exness forex tick data preprocessing with ClickHouse backend. Provides efficient storage with lossless precision and direct queryability."
 readme = { file = "README.md", content-type = "text/markdown" }
 authors = [
@@ -26,6 +26,7 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.15",
     "Topic :: Office/Business :: Financial :: Investment",
     "Topic :: Scientific/Engineering :: Information Analysis",
     "Topic :: Software Development :: Libraries :: Python Modules",
@@ -81,10 +82,6 @@ ignore = [
 [tool.ruff.lint.isort]
 known-first-party = ["exness_data_preprocess"]
 
-[tool.black]
-target-version = ['py311']
-line-length = 100
-
 [tool.mypy]
 python_version = "3.11"
 warn_return_any = true
@@ -97,10 +94,6 @@ no_implicit_optional = true
 strict_optional = true
 ignore_missing_imports = true
 
-[[tool.mypy.overrides]]
-module = "tests.*"
-ignore_errors = true
-
 [tool.pytest.ini_options]
 minversion = "7.0"
 addopts = "-ra -q --strict-markers"

{exness_data_preprocess-3.0.0 → exness_data_preprocess-3.0.1}/scripts/validate_clickhouse.py

@@ -233,6 +233,51 @@ def check_ties() -> None:
     print("✅ Tie pagination passed")
 
 
+def check_ohlc_counts() -> None:
+    """ohlc_1m tick counts and Standard spread match their definitions (per minute, per variant)."""
+    print("🔍 Validating OHLC tick counts and Standard spread...")
+    from exness_data_preprocess import get_clickhouse_client
+
+    client = get_clickhouse_client()
+    try:
+        sum_std, sum_raw, differing = client.query(
+            "SELECT sum(tick_count_standard), sum(tick_count_raw_spread),"
+            " countIf(tick_count_standard != tick_count_raw_spread)"
+            " FROM exness.ohlc_1m FINAL WHERE instrument = 'EURUSD'"
+        ).first_row
+        raw_total = client.query(
+            "SELECT count() FROM exness.raw_spread_ticks FINAL WHERE instrument = 'EURUSD'"
+        ).first_row[0]
+        std_in_bars = client.query(
+            "SELECT count() FROM exness.standard_ticks FINAL WHERE instrument = 'EURUSD'"
+            " AND toStartOfMinute(timestamp) IN"
+            " (SELECT timestamp FROM exness.ohlc_1m FINAL WHERE instrument = 'EURUSD')"
+        ).first_row[0]
+        minute, bar_avg, direct_avg = client.query(
+            "SELECT o.timestamp, o.standard_spread_avg, d.avg_spread FROM exness.ohlc_1m AS o FINAL"
+            " INNER JOIN (SELECT toStartOfMinute(timestamp) AS m, avg(ask - bid) AS avg_spread, count() AS n"
+            " FROM exness.standard_ticks FINAL WHERE instrument = 'EURUSD' GROUP BY m"
+            " ORDER BY n DESC, m LIMIT 1) AS d ON o.timestamp = d.m WHERE o.instrument = 'EURUSD'"
+        ).first_row
+    finally:
+        client.close()
+    assert sum_raw == raw_total, (
+        f"sum(tick_count_raw_spread)={sum_raw:,} != stored Raw_Spread ticks {raw_total:,}"
+    )
+    assert sum_std == std_in_bars, (
+        f"sum(tick_count_standard)={sum_std:,} != Standard ticks in bar minutes {std_in_bars:,}"
+    )
+    assert differing > 0, "tick_count_standard equals tick_count_raw_spread on every bar"
+    assert abs(bar_avg - direct_avg) < 1e-12, (
+        f"standard_spread_avg {bar_avg} != direct {direct_avg} at {minute}"
+    )
+    print(
+        f"  ✓ sum(tick_count_raw_spread)={sum_raw:,} = stored; sum(tick_count_standard)={sum_std:,} ="
+        f" Standard ticks in bar minutes; counts differ on {differing:,} bars; spread avg exact at {minute}"
+    )
+    print("✅ OHLC count validation passed")
+
+
 def check_data() -> None:
     print("🔍 Validating ClickHouse data operations...")
     from exness_data_preprocess import ClickHouseQueryEngine
@@ -318,6 +363,7 @@ CHECKS: dict[str, Callable[[], None]] = {
     "pagination": check_pagination,
     "ties": check_ties,
     "ohlc": check_ohlc,
+    "ohlc_counts": check_ohlc_counts,
 }
 
 

{exness_data_preprocess-3.0.0 → exness_data_preprocess-3.0.1}/src/exness_data_preprocess/clickhouse_manager.py

@@ -242,7 +242,7 @@ class ClickHouseManager(ClickHouseClientMixin):
             "standard_ticks",
             "Standard tick data from Exness. Columns: instrument (forex pair), timestamp "
             "(microsecond), seq (position among same-timestamp ticks in source order), bid/ask "
-            "(traditional quotes, Bid < Ask). ASOF merged with raw_spread_ticks.",
+            "(traditional quotes, Bid < Ask). Aggregated per minute into ohlc_1m.",
         )
 
     def _create_ticks_table(self, table: str, comment: str) -> None:

{exness_data_preprocess-3.0.0 → exness_data_preprocess-3.0.1}/src/exness_data_preprocess/clickhouse_ohlc_generator.py

@@ -5,15 +5,14 @@ ADR: /docs/adr/2025-12-09-exness-clickhouse-migration.md
 
 SLOs:
 - Availability: OHLC generation must succeed or raise exception (no silent failures)
-- Correctness: Phase7 30-column schema with dual variants, normalized metrics, holiday/session detection
+- Correctness: 27-column ohlc_1m with dual-variant spreads and tick counts, holiday/session detection
 - Observability: All OHLC operations logged via structured logging
 - Maintainability: Single module for all OHLC logic, off-the-shelf ClickHouse and exchange_calendars
 
 Handles:
-- Phase7 30-column OHLC schema
+- 27-column OHLC schema (ohlc_1m)
 - BID-only OHLC construction from raw_spread_ticks
-- ASOF JOIN with standard_ticks for dual spread tracking
-- Normalized metrics (range_per_spread, range_per_tick, body_per_spread, body_per_tick)
+- Per-minute Standard spread and tick count from standard_ticks, joined on the minute
 - Timezone/session tracking (NY and London sessions)
 - Holiday detection (US, UK, major holidays via exchange_calendars)
 - 10 global exchange session flags with trading hour detection
@@ -33,12 +32,11 @@ from exness_data_preprocess.session_detector import SessionDetector
 
 class ClickHouseOHLCGenerator(ClickHouseClientMixin):
     """
-    Generate Phase7 30-column OHLC from dual-variant tick data in ClickHouse.
+    Generate 27-column 1-minute OHLC from dual-variant tick data in ClickHouse.
 
     Responsibilities:
     - Delete existing OHLC data before regeneration (by instrument)
-    - Generate BID-only OHLC from raw_spread_ticks with ASOF JOIN to standard_ticks
-    - Calculate normalized spread metrics (NULL-safe)
+    - Generate BID-only OHLC from raw_spread_ticks plus per-minute Standard aggregates
     - Add timezone/session tracking for NY and London
     - Delegate holiday and session detection to SessionDetector
     - Insert OHLC data with holiday and session flags
@@ -69,11 +67,12 @@ class ClickHouseOHLCGenerator(ClickHouseClientMixin):
         end_date: str | None = None,
     ) -> int:
         """
-        Generate/update OHLC table with Phase7 schema (30 columns).
+        Generate/update ohlc_1m (27 columns).
 
-        Uses ASOF JOIN to combine raw_spread_ticks and standard_ticks.
-        Includes normalized spread metrics, timezone/session tracking,
-        holiday detection, and 10 global exchange session flags.
+        Raw_Spread and Standard ticks are each aggregated per minute and joined on the minute:
+        standard_spread_avg and tick_count_standard describe the Standard ticks in that minute
+        (NULL / 0 when there are none). Includes timezone/session tracking, holiday detection,
+        and 10 global exchange session flags.
 
         Supports three operation modes:
         1. Full regeneration (start_date=None, end_date=None): DELETE all + INSERT all
@@ -143,44 +142,42 @@ class ClickHouseOHLCGenerator(ClickHouseClientMixin):
             )
 
         where_sql = " AND ".join(where_parts)
-        standard_where = ["instrument = {instrument:String}"]
-        if start_date:
-            standard_where.append("timestamp >= {start_date:DateTime64(6, 'UTC')} - INTERVAL 7 DAY")
-        standard_where_sql = " AND ".join(standard_where)
 
         # Generate session column initializations from exchange registry
         session_columns = ", ".join([f"0 AS is_{name}_session" for name in EXCHANGES.keys()])
 
-        # Generate OHLC with 27-column schema (matches DATABASE_SCHEMA.md)
-        # Note: ClickHouse ASOF JOIN matches closest preceding timestamp from standard_ticks
+        # Each variant is aggregated per minute on its own and the two are joined on the minute.
+        # Bars exist for minutes with Raw_Spread ticks; standard_spread_avg / tick_count_standard
+        # describe the Standard ticks inside the same minute (NULL / 0 when there are none).
+        # Both subqueries apply the same instrument and date filter (alias r in each).
         insert_sql = f"""
             INSERT INTO {self.DATABASE}.ohlc_1m
             SELECT
                 {{instrument:String}} AS instrument,
-                toStartOfMinute(r.timestamp) AS timestamp,
-                argMin(r.bid, (r.timestamp, r.seq)) AS open,
-                max(r.bid) AS high,
-                min(r.bid) AS low,
-                argMax(r.bid, (r.timestamp, r.seq)) AS close,
-                avg(r.ask - r.bid) AS raw_spread_avg,
-                avg(s.ask - s.bid) AS standard_spread_avg,
-                toUInt32(count(r.timestamp)) AS tick_count_raw_spread,
-                toUInt32(countIf(s.timestamp IS NOT NULL)) AS tick_count_standard,
+                r.minute AS timestamp,
+                r.open AS open,
+                r.high AS high,
+                r.low AS low,
+                r.close AS close,
+                r.raw_spread_avg AS raw_spread_avg,
+                s.standard_spread_avg AS standard_spread_avg,
+                r.tick_count_raw_spread AS tick_count_raw_spread,
+                ifNull(s.tick_count_standard, toUInt32(0)) AS tick_count_standard,
                 -- Timezone-aware session columns
-                toUInt8(toHour(toTimezone(toStartOfMinute(r.timestamp), 'America/New_York'))) AS ny_hour,
-                toUInt8(toHour(toTimezone(toStartOfMinute(r.timestamp), 'Europe/London'))) AS london_hour,
+                toUInt8(toHour(toTimezone(r.minute, 'America/New_York'))) AS ny_hour,
+                toUInt8(toHour(toTimezone(r.minute, 'Europe/London'))) AS london_hour,
                 multiIf(
-                    toHour(toTimezone(toStartOfMinute(r.timestamp), 'America/New_York')) >= 9
-                    AND toHour(toTimezone(toStartOfMinute(r.timestamp), 'America/New_York')) <= 16,
+                    toHour(toTimezone(r.minute, 'America/New_York')) >= 9
+                    AND toHour(toTimezone(r.minute, 'America/New_York')) <= 16,
                     'NY_Session',
-                    toHour(toTimezone(toStartOfMinute(r.timestamp), 'America/New_York')) >= 17
-                    AND toHour(toTimezone(toStartOfMinute(r.timestamp), 'America/New_York')) <= 20,
+                    toHour(toTimezone(r.minute, 'America/New_York')) >= 17
+                    AND toHour(toTimezone(r.minute, 'America/New_York')) <= 20,
                     'NY_After_Hours',
                     'NY_Closed'
                 ) AS ny_session,
                 if(
-                    toHour(toTimezone(toStartOfMinute(r.timestamp), 'Europe/London')) >= 8
-                    AND toHour(toTimezone(toStartOfMinute(r.timestamp), 'Europe/London')) <= 16,
+                    toHour(toTimezone(r.minute, 'Europe/London')) >= 8
+                    AND toHour(toTimezone(r.minute, 'Europe/London')) <= 16,
                     'London_Session',
                     'London_Closed'
                 ) AS london_session,
@@ -190,20 +187,30 @@ class ClickHouseOHLCGenerator(ClickHouseClientMixin):
                 toUInt8(0) AS is_major_holiday,
                 -- Trading session flags (initialized to 0, updated via session_detector)
                 {session_columns}
-            FROM {self.DATABASE}.raw_spread_ticks AS r FINAL
-            ASOF LEFT JOIN (
-                -- Only this instrument, and only from a week before start_date: enough for the
-                -- closest preceding Standard tick across a weekend, without reading (and FINAL-
-                -- merging) every instrument and month on the join's build side.
-                SELECT instrument, timestamp, ask, bid
-                FROM {self.DATABASE}.standard_ticks FINAL
-                WHERE {standard_where_sql}
-            ) AS s
-                ON r.instrument = s.instrument
-                AND r.timestamp >= s.timestamp
-            WHERE {where_sql}
-            GROUP BY toStartOfMinute(r.timestamp)
+            FROM (
+                SELECT
+                    toStartOfMinute(r.timestamp) AS minute,
+                    argMin(r.bid, (r.timestamp, r.seq)) AS open,
+                    max(r.bid) AS high,
+                    min(r.bid) AS low,
+                    argMax(r.bid, (r.timestamp, r.seq)) AS close,
+                    avg(r.ask - r.bid) AS raw_spread_avg,
+                    toUInt32(count()) AS tick_count_raw_spread
+                FROM {self.DATABASE}.raw_spread_ticks AS r FINAL
+                WHERE {where_sql}
+                GROUP BY minute
+            ) AS r
+            LEFT JOIN (
+                SELECT
+                    toStartOfMinute(r.timestamp) AS minute,
+                    avg(r.ask - r.bid) AS standard_spread_avg,
+                    toUInt32(count()) AS tick_count_standard
+                FROM {self.DATABASE}.standard_ticks AS r FINAL
+                WHERE {where_sql}
+                GROUP BY minute
+            ) AS s ON r.minute = s.minute
             ORDER BY timestamp
+            SETTINGS join_use_nulls = 1
         """
 
         execute_command(self.client, insert_sql, parameters=params)

exness_data_preprocess-3.0.0/.python-version

@@ -1 +0,0 @@
-3.11