exness-data-preprocess 0.3.0__tar.gz → 0.3.1__tar.gz

{exness_data_preprocess-0.3.0 → exness_data_preprocess-0.3.1}/.cz.toml

@@ -1,6 +1,6 @@
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "0.2.0"
+version = "0.3.0"
 tag_format = "v$major.$minor.$patch"
 update_changelog_on_bump = true
 version_files = [

{exness_data_preprocess-0.3.0 → exness_data_preprocess-0.3.1}/.gitignore

@@ -72,6 +72,7 @@ data/
 *.zip
 *.csv
 temp/
+eurusd
 
 # macOS
 .DS_Store

{exness_data_preprocess-0.3.0 → exness_data_preprocess-0.3.1}/CHANGELOG.md

@@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
+### ♻️ Refactoring
+
+- **schema**: Centralize OHLC schema in schema.py module - Create schema.py with OHLCSchema class as single source of truth - Reduce coupling from 42 to 5 update locations (88% reduction) - Replace hardcoded column lists in docs with pointers to schema.py - Delete obsolete PYDANTIC_TEST_STRATEGY.md (tests implemented) - Delete add_schema_comments.py (functionality merged into processor) - Update tests to use OHLCSchema.get_required_columns() - Achieve maximum DRY: future column additions require 1-2 file updates BREAKING CHANGE: None (additive refactoring, no API changes)
+
+
 ### ✨ Features
 
 - Implement pydantic v2 models with dual-variant e2e testing - Add Pydantic v2 models (UpdateResult, CoverageInfo) for type-safe API - Update processor to use Pydantic types (PairType, TimeframeType, VariantType) - Remove deprecated api.py module (broken functionality) - Implement comprehensive test suite (48 tests, 100% passing): * test_models.py - Pydantic model validation (13 tests) * test_types.py - Type safety and helpers (15 tests) * test_processor_pydantic.py - Integration tests (6 tests) * test_functional_regression.py - v2.0.0 regression tests (10 tests) - Fix Standard variant downloads (variant="" not "Standard") - Add true end-to-end testing with real Exness downloads - Test data: EURUSD August 2024 (815K Raw_Spread + 877K Standard ticks) - Update documentation and add refactoring status tracking - Coverage: models.py 100%, __init__.py 100%, processor.py 45% BREAKING CHANGE: api.py removed (use ExnessDataProcessor directly)
@@ -20,6 +25,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - **release**: Bump version 0.1.0 → 0.2.0 [skip ci]
 
+- **release**: Bump version 0.2.0 → 0.3.0 [skip ci]
+
 
 ### ✨ Features
 

{exness_data_preprocess-0.3.0 → exness_data_preprocess-0.3.1}/CLAUDE.md

@@ -68,45 +68,105 @@ doppler run --project claude-config --config dev -- uv publish --token "$PYPI_TO
 
 ## Codebase Architecture
 
-### Module Structure
-
-**Core Components**:
-
-1. **`processor.py`** (`/Users/terryli/eon/exness-data-preprocess/src/exness_data_preprocess/processor.py`) - Main `ExnessDataProcessor` class
-   - **Lines 29-899**: Complete processor implementation
-   - **Lines 75-113**: `download_exness_zip()` - Downloads monthly ZIP files from ticks.ex2archive.com
-   - **Lines 115-232**: `_get_or_create_db()` - Database initialization with schema and COMMENT ON statements
-   - **Lines 234-245**: `_load_ticks_from_zip()` - CSV parsing from ZIP files
-   - **Lines 247-275**: `_append_ticks_to_db()` - Tick insertion with PRIMARY KEY duplicate prevention
-   - **Lines 277-377**: `_discover_missing_months()` - Gap detection for incremental updates
-   - **Lines 379-516**: `update_data()` - Main entry point for downloading and updating data
-   - **Lines 518-549**: `_regenerate_ohlc()` - Phase7 13-column OHLC (v1.2.0) generation with LEFT JOIN
-   - **Lines 551-603**: `query_ticks()` - Tick queries with date range and SQL filters
-   - **Lines 605-706**: `query_ohlc()` - OHLC queries with on-demand resampling (1m/5m/15m/1h/4h/1d)
-   - **Lines 708-821**: `add_schema_comments()` / `add_schema_comments_all()` - Retrofit self-documentation
-   - **Lines 823-899**: `get_data_coverage()` - Coverage statistics and metadata
-
-2. **`api.py`** (`/Users/terryli/eon/exness-data-preprocess/src/exness_data_preprocess/api.py`) - Simple wrapper functions
+### Module Structure (v1.3.0 - Facade Pattern)
+
+**Architecture**: Thin facade orchestrator with 7 focused modules (414 lines processor, 1,146 lines extracted)
+
+**Core Facade**:
+
+1. **`processor.py`** (`/Users/terryli/eon/exness-data-preprocess/src/exness_data_preprocess/processor.py`) - Thin orchestrator facade (414 lines)
+   - **Responsibility**: Coordinate workflow, delegate to specialized modules
+   - **Pattern**: Facade pattern - all public methods delegate to modules
+   - **Lines 76-110**: `__init__()` - Initialize 7 module dependencies
+   - **Lines 111-132**: `download_exness_zip()` - Delegates to downloader module
+   - **Lines 134-145**: `_get_or_create_db()` - Delegates to database_manager module
+   - **Lines 147-150**: `_load_ticks_from_zip()` - Delegates to tick_loader module
+   - **Lines 152-162**: `_append_ticks_to_db()` - Delegates to database_manager module
+   - **Lines 164-176**: `_discover_missing_months()` - Delegates to gap_detector module
+   - **Lines 178-315**: `update_data()` - Main workflow orchestrator
+   - **Lines 317-326**: `_regenerate_ohlc()` - Delegates to ohlc_generator module
+   - **Lines 328-357**: `query_ticks()` - Delegates to query_engine module
+   - **Lines 359-386**: `query_ohlc()` - Delegates to query_engine module
+   - **Lines 388-414**: `get_data_coverage()` - Delegates to query_engine module
+
+**Specialized Modules**:
+
+2. **`downloader.py`** (`/Users/terryli/eon/exness-data-preprocess/src/exness_data_preprocess/downloader.py`) - HTTP download operations (89 lines)
+   - **Responsibility**: Download Exness ZIP files from ticks.ex2archive.com
+   - **SLOs**: Availability (raise on failure), Correctness (URL patterns), Observability (logging), Maintainability (httpx library)
+   - **Class**: `ExnessDownloader`
+   - **Methods**: `download_zip(year, month, pair, variant)`
+
+3. **`tick_loader.py`** (`/Users/terryli/eon/exness-data-preprocess/src/exness_data_preprocess/tick_loader.py`) - CSV parsing (67 lines)
+   - **Responsibility**: Load tick data from ZIP files into pandas DataFrames
+   - **SLOs**: Availability (raise on failure), Correctness (timestamp parsing), Observability (logging), Maintainability (pandas library)
+   - **Class**: `TickLoader`
+   - **Methods**: `load_from_zip(zip_path)` (static method)
+
+4. **`database_manager.py`** (`/Users/terryli/eon/exness-data-preprocess/src/exness_data_preprocess/database_manager.py`) - Database operations (213 lines)
+   - **Responsibility**: Database initialization, schema creation, tick insertion with PRIMARY KEY duplicate prevention
+   - **SLOs**: Availability (raise on failure), Correctness (schema integrity), Observability (DuckDB logging), Maintainability (DuckDB library)
+   - **Class**: `DatabaseManager`
+   - **Methods**: `get_or_create_db(pair)`, `append_ticks(duckdb_path, df, table_name)`
+
+5. **`session_detector.py`** (`/Users/terryli/eon/exness-data-preprocess/src/exness_data_preprocess/session_detector.py`) - Holiday and session detection (121 lines)
+   - **Responsibility**: Detect holidays (US, UK, major) and trading sessions for 10 global exchanges using exchange_calendars
+   - **SLOs**: Availability (raise on failure), Correctness (official calendars), Observability (logging), Maintainability (exchange_calendars library)
+   - **Class**: `SessionDetector`
+   - **Methods**: `detect_sessions_and_holidays(dates_df)`
+
+6. **`gap_detector.py`** (`/Users/terryli/eon/exness-data-preprocess/src/exness_data_preprocess/gap_detector.py`) - Incremental update logic (163 lines)
+   - **Responsibility**: Discover missing months for incremental database updates
+   - **SLOs**: Availability (raise on failure), Correctness (gap detection), Observability (logging), Maintainability (DuckDB library)
+   - **Class**: `GapDetector`
+   - **Methods**: `discover_missing_months(pair, start_date)`
+
+7. **`ohlc_generator.py`** (`/Users/terryli/eon/exness-data-preprocess/src/exness_data_preprocess/ohlc_generator.py`) - OHLC generation (210 lines)
+   - **Responsibility**: Generate Phase7 30-column OHLC from dual-variant tick data with LEFT JOIN, normalized metrics, and exchange session detection
+   - **SLOs**: Availability (raise on failure), Correctness (Phase7 schema), Observability (logging), Maintainability (DuckDB + exchange_calendars)
+   - **Class**: `OHLCGenerator`
+   - **Methods**: `regenerate_ohlc(duckdb_path)`
+
+8. **`query_engine.py`** (`/Users/terryli/eon/exness-data-preprocess/src/exness_data_preprocess/query_engine.py`) - Query operations (283 lines)
+   - **Responsibility**: Query tick and OHLC data with date filtering, SQL filters, and on-demand resampling (1m/5m/15m/1h/4h/1d)
+   - **SLOs**: Availability (raise on failure), Correctness (SQL queries), Observability (DuckDB logging), Maintainability (DuckDB library)
+   - **Class**: `QueryEngine`
+   - **Methods**: `query_ticks()`, `query_ohlc()`, `get_data_coverage()`
+
+**Legacy Components**:
+
+9. **`api.py`** (`/Users/terryli/eon/exness-data-preprocess/src/exness_data_preprocess/api.py`) - Simple wrapper functions
    - Convenience functions wrapping `ExnessDataProcessor` methods
    - Legacy v1.0.0 API compatibility layer (to be deprecated)
 
-3. **`cli.py`** (`/Users/terryli/eon/exness-data-preprocess/src/exness_data_preprocess/cli.py`) - Command-line interface
-   - Entry point: `exness-preprocess` command
-   - Commands for download, query, coverage operations
+10. **`cli.py`** (`/Users/terryli/eon/exness-data-preprocess/src/exness_data_preprocess/cli.py`) - Command-line interface
+    - Entry point: `exness-preprocess` command
+    - Commands for download, query, coverage operations
 
 ### Key Design Patterns
 
+**Facade Pattern** (v1.3.0):
+- **processor.py** is a thin orchestrator (414 lines) coordinating 7 specialized modules
+- All public methods delegate to modules (no business logic in processor)
+- **Separation of concerns**: Each module has single responsibility
+- **SLO-based design**: All modules define SLOs (Availability, Correctness, Observability, Maintainability)
+- **Zero regressions**: All 48 tests pass after 7-module extraction (53% line reduction)
+- **Off-the-shelf libraries**: httpx, pandas, DuckDB, exchange_calendars (no custom implementations)
+
 **Unified Single-File Architecture** (v2.0.0):
 - ONE DuckDB file per instrument (e.g., `eurusd.duckdb`) containing all historical data
 - NO monthly file separation (major change from v1.0.0)
 - Incremental updates with automatic gap detection
 - PRIMARY KEY constraints prevent duplicates during updates
 
-**Phase7 13-Column OHLC Schema** (v1.2.0):
+**Phase7 30-Column OHLC Schema** (v1.5.0):
 - BID-only OHLC from Raw_Spread variant
 - Dual spreads: `raw_spread_avg` and `standard_spread_avg`
 - Dual tick counts: `tick_count_raw_spread` and `tick_count_standard`
 - Normalized metrics: `range_per_spread`, `range_per_tick`, `body_per_spread`, `body_per_tick`
+- Timezone/session tracking: `ny_hour`, `london_hour`, `ny_session`, `london_session` (v1.3.0+)
+- Holiday tracking: `is_us_holiday`, `is_uk_holiday`, `is_major_holiday` (v1.4.0+)
+- Global exchange sessions: 10 binary flags (`is_nyse_session`, `is_lse_session`, `is_xswx_session`, `is_xfra_session`, `is_xtse_session`, `is_xnze_session`, `is_xtks_session`, `is_xasx_session`, `is_xhkg_session`, `is_xses_session`) covering 24-hour forex trading (v1.5.0)
 - Generated via LEFT JOIN between Raw_Spread and Standard variants
 
 **Self-Documentation**:
@@ -114,14 +174,16 @@ doppler run --project claude-config --config dev -- uv publish --token "$PYPI_TO
 - Queryable via `duckdb_tables()` and `duckdb_columns()` system functions
 - Enables BI tools and IDEs to display inline help
 
-**Data Flow**:
+**Data Flow** (Module-Based Architecture v1.3.0):
 ```
 Exness Repository (monthly ZIPs)
-  → Automatic Gap Detection
-  → Download Missing Months (Raw_Spread + Standard)
-  → DuckDB Storage (PRIMARY KEY prevents duplicates)
-  → Phase7 OHLC Generation
-  → Query Interface (date ranges, SQL filters, resampling)
+  → gap_detector.py (discover missing months)
+  → downloader.py (download Raw_Spread + Standard)
+  → tick_loader.py (parse CSV from ZIP)
+  → database_manager.py (append to DuckDB, PRIMARY KEY prevents duplicates)
+  → ohlc_generator.py (Phase7 30-column OHLC with LEFT JOIN)
+  → session_detector.py (holidays + 10 exchange sessions)
+  → query_engine.py (tick/OHLC queries, date filters, resampling)
 ```
 
 ### Database Schema (per instrument)
@@ -139,8 +201,8 @@ Each `.duckdb` file contains:
    - Reference data for spread comparison
 
 3. **`ohlc_1m`** table:
-   - **Schema**: Phase7 13-column (v1.2.0) - See [`schema.py`](src/exness_data_preprocess/schema.py)
-   - **Details**: BID-only OHLC with dual-variant spreads, tick counts, and normalized metrics
+   - **Schema**: Phase7 30-column (v1.5.0) - See [`schema.py`](src/exness_data_preprocess/schema.py)
+   - **Details**: BID-only OHLC with dual-variant spreads, tick counts, normalized metrics, and 10 global exchange sessions
    - **Reference**: [`DATABASE_SCHEMA.md`](docs/DATABASE_SCHEMA.md)
 
 4. **`metadata`** table:
@@ -180,7 +242,7 @@ Each `.duckdb` file contains:
 - ✅ **Dual-variant storage**: Raw_Spread + Standard in same database
 - ✅ **PRIMARY KEY constraints**: Prevents duplicates during incremental updates
 - ✅ **Automatic gap detection**: Downloads only missing months
-- ✅ **Phase7 13-column OHLC (v1.2.0)**: Dual spreads + dual tick counts
+- ✅ **Phase7 30-column OHLC (v1.5.0)**: Dual spreads + dual tick counts + 10 global exchange sessions
 - ✅ **Date range queries**: Sub-15ms query performance
 - ✅ **On-demand resampling**: Any timeframe (5m, 1h, 1d) in <15ms
 
@@ -207,21 +269,29 @@ Each `.duckdb` file contains:
 - `/tmp/exness-duckdb-test/test_refactored_processor.py` - Validation test
 - `/tmp/exness-duckdb-test/test_queries_only.py` - Query validation test
 
-### Phase7 13-Column OHLC Schema v1.2.0 (✅ Implemented)
+### Phase7 30-Column OHLC Schema v1.5.0 (✅ Implemented)
 
-**Decision**: Dual-variant BID-only OHLC with 13 columns capturing both Raw_Spread and Standard characteristics plus normalized metrics
+**Decision**: Dual-variant BID-only OHLC with 30 columns capturing Raw_Spread and Standard characteristics, normalized metrics, and 10 global exchange sessions
 
-**Schema**: Phase7 13-column (v1.2.0)
+**Schema**: Phase7 30-column (v1.5.0)
 - **Definition**: See [`schema.py`](src/exness_data_preprocess/schema.py)
 - **Documentation**: See [`DATABASE_SCHEMA.md`](docs/DATABASE_SCHEMA.md)
+- **Architecture**: Exchange Registry Pattern for dynamic session column generation
 
 **Key Features**:
 - ✅ **BID-only OHLC**: Uses Raw_Spread Bid prices (execution prices)
 - ✅ **Dual spreads**: Tracks both Raw_Spread (zero-spreads) and Standard (market spreads)
 - ✅ **Dual tick counts**: Records tick counts from both variants
+- ✅ **Normalized metrics** (v1.2.0): range_per_spread, range_per_tick, body_per_spread, body_per_tick
+- ✅ **Timezone/session tracking** (v1.3.0): ny_hour, london_hour, ny_session, london_session with automatic DST handling
+- ✅ **Holiday tracking** (v1.4.0): is_us_holiday, is_uk_holiday, is_major_holiday via exchange_calendars
+- ✅ **10 Global Exchange Sessions** (v1.5.0): is_nyse_session, is_lse_session, is_xswx_session, is_xfra_session, is_xtse_session, is_xnze_session, is_xtks_session, is_xasx_session, is_xhkg_session, is_xses_session covering 24-hour forex trading
 - ✅ **LEFT JOIN methodology**: Raw_Spread primary, Standard reference
 
-**Implementation**: [`src/exness_data_preprocess/processor.py`](src/exness_data_preprocess/processor.py) (lines 518-549)
+**Implementation**:
+- **OHLC Generation**: [`src/exness_data_preprocess/ohlc_generator.py`](src/exness_data_preprocess/ohlc_generator.py) (210 lines)
+- **Session Detection**: [`src/exness_data_preprocess/session_detector.py`](src/exness_data_preprocess/session_detector.py) (121 lines)
+- **Orchestration**: [`src/exness_data_preprocess/processor.py`](src/exness_data_preprocess/processor.py) (lines 317-326, delegation)
 
 **Specification**: [`docs/research/eurusd-zero-spread-deviations/data/plan/phase7_bid_ohlc_construction_v1.1.0.md`](docs/research/eurusd-zero-spread-deviations/data/plan/phase7_bid_ohlc_construction_v1.1.0.md)
 
@@ -256,7 +326,7 @@ SELECT table_name, column_name, data_type, comment FROM duckdb_columns();
 - `add_schema_comments(pair)` - Add comments to single database
 - `add_schema_comments_all()` - Batch-update all databases
 
-**Implementation**: [`src/exness_data_preprocess/processor.py`](src/exness_data_preprocess/processor.py) (lines 138-229, 708-821)
+**Implementation**: [`src/exness_data_preprocess/database_manager.py`](src/exness_data_preprocess/database_manager.py) (schema comments in get_or_create_db method)
 
 **Usage Example**: [`examples/add_schema_comments.py`](examples/add_schema_comments.py)
 
@@ -314,7 +384,7 @@ SELECT table_name, column_name, data_type, comment FROM duckdb_columns();
 - ✅ **Dual-variant storage** - Raw_Spread + Standard in same database
 - ✅ **PRIMARY KEY constraints** - Prevents duplicates during incremental updates
 - ✅ **Automatic gap detection** - Downloads only missing months
-- ✅ **Phase7 13-column OHLC (v1.2.0)** - Dual spreads + dual tick counts
+- ✅ **Phase7 30-column OHLC (v1.5.0)** - Dual spreads + dual tick counts + 10 global exchange sessions
 - ✅ **Date range queries** - Sub-15ms query performance
 - ✅ **On-demand resampling** - Any timeframe in <15ms
 - ✅ **SQL filter support** - Direct SQL WHERE clauses on ticks
@@ -359,7 +429,7 @@ SELECT table_name, column_name, data_type, comment FROM duckdb_columns();
 eurusd.duckdb:
 ├── raw_spread_ticks   # Timestamp (PK), Bid, Ask
 ├── standard_ticks     # Timestamp (PK), Bid, Ask
-├── ohlc_1m            # Phase7 13-column schema (v1.2.0)
+├── ohlc_1m            # Phase7 30-column schema (v1.5.0)
 └── metadata           # Coverage tracking
 ```
 
@@ -399,6 +469,7 @@ eurusd.duckdb:
 
 ---
 
-**Version**: 2.0.0
-**Last Updated**: 2025-10-12
+**Version**: 2.0.0 (Architecture) + 1.3.0 (Implementation)
+**Last Updated**: 2025-10-15
 **Architecture**: Unified Single-File DuckDB Storage with Incremental Updates
+**Implementation**: Facade Pattern with 7 Specialized Modules (Phase 1-4 Complete)

{exness_data_preprocess-0.3.0 → exness_data_preprocess-0.3.1}/GITHUB_PYPI_SETUP.md

@@ -173,7 +173,7 @@ gh release create v0.1.0 \
 
 **Architecture v2.0.0:**
 - Unified single-file DuckDB storage per instrument
-- Phase7 9-column OHLC schema with dual-variant spreads
+- Phase7 13-column (v1.2.0) OHLC schema with dual-variant spreads and normalized metrics
 - Sub-15ms query performance with date range filtering
 - PRIMARY KEY constraints prevent duplicates
 - Automatic gap detection for incremental updates

{exness_data_preprocess-0.3.0 → exness_data_preprocess-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: exness-data-preprocess
-Version: 0.3.0
+Version: 0.3.1
 Summary: Professional Exness forex tick data preprocessing with optimal compression (Parquet Zstd-22) and DuckDB OHLC generation. Provides efficient storage (9% smaller than ZIP) with lossless precision and direct queryability.
 Project-URL: Homepage, https://github.com/terrylica/exness-data-preprocess
 Project-URL: Documentation, https://github.com/terrylica/exness-data-preprocess#readme
@@ -30,6 +30,7 @@ Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.9
 Requires-Dist: duckdb>=0.9.0
+Requires-Dist: exchange-calendars>=4.5.6
 Requires-Dist: httpx>=0.25.0
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: pyarrow>=16.0.0
@@ -55,14 +56,14 @@ Description-Content-Type: text/markdown
 [![Downloads](https://img.shields.io/pypi/dm/exness-data-preprocess.svg)](https://pypi.org/project/exness-data-preprocess/)
 [![Code style: ruff](https://img.shields.io/badge/code%20style-ruff-000000.svg)](https://github.com/astral-sh/ruff)
 
-Professional forex tick data preprocessing with unified single-file DuckDB storage. Provides incremental updates, dual-variant storage (Raw_Spread + Standard), and Phase7 13-column OHLC schema (v1.2.0) with sub-15ms query performance.
+Professional forex tick data preprocessing with unified single-file DuckDB storage. Provides incremental updates, dual-variant storage (Raw_Spread + Standard), and Phase7 30-column OHLC schema (v1.5.0) with 10 global exchange sessions and sub-15ms query performance.
 
 ## Features
 
 - **Unified Single-File Architecture**: One DuckDB file per instrument (eurusd.duckdb)
 - **Incremental Updates**: Automatic gap detection and download only missing months
 - **Dual-Variant Storage**: Raw_Spread (primary) + Standard (reference) in same database
-- **Phase7 OHLC Schema**: 13-column bars (v1.2.0) with dual spreads, tick counts, and normalized metrics
+- **Phase7 OHLC Schema**: 30-column bars (v1.5.0) with dual spreads, tick counts, normalized metrics, and 10 global exchange sessions
 - **Fast Queries**: Date range queries with sub-15ms performance
 - **On-Demand Resampling**: Any timeframe (5m, 1h, 1d) resampled in <15ms
 - **PRIMARY KEY Constraints**: Prevents duplicate data during incremental updates
@@ -138,7 +139,7 @@ Download Only Missing Months (Raw_Spread + Standard)
            ↓
 DuckDB Single-File Storage (PRIMARY KEY prevents duplicates)
            ↓
-Phase7 13-Column OHLC Generation (v1.2.0 - dual spreads, tick counts, normalized metrics)
+Phase7 30-Column OHLC Generation (v1.5.0 - dual spreads, tick counts, normalized metrics, 10 global exchange sessions)
            ↓
 Query Interface (date ranges, SQL filters, on-demand resampling)
 ```
@@ -150,13 +151,13 @@ Query Interface (date ranges, SQL filters, on-demand resampling)
 **Schema**:
 - `raw_spread_ticks` table: Timestamp (PK), Bid, Ask
 - `standard_ticks` table: Timestamp (PK), Bid, Ask
-- `ohlc_1m` table: Phase7 13-column schema (v1.2.0)
+- `ohlc_1m` table: Phase7 30-column schema (v1.5.0)
 - `metadata` table: Coverage tracking
 
-**Phase7 13-Column OHLC (v1.2.0)**:
+**Phase7 30-Column OHLC (v1.5.0)**:
 - **Column Definitions**: See [`schema.py`](src/exness_data_preprocess/schema.py) - Single source of truth
 - **Comprehensive Reference**: See [`DATABASE_SCHEMA.md`](docs/DATABASE_SCHEMA.md) - Query examples and usage patterns
-- **Key Features**: BID-only OHLC with dual spreads (Raw_Spread + Standard) and normalized spread metrics
+- **Key Features**: BID-only OHLC with dual spreads (Raw_Spread + Standard), normalized spread metrics, and 10 global exchange sessions (XNYS, XLON, XSWX, XFRA, XTSE, XNZE, XTKS, XASX, XHKG, XSES)
 
 ### Directory Structure
 

{exness_data_preprocess-0.3.0 → exness_data_preprocess-0.3.1}/PYDANTIC_REFACTORING_PLAN.md

@@ -159,7 +159,7 @@ class UpdateResult(BaseModel):
     )
     ohlc_bars: int = Field(
         ge=0,
-        description="Total 1-minute OHLC bars in database (Phase7 9-column schema)"
+        description="Total 1-minute OHLC bars in database (Phase7 13-column v1.2.0 schema)"
     )
     duckdb_size_mb: float = Field(
         ge=0,
@@ -347,7 +347,7 @@ def update_data(
     1. Discover missing months between start_date and current month
     2. Download missing months from ticks.ex2archive.com (both variants)
     3. Insert ticks into DuckDB with PRIMARY KEY duplicate prevention
-    4. Regenerate Phase7 9-column OHLC with dual spreads
+    4. Regenerate Phase7 13-column (v1.2.0) OHLC with dual spreads and normalized metrics
     5. Update metadata table with coverage statistics
 
     Args:
@@ -559,13 +559,12 @@ def query_ohlc(
     timeframes. All resampling is performed in-database using DuckDB's window
     functions for optimal performance (<15ms for most queries).
 
-    The returned DataFrame contains Phase7 9-column schema:
-    - Timestamp: Bar timestamp (UTC)
-    - Open, High, Low, Close: BID prices from Raw_Spread variant
-    - raw_spread_avg: Average spread from Raw_Spread variant
-    - standard_spread_avg: Average spread from Standard variant
-    - tick_count_raw_spread: Tick count from Raw_Spread variant
-    - tick_count_standard: Tick count from Standard variant
+    The returned DataFrame contains Phase7 13-column (v1.2.0) schema:
+    - See schema.py for complete 13-column definition
+    - Core: Timestamp, Open, High, Low, Close (BID prices from Raw_Spread)
+    - Dual Spreads: raw_spread_avg, standard_spread_avg
+    - Dual Tick Counts: tick_count_raw_spread, tick_count_standard
+    - Normalized Metrics: range_per_spread, range_per_tick, body_per_spread, body_per_tick
 
     Args:
         pair: Currency pair (EURUSD, GBPUSD, XAUUSD, USDJPY, AUDUSD)
@@ -574,7 +573,7 @@ def query_ohlc(
         end_date: Optional end date in YYYY-MM-DD format (inclusive)
 
     Returns:
-        DataFrame with Phase7 9-column OHLC schema. Empty if no data in range.
+        DataFrame with Phase7 13-column (v1.2.0) OHLC schema. Empty if no data in range.
 
     Raises:
         FileNotFoundError: If database doesn't exist (call update_data first)

{exness_data_preprocess-0.3.0 → exness_data_preprocess-0.3.1}/README.md

@@ -7,14 +7,14 @@
 [![Downloads](https://img.shields.io/pypi/dm/exness-data-preprocess.svg)](https://pypi.org/project/exness-data-preprocess/)
 [![Code style: ruff](https://img.shields.io/badge/code%20style-ruff-000000.svg)](https://github.com/astral-sh/ruff)
 
-Professional forex tick data preprocessing with unified single-file DuckDB storage. Provides incremental updates, dual-variant storage (Raw_Spread + Standard), and Phase7 13-column OHLC schema (v1.2.0) with sub-15ms query performance.
+Professional forex tick data preprocessing with unified single-file DuckDB storage. Provides incremental updates, dual-variant storage (Raw_Spread + Standard), and Phase7 30-column OHLC schema (v1.5.0) with 10 global exchange sessions and sub-15ms query performance.
 
 ## Features
 
 - **Unified Single-File Architecture**: One DuckDB file per instrument (eurusd.duckdb)
 - **Incremental Updates**: Automatic gap detection and download only missing months
 - **Dual-Variant Storage**: Raw_Spread (primary) + Standard (reference) in same database
-- **Phase7 OHLC Schema**: 13-column bars (v1.2.0) with dual spreads, tick counts, and normalized metrics
+- **Phase7 OHLC Schema**: 30-column bars (v1.5.0) with dual spreads, tick counts, normalized metrics, and 10 global exchange sessions
 - **Fast Queries**: Date range queries with sub-15ms performance
 - **On-Demand Resampling**: Any timeframe (5m, 1h, 1d) resampled in <15ms
 - **PRIMARY KEY Constraints**: Prevents duplicate data during incremental updates
@@ -90,7 +90,7 @@ Download Only Missing Months (Raw_Spread + Standard)
            ↓
 DuckDB Single-File Storage (PRIMARY KEY prevents duplicates)
            ↓
-Phase7 13-Column OHLC Generation (v1.2.0 - dual spreads, tick counts, normalized metrics)
+Phase7 30-Column OHLC Generation (v1.5.0 - dual spreads, tick counts, normalized metrics, 10 global exchange sessions)
            ↓
 Query Interface (date ranges, SQL filters, on-demand resampling)
 ```
@@ -102,13 +102,13 @@ Query Interface (date ranges, SQL filters, on-demand resampling)
 **Schema**:
 - `raw_spread_ticks` table: Timestamp (PK), Bid, Ask
 - `standard_ticks` table: Timestamp (PK), Bid, Ask
-- `ohlc_1m` table: Phase7 13-column schema (v1.2.0)
+- `ohlc_1m` table: Phase7 30-column schema (v1.5.0)
 - `metadata` table: Coverage tracking
 
-**Phase7 13-Column OHLC (v1.2.0)**:
+**Phase7 30-Column OHLC (v1.5.0)**:
 - **Column Definitions**: See [`schema.py`](src/exness_data_preprocess/schema.py) - Single source of truth
 - **Comprehensive Reference**: See [`DATABASE_SCHEMA.md`](docs/DATABASE_SCHEMA.md) - Query examples and usage patterns
-- **Key Features**: BID-only OHLC with dual spreads (Raw_Spread + Standard) and normalized spread metrics
+- **Key Features**: BID-only OHLC with dual spreads (Raw_Spread + Standard), normalized spread metrics, and 10 global exchange sessions (XNYS, XLON, XSWX, XFRA, XTSE, XNZE, XTKS, XASX, XHKG, XSES)
 
 ### Directory Structure
 

exness_data_preprocess-0.3.1/RELEASE_NOTES.md

@@ -0,0 +1,6 @@
+
+## 0.3.0 - 2025-10-15
+
+
+---
+**Full Changelog**: https://github.com/Eon-Labs/rangebar/compare/v0.2.0...v0.3.0

exness_data_preprocess-0.3.1/cliff-release-notes.toml

@@ -0,0 +1,131 @@
+# git-cliff configuration for GitHub release notes generation
+# Optimized for user-friendly, marketing-style release announcements
+
+[changelog]
+header = ""
+body = """
+## {% if version %}{{ version | trim_start_matches(pat="v") }}{% else %}Unreleased{% endif %} - {{ timestamp | date(format="%Y-%m-%d") }}
+
+{% for group, commits in commits | group_by(attribute="group") %}
+### {{ group }}
+{% for commit in commits | unique(attribute="message") %}
+- {% if commit.breaking %}⚠️ **BREAKING**: {% endif %}\
+{{ commit.message | upper_first }}\
+{% if commit.github.username %} (by @{{ commit.github.username }}){% endif %}
+{% endfor %}
+
+{% endfor %}
+---
+**Full Changelog**: https://github.com/Eon-Labs/rangebar/compare/{{ previous.version }}...{{ version }}
+"""
+footer = ""
+trim = true
+
+[git]
+conventional_commits = true
+filter_unconventional = false
+split_commits = false
+tag_pattern = "v[0-9]*"
+tag_prefix = "v"
+skip_tags = ""
+ignore_tags = ""
+
+[[git.commit_parsers]]
+# Breaking changes (most prominent)
+message = "^.*!:"
+group = "⚠️ Breaking Changes"
+
+[[git.commit_parsers]]
+body = ".*BREAKING[- ]CHANGE.*"
+group = "⚠️ Breaking Changes"
+
+# Features (user-facing semantic grouping)
+[[git.commit_parsers]]
+message = "^feat\\(sanitization\\)"
+group = "🧹 Codebase Sanitization"
+
+[[git.commit_parsers]]
+message = "^feat\\(security\\)"
+group = "🔒 Security Enhancements"
+
+[[git.commit_parsers]]
+message = "^feat\\(perf\\)"
+group = "🚀 Performance Improvements"
+
+[[git.commit_parsers]]
+message = "^feat\\(deps\\)"
+group = "📦 Dependency Management"
+
+[[git.commit_parsers]]
+message = "^feat"
+group = "✨ New Features"
+
+# Fixes (user-impacting only)
+[[git.commit_parsers]]
+message = "^fix\\(security\\)"
+group = "🔒 Security Fixes"
+
+[[git.commit_parsers]]
+message = "^fix\\(perf\\)"
+group = "🚀 Performance Fixes"
+
+[[git.commit_parsers]]
+message = "^fix"
+group = "🐛 Bug Fixes & Improvements"
+
+# Infrastructure (consolidated)
+[[git.commit_parsers]]
+message = "^(fix|feat)\\(ci\\)"
+group = "🔧 Infrastructure & CI/CD"
+
+[[git.commit_parsers]]
+message = "^(fix|feat)\\(fmt\\)"
+group = "🔧 Infrastructure & CI/CD"
+
+[[git.commit_parsers]]
+message = "^(fix|feat)\\(clippy\\)"
+group = "🔧 Infrastructure & CI/CD"
+
+# Skip non-user-facing commits (chore, docs, test, etc.)
+[[git.commit_parsers]]
+message = "^chore"
+skip = true
+
+[[git.commit_parsers]]
+message = "^docs"
+skip = true
+
+[[git.commit_parsers]]
+message = "^test"
+skip = true
+
+[[git.commit_parsers]]
+message = "^build"
+skip = true
+
+[[git.commit_parsers]]
+message = "^ci"
+skip = true
+
+[[git.commit_parsers]]
+message = "^style"
+skip = true
+
+[[git.commit_parsers]]
+message = "^refactor"
+skip = true
+
+# Catch-all for anything else
+[[git.commit_parsers]]
+message = ".*"
+group = "📝 Other Changes"
+
+[[git.commit_preprocessors]]
+# Remove gitmoji to avoid duplication with our emoji groups
+pattern = ':\w+:'
+replace = ""
+
+[[git.commit_preprocessors]]
+# Clean up extra whitespace
+pattern = '\s+'
+replace = " "