exness-data-preprocess 0.1.0__py3-none-any.whl

exness_data_preprocess/__init__.py

@@ -0,0 +1,51 @@
+"""
+Exness Data Preprocessing
+
+Professional forex tick data preprocessing with optimal compression and DuckDB OHLC generation.
+
+Features:
+- Optimal compression: Parquet with Zstd-22 (9% smaller than ZIP, lossless)
+- DuckDB OHLC generation with embedded metadata
+- Direct queryability (no decode step needed)
+- On-demand tick analysis workflow
+- Production-grade atomic operations
+
+Quick Start:
+    >>> import exness_data_preprocess as edp
+    >>>
+    >>> # Download and preprocess Exness data
+    >>> processor = edp.ExnessDataProcessor()
+    >>> result = processor.process_month(year=2024, month=8)
+    >>>
+    >>> # Query OHLC data
+    >>> df_ohlc = edp.query_ohlc(year=2024, month=8, timeframe='1m')
+    >>>
+    >>> # On-demand tick analysis
+    >>> df_ticks = edp.analyze_ticks(year=2024, month=8)
+"""
+
+__version__ = "0.1.0"
+__author__ = "Terry Li <terry@eonlabs.com>"
+__license__ = "MIT"
+
+from exness_data_preprocess.api import (
+    analyze_ticks,
+    get_storage_stats,
+    process_date_range,
+    process_month,
+    query_ohlc,
+)
+from exness_data_preprocess.processor import ExnessDataProcessor
+
+__all__ = [
+    # Main class
+    "ExnessDataProcessor",
+    # Simple API functions
+    "process_month",
+    "process_date_range",
+    "query_ohlc",
+    "analyze_ticks",
+    "get_storage_stats",
+    # Package metadata
+    "__version__",
+]

exness_data_preprocess/api.py

@@ -0,0 +1,216 @@
+"""
+Simple API for Exness data preprocessing.
+
+This module provides function-based wrappers around ExnessDataProcessor
+for common use cases.
+"""
+
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+import pandas as pd
+
+from exness_data_preprocess.processor import ExnessDataProcessor
+
+# Global processor instance (lazy initialization)
+_processor = None
+
+
+def _get_processor(base_dir: Optional[Path] = None) -> ExnessDataProcessor:
+    """Get or create global processor instance."""
+    global _processor
+    if _processor is None or base_dir is not None:
+        _processor = ExnessDataProcessor(base_dir=base_dir)
+    return _processor
+
+
+def process_month(
+    year: int,
+    month: int,
+    pair: str = "EURUSD",
+    delete_zip: bool = True,
+    base_dir: Optional[Path] = None,
+) -> Dict[str, Any]:
+    """
+    Process one month of Exness forex tick data.
+
+    This is the primary API for downloading and preprocessing Exness data.
+    Downloads monthly ZIP, converts to Parquet (Zstd-22), generates DuckDB OHLC.
+
+    Args:
+        year: Year (e.g., 2024)
+        month: Month (1-12)
+        pair: Currency pair (default: EURUSD)
+        delete_zip: Whether to delete ZIP after processing (default: True)
+        base_dir: Base directory for data storage (default: ~/eon/exness-data-preprocess/data)
+
+    Returns:
+        Dictionary with processing results:
+            - parquet_path: Path to Parquet file
+            - duckdb_path: Path to DuckDB file
+            - tick_count: Number of ticks
+            - ohlc_bar_count: Number of 1-minute bars
+            - parquet_size_mb: Parquet file size in MB
+            - duckdb_size_mb: DuckDB file size in MB
+            - compression_ratio: Parquet size / ZIP size
+
+    Example:
+        >>> import exness_data_preprocess as edp
+        >>> result = edp.process_month(2024, 8)
+        >>> print(f"Processed {result['tick_count']:,} ticks")
+        >>> print(f"Storage: {result['parquet_size_mb']:.2f} MB")
+    """
+    processor = _get_processor(base_dir)
+    return processor.process_month(year, month, pair, delete_zip)
+
+
+def process_date_range(
+    start_year: int,
+    start_month: int,
+    end_year: int,
+    end_month: int,
+    pair: str = "EURUSD",
+    delete_zip: bool = True,
+    base_dir: Optional[Path] = None,
+) -> List[Dict[str, Any]]:
+    """
+    Process multiple months of Exness data.
+
+    Downloads and processes all months in the specified range.
+
+    Args:
+        start_year: Start year
+        start_month: Start month (1-12)
+        end_year: End year
+        end_month: End month (1-12)
+        pair: Currency pair (default: EURUSD)
+        delete_zip: Whether to delete ZIPs after processing
+        base_dir: Base directory for data storage
+
+    Returns:
+        List of result dictionaries (one per month)
+
+    Example:
+        >>> import exness_data_preprocess as edp
+        >>> results = edp.process_date_range(2024, 1, 2024, 12)
+        >>> total_ticks = sum(r['tick_count'] for r in results)
+        >>> print(f"Processed {total_ticks:,} ticks across {len(results)} months")
+    """
+    processor = _get_processor(base_dir)
+    results = []
+
+    # Generate month sequence
+    current_year = start_year
+    current_month = start_month
+
+    while (current_year < end_year) or (current_year == end_year and current_month <= end_month):
+        try:
+            result = processor.process_month(current_year, current_month, pair, delete_zip)
+            results.append(result)
+        except Exception as e:
+            print(f"✗ Failed to process {current_year}-{current_month:02d}: {e}")
+
+        # Move to next month
+        if current_month == 12:
+            current_month = 1
+            current_year += 1
+        else:
+            current_month += 1
+
+    return results
+
+
+def query_ohlc(
+    year: int,
+    month: int,
+    pair: str = "EURUSD",
+    timeframe: str = "1m",
+    base_dir: Optional[Path] = None,
+) -> pd.DataFrame:
+    """
+    Query OHLC data from DuckDB.
+
+    Loads pre-computed OHLC data from DuckDB storage. Supports resampling
+    to higher timeframes on the fly.
+
+    Args:
+        year: Year
+        month: Month (1-12)
+        pair: Currency pair (default: EURUSD)
+        timeframe: Timeframe (1m, 5m, 15m, 30m, 1h, 4h, 1d)
+        base_dir: Base directory for data storage
+
+    Returns:
+        DataFrame with OHLC data (Timestamp, Open, High, Low, Close, spread_avg, tick_count)
+
+    Example:
+        >>> import exness_data_preprocess as edp
+        >>> # Query 1-hour OHLC bars
+        >>> df = edp.query_ohlc(2024, 8, timeframe='1h')
+        >>> print(df.head())
+        >>> print(f"Average spread: {df['spread_avg'].mean():.5f}")
+    """
+    processor = _get_processor(base_dir)
+    return processor.query_ohlc(year, month, pair, timeframe)
+
+
+def analyze_ticks(
+    year: int,
+    month: int,
+    pair: str = "EURUSD",
+    base_dir: Optional[Path] = None,
+) -> pd.DataFrame:
+    """
+    Load tick data for analysis.
+
+    Provides on-demand access to raw tick data from Parquet storage.
+    Useful for microstructure analysis, spread statistics, etc.
+
+    Args:
+        year: Year
+        month: Month (1-12)
+        pair: Currency pair (default: EURUSD)
+        base_dir: Base directory for data storage
+
+    Returns:
+        DataFrame with tick data (Timestamp, Bid, Ask)
+
+    Example:
+        >>> import exness_data_preprocess as edp
+        >>> df_ticks = edp.analyze_ticks(2024, 8)
+        >>> # Calculate spread statistics
+        >>> spreads = df_ticks['Ask'] - df_ticks['Bid']
+        >>> print(f"Mean spread: {spreads.mean():.5f}")
+        >>> print(f"Median spread: {spreads.median():.5f}")
+        >>> print(f"95th percentile: {spreads.quantile(0.95):.5f}")
+    """
+    processor = _get_processor(base_dir)
+    return processor.analyze_ticks(year, month, pair)
+
+
+def get_storage_stats(base_dir: Optional[Path] = None) -> Dict[str, Any]:
+    """
+    Get storage statistics for all processed data.
+
+    Returns counts and total sizes for Parquet and DuckDB files.
+
+    Args:
+        base_dir: Base directory for data storage
+
+    Returns:
+        Dictionary with storage stats:
+            - parquet_count: Number of Parquet files
+            - parquet_total_mb: Total Parquet storage (MB)
+            - duckdb_count: Number of DuckDB files
+            - duckdb_total_mb: Total DuckDB storage (MB)
+            - total_mb: Total storage (MB)
+
+    Example:
+        >>> import exness_data_preprocess as edp
+        >>> stats = edp.get_storage_stats()
+        >>> print(f"Total storage: {stats['total_mb']:.2f} MB")
+        >>> print(f"Parquet files: {stats['parquet_count']}")
+        >>> print(f"DuckDB files: {stats['duckdb_count']}")
+    """
+    processor = _get_processor(base_dir)
+    return processor.get_storage_stats()

exness_data_preprocess/cli.py

@@ -0,0 +1,199 @@
+"""
+Command-line interface for Exness data preprocessing.
+"""
+
+import argparse
+import sys
+from pathlib import Path
+
+from exness_data_preprocess import api
+
+
+def process_command(args):
+    """Handle 'process' command."""
+    if args.range:
+        # Process date range
+        start_year, start_month = map(int, args.range[0].split("-"))
+        end_year, end_month = map(int, args.range[1].split("-"))
+
+        results = api.process_date_range(
+            start_year=start_year,
+            start_month=start_month,
+            end_year=end_year,
+            end_month=end_month,
+            pair=args.pair,
+            delete_zip=not args.keep_zip,
+            base_dir=Path(args.base_dir) if args.base_dir else None,
+        )
+
+        print(f"\n{'=' * 70}")
+        print(f"Processed {len(results)} months")
+        total_ticks = sum(r["tick_count"] for r in results)
+        total_size = sum(r["parquet_size_mb"] + r["duckdb_size_mb"] for r in results)
+        print(f"Total ticks: {total_ticks:,}")
+        print(f"Total storage: {total_size:.2f} MB")
+        print(f"{'=' * 70}")
+    else:
+        # Process single month
+        result = api.process_month(
+            year=args.year,
+            month=args.month,
+            pair=args.pair,
+            delete_zip=not args.keep_zip,
+            base_dir=Path(args.base_dir) if args.base_dir else None,
+        )
+        print(f"\n✓ Successfully processed {args.pair} {args.year}-{args.month:02d}")
+
+
+def query_command(args):
+    """Handle 'query' command."""
+    df = api.query_ohlc(
+        year=args.year,
+        month=args.month,
+        pair=args.pair,
+        timeframe=args.timeframe,
+        base_dir=Path(args.base_dir) if args.base_dir else None,
+    )
+
+    if args.output:
+        df.to_csv(args.output, index=False)
+        print(f"✓ Saved {len(df)} bars to {args.output}")
+    else:
+        print(df.head(args.head))
+        print(f"\nTotal bars: {len(df)}")
+
+
+def analyze_command(args):
+    """Handle 'analyze' command."""
+    df = api.analyze_ticks(
+        year=args.year,
+        month=args.month,
+        pair=args.pair,
+        base_dir=Path(args.base_dir) if args.base_dir else None,
+    )
+
+    # Calculate spread statistics
+    spreads = df["Ask"] - df["Bid"]
+
+    print(f"\n{'=' * 70}")
+    print(f"Tick Analysis: {args.pair} {args.year}-{args.month:02d}")
+    print(f"{'=' * 70}")
+    print(f"Total ticks:      {len(df):,}")
+    print("\nSpread Statistics (base units):")
+    print(f"  Mean:           {spreads.mean():.5f}")
+    print(f"  Median:         {spreads.median():.5f}")
+    print(f"  Std Dev:        {spreads.std():.5f}")
+    print(f"  Min:            {spreads.min():.5f}")
+    print(f"  Max:            {spreads.max():.5f}")
+    print(f"  25th percentile: {spreads.quantile(0.25):.5f}")
+    print(f"  75th percentile: {spreads.quantile(0.75):.5f}")
+    print(f"  95th percentile: {spreads.quantile(0.95):.5f}")
+
+    if args.output:
+        df.to_csv(args.output, index=False)
+        print(f"\n✓ Saved {len(df)} ticks to {args.output}")
+
+
+def stats_command(args):
+    """Handle 'stats' command."""
+    stats = api.get_storage_stats(base_dir=Path(args.base_dir) if args.base_dir else None)
+
+    print(f"\n{'=' * 70}")
+    print("Storage Statistics")
+    print(f"{'=' * 70}")
+    print(f"Parquet files:    {stats['parquet_count']}")
+    print(f"Parquet storage:  {stats['parquet_total_mb']:.2f} MB")
+    print(f"DuckDB files:     {stats['duckdb_count']}")
+    print(f"DuckDB storage:   {stats['duckdb_total_mb']:.2f} MB")
+    print(f"{'=' * 70}")
+    print(f"Total storage:    {stats['total_mb']:.2f} MB")
+
+
+def main():
+    """Main entry point for CLI."""
+    parser = argparse.ArgumentParser(
+        description="Exness Forex Tick Data Preprocessing Tool",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Process single month
+  exness-preprocess process --year 2024 --month 8
+
+  # Process date range
+  exness-preprocess process --range 2024-01 2024-12
+
+  # Query 1-hour OHLC
+  exness-preprocess query --year 2024 --month 8 --timeframe 1h
+
+  # Analyze tick data
+  exness-preprocess analyze --year 2024 --month 8
+
+  # Show storage statistics
+  exness-preprocess stats
+        """,
+    )
+
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    # Process command
+    process_parser = subparsers.add_parser("process", help="Process Exness tick data")
+    process_parser.add_argument("--year", type=int, help="Year to process")
+    process_parser.add_argument("--month", type=int, help="Month to process (1-12)")
+    process_parser.add_argument(
+        "--range", nargs=2, metavar=("START", "END"), help="Date range (YYYY-MM YYYY-MM)"
+    )
+    process_parser.add_argument("--pair", default="EURUSD", help="Currency pair (default: EURUSD)")
+    process_parser.add_argument(
+        "--keep-zip", action="store_true", help="Keep ZIP files after processing"
+    )
+    process_parser.add_argument("--base-dir", help="Base directory for data storage")
+    process_parser.set_defaults(func=process_command)
+
+    # Query command
+    query_parser = subparsers.add_parser("query", help="Query OHLC data")
+    query_parser.add_argument("--year", type=int, required=True, help="Year")
+    query_parser.add_argument("--month", type=int, required=True, help="Month (1-12)")
+    query_parser.add_argument("--pair", default="EURUSD", help="Currency pair (default: EURUSD)")
+    query_parser.add_argument(
+        "--timeframe",
+        default="1m",
+        choices=["1m", "5m", "15m", "30m", "1h", "4h", "1d"],
+        help="Timeframe (default: 1m)",
+    )
+    query_parser.add_argument("--output", help="Output CSV file")
+    query_parser.add_argument(
+        "--head", type=int, default=10, help="Number of rows to display (default: 10)"
+    )
+    query_parser.add_argument("--base-dir", help="Base directory for data storage")
+    query_parser.set_defaults(func=query_command)
+
+    # Analyze command
+    analyze_parser = subparsers.add_parser("analyze", help="Analyze tick data")
+    analyze_parser.add_argument("--year", type=int, required=True, help="Year")
+    analyze_parser.add_argument("--month", type=int, required=True, help="Month (1-12)")
+    analyze_parser.add_argument("--pair", default="EURUSD", help="Currency pair (default: EURUSD)")
+    analyze_parser.add_argument("--output", help="Output CSV file")
+    analyze_parser.add_argument("--base-dir", help="Base directory for data storage")
+    analyze_parser.set_defaults(func=analyze_command)
+
+    # Stats command
+    stats_parser = subparsers.add_parser("stats", help="Show storage statistics")
+    stats_parser.add_argument("--base-dir", help="Base directory for data storage")
+    stats_parser.set_defaults(func=stats_command)
+
+    # Parse and execute
+    args = parser.parse_args()
+
+    if not args.command:
+        parser.print_help()
+        sys.exit(1)
+
+    try:
+        args.func(args)
+    except Exception as e:
+        print(f"✗ Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()