rangebar 11.6.1__cp313-cp313-macosx_11_0_arm64.whl

rangebar/hooks.py

@@ -0,0 +1,311 @@
+"""Event hook system for rangebar-py operations.
+
+This module provides a publish-subscribe event system for monitoring
+long-running cache operations, validation results, and population progress.
+
+Usage
+-----
+>>> from rangebar.hooks import register_hook, HookEvent
+>>>
+>>> def my_callback(payload):
+...     print(f"Event: {payload.event.value}, Symbol: {payload.symbol}")
+...
+>>> register_hook(HookEvent.CACHE_WRITE_COMPLETE, my_callback)
+>>> # Later, when bars are cached:
+>>> # Event: cache_write_complete, Symbol: BTCUSDT
+
+Events
+------
+- CACHE_WRITE_START: Cache write operation started
+- CACHE_WRITE_COMPLETE: Cache write operation completed successfully
+- CACHE_WRITE_FAILED: Cache write operation failed
+- VALIDATION_COMPLETE: Post-storage validation passed
+- VALIDATION_FAILED: Post-storage validation failed
+- CHECKPOINT_SAVED: Resumable population checkpoint saved
+- POPULATION_COMPLETE: Cache population completed successfully
+- POPULATION_FAILED: Cache population failed
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from collections import defaultdict
+from dataclasses import asdict, dataclass, field
+from datetime import UTC, datetime
+from enum import Enum
+from typing import TYPE_CHECKING, Any, Callable
+
+if TYPE_CHECKING:
+    pass
+
+logger = logging.getLogger(__name__)
+
+
+class HookEvent(Enum):
+    """Events that can trigger hook callbacks.
+
+    Attributes
+    ----------
+    CACHE_WRITE_START : str
+        Emitted when cache write operation begins.
+    CACHE_WRITE_COMPLETE : str
+        Emitted when cache write completes successfully.
+    CACHE_WRITE_FAILED : str
+        Emitted when cache write fails.
+    VALIDATION_COMPLETE : str
+        Emitted when post-storage validation passes.
+    VALIDATION_FAILED : str
+        Emitted when post-storage validation fails.
+    CHECKPOINT_SAVED : str
+        Emitted when a population checkpoint is saved.
+    POPULATION_COMPLETE : str
+        Emitted when cache population completes.
+    POPULATION_FAILED : str
+        Emitted when cache population fails.
+    """
+
+    CACHE_WRITE_START = "cache_write_start"
+    CACHE_WRITE_COMPLETE = "cache_write_complete"
+    CACHE_WRITE_FAILED = "cache_write_failed"
+    VALIDATION_COMPLETE = "validation_complete"
+    VALIDATION_FAILED = "validation_failed"
+    CHECKPOINT_SAVED = "checkpoint_saved"
+    POPULATION_COMPLETE = "population_complete"
+    POPULATION_FAILED = "population_failed"
+
+
+@dataclass
+class HookPayload:
+    """Payload delivered to hook callbacks.
+
+    Attributes
+    ----------
+    event : HookEvent
+        The event type that triggered this callback.
+    symbol : str
+        Trading symbol involved (e.g., "BTCUSDT").
+    timestamp : datetime
+        When the event occurred (UTC).
+    details : dict
+        Additional event-specific information.
+    is_failure : bool
+        True if this event represents a failure.
+    """
+
+    event: HookEvent
+    symbol: str
+    timestamp: datetime = field(default_factory=lambda: datetime.now(UTC))
+    details: dict[str, Any] = field(default_factory=dict)
+    is_failure: bool = False
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert payload to dictionary for serialization."""
+        d = asdict(self)
+        d["event"] = self.event.value
+        d["timestamp"] = self.timestamp.isoformat()
+        return d
+
+    def to_json(self) -> str:
+        """Convert payload to JSON string."""
+        return json.dumps(self.to_dict(), indent=2)
+
+
+# Global hook registry
+_hooks: dict[HookEvent, list[Callable[[HookPayload], None]]] = defaultdict(list)
+
+
+def register_hook(
+    event: HookEvent,
+    callback: Callable[[HookPayload], None],
+) -> None:
+    """Register a callback for a specific event.
+
+    Parameters
+    ----------
+    event : HookEvent
+        The event to subscribe to.
+    callback : Callable[[HookPayload], None]
+        Function to call when the event occurs. Receives a HookPayload.
+
+    Examples
+    --------
+    >>> def log_completion(payload):
+    ...     print(f"Completed: {payload.symbol} at {payload.timestamp}")
+    ...
+    >>> register_hook(HookEvent.CACHE_WRITE_COMPLETE, log_completion)
+    """
+    _hooks[event].append(callback)
+    logger.debug("Registered hook for %s: %s", event.value, callback.__name__)
+
+
+def unregister_hook(
+    event: HookEvent,
+    callback: Callable[[HookPayload], None],
+) -> bool:
+    """Unregister a callback for a specific event.
+
+    Parameters
+    ----------
+    event : HookEvent
+        The event to unsubscribe from.
+    callback : Callable[[HookPayload], None]
+        The callback to remove.
+
+    Returns
+    -------
+    bool
+        True if the callback was found and removed, False otherwise.
+    """
+    if callback in _hooks[event]:
+        _hooks[event].remove(callback)
+        logger.debug("Unregistered hook for %s: %s", event.value, callback.__name__)
+        return True
+    return False
+
+
+def clear_hooks(event: HookEvent | None = None) -> None:
+    """Clear all hooks for a specific event or all events.
+
+    Parameters
+    ----------
+    event : HookEvent | None
+        Event to clear hooks for. If None, clears all hooks.
+    """
+    if event is None:
+        _hooks.clear()
+        logger.debug("Cleared all hooks")
+    else:
+        _hooks[event].clear()
+        logger.debug("Cleared hooks for %s", event.value)
+
+
+def emit_hook(
+    event: HookEvent,
+    symbol: str,
+    **details: Any,
+) -> None:
+    """Emit an event to all registered callbacks.
+
+    Parameters
+    ----------
+    event : HookEvent
+        The event type to emit.
+    symbol : str
+        Trading symbol involved.
+    **details
+        Additional event-specific information.
+
+    Notes
+    -----
+    Callback exceptions are caught and logged but do not propagate.
+    This ensures one failing callback doesn't break the main operation.
+
+    Examples
+    --------
+    >>> emit_hook(
+    ...     HookEvent.CACHE_WRITE_COMPLETE,
+    ...     symbol="BTCUSDT",
+    ...     bars_written=1500,
+    ...     threshold_bps=250,
+    ... )
+    """
+    # Add memory snapshot to all hook payloads (Issue #49 T2.3)
+    try:
+        from rangebar.resource_guard import get_memory_info
+
+        mem = get_memory_info()
+        details["memory_rss_mb"] = mem.process_rss_mb
+        details["memory_pct"] = round(mem.usage_pct, 3)
+    except Exception:
+        logger.debug("Memory snapshot unavailable for hook payload", exc_info=True)
+
+    is_failure = "FAILED" in event.name
+    payload = HookPayload(
+        event=event,
+        symbol=symbol,
+        timestamp=datetime.now(UTC),
+        details=details,
+        is_failure=is_failure,
+    )
+
+    callbacks = _hooks.get(event, [])
+    if not callbacks:
+        logger.debug("No hooks registered for %s", event.value)
+        return
+
+    logger.debug(
+        "Emitting %s for %s to %d callback(s)",
+        event.value,
+        symbol,
+        len(callbacks),
+    )
+
+    for callback in callbacks:
+        try:
+            callback(payload)
+        except (OSError, RuntimeError, ValueError, TypeError) as e:
+            # Log but don't propagate - callbacks shouldn't break main flow
+            logger.warning(
+                "Hook callback %s failed for %s: %s",
+                callback.__name__,
+                event.value,
+                e,
+            )
+
+
+def register_for_failures(
+    callback: Callable[[HookPayload], None],
+) -> None:
+    """Register a callback for all failure events.
+
+    Convenience function to subscribe to all *_FAILED events.
+
+    Parameters
+    ----------
+    callback : Callable[[HookPayload], None]
+        Function to call when any failure event occurs.
+
+    Examples
+    --------
+    >>> def alert_on_failure(payload):
+    ...     send_slack_alert(f"FAILURE: {payload.event.value}")
+    ...
+    >>> register_for_failures(alert_on_failure)
+    """
+    for event in HookEvent:
+        if "FAILED" in event.name:
+            register_hook(event, callback)
+
+
+def register_for_all(
+    callback: Callable[[HookPayload], None],
+) -> None:
+    """Register a callback for all events.
+
+    Parameters
+    ----------
+    callback : Callable[[HookPayload], None]
+        Function to call when any event occurs.
+
+    Examples
+    --------
+    >>> def log_all(payload):
+    ...     print(f"{payload.event.value}: {payload.symbol}")
+    ...
+    >>> register_for_all(log_all)
+    """
+    for event in HookEvent:
+        register_hook(event, callback)
+
+
+__all__ = [
+    "HookEvent",
+    "HookPayload",
+    "clear_hooks",
+    "emit_hook",
+    "register_for_all",
+    "register_for_failures",
+    "register_hook",
+    "unregister_hook",
+]

rangebar/logging.py

@@ -0,0 +1,171 @@
+"""Centralized NDJSON logging configuration for rangebar-py.
+
+Implements GitHub Issue #43: Structured logging for checksum verification
+and other observability events.
+
+Logs are stored in the repository tree under `logs/` directory.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from loguru import Logger
+
+# Logs in repository tree (not platformdirs)
+PROJECT_ROOT = Path(__file__).parent.parent.parent  # rangebar-py/
+LOG_DIR = PROJECT_ROOT / "logs"
+NDJSON_FILE = LOG_DIR / "events.jsonl"
+CHECKSUM_REGISTRY_FILE = LOG_DIR / "checksum_registry.jsonl"
+
+# Lazy initialization flag
+_logger_initialized = False
+_logger: Logger | None = None
+
+
+def _get_context_extra() -> dict:
+    """Get context fields added to every log entry."""
+    import socket
+
+    return {
+        "service": "rangebar-py",
+        "environment": os.environ.get("RANGEBAR_ENV", "development"),
+        "git_sha": os.environ.get("RANGEBAR_GIT_SHA", "unknown"),
+        "pid": os.getpid(),
+        "host": socket.gethostname(),
+    }
+
+
+def get_logger() -> Logger:
+    """Get the configured logger instance.
+
+    Lazy initialization to avoid import-time side effects.
+    """
+    global _logger_initialized, _logger
+
+    if _logger_initialized and _logger is not None:
+        return _logger
+
+    try:
+        from loguru import logger
+    except ImportError:
+        # Fallback to standard logging if loguru not installed
+        import logging
+
+        logging.basicConfig(
+            level=logging.INFO,
+            format="%(asctime)s | %(levelname)-8s | %(name)s - %(message)s",
+        )
+        # Return a minimal logger-like object
+        return logging.getLogger("rangebar")  # type: ignore[return-value]
+
+    # Ensure log directory exists
+    LOG_DIR.mkdir(parents=True, exist_ok=True)
+
+    # Remove default handler
+    logger.remove()
+
+    # Add context to every log entry
+    context = _get_context_extra()
+    logger = logger.bind(**context)
+
+    # NDJSON file sink (in repo tree)
+    logger.add(
+        NDJSON_FILE,
+        format="{message}",
+        serialize=True,  # NDJSON output
+        rotation="10 MB",
+        retention="7 days",
+        compression="gz",
+        enqueue=True,
+        level="DEBUG",
+    )
+
+    # Console (human-readable, INFO+)
+    console_format = (
+        "<green>{time:HH:mm:ss}</green> | <level>{level: <8}</level> | "
+        "<cyan>{extra[component]}</cyan> - <level>{message}</level>"
+    )
+    logger.add(
+        sys.stderr,
+        format=console_format,
+        level="INFO",
+        colorize=True,
+        filter=lambda record: "component" in record["extra"],
+    )
+
+    _logger = logger
+    _logger_initialized = True
+    return logger
+
+
+def log_checksum_event(
+    event_type: str,
+    symbol: str,
+    date: str,
+    trace_id: str,
+    **kwargs: object,
+) -> None:
+    """Log a checksum-related event.
+
+    Args:
+        event_type: Type of event (checksum_fetch_start, checksum_verify_success, etc.)
+        symbol: Trading symbol (e.g., "BTCUSDT")
+        date: Date being processed (YYYY-MM-DD)
+        trace_id: Correlation ID for the request chain
+        **kwargs: Additional event-specific fields
+    """
+    logger = get_logger()
+    logger.bind(
+        component="checksum",
+        event_type=event_type,
+        symbol=symbol,
+        date=date,
+        trace_id=trace_id,
+        **kwargs,
+    ).info(f"{event_type}: {symbol} {date}")
+
+
+def log_download_event(
+    event_type: str,
+    symbol: str,
+    date: str,
+    trace_id: str,
+    **kwargs: object,
+) -> None:
+    """Log a download-related event.
+
+    Args:
+        event_type: Type of event (download_start, download_complete, etc.)
+        symbol: Trading symbol (e.g., "BTCUSDT")
+        date: Date being processed (YYYY-MM-DD)
+        trace_id: Correlation ID for the request chain
+        **kwargs: Additional event-specific fields
+    """
+    logger = get_logger()
+    logger.bind(
+        component="download",
+        event_type=event_type,
+        symbol=symbol,
+        date=date,
+        trace_id=trace_id,
+        **kwargs,
+    ).info(f"{event_type}: {symbol} {date}")
+
+
+def generate_trace_id(prefix: str = "rb") -> str:
+    """Generate a unique trace ID for request correlation.
+
+    Args:
+        prefix: Prefix for the trace ID (default: "rb" for rangebar)
+
+    Returns:
+        Trace ID in format "{prefix}-{hex8}" (e.g., "rb-a1b2c3d4")
+    """
+    import uuid
+
+    return f"{prefix}-{uuid.uuid4().hex[:8]}"

rangebar/notify/__init__.py

@@ -0,0 +1,15 @@
+"""Notification integrations for rangebar-py.
+
+This package provides notification backends for the hooks system.
+Currently supported: Telegram.
+
+Usage
+-----
+>>> from rangebar.notify.telegram import enable_telegram_notifications
+>>> enable_telegram_notifications()
+>>> # Now all hook events will be sent to Telegram
+"""
+
+from __future__ import annotations
+
+__all__: list[str] = []

rangebar/notify/pushover.py

@@ -0,0 +1,155 @@
+"""Pushover critical alerts for rangebar-py.
+
+Implements GitHub Issue #43: Loud alerting for checksum verification failures.
+
+Pushover alerts are used for CRITICAL data integrity issues that require
+immediate attention, such as:
+- SHA-256 checksum mismatches (data corruption detected)
+- Tier 1 cache integrity failures
+"""
+
+from __future__ import annotations
+
+import sys
+
+import requests
+
+# Pushover API configuration
+# These credentials are for the "RB Checksum Fail" app
+PUSHOVER_APP_TOKEN = "asxuepwiaqkwc5e749xj1qx2eg1e3b"
+PUSHOVER_USER_KEY = "ury88s1def6v16seeueoefqn1zbua1"
+PUSHOVER_API_URL = "https://api.pushover.net/1/messages.json"
+
+
+def send_critical_alert(
+    title: str,
+    message: str,
+    url: str | None = None,
+    url_title: str | None = None,
+) -> bool:
+    """Send LOUD critical alert via Pushover.
+
+    Uses:
+    - Priority 2 (emergency) - requires acknowledgment
+    - Dune custom sound for maximum attention
+    - Retry every 60s for 1 hour until acknowledged
+
+    Args:
+        title: Alert title (e.g., "🚨 CHECKSUM FAIL: BTCUSDT")
+        message: Alert body with details
+        url: Optional URL for more information
+        url_title: Display title for the URL
+
+    Returns:
+        True if alert was sent successfully, False otherwise
+    """
+    payload = {
+        "token": PUSHOVER_APP_TOKEN,
+        "user": PUSHOVER_USER_KEY,
+        "title": title,
+        "message": message,
+        "priority": 2,  # Emergency - requires acknowledgment
+        "retry": 60,  # Retry every 60 seconds
+        "expire": 3600,  # Stop retrying after 1 hour
+        "sound": "dune",  # Dune custom sound for maximum attention
+    }
+
+    if url:
+        payload["url"] = url
+        payload["url_title"] = url_title or "View Details"
+
+    try:
+        response = requests.post(PUSHOVER_API_URL, data=payload, timeout=10)
+        response.raise_for_status()
+        return True
+    except requests.RequestException as e:
+        # Log failure but don't crash - alerting is secondary
+        _log_alert_failure(str(e))
+        return False
+
+
+def _log_alert_failure(error: str) -> None:
+    """Log Pushover alert failure without crashing."""
+    try:
+        from ..logging import get_logger
+
+        logger = get_logger()
+        logger.bind(component="pushover").error(f"Pushover alert failed: {error}")
+    except ImportError:
+        # Fallback - print to stderr if logging module not available
+        print(f"Pushover alert failed: {error}", file=sys.stderr)
+
+
+def alert_checksum_failure(
+    symbol: str,
+    date: str,
+    expected_hash: str,
+    actual_hash: str,
+    data_source: str = "binance",
+) -> None:
+    """Alert on checksum mismatch - CRITICAL data integrity issue.
+
+    This function sends an emergency Pushover alert when a downloaded file's
+    SHA-256 hash does not match the expected value from Binance.
+
+    Args:
+        symbol: Trading symbol (e.g., "BTCUSDT")
+        date: Date of the corrupted data (YYYY-MM-DD)
+        expected_hash: Expected SHA-256 hash from Binance
+        actual_hash: Actual computed hash of downloaded data
+        data_source: Data source identifier (default: "binance")
+    """
+    title = f"🚨 CHECKSUM FAIL: {symbol}"
+    message = f"""Data corruption detected!
+
+Symbol: {symbol}
+Date: {date}
+Source: {data_source}
+Expected: {expected_hash[:16]}...
+Actual: {actual_hash[:16]}...
+
+ACTION REQUIRED: Investigate immediately.
+Data may be corrupted or tampered with."""
+
+    send_critical_alert(title, message)
+
+
+def alert_tier1_cache_unverified(
+    symbol: str,
+    date_range: str,
+    unverified_count: int,
+    total_count: int,
+) -> None:
+    """Alert when Tier 1 cache contains unverified files.
+
+    Args:
+        symbol: Trading symbol (e.g., "BTCUSDT")
+        date_range: Date range being audited (e.g., "2024-01-01 to 2024-01-07")
+        unverified_count: Number of unverified dates
+        total_count: Total number of dates in range
+    """
+    title = f"⚠️ CACHE AUDIT: {symbol}"
+    message = f"""Tier 1 cache audit found unverified files.
+
+Symbol: {symbol}
+Date Range: {date_range}
+Unverified: {unverified_count}/{total_count} dates
+
+Consider re-downloading with verify_checksum=True
+to ensure data integrity."""
+
+    # Use lower priority (1) for audit warnings vs checksum failures (2)
+    payload = {
+        "token": PUSHOVER_APP_TOKEN,
+        "user": PUSHOVER_USER_KEY,
+        "title": title,
+        "message": message,
+        "priority": 1,  # High priority but not emergency
+        "sound": "siren",
+    }
+
+    try:
+        response = requests.post(PUSHOVER_API_URL, data=payload, timeout=10)
+        response.raise_for_status()
+    except requests.RequestException as e:
+        _log_alert_failure(str(e))