signalflow-trading 0.2.1__py3-none-any.whl

signalflow/data/raw_data_factory.py

@@ -0,0 +1,225 @@
+from datetime import datetime
+from pathlib import Path
+
+import polars as pl
+
+from signalflow.core import RawData
+from signalflow.data.raw_store import DuckDbSpotStore
+
+
+class RawDataFactory:
+    """Factory for creating RawData instances from various sources.
+
+    Provides static methods to construct RawData objects from different
+    storage backends (DuckDB, Parquet, etc.) with proper validation
+    and schema normalization.
+
+    Key features:
+        - Automatic schema validation
+        - Duplicate detection
+        - Timezone normalization
+        - Column cleanup (remove unnecessary columns)
+        - Proper sorting by (pair, timestamp)
+
+    Example:
+        ```python
+        from signalflow.data import RawDataFactory
+        from pathlib import Path
+        from datetime import datetime
+
+        # Load spot data from DuckDB
+        raw_data = RawDataFactory.from_duckdb_spot_store(
+            spot_store_path=Path("data/binance_spot.duckdb"),
+            pairs=["BTCUSDT", "ETHUSDT"],
+            start=datetime(2024, 1, 1),
+            end=datetime(2024, 12, 31),
+            data_types=["spot"]
+        )
+
+        # Access loaded data
+        spot_df = raw_data["spot"]
+        print(f"Loaded {len(spot_df)} bars")
+        print(f"Pairs: {raw_data.pairs}")
+        print(f"Date range: {raw_data.datetime_start} to {raw_data.datetime_end}")
+
+        # Use in detector
+        from signalflow.detector import SmaCrossSignalDetector
+
+        detector = SmaCrossSignalDetector(fast_window=10, slow_window=20)
+        signals = detector.detect(raw_data)
+        ```
+
+    See Also:
+        RawData: Immutable container for raw market data.
+        DuckDbSpotStore: DuckDB storage backend for spot data.
+    """
+
+    @staticmethod
+    def from_duckdb_spot_store(
+        spot_store_path: Path,
+        pairs: list[str],
+        start: datetime,
+        end: datetime,
+        data_types: list[str] | None = None,
+    ) -> RawData:
+        """Create RawData from DuckDB spot store.
+
+        Loads spot trading data from DuckDB storage with validation,
+        deduplication checks, and schema normalization.
+
+        Processing steps:
+            1. Load data from DuckDB for specified pairs and date range
+            2. Validate required columns (pair, timestamp)
+            3. Remove unnecessary columns (timeframe)
+            4. Normalize timestamps (microseconds, timezone-naive)
+            5. Check for duplicates (pair, timestamp)
+            6. Sort by (pair, timestamp)
+            7. Package into RawData container
+
+        Args:
+            spot_store_path (Path): Path to DuckDB file.
+            pairs (list[str]): List of trading pairs to load (e.g., ["BTCUSDT", "ETHUSDT"]).
+            start (datetime): Start datetime (inclusive).
+            end (datetime): End datetime (inclusive).
+            data_types (list[str] | None): Data types to load. Default: None.
+                Currently supports: ["spot"].
+
+        Returns:
+            RawData: Immutable container with loaded and validated data.
+
+        Raises:
+            ValueError: If required columns missing (pair, timestamp).
+            ValueError: If duplicate (pair, timestamp) combinations detected.
+
+        Example:
+            ```python
+            from pathlib import Path
+            from datetime import datetime
+            from signalflow.data import RawDataFactory
+
+            # Load single pair
+            raw_data = RawDataFactory.from_duckdb_spot_store(
+                spot_store_path=Path("data/binance.duckdb"),
+                pairs=["BTCUSDT"],
+                start=datetime(2024, 1, 1),
+                end=datetime(2024, 1, 31),
+                data_types=["spot"]
+            )
+
+            # Load multiple pairs
+            raw_data = RawDataFactory.from_duckdb_spot_store(
+                spot_store_path=Path("data/binance.duckdb"),
+                pairs=["BTCUSDT", "ETHUSDT", "BNBUSDT"],
+                start=datetime(2024, 1, 1),
+                end=datetime(2024, 12, 31),
+                data_types=["spot"]
+            )
+
+            # Check loaded data
+            spot_df = raw_data["spot"]
+            print(f"Shape: {spot_df.shape}")
+            print(f"Columns: {spot_df.columns}")
+            print(f"Pairs: {spot_df['pair'].unique().to_list()}")
+
+            # Verify no duplicates
+            dup_check = (
+                spot_df.group_by(["pair", "timestamp"])
+                .len()
+                .filter(pl.col("len") > 1)
+            )
+            assert dup_check.is_empty()
+
+            # Use in pipeline
+            from signalflow.core import RawDataView
+            view = RawDataView(raw=raw_data)
+            spot_pandas = view.to_pandas("spot")
+            ```
+
+        Example:
+            ```python
+            # Handle missing data gracefully
+            try:
+                raw_data = RawDataFactory.from_duckdb_spot_store(
+                    spot_store_path=Path("data/binance.duckdb"),
+                    pairs=["BTCUSDT"],
+                    start=datetime(2024, 1, 1),
+                    end=datetime(2024, 1, 31),
+                    data_types=["spot"]
+                )
+            except ValueError as e:
+                if "missing columns" in str(e):
+                    print("Data schema invalid")
+                elif "Duplicate" in str(e):
+                    print("Data contains duplicates")
+                raise
+
+            # Validate date range
+            assert raw_data.datetime_start == datetime(2024, 1, 1)
+            assert raw_data.datetime_end == datetime(2024, 1, 31)
+
+            # Check data quality
+            spot_df = raw_data["spot"]
+            
+            # Verify timestamps are sorted
+            assert spot_df["timestamp"].is_sorted()
+            
+            # Verify timezone-naive
+            assert spot_df["timestamp"].dtype == pl.Datetime("us")
+            
+            # Verify no nulls in key columns
+            assert spot_df["pair"].null_count() == 0
+            assert spot_df["timestamp"].null_count() == 0
+            ```
+
+        Note:
+            Store connection is automatically closed via finally block.
+            Timestamps are normalized to timezone-naive microseconds.
+            Duplicate detection shows first 10 examples if found.
+            All data sorted by (pair, timestamp) for consistent ordering.
+        """
+        data: dict[str, pl.DataFrame] = {}
+        store = DuckDbSpotStore(spot_store_path)
+        try:
+            if "spot" in data_types:
+                spot = store.load_many(pairs=pairs, start=start, end=end)
+
+                required = {"pair", "timestamp"}
+                missing = required - set(spot.columns)
+                if missing:
+                    raise ValueError(f"Spot df missing columns: {sorted(missing)}")
+
+                if "timeframe" in spot.columns:
+                    spot = spot.drop("timeframe")
+
+                spot = spot.with_columns(
+                    pl.col("timestamp").cast(pl.Datetime("us")).dt.replace_time_zone(None)
+                )
+
+                dup_count = (
+                    spot.group_by(["pair", "timestamp"]).len()
+                    .filter(pl.col("len") > 1)
+                )
+                if dup_count.height > 0:
+                    dups = (
+                        spot.join(
+                            dup_count.select(["pair", "timestamp"]),
+                            on=["pair", "timestamp"],
+                        )
+                        .select(["pair", "timestamp"])
+                        .head(10)
+                    )
+                    raise ValueError(
+                        f"Duplicate (pair, timestamp) detected. Examples:\n{dups}"
+                    )
+
+                spot = spot.sort(["pair", "timestamp"])
+                data["spot"] = spot
+
+            return RawData(
+                datetime_start=start,
+                datetime_end=end,
+                pairs=pairs,
+                data=data,
+            )
+        finally:
+            store.close()

signalflow/data/raw_store/__init__.py

@@ -0,0 +1,7 @@
+from signalflow.data.raw_store.duckdb_stores import DuckDbSpotStore
+from signalflow.data.raw_store.base import RawDataStore
+
+__all__ = [
+    "DuckDbSpotStore",
+    "RawDataStore",
+]

signalflow/data/raw_store/base.py

@@ -0,0 +1,271 @@
+from abc import ABC, abstractmethod
+
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Optional, ClassVar
+from signalflow.core import SfComponentType
+import polars as pl
+import pandas as pd
+
+@dataclass
+class RawDataStore(ABC):
+    """Abstract base class for raw data storage backends.
+
+    Defines the interface for loading historical market data from storage.
+    Implementations provide specific storage backends (DuckDB, Parquet, etc.)
+    while maintaining a consistent API.
+
+    Key features:
+        - Single and batch loading (load, load_many)
+        - Flexible time filtering (hours, start/end)
+        - Multi-format output (Polars, Pandas)
+        - Resource management (close)
+
+    Supported operations:
+        - Load single pair with time filtering
+        - Load multiple pairs efficiently (batch query)
+        - Convert to Pandas for legacy compatibility
+        - Cleanup resources on shutdown
+
+    Attributes:
+        component_type (ClassVar[SfComponentType]): Always RAW_DATA_STORE for registry.
+
+    Example:
+        ```python
+        from signalflow.data.raw_store import DuckDbSpotStore
+        from datetime import datetime
+        from pathlib import Path
+
+        # Create store instance
+        store = DuckDbSpotStore(Path("data/binance_spot.duckdb"))
+
+        try:
+            # Load single pair
+            btc_df = store.load(
+                pair="BTCUSDT",
+                start=datetime(2024, 1, 1),
+                end=datetime(2024, 1, 31)
+            )
+
+            # Load multiple pairs
+            multi_df = store.load_many(
+                pairs=["BTCUSDT", "ETHUSDT"],
+                start=datetime(2024, 1, 1),
+                end=datetime(2024, 1, 31)
+            )
+
+            # Load as Pandas
+            pandas_df = store.load_many_pandas(
+                pairs=["BTCUSDT"],
+                start=datetime(2024, 1, 1),
+                end=datetime(2024, 1, 31)
+            )
+        finally:
+            store.close()
+
+        # Use with context manager (if implemented)
+        with DuckDbSpotStore(Path("data/binance_spot.duckdb")) as store:
+            df = store.load("BTCUSDT", hours=24)
+        ```
+
+    Note:
+        Subclasses must implement all abstract methods.
+        Always call close() or use context manager to cleanup resources.
+        Time filtering supports both relative (hours) and absolute (start/end).
+
+    See Also:
+        DuckDbSpotStore: DuckDB implementation for spot data.
+        RawDataFactory: Factory for creating RawData from stores.
+    """
+    component_type: ClassVar[SfComponentType] = SfComponentType.RAW_DATA_STORE
+    
+    @abstractmethod
+    def load(self, pair: str, hours: Optional[int] = None, start: Optional[datetime] = None, end: Optional[datetime] = None) -> pl.DataFrame:
+        """Load data for a single trading pair.
+
+        Loads historical market data with flexible time filtering.
+        Use either relative (hours) or absolute (start/end) time filtering.
+
+        Args:
+            pair (str): Trading pair (e.g., "BTCUSDT").
+            hours (int | None): Load last N hours of data. Mutually exclusive with start/end.
+            start (datetime | None): Start datetime (inclusive). Requires end parameter.
+            end (datetime | None): End datetime (inclusive). Requires start parameter.
+
+        Returns:
+            pl.DataFrame: Market data as Polars DataFrame.
+                Typically includes columns: pair, timestamp, open, high, low, close, volume.
+
+        Raises:
+            ValueError: If both hours and start/end are provided.
+            ValueError: If start provided without end or vice versa.
+            FileNotFoundError: If storage file/database doesn't exist.
+
+        Example:
+            ```python
+            from datetime import datetime
+
+            # Load last 24 hours
+            recent_df = store.load("BTCUSDT", hours=24)
+
+            # Load specific date range
+            historical_df = store.load(
+                "BTCUSDT",
+                start=datetime(2024, 1, 1),
+                end=datetime(2024, 1, 31)
+            )
+
+            # Check loaded data
+            print(f"Loaded {len(historical_df)} bars")
+            print(f"Date range: {historical_df['timestamp'].min()} to {historical_df['timestamp'].max()}")
+            ```
+
+        Note:
+            Implementation should handle timezone normalization.
+            Returned DataFrame should be sorted by timestamp.
+        """
+        pass
+
+    @abstractmethod
+    def load_many(self, pairs: list[str], hours: Optional[int] = None, start: Optional[datetime] = None, end: Optional[datetime] = None) -> pl.DataFrame:
+        """Load data for multiple trading pairs efficiently.
+
+        Batch loading is more efficient than calling load() repeatedly.
+        Returns combined DataFrame with all pairs.
+
+        Args:
+            pairs (list[str]): List of trading pairs (e.g., ["BTCUSDT", "ETHUSDT"]).
+            hours (int | None): Load last N hours of data. Mutually exclusive with start/end.
+            start (datetime | None): Start datetime (inclusive). Requires end parameter.
+            end (datetime | None): End datetime (inclusive). Requires start parameter.
+
+        Returns:
+            pl.DataFrame: Combined market data for all pairs as Polars DataFrame.
+                Includes pair column to distinguish between pairs.
+
+        Raises:
+            ValueError: If both hours and start/end are provided.
+            ValueError: If start provided without end or vice versa.
+            ValueError: If pairs list is empty.
+
+        Example:
+            ```python
+            from datetime import datetime
+
+            # Load multiple pairs
+            multi_df = store.load_many(
+                pairs=["BTCUSDT", "ETHUSDT", "BNBUSDT"],
+                start=datetime(2024, 1, 1),
+                end=datetime(2024, 1, 31)
+            )
+
+            # Analyze by pair
+            for pair in multi_df["pair"].unique():
+                pair_df = multi_df.filter(pl.col("pair") == pair)
+                print(f"{pair}: {len(pair_df)} bars")
+
+            # Last 24 hours for monitoring
+            recent_multi = store.load_many(
+                pairs=["BTCUSDT", "ETHUSDT"],
+                hours=24
+            )
+            ```
+
+        Note:
+            Returned DataFrame sorted by (pair, timestamp).
+            More efficient than multiple load() calls due to batch query.
+        """
+        pass
+
+    @abstractmethod
+    def load_many_pandas(self, pairs: list[str], start: Optional[datetime] = None, end: Optional[datetime] = None) -> pd.DataFrame:
+        """Load data for multiple pairs as Pandas DataFrame.
+
+        Convenience method for legacy code or libraries requiring Pandas.
+        Typically converts from Polars internally.
+
+        Args:
+            pairs (list[str]): List of trading pairs (e.g., ["BTCUSDT", "ETHUSDT"]).
+            start (datetime | None): Start datetime (inclusive). Requires end parameter.
+            end (datetime | None): End datetime (inclusive). Requires start parameter.
+
+        Returns:
+            pd.DataFrame: Combined market data as Pandas DataFrame.
+
+        Raises:
+            ValueError: If start provided without end or vice versa.
+            ValueError: If pairs list is empty.
+
+        Example:
+            ```python
+            from datetime import datetime
+            import pandas as pd
+
+            # Load as Pandas
+            df = store.load_many_pandas(
+                pairs=["BTCUSDT", "ETHUSDT"],
+                start=datetime(2024, 1, 1),
+                end=datetime(2024, 1, 31)
+            )
+
+            # Use with pandas-ta
+            import pandas_ta as ta
+            df["rsi"] = ta.rsi(df["close"], length=14)
+
+            # Use with legacy extractors
+            class LegacyExtractor:
+                def extract(self, df: pd.DataFrame) -> pd.DataFrame:
+                    df["sma_20"] = df.groupby("pair")["close"].rolling(20).mean()
+                    return df
+
+            extractor = LegacyExtractor()
+            df_with_features = extractor.extract(df)
+            ```
+
+        Note:
+            Prefer load_many() with Polars for better performance.
+            Use this only when Pandas is required.
+            Timestamps normalized to timezone-naive datetime64[ns].
+        """
+        pass
+
+    @abstractmethod
+    def close(self) -> None:
+        """Close storage connection and cleanup resources.
+
+        Releases database connections, file handles, or other resources.
+        Should be called when store is no longer needed.
+
+        Always call close() in a finally block or use context manager
+        to ensure cleanup even on errors.
+
+        Example:
+            ```python
+            # Manual cleanup
+            store = DuckDbSpotStore(Path("data/binance.duckdb"))
+            try:
+                df = store.load("BTCUSDT", hours=24)
+                # ... process data ...
+            finally:
+                store.close()
+
+            # With context manager (if implemented)
+            with DuckDbSpotStore(Path("data/binance.duckdb")) as store:
+                df = store.load("BTCUSDT", hours=24)
+                # ... process data ...
+            # Automatically closed
+
+            # In RawDataFactory
+            store = DuckDbSpotStore(store_path)
+            try:
+                data = store.load_many(pairs, start, end)
+                return RawData(data={"spot": data})
+            finally:
+                store.close()  # Always cleanup
+            ```
+
+        Note:
+            Idempotent - safe to call multiple times.
+            After close(), store should not be used for loading.
+        """
+        pass