mcli-framework 7.1.0__py3-none-any.whl → 7.1.2__py3-none-any.whl

mcli/ml/data_ingestion/data_pipeline.py

@@ -1,25 +1,27 @@
 """Complete data ingestion pipeline with validation and transformation"""
 
 import asyncio
-from typing import Dict, Any, Optional, List, Union, Callable
+import json
+import logging
+from abc import ABC, abstractmethod
 from dataclasses import dataclass, field
 from datetime import datetime, timedelta
-import pandas as pd
-import numpy as np
-import logging
 from pathlib import Path
-import json
-from abc import ABC, abstractmethod
+from typing import Any, Callable, Dict, List, Optional, Union
+
+import numpy as np
+import pandas as pd
 
 from .api_connectors import (
-    CongressionalDataAPI,
-    YahooFinanceConnector,
     AlphaVantageConnector,
+    CongressionalDataAPI,
+    DataAggregator,
     PolygonIOConnector,
     QuiverQuantConnector,
-    DataAggregator,
+    YahooFinanceConnector,
 )
-from .stream_processor import StreamProcessor, StreamConfig, DataAggregator as StreamAggregator
+from .stream_processor import DataAggregator as StreamAggregator
+from .stream_processor import StreamConfig, StreamProcessor
 
 logger = logging.getLogger(__name__)
 
@@ -27,6 +29,7 @@ logger = logging.getLogger(__name__)
 @dataclass
 class PipelineConfig:
     """Data pipeline configuration"""
+
     data_dir: Path = Path("data")
     batch_size: int = 1000
     enable_streaming: bool = True
@@ -43,20 +46,15 @@ class DataValidator:
 
     def __init__(self):
         self.validation_rules = {
-            'politician_trades': self._validate_politician_trade,
-            'stock_quotes': self._validate_stock_quote,
-            'market_data': self._validate_market_data
-        }
-        self.validation_stats = {
-            'total': 0,
-            'valid': 0,
-            'invalid': 0,
-            'errors': []
+            "politician_trades": self._validate_politician_trade,
+            "stock_quotes": self._validate_stock_quote,
+            "market_data": self._validate_market_data,
         }
+        self.validation_stats = {"total": 0, "valid": 0, "invalid": 0, "errors": []}
 
     def validate(self, data: Dict[str, Any], data_type: str) -> bool:
         """Validate data based on type"""
-        self.validation_stats['total'] += 1
+        self.validation_stats["total"] += 1
 
         if data_type not in self.validation_rules:
             logger.warning(f"Unknown data type: {data_type}")
@@ -65,19 +63,19 @@ class DataValidator:
         try:
             is_valid = self.validation_rules[data_type](data)
             if is_valid:
-                self.validation_stats['valid'] += 1
+                self.validation_stats["valid"] += 1
             else:
-                self.validation_stats['invalid'] += 1
+                self.validation_stats["invalid"] += 1
             return is_valid
         except Exception as e:
-            self.validation_stats['invalid'] += 1
-            self.validation_stats['errors'].append(str(e))
+            self.validation_stats["invalid"] += 1
+            self.validation_stats["errors"].append(str(e))
             logger.error(f"Validation error: {e}")
             return False
 
     def _validate_politician_trade(self, data: Dict[str, Any]) -> bool:
         """Validate politician trading data"""
-        required_fields = ['politician', 'ticker', 'transaction_type', 'amount', 'transaction_date']
+        required_fields = ["politician", "ticker", "transaction_type", "amount", "transaction_date"]
 
         # Check required fields
         for field in required_fields:
@@ -86,19 +84,19 @@ class DataValidator:
                 return False
 
         # Validate transaction type
-        if data['transaction_type'] not in ['buy', 'sell', 'exchange']:
+        if data["transaction_type"] not in ["buy", "sell", "exchange"]:
             logger.warning(f"Invalid transaction type: {data['transaction_type']}")
             return False
 
         # Validate amount
-        if not isinstance(data['amount'], (int, float)) or data['amount'] <= 0:
+        if not isinstance(data["amount"], (int, float)) or data["amount"] <= 0:
             logger.warning(f"Invalid amount: {data['amount']}")
             return False
 
         # Validate date
         try:
-            if isinstance(data['transaction_date'], str):
-                datetime.fromisoformat(data['transaction_date'])
+            if isinstance(data["transaction_date"], str):
+                datetime.fromisoformat(data["transaction_date"])
         except:
             logger.warning(f"Invalid date format: {data['transaction_date']}")
             return False
@@ -107,34 +105,34 @@ class DataValidator:
 
     def _validate_stock_quote(self, data: Dict[str, Any]) -> bool:
         """Validate stock quote data"""
-        required_fields = ['symbol', 'price', 'timestamp']
+        required_fields = ["symbol", "price", "timestamp"]
 
         for field in required_fields:
             if field not in data:
                 return False
 
         # Validate price
-        if not isinstance(data['price'], (int, float)) or data['price'] <= 0:
+        if not isinstance(data["price"], (int, float)) or data["price"] <= 0:
             return False
 
         return True
 
     def _validate_market_data(self, data: Dict[str, Any]) -> bool:
         """Validate market data"""
-        required_fields = ['symbol', 'close', 'volume', 'date']
+        required_fields = ["symbol", "close", "volume", "date"]
 
         for field in required_fields:
             if field not in data:
                 return False
 
         # Validate prices
-        for price_field in ['close', 'open', 'high', 'low']:
+        for price_field in ["close", "open", "high", "low"]:
             if price_field in data:
                 if not isinstance(data[price_field], (int, float)) or data[price_field] <= 0:
                     return False
 
         # Validate volume
-        if not isinstance(data['volume'], (int, float)) or data['volume'] < 0:
+        if not isinstance(data["volume"], (int, float)) or data["volume"] < 0:
             return False
 
         return True
@@ -149,13 +147,14 @@ class DataTransformer:
 
     def __init__(self):
         self.transformers = {
-            'politician_trades': self._transform_politician_trade,
-            'stock_quotes': self._transform_stock_quote,
-            'market_data': self._transform_market_data
+            "politician_trades": self._transform_politician_trade,
+            "stock_quotes": self._transform_stock_quote,
+            "market_data": self._transform_market_data,
         }
 
-    def transform(self, data: Union[Dict[str, Any], List[Dict[str, Any]]],
-                  data_type: str) -> Union[Dict[str, Any], pd.DataFrame]:
+    def transform(
+        self, data: Union[Dict[str, Any], List[Dict[str, Any]]], data_type: str
+    ) -> Union[Dict[str, Any], pd.DataFrame]:
         """Transform data based on type"""
         if data_type not in self.transformers:
             return data
@@ -171,30 +170,30 @@ class DataTransformer:
         transformed = data.copy()
 
         # Standardize politician name
-        transformed['politician_normalized'] = self._normalize_name(data.get('politician', ''))
+        transformed["politician_normalized"] = self._normalize_name(data.get("politician", ""))
 
         # Convert dates to datetime
-        if 'transaction_date' in data:
-            transformed['transaction_date'] = pd.to_datetime(data['transaction_date'])
+        if "transaction_date" in data:
+            transformed["transaction_date"] = pd.to_datetime(data["transaction_date"])
 
-        if 'disclosure_date' in data:
-            transformed['disclosure_date'] = pd.to_datetime(data['disclosure_date'])
+        if "disclosure_date" in data:
+            transformed["disclosure_date"] = pd.to_datetime(data["disclosure_date"])
 
             # Calculate disclosure delay
-            if 'transaction_date' in transformed:
-                delay = (transformed['disclosure_date'] - transformed['transaction_date']).days
-                transformed['disclosure_delay_days'] = max(0, delay)
+            if "transaction_date" in transformed:
+                delay = (transformed["disclosure_date"] - transformed["transaction_date"]).days
+                transformed["disclosure_delay_days"] = max(0, delay)
 
         # Normalize ticker
-        transformed['ticker'] = data.get('ticker', '').upper()
+        transformed["ticker"] = data.get("ticker", "").upper()
 
         # Categorize transaction amount
-        amount = data.get('amount', 0)
-        transformed['amount_category'] = self._categorize_amount(amount)
+        amount = data.get("amount", 0)
+        transformed["amount_category"] = self._categorize_amount(amount)
 
         # Add derived features
-        transformed['is_purchase'] = data.get('transaction_type') == 'buy'
-        transformed['is_sale'] = data.get('transaction_type') == 'sell'
+        transformed["is_purchase"] = data.get("transaction_type") == "buy"
+        transformed["is_sale"] = data.get("transaction_type") == "sell"
 
         return transformed
 
@@ -203,19 +202,19 @@ class DataTransformer:
         transformed = data.copy()
 
         # Normalize symbol
-        transformed['symbol'] = data.get('symbol', '').upper()
+        transformed["symbol"] = data.get("symbol", "").upper()
 
         # Convert timestamp
-        if 'timestamp' in data:
-            if isinstance(data['timestamp'], (int, float)):
-                transformed['timestamp'] = datetime.fromtimestamp(data['timestamp'])
+        if "timestamp" in data:
+            if isinstance(data["timestamp"], (int, float)):
+                transformed["timestamp"] = datetime.fromtimestamp(data["timestamp"])
             else:
-                transformed['timestamp'] = pd.to_datetime(data['timestamp'])
+                transformed["timestamp"] = pd.to_datetime(data["timestamp"])
 
         # Calculate spread if bid/ask available
-        if 'bid' in data and 'ask' in data:
-            transformed['spread'] = data['ask'] - data['bid']
-            transformed['spread_pct'] = (transformed['spread'] / data['ask']) * 100
+        if "bid" in data and "ask" in data:
+            transformed["spread"] = data["ask"] - data["bid"]
+            transformed["spread_pct"] = (transformed["spread"] / data["ask"]) * 100
 
         return transformed
 
@@ -224,34 +223,34 @@ class DataTransformer:
         transformed = data.copy()
 
         # Normalize symbol
-        transformed['symbol'] = data.get('symbol', '').upper()
+        transformed["symbol"] = data.get("symbol", "").upper()
 
         # Convert date
-        if 'date' in data:
-            transformed['date'] = pd.to_datetime(data['date'])
+        if "date" in data:
+            transformed["date"] = pd.to_datetime(data["date"])
 
         # Calculate OHLC metrics
-        if all(k in data for k in ['open', 'high', 'low', 'close']):
-            transformed['daily_range'] = data['high'] - data['low']
-            transformed['daily_return'] = (data['close'] - data['open']) / data['open']
-            transformed['intraday_volatility'] = transformed['daily_range'] / data['close']
+        if all(k in data for k in ["open", "high", "low", "close"]):
+            transformed["daily_range"] = data["high"] - data["low"]
+            transformed["daily_return"] = (data["close"] - data["open"]) / data["open"]
+            transformed["intraday_volatility"] = transformed["daily_range"] / data["close"]
 
         # Calculate volume metrics
-        if 'volume' in data and 'close' in data:
-            transformed['dollar_volume'] = data['volume'] * data['close']
+        if "volume" in data and "close" in data:
+            transformed["dollar_volume"] = data["volume"] * data["close"]
 
         return transformed
 
     def _normalize_name(self, name: str) -> str:
         """Normalize politician name"""
         # Remove titles
-        titles = ['Sen.', 'Senator', 'Rep.', 'Representative', 'Hon.', 'Dr.', 'Mr.', 'Mrs.', 'Ms.']
+        titles = ["Sen.", "Senator", "Rep.", "Representative", "Hon.", "Dr.", "Mr.", "Mrs.", "Ms."]
         normalized = name
         for title in titles:
-            normalized = normalized.replace(title, '')
+            normalized = normalized.replace(title, "")
 
         # Clean and standardize
-        normalized = ' '.join(normalized.split())  # Remove extra spaces
+        normalized = " ".join(normalized.split())  # Remove extra spaces
         normalized = normalized.strip()
 
         return normalized
@@ -259,17 +258,17 @@ class DataTransformer:
     def _categorize_amount(self, amount: float) -> str:
         """Categorize transaction amount"""
         if amount < 1000:
-            return 'micro'
+            return "micro"
         elif amount < 15000:
-            return 'small'
+            return "small"
         elif amount < 50000:
-            return 'medium'
+            return "medium"
         elif amount < 250000:
-            return 'large'
+            return "large"
         elif amount < 1000000:
-            return 'very_large'
+            return "very_large"
         else:
-            return 'mega'
+            return "mega"
 
 
 class DataLoader:
@@ -279,8 +278,9 @@ class DataLoader:
         self.data_dir = data_dir
         self.data_dir.mkdir(parents=True, exist_ok=True)
 
-    async def save_batch(self, data: pd.DataFrame, data_type: str,
-                         timestamp: Optional[datetime] = None):
+    async def save_batch(
+        self, data: pd.DataFrame, data_type: str, timestamp: Optional[datetime] = None
+    ):
         """Save batch of data"""
         if timestamp is None:
             timestamp = datetime.now()
@@ -294,13 +294,14 @@ class DataLoader:
         filepath = type_dir / filename
 
         # Save as parquet
-        data.to_parquet(filepath, compression='snappy')
+        data.to_parquet(filepath, compression="snappy")
         logger.info(f"Saved {len(data)} records to {filepath}")
 
         return filepath
 
-    async def save_json(self, data: Union[Dict, List], data_type: str,
-                        timestamp: Optional[datetime] = None):
+    async def save_json(
+        self, data: Union[Dict, List], data_type: str, timestamp: Optional[datetime] = None
+    ):
         """Save data as JSON"""
         if timestamp is None:
             timestamp = datetime.now()
@@ -314,7 +315,7 @@ class DataLoader:
         filepath = type_dir / filename
 
         # Save JSON
-        with open(filepath, 'w') as f:
+        with open(filepath, "w") as f:
             json.dump(data, f, indent=2, default=str)
 
         logger.info(f"Saved JSON to {filepath}")
@@ -357,13 +358,13 @@ class IngestionPipeline:
 
         # Pipeline metrics
         self.metrics = {
-            'records_processed': 0,
-            'records_validated': 0,
-            'records_transformed': 0,
-            'records_saved': 0,
-            'errors': 0,
-            'start_time': None,
-            'last_update': None
+            "records_processed": 0,
+            "records_validated": 0,
+            "records_transformed": 0,
+            "records_saved": 0,
+            "errors": 0,
+            "start_time": None,
+            "last_update": None,
         }
 
     def add_source(self, name: str, connector):
@@ -375,11 +376,11 @@ class IngestionPipeline:
         """Initialize all data sources"""
         # Congressional data
         congress_api = CongressionalDataAPI()
-        self.add_source('congress', congress_api)
+        self.add_source("congress", congress_api)
 
         # Stock data sources
         yahoo = YahooFinanceConnector()
-        self.add_source('yahoo', yahoo)
+        self.add_source("yahoo", yahoo)
 
         # Add more sources as needed
         logger.info(f"Initialized {len(self.sources)} data sources")
@@ -392,17 +393,17 @@ class IngestionPipeline:
             # Validate
             if self.config.enable_validation:
                 if not self.validator.validate(record, data_type):
-                    self.metrics['errors'] += 1
+                    self.metrics["errors"] += 1
                     continue
-                self.metrics['records_validated'] += 1
+                self.metrics["records_validated"] += 1
 
             # Transform
             if self.config.enable_transformation:
                 record = self.transformer.transform(record, data_type)
-                self.metrics['records_transformed'] += 1
+                self.metrics["records_transformed"] += 1
 
             processed_data.append(record)
-            self.metrics['records_processed'] += 1
+            self.metrics["records_processed"] += 1
 
         # Convert to DataFrame
         if processed_data:
@@ -410,7 +411,7 @@ class IngestionPipeline:
 
             # Save to storage
             await self.loader.save_batch(df, data_type)
-            self.metrics['records_saved'] += len(df)
+            self.metrics["records_saved"] += len(df)
 
             return df
 
@@ -418,7 +419,7 @@ class IngestionPipeline:
 
     async def fetch_politician_trades(self, days: int = 30) -> pd.DataFrame:
         """Fetch recent politician trades"""
-        congress_api = self.sources.get('congress')
+        congress_api = self.sources.get("congress")
         if not congress_api:
             logger.error("Congressional data source not available")
             return pd.DataFrame()
@@ -427,28 +428,30 @@ class IngestionPipeline:
         trades = await congress_api.fetch_recent_trades(days=days)
 
         # Process through pipeline
-        df = await self.process_batch(trades, 'politician_trades')
+        df = await self.process_batch(trades, "politician_trades")
 
         logger.info(f"Fetched {len(df)} politician trades")
         return df
 
-    async def fetch_stock_data(self, tickers: List[str], period: str = '1mo') -> Dict[str, pd.DataFrame]:
+    async def fetch_stock_data(
+        self, tickers: List[str], period: str = "1mo"
+    ) -> Dict[str, pd.DataFrame]:
         """Fetch stock data for multiple tickers"""
         stock_data = {}
 
         for ticker in tickers:
             # Try Yahoo Finance first
-            yahoo = self.sources.get('yahoo')
+            yahoo = self.sources.get("yahoo")
             if yahoo:
                 try:
                     df = await yahoo.fetch_historical(ticker, period)
                     if not df.empty:
                         # Process through pipeline
-                        records = df.to_dict('records')
+                        records = df.to_dict("records")
                         for record in records:
-                            record['symbol'] = ticker
+                            record["symbol"] = ticker
 
-                        processed = await self.process_batch(records, 'market_data')
+                        processed = await self.process_batch(records, "market_data")
                         stock_data[ticker] = processed
                 except Exception as e:
                     logger.error(f"Failed to fetch {ticker}: {e}")
@@ -463,16 +466,14 @@ class IngestionPipeline:
 
         # Initialize stream processor
         stream_config = StreamConfig(
-            buffer_size=self.config.batch_size,
-            batch_size=100,
-            flush_interval=5
+            buffer_size=self.config.batch_size, batch_size=100, flush_interval=5
         )
 
         self.stream_processor = StreamProcessor(stream_config)
 
         # Add processor for pipeline
         async def pipeline_processor(batch):
-            await self.process_batch(batch, 'streaming_data')
+            await self.process_batch(batch, "streaming_data")
 
         self.stream_processor.add_processor(pipeline_processor)
 
@@ -484,21 +485,21 @@ class IngestionPipeline:
         if self.stream_processor:
             await self.stream_processor.stop()
 
-    async def run(self, mode: str = 'batch'):
+    async def run(self, mode: str = "batch"):
         """Run ingestion pipeline"""
-        self.metrics['start_time'] = datetime.now()
+        self.metrics["start_time"] = datetime.now()
 
         try:
             # Initialize sources
             await self.initialize_sources()
 
-            if mode == 'batch':
+            if mode == "batch":
                 # Batch processing
                 await self.run_batch()
-            elif mode == 'streaming':
+            elif mode == "streaming":
                 # Streaming mode
                 await self.start_streaming()
-            elif mode == 'hybrid':
+            elif mode == "hybrid":
                 # Both batch and streaming
                 batch_task = asyncio.create_task(self.run_batch())
                 stream_task = asyncio.create_task(self.start_streaming())
@@ -506,10 +507,10 @@ class IngestionPipeline:
 
         except Exception as e:
             logger.error(f"Pipeline error: {e}")
-            self.metrics['errors'] += 1
+            self.metrics["errors"] += 1
             raise
         finally:
-            self.metrics['last_update'] = datetime.now()
+            self.metrics["last_update"] = datetime.now()
 
     async def run_batch(self):
         """Run batch processing"""
@@ -519,8 +520,8 @@ class IngestionPipeline:
         trades_df = await self.fetch_politician_trades(days=30)
 
         # Extract unique tickers
-        if not trades_df.empty and 'ticker' in trades_df.columns:
-            tickers = trades_df['ticker'].unique().tolist()
+        if not trades_df.empty and "ticker" in trades_df.columns:
+            tickers = trades_df["ticker"].unique().tolist()
 
             # Fetch stock data for those tickers
             stock_data = await self.fetch_stock_data(tickers[:20])  # Limit to 20 for demo
@@ -532,36 +533,37 @@ class IngestionPipeline:
         metrics = self.metrics.copy()
 
         # Calculate throughput
-        if metrics['start_time']:
-            elapsed = (datetime.now() - metrics['start_time']).total_seconds()
+        if metrics["start_time"]:
+            elapsed = (datetime.now() - metrics["start_time"]).total_seconds()
             if elapsed > 0:
-                metrics['throughput'] = metrics['records_processed'] / elapsed
+                metrics["throughput"] = metrics["records_processed"] / elapsed
 
         # Add validation stats
-        metrics['validation_stats'] = self.validator.get_stats()
+        metrics["validation_stats"] = self.validator.get_stats()
 
         return metrics
 
 
 # Example usage
 if __name__ == "__main__":
+
     async def main():
         # Configure pipeline
         config = PipelineConfig(
             data_dir=Path("data/ingestion"),
             enable_streaming=False,  # Batch mode for testing
             enable_validation=True,
-            enable_transformation=True
+            enable_transformation=True,
         )
 
         # Create pipeline
         pipeline = IngestionPipeline(config)
 
         # Run batch processing
-        await pipeline.run(mode='batch')
+        await pipeline.run(mode="batch")
 
         # Get metrics
         metrics = pipeline.get_metrics()
         print(f"Pipeline metrics: {json.dumps(metrics, indent=2, default=str)}")
 
-    asyncio.run(main())
+    asyncio.run(main())