mcli-framework 7.12.2__py3-none-any.whl → 7.12.4__py3-none-any.whl

mcli/ml/data_ingestion/api_connectors.py

@@ -1,4 +1,4 @@
-"""API connectors for real-time data ingestion"""
+"""API connectors for real-time data ingestion."""
 
 import asyncio
 import json
@@ -12,7 +12,6 @@ from urllib.parse import urljoin
 
 import aiohttp
 import pandas as pd
-import requests
 import websockets
 import yfinance as yf
 
@@ -21,7 +20,7 @@ logger = logging.getLogger(__name__)
 
 @dataclass
 class APIConfig:
-    """API configuration"""
+    """API configuration."""
 
     api_key: Optional[str] = None
     base_url: str = ""
@@ -32,7 +31,7 @@ class APIConfig:
 
 
 class BaseAPIConnector(ABC):
-    """Base class for API connectors"""
+    """Base class for API connectors."""
 
     def __init__(self, config: APIConfig):
         self.config = config
@@ -41,11 +40,10 @@ class BaseAPIConnector(ABC):
 
     @abstractmethod
     async def fetch_data(self, **kwargs) -> Dict[str, Any]:
-        """Fetch data from API"""
-        pass
+        """Fetch data from API."""
 
     async def _make_request(self, endpoint: str, params: Optional[Dict] = None) -> Dict[str, Any]:
-        """Make API request with rate limiting and retry logic"""
+        """Make API request with rate limiting and retry logic."""
         await self.rate_limiter.acquire()
 
         url = urljoin(self.config.base_url, endpoint)
@@ -74,13 +72,13 @@ class BaseAPIConnector(ABC):
                 await asyncio.sleep(self.config.retry_delay * retry_count)
 
     async def close(self):
-        """Close session"""
+        """Close session."""
         if self.session:
             await self.session.close()
 
 
 class RateLimiter:
-    """Rate limiter for API requests"""
+    """Rate limiter for API requests."""
 
     def __init__(self, rate_limit: int):
         self.rate_limit = rate_limit
@@ -89,7 +87,7 @@ class RateLimiter:
         self.lock = asyncio.Lock()
 
     async def acquire(self):
-        """Acquire rate limit token"""
+        """Acquire rate limit token."""
         async with self.lock:
             while self.tokens <= 0:
                 now = time.time()
@@ -105,7 +103,7 @@ class RateLimiter:
 
 
 class CongressionalDataAPI(BaseAPIConnector):
-    """Congressional trading data API connector"""
+    """Congressional trading data API connector."""
 
     def __init__(self, config: Optional[APIConfig] = None):
         if not config:
@@ -113,7 +111,7 @@ class CongressionalDataAPI(BaseAPIConnector):
         super().__init__(config)
 
     async def fetch_recent_trades(self, days: int = 30) -> List[Dict[str, Any]]:
-        """Fetch recent congressional trades"""
+        """Fetch recent congressional trades."""
         params = {
             "from_date": (datetime.now() - timedelta(days=days)).isoformat(),
             "to_date": datetime.now().isoformat(),
@@ -128,7 +126,7 @@ class CongressionalDataAPI(BaseAPIConnector):
             return self._generate_mock_trades()
 
     async def fetch_politician_info(self, politician_id: str) -> Dict[str, Any]:
-        """Fetch politician information"""
+        """Fetch politician information."""
         try:
             return await self._make_request(f"politicians/{politician_id}")
         except Exception as e:
@@ -136,7 +134,7 @@ class CongressionalDataAPI(BaseAPIConnector):
             return self._generate_mock_politician_info(politician_id)
 
     def _generate_mock_trades(self) -> List[Dict[str, Any]]:
-        """Generate mock trades for testing"""
+        """Generate mock trades for testing."""
         import random
 
         trades = []
@@ -160,7 +158,7 @@ class CongressionalDataAPI(BaseAPIConnector):
         return trades
 
     def _generate_mock_politician_info(self, politician_id: str) -> Dict[str, Any]:
-        """Generate mock politician info"""
+        """Generate mock politician info."""
         return {
             "id": politician_id,
             "name": "Mock Politician",
@@ -172,23 +170,20 @@ class CongressionalDataAPI(BaseAPIConnector):
 
 
 class StockMarketAPI(BaseAPIConnector):
-    """Base class for stock market APIs"""
+    """Base class for stock market APIs."""
 
     async def fetch_quote(self, symbol: str) -> Dict[str, Any]:
-        """Fetch current stock quote"""
-        pass
+        """Fetch current stock quote."""
 
     async def fetch_historical(self, symbol: str, period: str = "1mo") -> pd.DataFrame:
-        """Fetch historical stock data"""
-        pass
+        """Fetch historical stock data."""
 
     async def stream_quotes(self, symbols: List[str]) -> AsyncIterator[Dict[str, Any]]:
-        """Stream real-time quotes"""
-        pass
+        """Stream real-time quotes."""
 
 
 class AlphaVantageConnector(StockMarketAPI):
-    """Alpha Vantage API connector"""
+    """Alpha Vantage API connector."""
 
     def __init__(self, api_key: str):
         config = APIConfig(
@@ -199,14 +194,14 @@ class AlphaVantageConnector(StockMarketAPI):
         super().__init__(config)
 
     async def fetch_quote(self, symbol: str) -> Dict[str, Any]:
-        """Fetch current quote from Alpha Vantage"""
+        """Fetch current quote from Alpha Vantage."""
         params = {"function": "GLOBAL_QUOTE", "symbol": symbol, "apikey": self.config.api_key}
 
         data = await self._make_request("", params)
         return self._parse_quote(data.get("Global Quote", {}))
 
     async def fetch_historical(self, symbol: str, period: str = "1mo") -> pd.DataFrame:
-        """Fetch historical data from Alpha Vantage"""
+        """Fetch historical data from Alpha Vantage."""
         params = {
             "function": "TIME_SERIES_DAILY",
             "symbol": symbol,
@@ -226,7 +221,7 @@ class AlphaVantageConnector(StockMarketAPI):
         return df.sort_index()
 
     def _parse_quote(self, quote_data: Dict[str, Any]) -> Dict[str, Any]:
-        """Parse Alpha Vantage quote"""
+        """Parse Alpha Vantage quote."""
         return {
             "symbol": quote_data.get("01. symbol", ""),
             "price": float(quote_data.get("05. price", 0)),
@@ -238,14 +233,14 @@ class AlphaVantageConnector(StockMarketAPI):
 
 
 class YahooFinanceConnector(StockMarketAPI):
-    """Yahoo Finance connector using yfinance"""
+    """Yahoo Finance connector using yfinance."""
 
     def __init__(self):
         config = APIConfig(rate_limit=2000)  # Yahoo Finance is generous
         super().__init__(config)
 
     async def fetch_quote(self, symbol: str) -> Dict[str, Any]:
-        """Fetch current quote from Yahoo Finance"""
+        """Fetch current quote from Yahoo Finance."""
         try:
             ticker = yf.Ticker(symbol)
             info = ticker.info
@@ -263,7 +258,7 @@ class YahooFinanceConnector(StockMarketAPI):
             return {}
 
     async def fetch_historical(self, symbol: str, period: str = "1mo") -> pd.DataFrame:
-        """Fetch historical data from Yahoo Finance"""
+        """Fetch historical data from Yahoo Finance."""
         try:
             ticker = yf.Ticker(symbol)
             df = ticker.history(period=period)
@@ -274,14 +269,14 @@ class YahooFinanceConnector(StockMarketAPI):
 
 
 class PolygonIOConnector(StockMarketAPI):
-    """Polygon.io API connector"""
+    """Polygon.io API connector."""
 
     def __init__(self, api_key: str):
         config = APIConfig(api_key=api_key, base_url="https://api.polygon.io/", rate_limit=100)
         super().__init__(config)
 
     async def fetch_quote(self, symbol: str) -> Dict[str, Any]:
-        """Fetch current quote from Polygon.io"""
+        """Fetch current quote from Polygon.io."""
         endpoint = f"v2/last/nbbo/{symbol}"
         params = {"apiKey": self.config.api_key}
 
@@ -291,7 +286,7 @@ class PolygonIOConnector(StockMarketAPI):
     async def fetch_aggregates(
         self, symbol: str, from_date: str, to_date: str, timespan: str = "day"
     ) -> pd.DataFrame:
-        """Fetch aggregate bars from Polygon.io"""
+        """Fetch aggregate bars from Polygon.io."""
         endpoint = f"v2/aggs/ticker/{symbol}/range/1/{timespan}/{from_date}/{to_date}"
         params = {"apiKey": self.config.api_key, "adjusted": "true", "sort": "asc"}
 
@@ -308,7 +303,7 @@ class PolygonIOConnector(StockMarketAPI):
         return df.set_index("timestamp")
 
     def _parse_polygon_quote(self, data: Dict[str, Any]) -> Dict[str, Any]:
-        """Parse Polygon.io quote"""
+        """Parse Polygon.io quote."""
         results = data.get("results", {})
         return {
             "symbol": results.get("T", ""),
@@ -319,7 +314,7 @@ class PolygonIOConnector(StockMarketAPI):
 
 
 class QuiverQuantConnector(BaseAPIConnector):
-    """QuiverQuant API for congressional trading data"""
+    """QuiverQuant API for congressional trading data."""
 
     def __init__(self, api_key: str):
         config = APIConfig(
@@ -328,7 +323,7 @@ class QuiverQuantConnector(BaseAPIConnector):
         super().__init__(config)
 
     async def fetch_congress_trades(self) -> List[Dict[str, Any]]:
-        """Fetch congressional trading data"""
+        """Fetch congressional trading data."""
         headers = {"Authorization": f"Bearer {self.config.api_key}", "Accept": "application/json"}
 
         try:
@@ -346,7 +341,7 @@ class QuiverQuantConnector(BaseAPIConnector):
             return []
 
     async def fetch_lobbying(self, ticker: str) -> List[Dict[str, Any]]:
-        """Fetch lobbying data for a ticker"""
+        """Fetch lobbying data for a ticker."""
         headers = {"Authorization": f"Bearer {self.config.api_key}", "Accept": "application/json"}
 
         try:
@@ -365,7 +360,7 @@ class QuiverQuantConnector(BaseAPIConnector):
 
 
 class WebSocketDataStream:
-    """WebSocket stream for real-time data"""
+    """WebSocket stream for real-time data."""
 
     def __init__(self, url: str, api_key: Optional[str] = None):
         self.url = url
@@ -374,11 +369,11 @@ class WebSocketDataStream:
         self.handlers = []
 
     def add_handler(self, handler: Callable):
-        """Add message handler"""
+        """Add message handler."""
         self.handlers.append(handler)
 
     async def connect(self):
-        """Connect to WebSocket"""
+        """Connect to WebSocket."""
         headers = {}
         if self.api_key:
             headers["Authorization"] = f"Bearer {self.api_key}"
@@ -387,7 +382,7 @@ class WebSocketDataStream:
         logger.info(f"Connected to WebSocket: {self.url}")
 
     async def subscribe(self, symbols: List[str]):
-        """Subscribe to symbols"""
+        """Subscribe to symbols."""
         if not self.websocket:
             await self.connect()
 
@@ -395,7 +390,7 @@ class WebSocketDataStream:
         await self.websocket.send(json.dumps(message))
 
     async def stream(self):
-        """Stream messages"""
+        """Stream messages."""
         if not self.websocket:
             await self.connect()
 
@@ -413,13 +408,13 @@ class WebSocketDataStream:
                     logger.error(f"Handler error: {e}")
 
     async def close(self):
-        """Close WebSocket connection"""
+        """Close WebSocket connection."""
         if self.websocket:
             await self.websocket.close()
 
 
 class DataAggregator:
-    """Aggregate data from multiple sources"""
+    """Aggregate data from multiple sources."""
 
     def __init__(self):
         self.sources = {}
@@ -427,12 +422,12 @@ class DataAggregator:
         self.cache_ttl = 300  # 5 minutes
 
     def add_source(self, name: str, connector: BaseAPIConnector):
-        """Add data source"""
+        """Add data source."""
         self.sources[name] = connector
         logger.info(f"Added data source: {name}")
 
     async def fetch_all(self, symbol: str) -> Dict[str, Any]:
-        """Fetch data from all sources"""
+        """Fetch data from all sources."""
         results = {}
 
         # Check cache
@@ -466,6 +461,6 @@ class DataAggregator:
         return results
 
     async def _fetch_with_name(self, name: str, coro):
-        """Helper to fetch with source name"""
+        """Helper to fetch with source name."""
         result = await coro
         return name, result

mcli/ml/data_ingestion/data_pipeline.py

@@ -1,26 +1,16 @@
-"""Complete data ingestion pipeline with validation and transformation"""
+"""Complete data ingestion pipeline with validation and transformation."""
 
 import asyncio
 import json
 import logging
-from abc import ABC, abstractmethod
-from dataclasses import dataclass, field
-from datetime import datetime, timedelta
+from dataclasses import dataclass
+from datetime import datetime
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
-import numpy as np
 import pandas as pd
 
-from .api_connectors import (
-    AlphaVantageConnector,
-    CongressionalDataAPI,
-    DataAggregator,
-    PolygonIOConnector,
-    QuiverQuantConnector,
-    YahooFinanceConnector,
-)
-from .stream_processor import DataAggregator as StreamAggregator
+from .api_connectors import CongressionalDataAPI, YahooFinanceConnector
 from .stream_processor import StreamConfig, StreamProcessor
 
 logger = logging.getLogger(__name__)
@@ -28,7 +18,7 @@ logger = logging.getLogger(__name__)
 
 @dataclass
 class PipelineConfig:
-    """Data pipeline configuration"""
+    """Data pipeline configuration."""
 
     data_dir: Path = Path("data")
     batch_size: int = 1000
@@ -42,7 +32,7 @@ class PipelineConfig:
 
 
 class DataValidator:
-    """Validate incoming data"""
+    """Validate incoming data."""
 
     def __init__(self):
         self.validation_rules = {
@@ -53,7 +43,7 @@ class DataValidator:
         self.validation_stats = {"total": 0, "valid": 0, "invalid": 0, "errors": []}
 
     def validate(self, data: Dict[str, Any], data_type: str) -> bool:
-        """Validate data based on type"""
+        """Validate data based on type."""
         self.validation_stats["total"] += 1
 
         if data_type not in self.validation_rules:
@@ -74,7 +64,7 @@ class DataValidator:
             return False
 
     def _validate_politician_trade(self, data: Dict[str, Any]) -> bool:
-        """Validate politician trading data"""
+        """Validate politician trading data."""
         required_fields = ["politician", "ticker", "transaction_type", "amount", "transaction_date"]
 
         # Check required fields
@@ -97,14 +87,14 @@ class DataValidator:
         try:
             if isinstance(data["transaction_date"], str):
                 datetime.fromisoformat(data["transaction_date"])
-        except:
+        except Exception:
             logger.warning(f"Invalid date format: {data['transaction_date']}")
             return False
 
         return True
 
     def _validate_stock_quote(self, data: Dict[str, Any]) -> bool:
-        """Validate stock quote data"""
+        """Validate stock quote data."""
         required_fields = ["symbol", "price", "timestamp"]
 
         for field in required_fields:
@@ -118,7 +108,7 @@ class DataValidator:
         return True
 
     def _validate_market_data(self, data: Dict[str, Any]) -> bool:
-        """Validate market data"""
+        """Validate market data."""
         required_fields = ["symbol", "close", "volume", "date"]
 
         for field in required_fields:
@@ -127,7 +117,7 @@ class DataValidator:
 
         # Validate prices
         for price_field in ["close", "open", "high", "low"]:
-            if price_field in data:
+            if price_field in data:  # noqa: SIM102
                 if not isinstance(data[price_field], (int, float)) or data[price_field] <= 0:
                     return False
 
@@ -138,12 +128,12 @@ class DataValidator:
         return True
 
     def get_stats(self) -> Dict[str, Any]:
-        """Get validation statistics"""
+        """Get validation statistics."""
         return self.validation_stats.copy()
 
 
 class DataTransformer:
-    """Transform and normalize data"""
+    """Transform and normalize data."""
 
     def __init__(self):
         self.transformers = {
@@ -155,7 +145,7 @@ class DataTransformer:
     def transform(
         self, data: Union[Dict[str, Any], List[Dict[str, Any]]], data_type: str
     ) -> Union[Dict[str, Any], pd.DataFrame]:
-        """Transform data based on type"""
+        """Transform data based on type."""
         if data_type not in self.transformers:
             return data
 
@@ -166,7 +156,7 @@ class DataTransformer:
             return self.transformers[data_type](data)
 
     def _transform_politician_trade(self, data: Dict[str, Any]) -> Dict[str, Any]:
-        """Transform politician trading data"""
+        """Transform politician trading data."""
         transformed = data.copy()
 
         # Standardize politician name
@@ -198,7 +188,7 @@ class DataTransformer:
         return transformed
 
     def _transform_stock_quote(self, data: Dict[str, Any]) -> Dict[str, Any]:
-        """Transform stock quote data"""
+        """Transform stock quote data."""
         transformed = data.copy()
 
         # Normalize symbol
@@ -219,7 +209,7 @@ class DataTransformer:
         return transformed
 
     def _transform_market_data(self, data: Dict[str, Any]) -> Dict[str, Any]:
-        """Transform market data"""
+        """Transform market data."""
         transformed = data.copy()
 
         # Normalize symbol
@@ -242,7 +232,7 @@ class DataTransformer:
         return transformed
 
     def _normalize_name(self, name: str) -> str:
-        """Normalize politician name"""
+        """Normalize politician name."""
         # Remove titles
         titles = ["Sen.", "Senator", "Rep.", "Representative", "Hon.", "Dr.", "Mr.", "Mrs.", "Ms."]
         normalized = name
@@ -256,7 +246,7 @@ class DataTransformer:
         return normalized
 
     def _categorize_amount(self, amount: float) -> str:
-        """Categorize transaction amount"""
+        """Categorize transaction amount."""
         if amount < 1000:
             return "micro"
         elif amount < 15000:
@@ -272,7 +262,7 @@ class DataTransformer:
 
 
 class DataLoader:
-    """Load data to storage"""
+    """Load data to storage."""
 
     def __init__(self, data_dir: Path):
         self.data_dir = data_dir
@@ -281,7 +271,7 @@ class DataLoader:
     async def save_batch(
         self, data: pd.DataFrame, data_type: str, timestamp: Optional[datetime] = None
     ):
-        """Save batch of data"""
+        """Save batch of data."""
         if timestamp is None:
             timestamp = datetime.now()
 
@@ -302,7 +292,7 @@ class DataLoader:
     async def save_json(
         self, data: Union[Dict, List], data_type: str, timestamp: Optional[datetime] = None
     ):
-        """Save data as JSON"""
+        """Save data as JSON."""
         if timestamp is None:
             timestamp = datetime.now()
 
@@ -322,7 +312,7 @@ class DataLoader:
         return filepath
 
     def load_latest(self, data_type: str, n_files: int = 1) -> pd.DataFrame:
-        """Load latest data files"""
+        """Load latest data files."""
         type_dir = self.data_dir / data_type
 
         if not type_dir.exists():
@@ -344,7 +334,7 @@ class DataLoader:
 
 
 class IngestionPipeline:
-    """Complete data ingestion pipeline"""
+    """Complete data ingestion pipeline."""
 
     def __init__(self, config: PipelineConfig):
         self.config = config
@@ -368,12 +358,12 @@ class IngestionPipeline:
         }
 
     def add_source(self, name: str, connector):
-        """Add data source"""
+        """Add data source."""
         self.sources[name] = connector
         logger.info(f"Added data source: {name}")
 
     async def initialize_sources(self):
-        """Initialize all data sources"""
+        """Initialize all data sources."""
         # Congressional data
         congress_api = CongressionalDataAPI()
         self.add_source("congress", congress_api)
@@ -386,7 +376,7 @@ class IngestionPipeline:
         logger.info(f"Initialized {len(self.sources)} data sources")
 
     async def process_batch(self, data: List[Dict[str, Any]], data_type: str) -> pd.DataFrame:
-        """Process batch of data through pipeline"""
+        """Process batch of data through pipeline."""
         processed_data = []
 
         for record in data:
@@ -418,7 +408,7 @@ class IngestionPipeline:
         return pd.DataFrame()
 
     async def fetch_politician_trades(self, days: int = 30) -> pd.DataFrame:
-        """Fetch recent politician trades"""
+        """Fetch recent politician trades."""
         congress_api = self.sources.get("congress")
         if not congress_api:
             logger.error("Congressional data source not available")
@@ -436,7 +426,7 @@ class IngestionPipeline:
     async def fetch_stock_data(
         self, tickers: List[str], period: str = "1mo"
     ) -> Dict[str, pd.DataFrame]:
-        """Fetch stock data for multiple tickers"""
+        """Fetch stock data for multiple tickers."""
         stock_data = {}
 
         for ticker in tickers:
@@ -459,7 +449,7 @@ class IngestionPipeline:
         return stock_data
 
     async def start_streaming(self):
-        """Start real-time streaming"""
+        """Start real-time streaming."""
         if not self.config.enable_streaming:
             logger.info("Streaming disabled")
             return
@@ -481,12 +471,12 @@ class IngestionPipeline:
         await self.stream_processor.start()
 
     async def stop_streaming(self):
-        """Stop streaming"""
+        """Stop streaming."""
         if self.stream_processor:
             await self.stream_processor.stop()
 
     async def run(self, mode: str = "batch"):
-        """Run ingestion pipeline"""
+        """Run ingestion pipeline."""
         self.metrics["start_time"] = datetime.now()
 
         try:
@@ -513,7 +503,7 @@ class IngestionPipeline:
             self.metrics["last_update"] = datetime.now()
 
     async def run_batch(self):
-        """Run batch processing"""
+        """Run batch processing."""
         logger.info("Starting batch processing...")
 
         # Fetch politician trades
@@ -529,7 +519,7 @@ class IngestionPipeline:
             logger.info(f"Processed {len(trades_df)} trades and {len(stock_data)} stocks")
 
     def get_metrics(self) -> Dict[str, Any]:
-        """Get pipeline metrics"""
+        """Get pipeline metrics."""
         metrics = self.metrics.copy()
 
         # Calculate throughput