mcli-framework 7.12.2__py3-none-any.whl → 7.12.4__py3-none-any.whl

mcli/ml/data_ingestion/stream_processor.py

@@ -1,4 +1,4 @@
-"""Real-time stream processing for financial data"""
+"""Real-time stream processing for financial data."""
 
 import asyncio
 import json
@@ -7,21 +7,20 @@ import time
 from abc import ABC, abstractmethod
 from collections import deque
 from dataclasses import dataclass
-from datetime import datetime, timedelta
+from datetime import datetime
 from typing import Any, AsyncIterator, Callable, Dict, List, Optional
 
 import numpy as np
-import pandas as pd
 import websockets
-from kafka import KafkaConsumer, KafkaProducer
-from kafka.errors import KafkaError
+
+# KafkaConsumer from kafka library not needed - we have our own implementation below
 
 logger = logging.getLogger(__name__)
 
 
 @dataclass
 class StreamConfig:
-    """Stream processing configuration"""
+    """Stream processing configuration."""
 
     buffer_size: int = 1000
     batch_size: int = 100
@@ -33,7 +32,7 @@ class StreamConfig:
 
 
 class DataStream(ABC):
-    """Base class for data streams"""
+    """Base class for data streams."""
 
     def __init__(self, config: StreamConfig):
         self.config = config
@@ -44,20 +43,18 @@ class DataStream(ABC):
 
     @abstractmethod
     async def connect(self):
-        """Connect to data source"""
-        pass
+        """Connect to data source."""
 
     @abstractmethod
     async def consume(self) -> AsyncIterator[Dict[str, Any]]:
-        """Consume data from stream"""
-        pass
+        """Consume data from stream."""
 
     def add_handler(self, handler: Callable):
-        """Add data handler"""
+        """Add data handler."""
         self.handlers.append(handler)
 
     async def process_message(self, message: Dict[str, Any]):
-        """Process single message"""
+        """Process single message."""
         # Add to buffer
         self.buffer.append(message)
 
@@ -70,7 +67,7 @@ class DataStream(ABC):
             await self.flush_buffer()
 
     async def flush_buffer(self):
-        """Flush buffer and process batch"""
+        """Flush buffer and process batch."""
         if not self.buffer:
             return
 
@@ -89,7 +86,7 @@ class DataStream(ABC):
                 logger.error(f"Handler error: {e}")
 
     async def start(self):
-        """Start consuming stream"""
+        """Start consuming stream."""
         self.is_running = True
         await self.connect()
 
@@ -102,13 +99,13 @@ class DataStream(ABC):
             await self.flush_buffer()
 
     async def stop(self):
-        """Stop consuming stream"""
+        """Stop consuming stream."""
         self.is_running = False
         await self.flush_buffer()
 
 
 class KafkaStream(DataStream):
-    """Kafka stream consumer"""
+    """Kafka stream consumer."""
 
     def __init__(
         self,
@@ -124,7 +121,7 @@ class KafkaStream(DataStream):
         self.consumer = None
 
     async def connect(self):
-        """Connect to Kafka"""
+        """Connect to Kafka."""
         self.consumer = KafkaConsumer(
             self.topic,
             bootstrap_servers=self.bootstrap_servers,
@@ -136,20 +133,20 @@ class KafkaStream(DataStream):
         logger.info(f"Connected to Kafka topic: {self.topic}")
 
     async def consume(self) -> AsyncIterator[Dict[str, Any]]:
-        """Consume from Kafka"""
+        """Consume from Kafka."""
         loop = asyncio.get_event_loop()
 
         while self.is_running:
             # Poll messages
             messages = await loop.run_in_executor(None, self.consumer.poll, 1000)  # timeout ms
 
-            for topic_partition, records in messages.items():
+            for _topic_partition, records in messages.items():
                 for record in records:
                     yield record.value
 
 
 class WebSocketStream(DataStream):
-    """WebSocket stream consumer"""
+    """WebSocket stream consumer."""
 
     def __init__(self, config: StreamConfig, url: str):
         super().__init__(config)
@@ -157,12 +154,12 @@ class WebSocketStream(DataStream):
         self.websocket = None
 
     async def connect(self):
-        """Connect to WebSocket"""
+        """Connect to WebSocket."""
         self.websocket = await websockets.connect(self.url)
         logger.info(f"Connected to WebSocket: {self.url}")
 
     async def consume(self) -> AsyncIterator[Dict[str, Any]]:
-        """Consume from WebSocket"""
+        """Consume from WebSocket."""
         async for message in self.websocket:
             try:
                 data = json.loads(message)
@@ -172,7 +169,7 @@ class WebSocketStream(DataStream):
 
 
 class StreamProcessor:
-    """Process real-time data streams"""
+    """Process real-time data streams."""
 
     def __init__(self, config: StreamConfig):
         self.config = config
@@ -181,7 +178,7 @@ class StreamProcessor:
         self.metrics = StreamMetrics()
 
     def add_stream(self, name: str, stream: DataStream):
-        """Add data stream"""
+        """Add data stream."""
         self.streams[name] = stream
 
         # Add metrics handler
@@ -192,7 +189,7 @@ class StreamProcessor:
             stream.add_handler(processor)
 
     def add_processor(self, processor: Callable):
-        """Add data processor"""
+        """Add data processor."""
         self.processors.append(processor)
 
         # Add to existing streams
@@ -200,7 +197,7 @@ class StreamProcessor:
             stream.add_handler(processor)
 
     async def update_metrics(self, batch: List[Dict[str, Any]]):
-        """Update stream metrics"""
+        """Update stream metrics."""
         self.metrics.messages_processed += len(batch)
         self.metrics.last_update = datetime.now()
 
@@ -214,7 +211,7 @@ class StreamProcessor:
             self.metrics.throughput = self.metrics.messages_processed / elapsed
 
     async def start(self):
-        """Start all streams"""
+        """Start all streams."""
         tasks = []
         for name, stream in self.streams.items():
             logger.info(f"Starting stream: {name}")
@@ -224,13 +221,13 @@ class StreamProcessor:
         await asyncio.gather(*tasks)
 
     async def stop(self):
-        """Stop all streams"""
+        """Stop all streams."""
         for name, stream in self.streams.items():
             logger.info(f"Stopping stream: {name}")
             await stream.stop()
 
     def get_metrics(self) -> Dict[str, Any]:
-        """Get stream metrics"""
+        """Get stream metrics."""
         return {
             "messages_processed": self.metrics.messages_processed,
             "throughput": self.metrics.throughput,
@@ -244,7 +241,7 @@ class StreamProcessor:
 
 @dataclass
 class StreamMetrics:
-    """Stream processing metrics"""
+    """Stream processing metrics."""
 
     messages_processed: int = 0
     throughput: float = 0  # messages per second
@@ -254,7 +251,7 @@ class StreamMetrics:
 
 
 class DataAggregator:
-    """Aggregate data from multiple streams"""
+    """Aggregate data from multiple streams."""
 
     def __init__(self, window_size: int = 60):
         self.window_size = window_size
@@ -263,7 +260,7 @@ class DataAggregator:
         self.last_aggregation = time.time()
 
     async def process_batch(self, batch: List[Dict[str, Any]]):
-        """Process batch of messages"""
+        """Process batch of messages."""
         for message in batch:
             # Extract key fields
             symbol = message.get("symbol") or message.get("ticker")
@@ -280,7 +277,7 @@ class DataAggregator:
             await self.aggregate()
 
     async def aggregate(self):
-        """Aggregate buffered data"""
+        """Aggregate buffered data."""
         self.last_aggregation = time.time()
 
         for symbol, data_points in self.data_buffer.items():
@@ -318,21 +315,21 @@ class DataAggregator:
         logger.info(f"Aggregated data for {len(self.aggregated_data)} symbols")
 
     def get_aggregated_data(self, symbol: Optional[str] = None) -> Dict[str, Any]:
-        """Get aggregated data"""
+        """Get aggregated data."""
         if symbol:
             return self.aggregated_data.get(symbol, {})
         return self.aggregated_data
 
 
 class StreamEnricher:
-    """Enrich streaming data with additional context"""
+    """Enrich streaming data with additional context."""
 
     def __init__(self):
         self.enrichment_cache = {}
         self.cache_ttl = 300  # 5 minutes
 
     async def enrich_batch(self, batch: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
-        """Enrich batch of messages"""
+        """Enrich batch of messages."""
         enriched = []
 
         for message in batch:
@@ -342,7 +339,7 @@ class StreamEnricher:
         return enriched
 
     async def enrich_message(self, message: Dict[str, Any]) -> Dict[str, Any]:
-        """Enrich single message"""
+        """Enrich single message."""
         enriched = message.copy()
 
         # Add processing metadata
@@ -359,7 +356,7 @@ class StreamEnricher:
         return enriched
 
     async def enrich_political_data(self, message: Dict[str, Any]) -> Dict[str, Any]:
-        """Enrich political trading data"""
+        """Enrich political trading data."""
         politician = message.get("politician")
 
         if politician:
@@ -388,7 +385,7 @@ class StreamEnricher:
         return message
 
     async def enrich_market_data(self, message: Dict[str, Any]) -> Dict[str, Any]:
-        """Enrich market data"""
+        """Enrich market data."""
         symbol = message.get("ticker") or message.get("symbol")
 
         if symbol:
@@ -418,7 +415,7 @@ class StreamEnricher:
 
 
 class KafkaConsumer:
-    """Kafka consumer for real-time data"""
+    """Kafka consumer for real-time data."""
 
     def __init__(self, bootstrap_servers: str, topics: List[str]):
         self.bootstrap_servers = bootstrap_servers
@@ -426,7 +423,7 @@ class KafkaConsumer:
         self.consumer = None
 
     async def connect(self):
-        """Connect to Kafka"""
+        """Connect to Kafka."""
         self.consumer = KafkaConsumer(
             *self.topics,
             bootstrap_servers=self.bootstrap_servers,
@@ -435,7 +432,7 @@ class KafkaConsumer:
         )
 
     async def consume(self, handler: Callable):
-        """Consume messages"""
+        """Consume messages."""
         for message in self.consumer:
             try:
                 await handler(message.value)
@@ -444,18 +441,18 @@ class KafkaConsumer:
 
 
 class WebSocketConsumer:
-    """WebSocket consumer for real-time data"""
+    """WebSocket consumer for real-time data."""
 
     def __init__(self, url: str):
         self.url = url
         self.websocket = None
 
     async def connect(self):
-        """Connect to WebSocket"""
+        """Connect to WebSocket."""
         self.websocket = await websockets.connect(self.url)
 
     async def consume(self, handler: Callable):
-        """Consume messages"""
+        """Consume messages."""
         async for message in self.websocket:
             try:
                 data = json.loads(message)

mcli/ml/database/__init__.py

@@ -1,4 +1,4 @@
-"""Database models and utilities"""
+"""Database models and utilities."""
 
 from .models import (
     Alert,

mcli/ml/database/migrations/env.py

@@ -1,4 +1,4 @@
-"""Alembic environment configuration"""
+"""Alembic environment configuration."""
 
 import os
 import sys
@@ -27,7 +27,7 @@ target_metadata = Base.metadata
 
 # Override database URL from settings
 def get_url():
-    """Get database URL from settings or environment"""
+    """Get database URL from settings or environment."""
     # First try environment variable
     url = os.getenv("DATABASE_URL")
     if url:

mcli/ml/database/models.py

@@ -1,8 +1,7 @@
-"""Database models for ML system"""
+"""Database models for ML system."""
 
 from datetime import datetime
 from enum import Enum as PyEnum
-from typing import Any, Dict, List, Optional
 from uuid import uuid4
 
 from sqlalchemy import (
@@ -26,7 +25,6 @@ from sqlalchemy import (
 from sqlalchemy.dialects.postgresql import ARRAY, UUID
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import declarative_base, relationship, validates
-from sqlalchemy.sql import func
 
 Base = declarative_base()
 
@@ -52,7 +50,7 @@ experiment_models = Table(
 
 
 class UserRole(PyEnum):
-    """User role enumeration"""
+    """User role enumeration."""
 
     ADMIN = "admin"
     USER = "user"
@@ -61,7 +59,7 @@ class UserRole(PyEnum):
 
 
 class ModelStatus(PyEnum):
-    """Model status enumeration"""
+    """Model status enumeration."""
 
     TRAINING = "training"
     TRAINED = "trained"
@@ -71,7 +69,7 @@ class ModelStatus(PyEnum):
 
 
 class TradeType(PyEnum):
-    """Trade type enumeration"""
+    """Trade type enumeration."""
 
     BUY = "buy"
     SELL = "sell"
@@ -79,7 +77,7 @@ class TradeType(PyEnum):
 
 
 class AlertType(PyEnum):
-    """Alert type enumeration"""
+    """Alert type enumeration."""
 
     POLITICIAN_TRADE = "politician_trade"
     PRICE_MOVEMENT = "price_movement"
@@ -89,7 +87,7 @@ class AlertType(PyEnum):
 
 
 class User(Base):
-    """User model for authentication and authorization"""
+    """User model for authentication and authorization."""
 
     __tablename__ = "users"
 
@@ -127,7 +125,7 @@ class User(Base):
 
     @validates("email")
     def validate_email(self, key, email):
-        """Validate email format"""
+        """Validate email format."""
         import re
 
         if not re.match(r"^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}$", email):
@@ -136,7 +134,7 @@ class User(Base):
 
 
 class Politician(Base):
-    """Politician information"""
+    """Politician information."""
 
     __tablename__ = "politicians"
 
@@ -169,7 +167,7 @@ class Politician(Base):
 
 
 class Trade(Base):
-    """Political trading records"""
+    """Political trading records."""
 
     __tablename__ = "trades"
 
@@ -206,14 +204,14 @@ class Trade(Base):
 
     @hybrid_property
     def estimated_amount(self):
-        """Get estimated trade amount (midpoint)"""
+        """Get estimated trade amount (midpoint)."""
         if self.amount_min and self.amount_max:
             return (self.amount_min + self.amount_max) / 2
         return self.amount_min or self.amount_max
 
 
 class StockData(Base):
-    """Stock market data"""
+    """Stock market data."""
 
     __tablename__ = "stock_data"
 
@@ -264,7 +262,7 @@ class StockData(Base):
 
 
 class Prediction(Base):
-    """Model predictions"""
+    """Model predictions."""
 
     __tablename__ = "predictions"
 
@@ -313,7 +311,7 @@ class Prediction(Base):
 
 
 class Portfolio(Base):
-    """User portfolios"""
+    """User portfolios."""
 
     __tablename__ = "portfolios"
 
@@ -358,7 +356,7 @@ class Portfolio(Base):
 
 
 class Alert(Base):
-    """User alerts and notifications"""
+    """User alerts and notifications."""
 
     __tablename__ = "alerts"
 
@@ -396,7 +394,7 @@ class Alert(Base):
 
 
 class BacktestResult(Base):
-    """Backtesting results"""
+    """Backtesting results."""
 
     __tablename__ = "backtest_results"
 
@@ -448,7 +446,7 @@ class BacktestResult(Base):
 
 
 class Experiment(Base):
-    """ML experiments tracking"""
+    """ML experiments tracking."""
 
     __tablename__ = "experiments"
 
@@ -496,7 +494,7 @@ class Experiment(Base):
 
 
 class Model(Base):
-    """ML models registry"""
+    """ML models registry."""
 
     __tablename__ = "models"
 
@@ -578,7 +576,7 @@ class Model(Base):
 
 
 class FeatureSet(Base):
-    """Feature sets for models"""
+    """Feature sets for models."""
 
     __tablename__ = "feature_sets"
 
@@ -617,7 +615,7 @@ class FeatureSet(Base):
 
 
 class DataVersion(Base):
-    """Data versioning for DVC"""
+    """Data versioning for DVC."""
 
     __tablename__ = "data_versions"
 
@@ -658,7 +656,7 @@ class DataVersion(Base):
 # Database events and triggers
 @event.listens_for(User, "before_insert")
 def generate_api_key(mapper, connection, target):
-    """Generate API key for new users"""
+    """Generate API key for new users."""
     if not target.api_key:
         import secrets
 
@@ -668,7 +666,7 @@ def generate_api_key(mapper, connection, target):
 
 @event.listens_for(Portfolio, "before_update")
 def update_portfolio_metrics(mapper, connection, target):
-    """Update portfolio performance metrics"""
+    """Update portfolio performance metrics."""
     if target.current_value and target.initial_capital:
         target.total_return = (
             (target.current_value - target.initial_capital) / target.initial_capital
@@ -677,7 +675,7 @@ def update_portfolio_metrics(mapper, connection, target):
 
 # Create indexes for better query performance
 def create_indexes(engine):
-    """Create additional database indexes"""
+    """Create additional database indexes."""
     from sqlalchemy import text
 
     indexes = [

mcli/ml/database/session.py

@@ -1,4 +1,4 @@
-"""Database session management"""
+"""Database session management."""
 
 # Synchronous database setup
 # Prioritize DATABASE_URL environment variable over settings
@@ -9,7 +9,7 @@ from typing import AsyncGenerator, Generator
 from sqlalchemy import create_engine
 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
 from sqlalchemy.orm import Session, sessionmaker
-from sqlalchemy.pool import NullPool, StaticPool
+from sqlalchemy.pool import StaticPool
 
 from mcli.ml.config import settings
 
@@ -28,7 +28,7 @@ if not database_url:
         settings_url = settings.database.url
         if settings_url and "sqlite" not in settings_url:
             database_url = settings_url
-    except (AttributeError, Exception):
+    except Exception:
         pass  # Continue with database_url=None
 
 # If still no valid DATABASE_URL, try to use Supabase REST API via connection pooler
@@ -103,7 +103,7 @@ import logging
 logger = logging.getLogger(__name__)
 
 if "pooler.supabase.com" in database_url:
-    logger.info(f"🔗 Using Supabase connection pooler")
+    logger.info("🔗 Using Supabase connection pooler")
 elif "sqlite" in database_url:
     logger.warning("📁 Using SQLite fallback (database features limited)")
 else:
@@ -157,7 +157,7 @@ try:
         pool_pre_ping=True,
         echo=settings.debug,
     )
-except (AttributeError, Exception):
+except Exception:
     # Fallback for async engine
     import os
 
@@ -284,7 +284,7 @@ async def get_async_session() -> AsyncGenerator[AsyncSession, None]:
 
 
 def init_db() -> None:
-    """Initialize database tables"""
+    """Initialize database tables."""
     Base.metadata.create_all(bind=engine)
 
     # Create additional indexes
@@ -294,25 +294,25 @@ def init_db() -> None:
 
 
 async def init_async_db() -> None:
-    """Initialize database tables asynchronously"""
+    """Initialize database tables asynchronously."""
     async with async_engine.begin() as conn:
         await conn.run_sync(Base.metadata.create_all)
 
 
 def drop_db() -> None:
-    """Drop all database tables (use with caution!)"""
+    """Drop all database tables (use with caution!)."""
     Base.metadata.drop_all(bind=engine)
 
 
 async def drop_async_db() -> None:
-    """Drop all database tables asynchronously"""
+    """Drop all database tables asynchronously."""
     async with async_engine.begin() as conn:
         await conn.run_sync(Base.metadata.drop_all)
 
 
 # Test database setup (for unit tests)
 def get_test_engine():
-    """Create test database engine with in-memory SQLite"""
+    """Create test database engine with in-memory SQLite."""
     from sqlalchemy import create_engine
 
     engine = create_engine(
@@ -326,7 +326,7 @@ def get_test_engine():
 
 
 def get_test_session():
-    """Create test database session"""
+    """Create test database session."""
     engine = get_test_engine()
     TestSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
     return TestSessionLocal()
@@ -334,7 +334,7 @@ def get_test_session():
 
 # Database health check
 async def check_database_health() -> bool:
-    """Check if database is accessible and healthy"""
+    """Check if database is accessible and healthy."""
     try:
         async with get_async_session() as session:
             await session.execute("SELECT 1")
@@ -346,14 +346,14 @@ async def check_database_health() -> bool:
 
 # Utility functions for bulk operations
 async def bulk_insert(model_class, data: list) -> None:
-    """Bulk insert data into database"""
+    """Bulk insert data into database."""
     async with get_async_session() as session:
         session.add_all([model_class(**item) for item in data])
         await session.commit()
 
 
 async def bulk_update(model_class, data: list, key_field: str = "id") -> None:
-    """Bulk update data in database"""
+    """Bulk update data in database."""
     async with get_async_session() as session:
         for item in data:
             key_value = item.pop(key_field)

mcli/ml/experimentation/__init__.py

@@ -1,4 +1,4 @@
-"""ML Experimentation and A/B Testing Framework"""
+"""ML Experimentation and A/B Testing Framework."""
 
 from .ab_testing import (
     ABTestingFramework,