mcli-framework 7.12.2__py3-none-any.whl → 7.12.4__py3-none-any.whl

mcli/ml/preprocessing/ml_pipeline.py

@@ -1,12 +1,12 @@
-"""ML Data Pipeline Integration"""
+"""ML Data Pipeline Integration."""
 
 import asyncio
 import json
 import logging
 from dataclasses import asdict, dataclass
-from datetime import datetime, timezone
+from datetime import datetime
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional
 
 import pandas as pd
 
@@ -25,7 +25,7 @@ logger = logging.getLogger(__name__)
 
 @dataclass
 class MLDataPipelineConfig:
-    """Configuration for ML data pipeline"""
+    """Configuration for ML data pipeline."""
 
     # Data ingestion
     batch_size: int = 50
@@ -53,7 +53,7 @@ class MLDataPipelineConfig:
 
 
 class MLDataPipeline:
-    """ML-focused data pipeline for politician trading data"""
+    """ML-focused data pipeline for politician trading data."""
 
     def __init__(self, lsh_client: LSHClient, config: Optional[MLDataPipelineConfig] = None):
         self.lsh_client = lsh_client
@@ -77,13 +77,13 @@ class MLDataPipeline:
         self._setup_ml_handlers()
 
     def _setup_ml_handlers(self):
-        """Setup ML-specific event handlers"""
+        """Setup ML-specific event handlers."""
         self.lsh_client.on("trading.data.received", self._handle_trading_data_for_ml)
         self.lsh_client.on("politician.data.updated", self._handle_politician_update)
         self.lsh_client.on("market.data.sync", self._handle_market_data)
 
     async def start(self):
-        """Start the ML data pipeline"""
+        """Start the ML data pipeline."""
         if self._is_running:
             logger.warning("ML pipeline already running")
             return
@@ -109,7 +109,7 @@ class MLDataPipeline:
         asyncio.create_task(self._periodic_processing())
 
     async def stop(self):
-        """Stop the ML data pipeline"""
+        """Stop the ML data pipeline."""
         if not self._is_running:
             return
 
@@ -127,7 +127,7 @@ class MLDataPipeline:
             self.mlops_manager.end_run()
 
     async def _handle_trading_data_for_ml(self, event_data: Dict[str, Any]):
-        """Handle trading data for ML processing"""
+        """Handle trading data for ML processing."""
         records = event_data.get("records", [])
 
         if not records:
@@ -146,7 +146,7 @@ class MLDataPipeline:
             await self._process_accumulated_data()
 
     async def _handle_politician_update(self, event_data: Dict[str, Any]):
-        """Handle politician metadata updates"""
+        """Handle politician metadata updates."""
         politician_data = event_data.get("politician", {})
         logger.info(f"Received politician update: {politician_data.get('name', 'unknown')}")
 
@@ -154,15 +154,15 @@ class MLDataPipeline:
         # For now, just log the update
 
     async def _handle_market_data(self, event_data: Dict[str, Any]):
-        """Handle market data updates"""
-        market_data = event_data.get("market", {})
-        logger.info(f"Received market data update")
+        """Handle market data updates."""
+        event_data.get("market", {})
+        logger.info("Received market data update")
 
         # This could be used to enrich existing records
         # For now, just log the update
 
     async def _periodic_processing(self):
-        """Periodic processing of accumulated data"""
+        """Periodic processing of accumulated data."""
         while self._is_running:
             try:
                 # Wait for timeout period
@@ -180,7 +180,7 @@ class MLDataPipeline:
                 logger.error(f"Error in periodic processing: {e}")
 
     async def _process_accumulated_data(self):
-        """Process accumulated raw data through ML preprocessing"""
+        """Process accumulated raw data through ML preprocessing."""
         if not self.raw_data_buffer:
             return
 
@@ -216,7 +216,7 @@ class MLDataPipeline:
     async def _run_preprocessing(
         self, records: List[Dict[str, Any]]
     ) -> Optional[PreprocessingResults]:
-        """Run the preprocessing pipeline"""
+        """Run the preprocessing pipeline."""
         if not records:
             return None
 
@@ -238,7 +238,7 @@ class MLDataPipeline:
             return None
 
     async def _save_processed_data(self, results: PreprocessingResults):
-        """Save processed data to files"""
+        """Save processed data to files."""
         timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
 
         # Save train/val/test splits
@@ -275,7 +275,7 @@ class MLDataPipeline:
         logger.info(f"Saved processed data to {processed_path}")
 
     async def _log_preprocessing_metrics(self, results: PreprocessingResults):
-        """Log preprocessing metrics to MLOps"""
+        """Log preprocessing metrics to MLOps."""
         try:
             # Log parameters
             params = {
@@ -307,7 +307,7 @@ class MLDataPipeline:
             logger.error(f"Failed to log preprocessing metrics: {e}")
 
     def _should_trigger_retraining(self) -> bool:
-        """Check if we should trigger model retraining"""
+        """Check if we should trigger model retraining."""
         if self._total_records_processed >= self.config.auto_retrain_threshold:
             # Reset counter
             self._total_records_processed = 0
@@ -315,7 +315,7 @@ class MLDataPipeline:
         return False
 
     async def _trigger_model_retraining(self):
-        """Trigger model retraining"""
+        """Trigger model retraining."""
         logger.info("Triggering model retraining due to data threshold")
 
         # This would integrate with the model training pipeline
@@ -330,7 +330,7 @@ class MLDataPipeline:
         )
 
     async def get_processing_stats(self) -> Dict[str, Any]:
-        """Get pipeline processing statistics"""
+        """Get pipeline processing statistics."""
         return {
             "is_running": self._is_running,
             "raw_buffer_size": len(self.raw_data_buffer),
@@ -347,7 +347,7 @@ class MLDataPipeline:
         }
 
     async def force_preprocessing(self) -> bool:
-        """Force preprocessing of current buffer"""
+        """Force preprocessing of current buffer."""
         if not self.raw_data_buffer:
             logger.warning("No data in buffer to process")
             return False
@@ -356,7 +356,7 @@ class MLDataPipeline:
         return True
 
     async def load_historical_data(self, data_path: Path) -> bool:
-        """Load and process historical data"""
+        """Load and process historical data."""
         try:
             if data_path.suffix == ".parquet":
                 df = pd.read_parquet(data_path)

mcli/ml/preprocessing/test_preprocessing.py

@@ -1,4 +1,4 @@
-"""Test script for the ML preprocessing pipeline"""
+"""Test script for the ML preprocessing pipeline."""
 
 import logging
 from datetime import datetime, timedelta
@@ -6,7 +6,6 @@ from pathlib import Path
 
 import numpy as np
 import pandas as pd
-from ml_pipeline import MLDataPipeline, MLDataPipelineConfig
 from politician_trading_preprocessor import PoliticianTradingPreprocessor, PreprocessingConfig
 
 # Setup logging
@@ -15,7 +14,7 @@ logger = logging.getLogger(__name__)
 
 
 def generate_sample_data(n_records: int = 100) -> pd.DataFrame:
-    """Generate sample politician trading data for testing"""
+    """Generate sample politician trading data for testing."""
     np.random.seed(42)
 
     # Sample politicians
@@ -94,7 +93,7 @@ def generate_sample_data(n_records: int = 100) -> pd.DataFrame:
 
 
 def test_data_cleaning():
-    """Test data cleaning functionality"""
+    """Test data cleaning functionality."""
     logger.info("Testing data cleaning...")
 
     # Generate sample data with issues
@@ -140,7 +139,7 @@ def test_data_cleaning():
 
 
 def test_feature_extraction():
-    """Test feature extraction functionality"""
+    """Test feature extraction functionality."""
     logger.info("Testing feature extraction...")
 
     data = generate_sample_data(100)
@@ -185,7 +184,7 @@ def test_feature_extraction():
 
 
 def test_full_preprocessing():
-    """Test full preprocessing pipeline"""
+    """Test full preprocessing pipeline."""
     logger.info("Testing full preprocessing pipeline...")
 
     data = generate_sample_data(200)
@@ -236,7 +235,7 @@ def test_full_preprocessing():
 
 
 def test_transform_new_data():
-    """Test transforming new data with fitted preprocessor"""
+    """Test transforming new data with fitted preprocessor."""
     logger.info("Testing new data transformation...")
 
     # Train on initial data
@@ -274,7 +273,7 @@ def test_transform_new_data():
 
 
 def main():
-    """Run all tests"""
+    """Run all tests."""
     logger.info("Starting preprocessing pipeline tests...")
 
     try:

mcli/ml/scripts/populate_sample_data.py

@@ -1,12 +1,8 @@
 """Populate database with sample data for dashboard testing."""
 
-import asyncio
 import random
 from datetime import datetime, timedelta
 
-import numpy as np
-
-from mcli.ml.config import settings
 from mcli.ml.database.models import (
     BacktestResult,
     Model,

mcli/ml/serving/serve.py

@@ -3,13 +3,12 @@
 
 import click
 
-from mcli.lib.ui.styling import error, info, success
+from mcli.lib.ui.styling import error, info
 
 
 @click.group(name="mcli-serve", help="Model serving CLI for MCLI ML models")
 def cli():
     """Main CLI group for model serving."""
-    pass
 
 
 @cli.command(name="start", help="Start model serving server")

mcli/ml/tasks.py

@@ -1,4 +1,4 @@
-"""Celery background tasks for ML system"""
+"""Celery background tasks for ML system."""
 
 import asyncio
 from datetime import datetime, timedelta
@@ -61,24 +61,24 @@ celery_app.conf.beat_schedule = {
 
 
 class MLTask(Task):
-    """Base task with error handling"""
+    """Base task with error handling."""
 
     def on_failure(self, exc, task_id, args, kwargs, einfo):
-        """Log task failure"""
+        """Log task failure."""
         logger.error(f"Task {self.name} failed: {exc}", exc_info=True)
 
     def on_retry(self, exc, task_id, args, kwargs, einfo):
-        """Log task retry"""
+        """Log task retry."""
         logger.warning(f"Task {self.name} retrying: {exc}")
 
     def on_success(self, retval, task_id, args, kwargs):
-        """Log task success"""
+        """Log task success."""
         logger.info(f"Task {self.name} completed successfully")
 
 
 @celery_app.task(base=MLTask, bind=True, max_retries=3)
 def train_model_task(self, model_id: str, retrain: bool = False) -> Dict[str, Any]:
-    """Train or retrain a model"""
+    """Train or retrain a model."""
     try:
         logger.info(f"Starting training for model {model_id}")
 
@@ -127,7 +127,7 @@ def train_model_task(self, model_id: str, retrain: bool = False) -> Dict[str, An
 
 @celery_app.task(base=MLTask, bind=True)
 def update_stock_data_task(self, ticker: str = None) -> Dict[str, Any]:
-    """Update stock data from external APIs"""
+    """Update stock data from external APIs."""
     try:
         logger.info(f"Updating stock data{f' for {ticker}' if ticker else ''}")
 
@@ -180,7 +180,7 @@ def update_stock_data_task(self, ticker: str = None) -> Dict[str, Any]:
 
 @celery_app.task(base=MLTask)
 def check_model_drift_task() -> Dict[str, Any]:
-    """Check for model drift"""
+    """Check for model drift."""
     try:
         logger.info("Checking for model drift")
 
@@ -215,7 +215,7 @@ def check_model_drift_task() -> Dict[str, Any]:
 
 @celery_app.task(base=MLTask)
 def cleanup_predictions_task() -> Dict[str, Any]:
-    """Clean up old predictions"""
+    """Clean up old predictions."""
     try:
         logger.info("Cleaning up old predictions")
 
@@ -241,7 +241,7 @@ def cleanup_predictions_task() -> Dict[str, Any]:
 
 @celery_app.task(base=MLTask)
 def retrain_models_task() -> Dict[str, Any]:
-    """Retrain models on schedule"""
+    """Retrain models on schedule."""
     try:
         logger.info("Starting scheduled model retraining")
 
@@ -278,7 +278,7 @@ def retrain_models_task() -> Dict[str, Any]:
 
 @celery_app.task(base=MLTask)
 def generate_daily_report_task() -> Dict[str, Any]:
-    """Generate daily performance report"""
+    """Generate daily performance report."""
     try:
         logger.info("Generating daily report")
 
@@ -294,7 +294,7 @@ def generate_daily_report_task() -> Dict[str, Any]:
             .count()
         )
 
-        active_portfolios = db.query(Portfolio).filter(Portfolio.is_active == True).count()
+        active_portfolios = db.query(Portfolio).filter(Portfolio.is_active is True).count()
 
         active_users = (
             db.query(User)
@@ -324,12 +324,12 @@ def generate_daily_report_task() -> Dict[str, Any]:
 
 @celery_app.task(base=MLTask)
 def fetch_politician_trades_task() -> Dict[str, Any]:
-    """Fetch latest politician trades"""
+    """Fetch latest politician trades."""
     try:
         logger.info("Fetching politician trades")
 
         from mcli.ml.data_ingestion.api_connectors import CongressionalTradingConnector
-        from mcli.ml.database.models import Politician, Trade
+        from mcli.ml.database.models import Trade
         from mcli.ml.database.session import SessionLocal
 
         connector = CongressionalTradingConnector()
@@ -369,7 +369,7 @@ def fetch_politician_trades_task() -> Dict[str, Any]:
 
 @celery_app.task(base=MLTask, bind=True)
 def process_batch_predictions_task(self, predictions: list) -> Dict[str, Any]:
-    """Process batch predictions asynchronously"""
+    """Process batch predictions asynchronously."""
     try:
         logger.info(f"Processing batch of {len(predictions)} predictions")
 
@@ -384,7 +384,7 @@ def process_batch_predictions_task(self, predictions: list) -> Dict[str, Any]:
             result = model.predict(features)
             results.append({"ticker": pred["ticker"], "prediction": float(result[0])})
 
-        logger.info(f"Batch predictions completed")
+        logger.info("Batch predictions completed")
         return {"predictions": results}
 
     except Exception as e:
@@ -395,5 +395,5 @@ def process_batch_predictions_task(self, predictions: list) -> Dict[str, Any]:
 # Worker health check
 @celery_app.task(name="health_check")
 def health_check():
-    """Health check for Celery worker"""
+    """Health check for Celery worker."""
     return {"status": "healthy", "timestamp": datetime.utcnow().isoformat()}

mcli/ml/tests/test_integration.py

@@ -1,4 +1,4 @@
-"""Integration tests for the complete ML pipeline"""
+"""Integration tests for the complete ML pipeline."""
 
 import os
 import sys
@@ -12,14 +12,13 @@ from pathlib import Path
 
 import numpy as np
 import pandas as pd
-import pytest
 from ml.backtesting.backtest_engine import BacktestConfig, BacktestEngine, TradingStrategy
 from ml.backtesting.performance_metrics import PerformanceAnalyzer
 from ml.features.ensemble_features import EnsembleFeatureBuilder
 from ml.features.political_features import PoliticalInfluenceFeatures
 from ml.features.stock_features import StockRecommendationFeatures
-from ml.mlops.experiment_tracker import ExperimentTracker, MLflowConfig, ModelRegistry
-from ml.mlops.pipeline_orchestrator import MLPipeline, PipelineConfig, PipelineExecutor
+from ml.mlops.experiment_tracker import MLflowConfig
+from ml.mlops.pipeline_orchestrator import MLPipeline, PipelineConfig
 from ml.models.ensemble_models import DeepEnsembleModel, EnsembleConfig, ModelConfig
 from ml.models.recommendation_models import RecommendationConfig, StockRecommendationModel
 
@@ -31,13 +30,13 @@ logger = logging.getLogger(__name__)
 
 
 class TestDataIntegration:
-    """Test data processing integration"""
+    """Test data processing integration."""
 
     def test_data_pipeline(self):
-        """Test complete data processing pipeline"""
+        """Test complete data processing pipeline."""
         # Generate mock data
         trading_data = self._generate_mock_trading_data()
-        stock_data = self._generate_mock_stock_data()
+        self._generate_mock_stock_data()
 
         # Initialize processor
         config = ProcessingConfig()
@@ -53,7 +52,7 @@ class TestDataIntegration:
         assert cleaned_data.isnull().sum().sum() == 0  # No nulls
 
     def test_feature_extraction_pipeline(self):
-        """Test feature extraction pipeline"""
+        """Test feature extraction pipeline."""
         # Generate mock data
         trading_data = self._generate_mock_trading_data()
         stock_data = self._generate_mock_stock_data()
@@ -77,7 +76,7 @@ class TestDataIntegration:
         assert ensemble_features.shape[1] > combined.shape[1]  # More features
 
     def _generate_mock_trading_data(self):
-        """Generate mock trading data"""
+        """Generate mock trading data."""
         n_records = 100
         data = []
         for _ in range(n_records):
@@ -94,7 +93,7 @@ class TestDataIntegration:
         return pd.DataFrame(data)
 
     def _generate_mock_stock_data(self):
-        """Generate mock stock data"""
+        """Generate mock stock data."""
         dates = pd.date_range(end=datetime.now(), periods=100)
         tickers = ["AAPL", "MSFT", "GOOGL"]
         data = []
@@ -119,15 +118,15 @@ class TestDataIntegration:
 
 
 class TestModelIntegration:
-    """Test model training and prediction integration"""
+    """Test model training and prediction integration."""
 
     def test_model_training_pipeline(self):
-        """Test complete model training pipeline"""
+        """Test complete model training pipeline."""
         # Generate data
         X = np.random.randn(200, 50)
-        y = np.random.randint(0, 2, 200)
-        returns = np.random.normal(0.05, 0.15, 200)
-        risk_labels = np.random.choice([0, 1, 2], 200)
+        np.random.randint(0, 2, 200)
+        np.random.normal(0.05, 0.15, 200)
+        np.random.choice([0, 1, 2], 200)
 
         # Configure model
         model_configs = [
@@ -154,7 +153,7 @@ class TestModelIntegration:
         assert len(predictions) == 10
 
     def test_model_serving(self):
-        """Test model serving capabilities"""
+        """Test model serving capabilities."""
         from ml.mlops.model_serving import ModelEndpoint, PredictionRequest
 
         # Create endpoint
@@ -172,10 +171,10 @@ class TestModelIntegration:
 
 
 class TestPipelineIntegration:
-    """Test complete ML pipeline integration"""
+    """Test complete ML pipeline integration."""
 
     def test_end_to_end_pipeline(self):
-        """Test complete end-to-end pipeline"""
+        """Test complete end-to-end pipeline."""
         with tempfile.TemporaryDirectory() as tmpdir:
             # Configure pipeline
             config = PipelineConfig(
@@ -195,7 +194,7 @@ class TestPipelineIntegration:
             assert result["model"] is not None
 
     def test_pipeline_with_mlflow(self):
-        """Test pipeline with MLflow tracking"""
+        """Test pipeline with MLflow tracking."""
         with tempfile.TemporaryDirectory() as tmpdir:
             # Configure MLflow
             mlflow_config = MLflowConfig(
@@ -220,10 +219,10 @@ class TestPipelineIntegration:
 
 
 class TestBacktestIntegration:
-    """Test backtesting framework integration"""
+    """Test backtesting framework integration."""
 
     def test_backtesting_pipeline(self):
-        """Test complete backtesting pipeline"""
+        """Test complete backtesting pipeline."""
         # Generate mock price data
         dates = pd.date_range(end=datetime.now(), periods=252)
         price_data = []
@@ -265,7 +264,7 @@ class TestBacktestIntegration:
         assert result.metrics["total_return"] is not None
 
     def test_performance_analysis(self):
-        """Test performance analysis"""
+        """Test performance analysis."""
         # Generate mock returns
         returns = pd.Series(np.random.normal(0.001, 0.02, 252))
         benchmark_returns = pd.Series(np.random.normal(0.0008, 0.015, 252))
@@ -280,11 +279,11 @@ class TestBacktestIntegration:
 
 
 class TestSystemIntegration:
-    """Test full system integration"""
+    """Test full system integration."""
 
     def test_complete_workflow(self):
-        """Test complete ML workflow from data to backtest"""
-        with tempfile.TemporaryDirectory() as tmpdir:
+        """Test complete ML workflow from data to backtest."""
+        with tempfile.TemporaryDirectory() as tmpdir:  # noqa: F841
             logger.info("Starting complete workflow test...")
 
             # Step 1: Data Processing
@@ -305,7 +304,7 @@ class TestSystemIntegration:
             # Step 3: Model Training
             logger.info("Step 3: Training model...")
             X = np.random.randn(100, 50)
-            y = np.random.randint(0, 2, 100)
+            np.random.randint(0, 2, 100)
 
             model_configs = [
                 ModelConfig(
@@ -344,7 +343,7 @@ class TestSystemIntegration:
             logger.info("Complete workflow test successful!")
 
     def _generate_trading_data(self):
-        """Generate comprehensive trading data"""
+        """Generate comprehensive trading data."""
         n_records = 500
         data = []
 
@@ -364,7 +363,7 @@ class TestSystemIntegration:
         return pd.DataFrame(data)
 
     def _generate_stock_data(self):
-        """Generate comprehensive stock data"""
+        """Generate comprehensive stock data."""
         dates = pd.date_range(end=datetime.now(), periods=365)
         tickers = ["AAPL", "MSFT", "GOOGL", "AMZN", "TSLA", "SPY"]
         data = []
@@ -373,7 +372,7 @@ class TestSystemIntegration:
             base_price = np.random.uniform(50, 500)
             prices = [base_price]
 
-            for i, date in enumerate(dates):
+            for _i, date in enumerate(dates):
                 # Random walk with momentum
                 change = np.random.normal(0.001, 0.02)
                 new_price = prices[-1] * (1 + change)
@@ -395,7 +394,7 @@ class TestSystemIntegration:
 
 
 def test_smoke():
-    """Smoke test to ensure all imports work"""
+    """Smoke test to ensure all imports work."""
     assert DataProcessor is not None
     assert StockRecommendationFeatures is not None
     assert DeepEnsembleModel is not None