mcli-framework 7.0.0__py3-none-any.whl

mcli/ml/preprocessing/test_preprocessing.py

@@ -0,0 +1,294 @@
+"""Test script for the ML preprocessing pipeline"""
+
+import pandas as pd
+import numpy as np
+from datetime import datetime, timedelta
+from pathlib import Path
+import logging
+
+from politician_trading_preprocessor import PoliticianTradingPreprocessor, PreprocessingConfig
+from ml_pipeline import MLDataPipeline, MLDataPipelineConfig
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def generate_sample_data(n_records: int = 100) -> pd.DataFrame:
+    """Generate sample politician trading data for testing"""
+    np.random.seed(42)
+
+    # Sample politicians
+    politicians = [
+        "Nancy Pelosi",
+        "Mitch McConnell",
+        "Chuck Schumer",
+        "Kevin McCarthy",
+        "Alexandria Ocasio-Cortez",
+        "Ted Cruz",
+        "Elizabeth Warren",
+        "Marco Rubio",
+        "Bernie Sanders",
+        "Mitt Romney",
+    ]
+
+    # Sample stocks
+    tickers = ["AAPL", "MSFT", "GOOGL", "AMZN", "TSLA", "META", "NVDA", "JPM", "BAC", "XOM"]
+    companies = [
+        "Apple Inc",
+        "Microsoft Corp",
+        "Alphabet Inc",
+        "Amazon.com Inc",
+        "Tesla Inc",
+        "Meta Platforms Inc",
+        "Nvidia Corp",
+        "JPMorgan Chase",
+        "Bank of America",
+        "Exxon Mobil",
+    ]
+
+    # Generate random data
+    records = []
+    start_date = datetime.now() - timedelta(days=365)
+
+    for i in range(n_records):
+        # Random date within last year
+        days_ago = np.random.randint(0, 365)
+        trade_date = start_date + timedelta(days=days_ago)
+
+        # Random amounts with some outliers
+        if np.random.random() < 0.1:  # 10% outliers
+            amount = np.random.uniform(100000, 5000000)
+        else:
+            amount = np.random.uniform(1000, 50000)
+
+        # Random transaction type
+        transaction_type = np.random.choice(["buy", "sell"], p=[0.6, 0.4])
+
+        # Random stock
+        ticker_idx = np.random.randint(0, len(tickers))
+
+        record = {
+            "politician_name": np.random.choice(politicians),
+            "transaction_date": trade_date.strftime("%Y-%m-%d"),
+            "transaction_amount": amount,
+            "transaction_type": transaction_type,
+            "asset_name": companies[ticker_idx],
+            "stock_symbol": tickers[ticker_idx],
+            "disclosure_date": (trade_date + timedelta(days=np.random.randint(1, 45))).strftime(
+                "%Y-%m-%d"
+            ),
+            "transaction_id": f"T{i:06d}",
+            "source": "test_data",
+        }
+
+        # Add some missing values randomly
+        if np.random.random() < 0.1:
+            del record["transaction_amount"]
+        if np.random.random() < 0.05:
+            del record["stock_symbol"]
+
+        records.append(record)
+
+    return pd.DataFrame(records)
+
+
+def test_data_cleaning():
+    """Test data cleaning functionality"""
+    logger.info("Testing data cleaning...")
+
+    # Generate sample data with issues
+    data = generate_sample_data(50)
+
+    # Add some problematic records
+    problematic_records = [
+        {
+            "politician_name": "  john DOE jr. ",
+            "transaction_amount": "$15,000.00",
+            "transaction_date": "2023-12-01",
+        },
+        {
+            "politician_name": "Jane Smith",
+            "transaction_amount": "1K - 15K",
+            "transaction_date": "12/01/2023",
+        },
+        {
+            "politician_name": "Bob Johnson",
+            "transaction_amount": "",
+            "transaction_date": "invalid-date",
+        },
+    ]
+
+    data = pd.concat([data, pd.DataFrame(problematic_records)], ignore_index=True)
+
+    # Initialize preprocessor
+    config = PreprocessingConfig(enable_data_cleaning=True)
+    preprocessor = PoliticianTradingPreprocessor(config)
+
+    # Test cleaning
+    records = data.to_dict("records")
+    cleaned_records, cleaning_stats = preprocessor.data_cleaner.clean_trading_records(records)
+
+    logger.info(f"Original records: {len(records)}")
+    logger.info(f"Cleaned records: {len(cleaned_records)}")
+    logger.info(f"Cleaning operations: {cleaning_stats.cleaning_operations}")
+
+    assert len(cleaned_records) > 0, "Should have some cleaned records"
+    assert cleaning_stats.cleaned_records > 0, "Should have cleaned some records"
+
+    logger.info("✅ Data cleaning test passed")
+
+
+def test_feature_extraction():
+    """Test feature extraction functionality"""
+    logger.info("Testing feature extraction...")
+
+    data = generate_sample_data(100)
+
+    config = PreprocessingConfig(
+        enable_politician_features=True,
+        enable_market_features=True,
+        enable_temporal_features=True,
+        enable_sentiment_features=True,
+    )
+    preprocessor = PoliticianTradingPreprocessor(config)
+
+    # Clean data first
+    records = data.to_dict("records")
+    cleaned_records, _ = preprocessor.data_cleaner.clean_trading_records(records)
+    df = pd.DataFrame(cleaned_records)
+
+    # Extract features
+    df_with_features = preprocessor._extract_features(df)
+
+    logger.info(f"Original columns: {len(df.columns)}")
+    logger.info(f"Columns after feature extraction: {len(df_with_features.columns)}")
+
+    # Check for expected features
+    expected_features = [
+        "politician_name_length",
+        "politician_trading_frequency",
+        "asset_trading_frequency",
+        "transaction_day_of_week",
+        "asset_name_sentiment_score",
+    ]
+
+    for feature in expected_features:
+        if feature in df_with_features.columns:
+            logger.info(f"✅ Found expected feature: {feature}")
+        else:
+            logger.warning(f"⚠️ Missing expected feature: {feature}")
+
+    assert len(df_with_features.columns) > len(df.columns), "Should have extracted new features"
+
+    logger.info("✅ Feature extraction test passed")
+
+
+def test_full_preprocessing():
+    """Test full preprocessing pipeline"""
+    logger.info("Testing full preprocessing pipeline...")
+
+    data = generate_sample_data(200)
+
+    config = PreprocessingConfig(
+        enable_data_cleaning=True,
+        enable_outlier_detection=True,
+        enable_missing_value_handling=True,
+        train_split_ratio=0.7,
+        val_split_ratio=0.15,
+        test_split_ratio=0.15,
+        save_preprocessing_artifacts=True,
+        artifacts_dir=Path("./test_artifacts"),
+    )
+
+    preprocessor = PoliticianTradingPreprocessor(config)
+
+    # Run full preprocessing
+    results = preprocessor.preprocess(data)
+
+    # Check results
+    logger.info(f"Original shape: {results.original_shape}")
+    logger.info(f"Final shape: {results.final_shape}")
+    logger.info(f"Features: {results.feature_count}")
+    logger.info(f"Train size: {len(results.train_data)}")
+    logger.info(f"Val size: {len(results.val_data)}")
+    logger.info(f"Test size: {len(results.test_data)}")
+    logger.info(f"Target columns: {results.target_columns}")
+
+    # Validate results
+    assert results.feature_count > 0, "Should have extracted features"
+    assert len(results.train_data) > 0, "Should have training data"
+    assert len(results.val_data) > 0, "Should have validation data"
+    assert len(results.test_data) > 0, "Should have test data"
+    assert len(results.target_columns) > 0, "Should have target columns"
+
+    # Check that splits sum to original
+    total_split_size = len(results.train_data) + len(results.val_data) + len(results.test_data)
+    assert total_split_size == results.final_shape[0], "Split sizes should sum to final shape"
+
+    # Check for target variables
+    expected_targets = ["target_profitable", "target_recommendation_score", "target_risk_level"]
+    for target in expected_targets:
+        if target in results.target_columns:
+            logger.info(f"✅ Found expected target: {target}")
+
+    logger.info("✅ Full preprocessing test passed")
+
+
+def test_transform_new_data():
+    """Test transforming new data with fitted preprocessor"""
+    logger.info("Testing new data transformation...")
+
+    # Train on initial data
+    train_data = generate_sample_data(100)
+
+    config = PreprocessingConfig(
+        save_preprocessing_artifacts=True, artifacts_dir=Path("./test_artifacts")
+    )
+
+    preprocessor = PoliticianTradingPreprocessor(config)
+    results = preprocessor.preprocess(train_data)
+
+    # Generate new data
+    new_data = generate_sample_data(20)
+
+    # Transform new data
+    transformed_data = preprocessor.transform_new_data(new_data)
+
+    logger.info(f"New data original shape: {new_data.shape}")
+    logger.info(f"Transformed data shape: {transformed_data.shape}")
+
+    # Should have same number of features as training
+    expected_feature_cols = [
+        col for col in results.train_data.columns if not col.startswith("target_")
+    ]
+    actual_feature_cols = [col for col in transformed_data.columns if not col.startswith("target_")]
+
+    logger.info(f"Expected feature columns: {len(expected_feature_cols)}")
+    logger.info(f"Actual feature columns: {len(actual_feature_cols)}")
+
+    # Note: Some features might be missing due to different data patterns
+    assert len(transformed_data) > 0, "Should have transformed some data"
+
+    logger.info("✅ New data transformation test passed")
+
+
+def main():
+    """Run all tests"""
+    logger.info("Starting preprocessing pipeline tests...")
+
+    try:
+        test_data_cleaning()
+        test_feature_extraction()
+        test_full_preprocessing()
+        test_transform_new_data()
+
+        logger.info("🎉 All preprocessing tests passed!")
+
+    except Exception as e:
+        logger.error(f"❌ Test failed: {e}")
+        raise
+
+
+if __name__ == "__main__":
+    main()

mcli/ml/scripts/populate_sample_data.py

@@ -0,0 +1,200 @@
+"""Populate database with sample data for dashboard testing."""
+
+import asyncio
+from datetime import datetime, timedelta
+import random
+import numpy as np
+
+from mcli.ml.database.session import SessionLocal, init_db
+from mcli.ml.database.models import (
+    User, Model, Prediction, Portfolio, Trade, StockData,
+    BacktestResult, ModelStatus
+)
+from mcli.ml.config import settings
+
+def populate_sample_data():
+    """Populate database with sample data."""
+
+    # Initialize database
+    init_db()
+
+    db = SessionLocal()
+
+    try:
+        # Clear existing data (optional)
+        print("Clearing existing data...")
+        db.query(Prediction).delete()
+        db.query(Trade).delete()
+        db.query(BacktestResult).delete()
+        db.query(Portfolio).delete()
+        db.query(Model).delete()
+        db.query(User).delete()
+        db.query(StockData).delete()
+        db.commit()
+
+        # Create sample users
+        print("Creating sample users...")
+        users = []
+        for i in range(5):
+            user = User(
+                username=f"user_{i+1}",
+                email=f"user{i+1}@example.com",
+                role="user" if i > 0 else "admin",
+                is_active=True,
+                last_login_at=datetime.utcnow() - timedelta(hours=random.randint(1, 48))
+            )
+            users.append(user)
+            db.add(user)
+
+        db.commit()
+
+        # Create sample models
+        print("Creating sample models...")
+        models = []
+        model_names = ["LSTM Predictor", "Transformer Model", "Ensemble Model", "CNN Extractor", "Attention Model"]
+        for i, name in enumerate(model_names):
+            model = Model(
+                name=name,
+                version=f"v1.{i}",
+                model_type="pytorch",
+                status=ModelStatus.DEPLOYED if i < 3 else ModelStatus.TRAINING,
+                test_accuracy=random.uniform(0.65, 0.95),
+                test_sharpe_ratio=random.uniform(1.2, 2.5),
+                test_max_drawdown=random.uniform(0.05, 0.15),
+                created_at=datetime.utcnow() - timedelta(days=random.randint(1, 30)),
+                updated_at=datetime.utcnow() - timedelta(hours=random.randint(1, 24)),
+                created_by_id=random.choice(users).id
+            )
+            models.append(model)
+            db.add(model)
+
+        db.commit()
+
+        # Create sample portfolios
+        print("Creating sample portfolios...")
+        portfolios = []
+        portfolio_names = ["Growth Portfolio", "Value Portfolio", "AI Picks", "Risk Parity", "Momentum Strategy"]
+        for i, name in enumerate(portfolio_names):
+            portfolio = Portfolio(
+                name=name,
+                description=f"Strategy based on {name.lower()}",
+                initial_capital=100000,
+                current_value=100000 * random.uniform(0.9, 1.3),
+                total_return=random.uniform(-0.1, 0.3),
+                sharpe_ratio=random.uniform(0.8, 2.0),
+                max_drawdown=random.uniform(0.05, 0.20),
+                is_active=i < 4,
+                created_by_id=random.choice(users).id
+            )
+            portfolios.append(portfolio)
+            db.add(portfolio)
+
+        db.commit()
+
+        # Create sample predictions
+        print("Creating sample predictions...")
+        tickers = ["AAPL", "GOOGL", "MSFT", "TSLA", "NVDA", "META", "AMZN", "SPY", "QQQ", "DIA"]
+
+        for _ in range(100):
+            prediction = Prediction(
+                ticker=random.choice(tickers),
+                prediction_date=datetime.utcnow().date() - timedelta(days=random.randint(0, 7)),
+                target_date=datetime.utcnow().date() + timedelta(days=random.randint(1, 30)),
+                predicted_return=random.uniform(-0.05, 0.05),
+                confidence_score=random.uniform(0.5, 0.95),
+                model_id=random.choice(models).id
+            )
+            db.add(prediction)
+
+        # Add some predictions for today
+        for ticker in tickers[:5]:
+            prediction = Prediction(
+                ticker=ticker,
+                prediction_date=datetime.utcnow().date(),
+                target_date=datetime.utcnow().date() + timedelta(days=7),
+                predicted_return=random.uniform(-0.03, 0.03),
+                confidence_score=random.uniform(0.6, 0.9),
+                model_id=random.choice(models).id
+            )
+            db.add(prediction)
+
+        db.commit()
+
+        # Create sample stock data
+        print("Creating sample stock data...")
+        for ticker in tickers:
+            base_price = random.uniform(50, 500)
+            for i in range(30):
+                date = datetime.utcnow().date() - timedelta(days=i)
+                stock_data = StockData(
+                    ticker=ticker,
+                    date=date,
+                    open=base_price * random.uniform(0.98, 1.02),
+                    high=base_price * random.uniform(1.01, 1.03),
+                    low=base_price * random.uniform(0.97, 0.99),
+                    close=base_price * random.uniform(0.98, 1.02),
+                    volume=random.randint(1000000, 50000000),
+                    adjusted_close=base_price * random.uniform(0.98, 1.02)
+                )
+                db.add(stock_data)
+                base_price = stock_data.close  # Random walk
+
+        db.commit()
+
+        # Create sample trades
+        print("Creating sample trades...")
+        for portfolio in portfolios:
+            if portfolio.is_active:
+                for _ in range(random.randint(5, 15)):
+                    trade = Trade(
+                        portfolio_id=portfolio.id,
+                        ticker=random.choice(tickers),
+                        trade_type=random.choice(["buy", "sell"]),
+                        quantity=random.randint(10, 100),
+                        price=random.uniform(50, 500),
+                        executed_at=datetime.utcnow() - timedelta(days=random.randint(0, 30))
+                    )
+                    db.add(trade)
+
+        db.commit()
+
+        # Create sample backtest results
+        print("Creating sample backtest results...")
+        for model in models:
+            if model.status == ModelStatus.DEPLOYED:
+                backtest = BacktestResult(
+                    model_id=model.id,
+                    start_date=datetime.utcnow().date() - timedelta(days=180),
+                    end_date=datetime.utcnow().date() - timedelta(days=1),
+                    initial_capital=100000,
+                    final_capital=100000 * random.uniform(0.9, 1.4),
+                    total_return=random.uniform(-0.1, 0.4),
+                    sharpe_ratio=random.uniform(0.5, 2.5),
+                    max_drawdown=random.uniform(0.05, 0.25),
+                    win_rate=random.uniform(0.45, 0.65),
+                    profit_factor=random.uniform(0.9, 2.0),
+                    total_trades=random.randint(50, 200)
+                )
+                db.add(backtest)
+
+        db.commit()
+
+        print("✅ Sample data populated successfully!")
+        print(f"   Users: {len(users)}")
+        print(f"   Models: {len(models)}")
+        print(f"   Portfolios: {len(portfolios)}")
+        print(f"   Predictions: {db.query(Prediction).count()}")
+        print(f"   Stock Data: {db.query(StockData).count()}")
+        print(f"   Trades: {db.query(Trade).count()}")
+        print(f"   Backtest Results: {db.query(BacktestResult).count()}")
+
+    except Exception as e:
+        print(f"❌ Error populating data: {e}")
+        db.rollback()
+        raise
+    finally:
+        db.close()
+
+
+if __name__ == "__main__":
+    populate_sample_data()