mcli-framework 7.1.1__py3-none-any.whl → 7.1.2__py3-none-any.whl

mcli/ml/features/recommendation_engine.py

@@ -1,28 +1,29 @@
 """Stock recommendation engine that combines all feature engineering components"""
 
-import numpy as np
-import pandas as pd
-from datetime import datetime, timedelta
-from typing import Any, Dict, List, Optional, Tuple, Union
-from dataclasses import dataclass, asdict
 import logging
+from dataclasses import asdict, dataclass
+from datetime import datetime, timedelta
 from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple, Union
+
 import joblib
+import numpy as np
+import pandas as pd
 
-from .stock_features import (
-    StockRecommendationFeatures,
-    TechnicalIndicatorFeatures,
-    MarketRegimeFeatures,
+from .ensemble_features import (
+    DynamicFeatureSelector,
+    EnsembleFeatureBuilder,
+    FeatureInteractionEngine,
 )
 from .political_features import (
-    PoliticalInfluenceFeatures,
     CongressionalTrackingFeatures,
     PolicyImpactFeatures,
+    PoliticalInfluenceFeatures,
 )
-from .ensemble_features import (
-    EnsembleFeatureBuilder,
-    FeatureInteractionEngine,
-    DynamicFeatureSelector,
+from .stock_features import (
+    MarketRegimeFeatures,
+    StockRecommendationFeatures,
+    TechnicalIndicatorFeatures,
 )
 
 logger = logging.getLogger(__name__)

mcli/ml/features/stock_features.py

@@ -1,13 +1,14 @@
 """Stock-specific feature engineering for recommendation models"""
 
-import numpy as np
-import pandas as pd
-from datetime import datetime, timedelta
-from typing import Any, Dict, List, Optional, Tuple, Union
-from dataclasses import dataclass
 import logging
-from collections import defaultdict
 import warnings
+from collections import defaultdict
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+import numpy as np
+import pandas as pd
 
 logger = logging.getLogger(__name__)
 

mcli/ml/features/test_feature_engineering.py

@@ -1,14 +1,15 @@
 """Test script for feature engineering system"""
 
-import sys
 import os
+import sys
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../.."))
 
-import pandas as pd
-import numpy as np
-from datetime import datetime, timedelta
 import logging
+from datetime import datetime, timedelta
+
+import numpy as np
+import pandas as pd
 
 # Set up logging
 logging.basicConfig(level=logging.INFO)
@@ -209,7 +210,7 @@ def test_recommendation_engine():
     """Test the full recommendation engine"""
     logger.info("Testing recommendation engine...")
 
-    from recommendation_engine import StockRecommendationEngine, RecommendationConfig
+    from recommendation_engine import RecommendationConfig, StockRecommendationEngine
 
     # Generate comprehensive test data
     trading_data = generate_mock_trading_data(100)
@@ -259,9 +260,9 @@ def test_feature_integration():
     """Test integration of all feature components"""
     logger.info("Testing feature integration...")
 
-    from stock_features import StockRecommendationFeatures
-    from political_features import PoliticalInfluenceFeatures
     from ensemble_features import EnsembleFeatureBuilder
+    from political_features import PoliticalInfluenceFeatures
+    from stock_features import StockRecommendationFeatures
 
     # Generate test data
     trading_data = generate_mock_trading_data(30)

mcli/ml/logging.py

@@ -1,11 +1,11 @@
 """Logging configuration for ML system"""
 
+import json
 import logging
 import sys
-from pathlib import Path
-from logging.handlers import RotatingFileHandler, TimedRotatingFileHandler
-import json
 from datetime import datetime
+from logging.handlers import RotatingFileHandler, TimedRotatingFileHandler
+from pathlib import Path
 
 from mcli.ml.config import settings
 
@@ -24,11 +24,11 @@ class StructuredFormatter(logging.Formatter):
             "line": record.lineno,
         }
 
-        if hasattr(record, 'request_id'):
-            log_obj['request_id'] = record.request_id
+        if hasattr(record, "request_id"):
+            log_obj["request_id"] = record.request_id
 
         if record.exc_info:
-            log_obj['exception'] = self.formatException(record.exc_info)
+            log_obj["exception"] = self.formatException(record.exc_info)
 
         return json.dumps(log_obj)
 
@@ -52,7 +52,7 @@ def setup_logging():
         console_formatter = StructuredFormatter()
     else:
         console_formatter = logging.Formatter(
-            '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+            "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
         )
 
     console_handler.setFormatter(console_formatter)
@@ -60,10 +60,7 @@ def setup_logging():
 
     # File handler for all logs
     file_handler = TimedRotatingFileHandler(
-        log_dir / "mcli_ml.log",
-        when="midnight",
-        interval=1,
-        backupCount=30
+        log_dir / "mcli_ml.log", when="midnight", interval=1, backupCount=30
     )
     file_handler.setLevel(logging.DEBUG)
     file_handler.setFormatter(StructuredFormatter())
@@ -71,9 +68,7 @@ def setup_logging():
 
     # Error file handler
     error_handler = RotatingFileHandler(
-        log_dir / "mcli_ml_errors.log",
-        maxBytes=10 * 1024 * 1024,  # 10MB
-        backupCount=5
+        log_dir / "mcli_ml_errors.log", maxBytes=10 * 1024 * 1024, backupCount=5  # 10MB
     )
     error_handler.setLevel(logging.ERROR)
     error_handler.setFormatter(StructuredFormatter())
@@ -82,4 +77,4 @@ def setup_logging():
 
 def get_logger(name: str) -> logging.Logger:
     """Get a logger instance"""
-    return logging.getLogger(name)
+    return logging.getLogger(name)

mcli/ml/mlops/data_versioning.py

@@ -1,16 +1,17 @@
 """DVC integration for data versioning and pipeline management"""
 
-import subprocess
-import json
-import yaml
-from pathlib import Path
-from typing import Dict, Any, Optional, List, Union
-from dataclasses import dataclass
 import hashlib
+import json
 import logging
+import os
+import subprocess
+from dataclasses import dataclass
 from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Union
+
 import pandas as pd
-import os
+import yaml
 
 logger = logging.getLogger(__name__)
 
@@ -18,6 +19,7 @@ logger = logging.getLogger(__name__)
 @dataclass
 class DVCConfig:
     """DVC configuration"""
+
     project_root: Path = Path(".")
     remote_storage: str = "s3://my-bucket/dvc-storage"  # or local path
     cache_dir: Path = Path(".dvc/cache")
@@ -49,10 +51,7 @@ class DataVersionControl:
         """Run DVC command"""
         try:
             result = subprocess.run(
-                command.split(),
-                capture_output=True,
-                text=True,
-                cwd=self.project_root
+                command.split(), capture_output=True, text=True, cwd=self.project_root
             )
 
             if result.returncode != 0:
@@ -67,8 +66,7 @@ class DataVersionControl:
             logger.error(f"Failed to run DVC command: {e}")
             raise
 
-    def add_data(self, data_path: Union[str, Path],
-                 description: Optional[str] = None) -> str:
+    def add_data(self, data_path: Union[str, Path], description: Optional[str] = None) -> str:
         """Add data file or directory to DVC tracking"""
         data_path = Path(data_path)
 
@@ -82,7 +80,7 @@ class DataVersionControl:
         metadata = self._generate_metadata(data_path, description)
         metadata_path = data_path.with_suffix(".meta.json")
 
-        with open(metadata_path, 'w') as f:
+        with open(metadata_path, "w") as f:
             json.dump(metadata, f, indent=2)
 
         # Commit if auto-commit enabled
@@ -115,13 +113,9 @@ class DataVersionControl:
         status_output = self._run_command("dvc status")
 
         # Parse status
-        status = {
-            "modified": [],
-            "not_in_cache": [],
-            "deleted": []
-        }
+        status = {"modified": [], "not_in_cache": [], "deleted": []}
 
-        for line in status_output.split('\n'):
+        for line in status_output.split("\n"):
             if "modified:" in line:
                 status["modified"].append(line.split(":")[-1].strip())
             elif "not in cache:" in line:
@@ -131,8 +125,9 @@ class DataVersionControl:
 
         return status
 
-    def _generate_metadata(self, data_path: Path,
-                          description: Optional[str] = None) -> Dict[str, Any]:
+    def _generate_metadata(
+        self, data_path: Path, description: Optional[str] = None
+    ) -> Dict[str, Any]:
         """Generate metadata for data file"""
         stat = data_path.stat()
 
@@ -143,13 +138,17 @@ class DataVersionControl:
             "modified": datetime.fromtimestamp(stat.st_mtime).isoformat(),
             "hash": self._calculate_hash(data_path),
             "description": description or "",
-            "type": "directory" if data_path.is_dir() else "file"
+            "type": "directory" if data_path.is_dir() else "file",
         }
 
         # Add data-specific metadata
-        if data_path.suffix in ['.csv', '.parquet']:
+        if data_path.suffix in [".csv", ".parquet"]:
             try:
-                df = pd.read_csv(data_path) if data_path.suffix == '.csv' else pd.read_parquet(data_path)
+                df = (
+                    pd.read_csv(data_path)
+                    if data_path.suffix == ".csv"
+                    else pd.read_parquet(data_path)
+                )
                 metadata["rows"] = len(df)
                 metadata["columns"] = len(df.columns)
                 metadata["column_names"] = df.columns.tolist()
@@ -196,32 +195,36 @@ class DVCPipeline:
                 "params": stage.get("params", []),
                 "outs": stage.get("outs", []),
                 "metrics": stage.get("metrics", []),
-                "plots": stage.get("plots", [])
+                "plots": stage.get("plots", []),
             }
 
         # Save pipeline
-        with open(self.pipeline_file, 'w') as f:
+        with open(self.pipeline_file, "w") as f:
             yaml.dump(pipeline, f, default_flow_style=False)
 
         logger.info(f"Created DVC pipeline with {len(stages)} stages")
 
-    def add_stage(self, name: str, cmd: str,
-                  deps: Optional[List[str]] = None,
-                  params: Optional[List[str]] = None,
-                  outs: Optional[List[str]] = None,
-                  metrics: Optional[List[str]] = None):
+    def add_stage(
+        self,
+        name: str,
+        cmd: str,
+        deps: Optional[List[str]] = None,
+        params: Optional[List[str]] = None,
+        outs: Optional[List[str]] = None,
+        metrics: Optional[List[str]] = None,
+    ):
         """Add stage to pipeline"""
         stage_config = {
             "cmd": cmd,
             "deps": deps or [],
             "params": params or [],
             "outs": outs or [],
-            "metrics": metrics or []
+            "metrics": metrics or [],
         }
 
         # Load existing pipeline
         if self.pipeline_file.exists():
-            with open(self.pipeline_file, 'r') as f:
+            with open(self.pipeline_file, "r") as f:
                 pipeline = yaml.safe_load(f) or {"stages": {}}
         else:
             pipeline = {"stages": {}}
@@ -230,7 +233,7 @@ class DVCPipeline:
         pipeline["stages"][name] = stage_config
 
         # Save pipeline
-        with open(self.pipeline_file, 'w') as f:
+        with open(self.pipeline_file, "w") as f:
             yaml.dump(pipeline, f, default_flow_style=False)
 
         logger.info(f"Added stage '{name}' to pipeline")
@@ -251,9 +254,9 @@ class DVCPipeline:
 
         # Parse metrics (simplified)
         metrics = {}
-        for line in metrics_output.split('\n'):
-            if ':' in line:
-                key, value = line.split(':', 1)
+        for line in metrics_output.split("\n"):
+            if ":" in line:
+                key, value = line.split(":", 1)
                 try:
                     metrics[key.strip()] = float(value.strip())
                 except:
@@ -269,14 +272,14 @@ class DVCPipeline:
                 "cmd": "python src/prepare_data.py",
                 "deps": ["data/raw"],
                 "outs": ["data/processed"],
-                "params": ["prepare.test_split", "prepare.seed"]
+                "params": ["prepare.test_split", "prepare.seed"],
             },
             {
                 "name": "feature_engineering",
                 "cmd": "python src/featurize.py",
                 "deps": ["data/processed"],
                 "outs": ["data/features"],
-                "params": ["featurize.max_features", "featurize.ngrams"]
+                "params": ["featurize.max_features", "featurize.ngrams"],
             },
             {
                 "name": "train",
@@ -284,36 +287,27 @@ class DVCPipeline:
                 "deps": ["data/features"],
                 "outs": ["models/model.pkl"],
                 "params": ["train.epochs", "train.learning_rate"],
-                "metrics": [{"metrics.json": {"cache": False}}]
+                "metrics": [{"metrics.json": {"cache": False}}],
             },
             {
                 "name": "evaluate",
                 "cmd": "python src/evaluate.py",
                 "deps": ["models/model.pkl", "data/features"],
                 "metrics": [{"eval/metrics.json": {"cache": False}}],
-                "plots": [{"eval/plots/roc.json": {"x": "fpr", "y": "tpr"}}]
-            }
+                "plots": [{"eval/plots/roc.json": {"x": "fpr", "y": "tpr"}}],
+            },
         ]
 
         self.create_pipeline(stages)
 
         # Create default params file
         params = {
-            "prepare": {
-                "test_split": 0.2,
-                "seed": 42
-            },
-            "featurize": {
-                "max_features": 100,
-                "ngrams": 2
-            },
-            "train": {
-                "epochs": 10,
-                "learning_rate": 0.001
-            }
+            "prepare": {"test_split": 0.2, "seed": 42},
+            "featurize": {"max_features": 100, "ngrams": 2},
+            "train": {"epochs": 10, "learning_rate": 0.001},
         }
 
-        with open(self.params_file, 'w') as f:
+        with open(self.params_file, "w") as f:
             yaml.dump(params, f, default_flow_style=False)
 
         logger.info("Created ML pipeline with DVC")
@@ -329,17 +323,16 @@ class DataRegistry:
     def _load_registry(self) -> Dict[str, Any]:
         """Load data registry"""
         if self.registry_path.exists():
-            with open(self.registry_path, 'r') as f:
+            with open(self.registry_path, "r") as f:
                 return json.load(f)
         return {"datasets": {}}
 
     def _save_registry(self):
         """Save data registry"""
-        with open(self.registry_path, 'w') as f:
+        with open(self.registry_path, "w") as f:
             json.dump(self.registry, f, indent=2)
 
-    def register_dataset(self, name: str, path: str,
-                        version: str, metadata: Dict[str, Any]):
+    def register_dataset(self, name: str, path: str, version: str, metadata: Dict[str, Any]):
         """Register new dataset version"""
         if name not in self.registry["datasets"]:
             self.registry["datasets"][name] = {"versions": {}}
@@ -347,7 +340,7 @@ class DataRegistry:
         self.registry["datasets"][name]["versions"][version] = {
             "path": path,
             "metadata": metadata,
-            "registered": datetime.now().isoformat()
+            "registered": datetime.now().isoformat(),
         }
 
         self.registry["datasets"][name]["latest"] = version
@@ -487,7 +480,7 @@ evaluate:
         ".dvc/.gitignore": dvc_gitignore,
         ".dvcignore": dvcignore,
         "dvc.yaml": dvc_yaml,
-        "params.yaml": params_yaml
+        "params.yaml": params_yaml,
     }
 
 
@@ -508,11 +501,11 @@ if __name__ == "__main__":
         name="politician_trades",
         path="data/politician_trades.csv",
         version="v1.0",
-        metadata={"source": "congress", "records": 10000}
+        metadata={"source": "congress", "records": 10000},
     )
 
     # Create ML pipeline
     pipeline = DVCPipeline(config)
     pipeline.create_ml_pipeline()
 
-    logger.info("DVC setup complete")
+    logger.info("DVC setup complete")

mcli/ml/mlops/experiment_tracker.py

@@ -1,19 +1,20 @@
 """MLflow experiment tracking and model registry"""
 
+import json
+import logging
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Union
+
 import mlflow
 import mlflow.pytorch
 import mlflow.sklearn
-from mlflow.tracking import MlflowClient
-from mlflow.models.signature import ModelSignature, infer_signature
-import torch
 import numpy as np
 import pandas as pd
-from typing import Dict, Any, Optional, List, Union
-from dataclasses import dataclass
-from pathlib import Path
-import json
-import logging
-from datetime import datetime
+import torch
+from mlflow.models.signature import ModelSignature, infer_signature
+from mlflow.tracking import MlflowClient
 
 logger = logging.getLogger(__name__)
 
@@ -21,6 +22,7 @@ logger = logging.getLogger(__name__)
 @dataclass
 class MLflowConfig:
     """Configuration for MLflow tracking"""
+
     tracking_uri: str = "sqlite:///mlruns.db"
     experiment_name: str = "politician-trading-predictions"
     artifact_location: Optional[str] = None
@@ -32,13 +34,14 @@ class MLflowConfig:
             self.tags = {
                 "project": "politician-trading",
                 "framework": "pytorch",
-                "type": "stock-recommendation"
+                "type": "stock-recommendation",
             }
 
 
 @dataclass
 class ExperimentRun:
     """Container for experiment run information"""
+
     run_id: str
     experiment_id: str
     run_name: str
@@ -73,7 +76,7 @@ class ExperimentTracker:
             experiment_id = mlflow.create_experiment(
                 self.config.experiment_name,
                 artifact_location=self.config.artifact_location,
-                tags=self.config.tags
+                tags=self.config.tags,
             )
         else:
             experiment_id = experiment.experiment_id
@@ -105,7 +108,7 @@ class ExperimentTracker:
             metrics={},
             params={},
             artifacts=[],
-            start_time=datetime.now()
+            start_time=datetime.now(),
         )
 
         logger.info(f"Started MLflow run: {run_name} (ID: {run.info.run_id})")
@@ -156,11 +159,15 @@ class ExperimentTracker:
         self.current_run.artifacts.append(str(artifact_path))
         logger.debug(f"Logged artifact: {artifact_path}")
 
-    def log_model(self, model: Any, model_name: str,
-                  input_example: Optional[Union[np.ndarray, pd.DataFrame]] = None,
-                  signature: Optional[ModelSignature] = None,
-                  conda_env: Optional[Dict] = None,
-                  pip_requirements: Optional[List[str]] = None):
+    def log_model(
+        self,
+        model: Any,
+        model_name: str,
+        input_example: Optional[Union[np.ndarray, pd.DataFrame]] = None,
+        signature: Optional[ModelSignature] = None,
+        conda_env: Optional[Dict] = None,
+        pip_requirements: Optional[List[str]] = None,
+    ):
         """Log model to current run"""
         if not self.current_run:
             raise ValueError("No active MLflow run. Call start_run() first.")
@@ -196,7 +203,7 @@ class ExperimentTracker:
                 signature=signature,
                 input_example=input_example,
                 conda_env=conda_env,
-                pip_requirements=pip_requirements
+                pip_requirements=pip_requirements,
             )
             framework = "pytorch"
         else:
@@ -207,7 +214,7 @@ class ExperimentTracker:
                 signature=signature,
                 input_example=input_example,
                 conda_env=conda_env,
-                pip_requirements=pip_requirements
+                pip_requirements=pip_requirements,
             )
             framework = "sklearn"
 
@@ -245,8 +252,10 @@ class ExperimentTracker:
         mlflow.end_run(status=status)
 
         duration = (self.current_run.end_time - self.current_run.start_time).total_seconds()
-        logger.info(f"Ended MLflow run {self.current_run.run_name} "
-                   f"(Duration: {duration:.2f}s, Status: {status})")
+        logger.info(
+            f"Ended MLflow run {self.current_run.run_name} "
+            f"(Duration: {duration:.2f}s, Status: {status})"
+        )
 
         current_run = self.current_run
         self.current_run = None
@@ -256,17 +265,17 @@ class ExperimentTracker:
         """Get run by ID"""
         return self.client.get_run(run_id)
 
-    def search_runs(self, filter_string: str = "",
-                   max_results: int = 100) -> List[mlflow.entities.Run]:
+    def search_runs(
+        self, filter_string: str = "", max_results: int = 100
+    ) -> List[mlflow.entities.Run]:
         """Search for runs in experiment"""
         return self.client.search_runs(
             experiment_ids=[self.experiment_id],
             filter_string=filter_string,
-            max_results=max_results
+            max_results=max_results,
         )
 
-    def compare_runs(self, run_ids: List[str],
-                     metrics: Optional[List[str]] = None) -> pd.DataFrame:
+    def compare_runs(self, run_ids: List[str], metrics: Optional[List[str]] = None) -> pd.DataFrame:
         """Compare multiple runs"""
         runs_data = []
 
@@ -307,15 +316,14 @@ class ModelRegistry:
         if config.registry_uri:
             mlflow.set_registry_uri(config.registry_uri)
 
-    def register_model(self, model_uri: str, model_name: str,
-                      tags: Optional[Dict[str, str]] = None) -> str:
+    def register_model(
+        self, model_uri: str, model_name: str, tags: Optional[Dict[str, str]] = None
+    ) -> str:
         """Register model in MLflow registry"""
         try:
             # Create registered model if it doesn't exist
             self.client.create_registered_model(
-                model_name,
-                tags=tags or {},
-                description=f"Model for {model_name}"
+                model_name, tags=tags or {}, description=f"Model for {model_name}"
             )
         except Exception as e:
             logger.debug(f"Model {model_name} already exists: {e}")
@@ -325,27 +333,28 @@ class ModelRegistry:
             name=model_name,
             source=model_uri,
             run_id=model_uri.split("/")[1] if "runs:/" in model_uri else None,
-            tags=tags or {}
+            tags=tags or {},
         )
 
         logger.info(f"Registered model {model_name} version {model_version.version}")
         return f"models:/{model_name}/{model_version.version}"
 
-    def transition_model_stage(self, model_name: str, version: int,
-                              stage: str, archive_existing: bool = True):
+    def transition_model_stage(
+        self, model_name: str, version: int, stage: str, archive_existing: bool = True
+    ):
         """Transition model version to new stage"""
         self.client.transition_model_version_stage(
             name=model_name,
             version=version,
             stage=stage,
-            archive_existing_versions=archive_existing
+            archive_existing_versions=archive_existing,
         )
 
         logger.info(f"Transitioned {model_name} v{version} to {stage}")
 
-    def load_model(self, model_name: str,
-                   version: Optional[int] = None,
-                   stage: Optional[str] = None) -> Any:
+    def load_model(
+        self, model_name: str, version: Optional[int] = None, stage: Optional[str] = None
+    ) -> Any:
         """Load model from registry"""
         if version:
             model_uri = f"models:/{model_name}/{version}"
@@ -362,8 +371,7 @@ class ModelRegistry:
         """Get specific model version details"""
         return self.client.get_model_version(model_name, version)
 
-    def get_latest_versions(self, model_name: str,
-                           stages: Optional[List[str]] = None):
+    def get_latest_versions(self, model_name: str, stages: Optional[List[str]] = None):
         """Get latest model versions for given stages"""
         return self.client.get_latest_versions(model_name, stages=stages)
 
@@ -374,4 +382,4 @@ class ModelRegistry:
 
     def search_models(self, filter_string: str = "") -> List:
         """Search registered models"""
-        return self.client.search_registered_models(filter_string=filter_string)
+        return self.client.search_registered_models(filter_string=filter_string)