QuickLab-MCP 0.1.0__py3-none-any.whl

ml_mcp/application/datasets/validator.py

@@ -0,0 +1,156 @@
+"""Dataset validation, schema extraction, and canonicalization."""
+
+import csv
+import io
+import warnings
+
+import pandas as pd
+
+from ml_mcp.config import Settings, get_settings
+from ml_mcp.domain.errors import DatasetValidationError
+from ml_mcp.domain.value_objects import (
+    ColumnSchema,
+    DatasetFormat,
+    DatasetSchema,
+    FeatureType,
+)
+from ml_mcp.infrastructure.object_storage.s3 import S3StorageService
+
+
+class DatasetValidator:
+    """Performs strict validation and schema extraction on untrusted tabular datasets."""
+
+    def __init__(self, settings: Settings | None = None) -> None:
+        self.settings = settings or get_settings()
+
+    def _infer_feature_type(self, series: pd.Series) -> FeatureType:
+        """Map pandas/pyarrow series to standard FeatureType."""
+        if pd.api.types.is_bool_dtype(series):
+            return FeatureType.BOOLEAN
+        elif pd.api.types.is_numeric_dtype(series):
+            return FeatureType.NUMERIC
+        elif pd.api.types.is_datetime64_any_dtype(series):
+            return FeatureType.DATETIME
+        elif pd.api.types.is_categorical_dtype(series):
+            return FeatureType.CATEGORICAL
+        else:
+            # String / object column
+            non_null = series.dropna()
+            if len(non_null) > 0:
+                unique_ratio = len(non_null.unique()) / len(non_null)
+                # If unique ratio is low, treat as categorical; otherwise text
+                if unique_ratio < 0.2 or len(non_null.unique()) <= 50:
+                    return FeatureType.CATEGORICAL
+            return FeatureType.TEXT
+
+    def parse_dataframe(self, raw_bytes: bytes, data_format: DatasetFormat | str) -> pd.DataFrame:
+        """Parse raw bytes into a pandas DataFrame, rejecting malformed inputs."""
+        # 1. Size constraint
+        if len(raw_bytes) > self.settings.security.max_dataset_upload_bytes:
+            raise DatasetValidationError(
+                f"Dataset size ({len(raw_bytes)} bytes) exceeds maximum permitted limit ({self.settings.security.max_dataset_upload_bytes} bytes)"
+            )
+        if len(raw_bytes) == 0:
+            raise DatasetValidationError("Dataset upload is empty")
+
+        fmt = str(data_format).lower()
+        try:
+            bio = io.BytesIO(raw_bytes)
+            if fmt == DatasetFormat.CSV.value:
+                # Deterministic check for consistent column counts across all rows
+                try:
+                    text_content = raw_bytes.decode("utf-8")
+                except UnicodeDecodeError as exc:
+                    raise DatasetValidationError(f"Malformed encoding: dataset must be valid UTF-8: {exc}") from exc
+
+                reader = csv.reader(io.StringIO(text_content))
+                try:
+                    header = next(reader, None)
+                except Exception as exc:
+                    raise DatasetValidationError(f"Failed to parse dataset as csv: {exc}") from exc
+
+                if header is None or len(header) == 0:
+                    raise DatasetValidationError("Dataset upload is empty or has no header")
+
+                expected_cols = len(header)
+                for line_idx, row in enumerate(reader, start=2):
+                    if not row:
+                        continue
+                    if len(row) != expected_cols:
+                        raise DatasetValidationError(
+                            f"Failed to parse dataset as csv: Row {line_idx} has {len(row)} columns, expected {expected_cols}"
+                        )
+
+                with warnings.catch_warnings():
+                    warnings.simplefilter("error", pd.errors.ParserWarning)
+                    df = pd.read_csv(bio, index_col=False)
+            elif fmt == DatasetFormat.PARQUET.value:
+                df = pd.read_parquet(bio)
+            elif fmt == DatasetFormat.JSONL.value:
+                df = pd.read_json(bio, lines=True)
+            else:
+                raise DatasetValidationError(f"Unsupported dataset format '{data_format}'. Allowed: csv, parquet, jsonl")
+        except Exception as exc:
+            raise DatasetValidationError(f"Failed to parse dataset as {fmt}: {exc}") from exc
+
+        # 2. Structural constraints
+        if df.empty or len(df) == 0:
+            raise DatasetValidationError("Dataset contains 0 rows")
+
+        if len(df.columns) > self.settings.security.max_dataset_columns:
+            raise DatasetValidationError(
+                f"Dataset column count ({len(df.columns)}) exceeds maximum allowed ({self.settings.security.max_dataset_columns})"
+            )
+
+        if len(df) > self.settings.security.max_dataset_rows:
+            raise DatasetValidationError(
+                f"Dataset row count ({len(df)}) exceeds maximum allowed ({self.settings.security.max_dataset_rows})"
+            )
+
+        return df
+
+    def extract_schema(self, df: pd.DataFrame, canonical_bytes: bytes) -> DatasetSchema:
+        """Profile dataset columns and construct validated DatasetSchema."""
+        columns: list[ColumnSchema] = []
+        for col_name in df.columns:
+            series = df[col_name]
+            ft = self._infer_feature_type(series)
+            null_cnt = int(series.isna().sum())
+            unique_cnt = int(series.nunique(dropna=True))
+
+            # Grab up to 3 non-null sample values
+            samples = [str(x) for x in series.dropna().iloc[:3].tolist()]
+
+            columns.append(
+                ColumnSchema(
+                    name=str(col_name),
+                    feature_type=ft,
+                    nullable=null_cnt > 0,
+                    null_count=null_cnt,
+                    unique_count=unique_cnt,
+                    sample_values=samples,
+                )
+            )
+
+        content_hash = S3StorageService.compute_sha256(canonical_bytes)
+
+        return DatasetSchema(
+            columns=columns,
+            row_count=len(df),
+            column_count=len(df.columns),
+            size_bytes=len(canonical_bytes),
+            content_hash=f"sha256:{content_hash}",
+        )
+
+    def canonicalize_to_parquet(self, df: pd.DataFrame) -> bytes:
+        """Convert DataFrame to standardized, compressed Apache Parquet format."""
+        out_buf = io.BytesIO()
+        df.to_parquet(out_buf, index=False, engine="pyarrow", compression="snappy")
+        return out_buf.getvalue()
+
+    def process(self, raw_bytes: bytes, data_format: DatasetFormat | str) -> tuple[pd.DataFrame, bytes, DatasetSchema]:
+        """Validate, convert to canonical parquet, and extract complete schema."""
+        df = self.parse_dataframe(raw_bytes, data_format)
+        parquet_bytes = self.canonicalize_to_parquet(df)
+        schema = self.extract_schema(df, parquet_bytes)
+        return df, parquet_bytes, schema

ml_mcp/application/experiments/__init__.py

@@ -0,0 +1,5 @@
+"""Experiments application module."""
+
+from ml_mcp.application.experiments.service import ExperimentService
+
+__all__ = ["ExperimentService"]

ml_mcp/application/experiments/service.py

@@ -0,0 +1,254 @@
+"""Experiment application service managing lifecycle, validation, idempotency, and outbox publication."""
+
+import logging
+from typing import Any
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ml_mcp.application.models.service import ModelService
+from ml_mcp.domain.errors import (
+    ExperimentNotCancellableError,
+    InvalidExperimentError,
+    ResourceNotFoundError,
+)
+from ml_mcp.domain.value_objects import (
+    EvaluationConfig,
+    ExperimentSpec,
+    ResourcePolicy,
+    TaskType,
+)
+from ml_mcp.infrastructure.postgres.base import generate_uuid7
+from ml_mcp.infrastructure.postgres.models import ExperimentOrm, OutboxEventOrm
+from ml_mcp.infrastructure.postgres.repositories.datasets import DatasetRepository
+from ml_mcp.infrastructure.postgres.repositories.experiments import ExperimentRepository
+
+logger = logging.getLogger(__name__)
+
+
+class ExperimentService:
+    """Encapsulates experiment creation, validation, querying, cancellation, and outbox emission."""
+
+    def __init__(self, session: AsyncSession) -> None:
+        self.session = session
+        self.repo = ExperimentRepository(session)
+        self.model_svc = ModelService(session)
+        self.dataset_repo = DatasetRepository(session)
+
+    async def create_experiment(
+        self,
+        tenant_id: str,
+        project_id: str,
+        dataset_version_id: str,
+        model_version_id: str,
+        task_type: str,
+        target_column: str,
+        principal_id: str,
+        feature_columns: list[str] | None = None,
+        primary_metric: str = "roc_auc",
+        additional_metrics: list[str] | None = None,
+        hyperparameters: dict[str, Any] | None = None,
+        random_seed: int = 42,
+        idempotency_key: str | None = None,
+        resource_policy: ResourcePolicy | None = None,
+    ) -> dict[str, Any]:
+        """Validate model approval, create experiment record and transactional outbox event."""
+        # 1. Enforce Model Approval and bounds validation
+        await self.model_svc.validate_model_for_experiment(
+            task_type=task_type,
+            hyperparameters=hyperparameters or {},
+            model_version_id=model_version_id,
+        )
+
+        # 2. Validate Dataset Version and Column Existence upfront
+        dsv = await self.dataset_repo.get_version_by_id(tenant_id, dataset_version_id)
+        if not dsv:
+            raise ResourceNotFoundError("DatasetVersion", dataset_version_id)
+
+        available_columns = [
+            col["name"]
+            for col in dsv.schema_json.get("columns", [])
+            if isinstance(col, dict) and "name" in col
+        ]
+        if available_columns:
+            if target_column not in available_columns:
+                raise InvalidExperimentError(
+                    f"Target column '{target_column}' not found in dataset '{dsv.dataset_id}' version '{dsv.version}'. Available columns: {available_columns}"
+                )
+            if feature_columns is not None:
+                if len(feature_columns) == 0:
+                    raise InvalidExperimentError("feature_columns cannot be empty when specified")
+                missing_features = [c for c in feature_columns if c not in available_columns]
+                if missing_features:
+                    raise InvalidExperimentError(
+                        f"Feature columns {missing_features} not found in dataset '{dsv.dataset_id}' version '{dsv.version}'. Available columns: {available_columns}"
+                    )
+                if target_column in feature_columns:
+                    raise InvalidExperimentError(
+                        f"Target column '{target_column}' cannot be included in feature_columns"
+                    )
+
+        # 3. Check Idempotency
+        if idempotency_key:
+            existing = await self.repo.get_by_idempotency_key(
+                tenant_id, project_id, idempotency_key
+            )
+            if existing:
+                return {
+                    "experiment_id": existing.id,
+                    "status": existing.status,
+                    "idempotent_replay": True,
+                    "created_at": existing.created_at.isoformat(),
+                }
+
+        # 3. Build ExperimentSpec
+        spec = ExperimentSpec(
+            tenant_id=tenant_id,
+            project_id=project_id,
+            dataset_version_id=dataset_version_id,
+            model_version_id=model_version_id,
+            task_type=TaskType(task_type),
+            target_column=target_column,
+            feature_columns=feature_columns,
+            evaluation_config=EvaluationConfig(
+                primary_metric=primary_metric,
+                additional_metrics=additional_metrics or [],
+            ),
+            hyperparameters=hyperparameters or {},
+            resource_policy=resource_policy or ResourcePolicy(),
+            random_seed=random_seed,
+            idempotency_key=idempotency_key,
+            created_by=principal_id,
+        )
+
+        # 4. Create ORM entities
+        experiment_id = generate_uuid7()
+        exp_orm = ExperimentOrm(
+            id=experiment_id,
+            tenant_id=tenant_id,
+            project_id=project_id,
+            dataset_version_id=dataset_version_id,
+            model_version_id=model_version_id,
+            spec_json=spec.model_dump(),
+            status="QUEUED",
+            idempotency_key=idempotency_key,
+            created_by=principal_id,
+        )
+
+        outbox_event = OutboxEventOrm(
+            id=generate_uuid7(),
+            event_type="EXPERIMENT_SUBMITTED",
+            aggregate_type="EXPERIMENT",
+            aggregate_id=experiment_id,
+            payload_json={
+                "experiment_id": experiment_id,
+                "tenant_id": tenant_id,
+                "project_id": project_id,
+            },
+        )
+
+        await self.repo.create_experiment(exp_orm, initial_outbox_event=outbox_event)
+
+        # 5. Direct enqueue to Redis for immediate execution
+        try:
+            from ml_mcp.infrastructure.queue.task_queue import RedisTaskQueue
+
+            queue = RedisTaskQueue()
+            await queue.enqueue(
+                experiment_id=experiment_id,
+                tenant_id=tenant_id,
+                payload={
+                    "experiment_id": experiment_id,
+                    "tenant_id": tenant_id,
+                    "project_id": project_id,
+                },
+            )
+        except Exception as q_exc:
+            logger.debug("Direct Redis enqueue deferred to outbox processor: %s", q_exc)
+
+        return {
+            "experiment_id": experiment_id,
+            "status": "QUEUED",
+            "idempotent_replay": False,
+            "task_type": task_type,
+            "target_column": target_column,
+            "created_at": exp_orm.created_at.isoformat(),
+        }
+
+    async def get_experiment(self, tenant_id: str, experiment_id: str) -> dict[str, Any]:
+        """Fetch experiment by tenant-scoped ID with full run details."""
+        exp = await self.repo.get_experiment(tenant_id, experiment_id)
+        if not exp:
+            raise ResourceNotFoundError("Experiment", experiment_id)
+
+        sorted_runs = sorted(
+            exp.runs, key=lambda r: (r.run_number or 0, r.created_at), reverse=True
+        )
+        latest_run = sorted_runs[0] if sorted_runs else None
+
+        error_message = (
+            latest_run.failure_reason if latest_run and latest_run.failure_reason else None
+        )
+        error_type = None
+        if error_message:
+            if ":" in error_message:
+                error_type = error_message.split(":")[0].strip()
+            else:
+                error_type = "ExecutionError"
+
+        return {
+            "experiment_id": exp.id,
+            "project_id": exp.project_id,
+            "status": exp.status,
+            "spec": exp.spec_json,
+            "runs_count": len(exp.runs),
+            "latest_run_status": latest_run.status if latest_run else None,
+            "error_message": error_message,
+            "error_type": error_type,
+            "metrics_count": len(exp.metrics),
+            "artifacts_count": len(exp.artifacts),
+            "created_at": exp.created_at.isoformat(),
+            "updated_at": exp.updated_at.isoformat(),
+        }
+
+    async def cancel_experiment(self, tenant_id: str, experiment_id: str) -> dict[str, Any]:
+        """Cancel an in-flight or queued experiment."""
+        exp = await self.repo.get_experiment(tenant_id, experiment_id)
+        if not exp:
+            raise ResourceNotFoundError("Experiment", experiment_id)
+
+        if exp.status in ("SUCCEEDED", "FAILED", "CANCELLED"):
+            raise ExperimentNotCancellableError(experiment_id, exp.status)
+
+        await self.repo.update_status(tenant_id, experiment_id, "CANCELLED")
+        return {
+            "experiment_id": experiment_id,
+            "status": "CANCELLED",
+            "message": f"Experiment {experiment_id} successfully cancelled.",
+        }
+
+    async def list_experiments(
+        self,
+        tenant_id: str,
+        project_id: str | None = None,
+        status: str | None = None,
+        limit: int = 50,
+        offset: int = 0,
+    ) -> list[dict[str, Any]]:
+        """List experiments for tenant."""
+        experiments = await self.repo.list_experiments(
+            tenant_id=tenant_id,
+            project_id=project_id,
+            status=status,
+            limit=limit,
+            offset=offset,
+        )
+        return [
+            {
+                "experiment_id": e.id,
+                "project_id": e.project_id,
+                "status": e.status,
+                "primary_metric": e.spec_json.get("evaluation_config", {}).get("primary_metric"),
+                "created_at": e.created_at.isoformat(),
+            }
+            for e in experiments
+        ]

ml_mcp/application/models/__init__.py

@@ -0,0 +1,6 @@
+"""Model catalog package."""
+
+from ml_mcp.application.models.catalog_seed import APPROVED_MODELS
+from ml_mcp.application.models.service import ModelService
+
+__all__ = ["APPROVED_MODELS", "ModelService"]

ml_mcp/application/models/catalog_seed.py

@@ -0,0 +1,131 @@
+"""Catalog seed definitions for all 7 approved tabular machine learning models."""
+
+import hashlib
+
+from ml_mcp.domain.value_objects import (
+    ApprovalStatus,
+    HyperparameterConstraint,
+    ModelFamily,
+    ModelMetadata,
+    TaskType,
+)
+
+
+def _make_digest(model_id: str) -> str:
+    """Generate deterministic SHA-256 container image digest for model family."""
+    return f"sha256:{hashlib.sha256(f'modellab/trainer-{model_id}:1.0.0'.encode()).hexdigest()}"
+
+
+APPROVED_MODELS: list[ModelMetadata] = [
+    # 1. Logistic Regression
+    ModelMetadata(
+        model_id="logistic_regression",
+        version="1.0.0",
+        name="Logistic Regression",
+        description="Regularized linear model for binary and multiclass classification",
+        family=ModelFamily.LOGISTIC_REGRESSION,
+        supported_task_types=[TaskType.BINARY_CLASSIFICATION, TaskType.MULTICLASS_CLASSIFICATION],
+        approval_status=ApprovalStatus.APPROVED,
+        container_image_digest=_make_digest("logistic_regression"),
+        supported_hyperparameters=[
+            HyperparameterConstraint(name="C", type="float", default=1.0, min_value=0.0001, max_value=10000.0, description="Inverse of regularization strength"),
+            HyperparameterConstraint(name="penalty", type="str", default="l2", allowed_values=["l1", "l2", "elasticnet", "none"], description="Norm used in penalization"),
+            HyperparameterConstraint(name="max_iter", type="int", default=200, min_value=10, max_value=5000, description="Maximum iterations for solver convergence"),
+        ],
+    ),
+    # 2. Random Forest
+    ModelMetadata(
+        model_id="random_forest",
+        version="1.0.0",
+        name="Random Forest",
+        description="Ensemble of decision trees trained with bagging and feature subsampling",
+        family=ModelFamily.RANDOM_FOREST,
+        supported_task_types=[TaskType.BINARY_CLASSIFICATION, TaskType.MULTICLASS_CLASSIFICATION, TaskType.REGRESSION],
+        approval_status=ApprovalStatus.APPROVED,
+        container_image_digest=_make_digest("random_forest"),
+        supported_hyperparameters=[
+            HyperparameterConstraint(name="n_estimators", type="int", default=100, min_value=10, max_value=1000, description="Number of trees in the forest"),
+            HyperparameterConstraint(name="max_depth", type="int", default=10, min_value=1, max_value=50, description="Maximum tree depth"),
+            HyperparameterConstraint(name="min_samples_split", type="int", default=2, min_value=2, max_value=20, description="Minimum samples to split an internal node"),
+        ],
+    ),
+    # 3. XGBoost
+    ModelMetadata(
+        model_id="xgboost",
+        version="1.0.0",
+        name="XGBoost",
+        description="Extreme Gradient Boosting decision trees optimized for speed and performance",
+        family=ModelFamily.XGBOOST,
+        supported_task_types=[TaskType.BINARY_CLASSIFICATION, TaskType.MULTICLASS_CLASSIFICATION, TaskType.REGRESSION],
+        approval_status=ApprovalStatus.APPROVED,
+        container_image_digest=_make_digest("xgboost"),
+        supported_hyperparameters=[
+            HyperparameterConstraint(name="n_estimators", type="int", default=100, min_value=10, max_value=2000, description="Number of boosting rounds"),
+            HyperparameterConstraint(name="max_depth", type="int", default=6, min_value=1, max_value=20, description="Maximum tree depth"),
+            HyperparameterConstraint(name="learning_rate", type="float", default=0.1, min_value=0.001, max_value=1.0, description="Step size shrinkage"),
+            HyperparameterConstraint(name="subsample", type="float", default=1.0, min_value=0.1, max_value=1.0, description="Subsample ratio of the training instances"),
+        ],
+    ),
+    # 4. LightGBM
+    ModelMetadata(
+        model_id="lightgbm",
+        version="1.0.0",
+        name="LightGBM",
+        description="Fast, distributed, high-performance gradient boosting based on decision tree algorithms",
+        family=ModelFamily.LIGHTGBM,
+        supported_task_types=[TaskType.BINARY_CLASSIFICATION, TaskType.MULTICLASS_CLASSIFICATION, TaskType.REGRESSION],
+        approval_status=ApprovalStatus.APPROVED,
+        container_image_digest=_make_digest("lightgbm"),
+        supported_hyperparameters=[
+            HyperparameterConstraint(name="n_estimators", type="int", default=100, min_value=10, max_value=2000, description="Number of boosting iterations"),
+            HyperparameterConstraint(name="num_leaves", type="int", default=31, min_value=2, max_value=256, description="Max tree leaves"),
+            HyperparameterConstraint(name="learning_rate", type="float", default=0.1, min_value=0.001, max_value=1.0, description="Boosting learning rate"),
+        ],
+    ),
+    # 5. CatBoost
+    ModelMetadata(
+        model_id="catboost",
+        version="1.0.0",
+        name="CatBoost",
+        description="Gradient boosting on decision trees with native categorical feature support",
+        family=ModelFamily.CATBOOST,
+        supported_task_types=[TaskType.BINARY_CLASSIFICATION, TaskType.MULTICLASS_CLASSIFICATION, TaskType.REGRESSION],
+        approval_status=ApprovalStatus.APPROVED,
+        container_image_digest=_make_digest("catboost"),
+        supported_hyperparameters=[
+            HyperparameterConstraint(name="iterations", type="int", default=100, min_value=10, max_value=2000, description="Number of boosting iterations"),
+            HyperparameterConstraint(name="depth", type="int", default=6, min_value=1, max_value=16, description="Depth of the trees"),
+            HyperparameterConstraint(name="learning_rate", type="float", default=0.1, min_value=0.001, max_value=1.0, description="Learning rate"),
+        ],
+    ),
+    # 6. Linear SVM
+    ModelMetadata(
+        model_id="linear_svm",
+        version="1.0.0",
+        name="Linear Support Vector Machine",
+        description="Linear support vector machine classifier and regressor. Note: LinearSVC uses decision boundaries without native probability estimates, so probability-dependent metrics (roc_auc, pr_auc, log_loss, brier_score) are omitted.",
+        family=ModelFamily.LINEAR_SVM,
+        supported_task_types=[TaskType.BINARY_CLASSIFICATION, TaskType.MULTICLASS_CLASSIFICATION, TaskType.REGRESSION],
+        approval_status=ApprovalStatus.APPROVED,
+        container_image_digest=_make_digest("linear_svm"),
+        supported_hyperparameters=[
+            HyperparameterConstraint(name="C", type="float", default=1.0, min_value=0.001, max_value=1000.0, description="Regularization parameter"),
+            HyperparameterConstraint(name="max_iter", type="int", default=1000, min_value=100, max_value=10000, description="Max iterations"),
+        ],
+    ),
+    # 7. MLP (Multi-Layer Perceptron)
+    ModelMetadata(
+        model_id="mlp",
+        version="1.0.0",
+        name="Multi-Layer Perceptron",
+        description="Feedforward artificial neural network with backpropagation training",
+        family=ModelFamily.MLP,
+        supported_task_types=[TaskType.BINARY_CLASSIFICATION, TaskType.MULTICLASS_CLASSIFICATION, TaskType.REGRESSION],
+        approval_status=ApprovalStatus.APPROVED,
+        container_image_digest=_make_digest("mlp"),
+        supported_hyperparameters=[
+            HyperparameterConstraint(name="alpha", type="float", default=0.0001, min_value=0.000001, max_value=1.0, description="L2 penalty regularization"),
+            HyperparameterConstraint(name="max_iter", type="int", default=200, min_value=50, max_value=2000, description="Maximum iterations"),
+        ],
+    ),
+]