QuickLab-MCP 0.1.0__py3-none-any.whl

ml_mcp/__init__.py

@@ -0,0 +1,3 @@
+"""Agentic ML MCP Server (ModelLab)."""
+
+__version__ = "0.1.0"

ml_mcp/application/__init__.py

@@ -0,0 +1 @@
+"""Application layer package."""

ml_mcp/application/analysis/__init__.py

@@ -0,0 +1,15 @@
+"""Structured analysis package."""
+
+from ml_mcp.application.analysis.diagnostics import (
+    DataLeakageDetector,
+    ErrorPatternAnalyzer,
+    OverfittingDetector,
+)
+from ml_mcp.application.analysis.service import AnalysisService
+
+__all__ = [
+    "OverfittingDetector",
+    "DataLeakageDetector",
+    "ErrorPatternAnalyzer",
+    "AnalysisService",
+]

ml_mcp/application/analysis/diagnostics.py

@@ -0,0 +1,122 @@
+"""Diagnostic evaluators for overfitting, data leakage, and error patterns."""
+
+from typing import Any
+
+import numpy as np
+
+
+class OverfittingDetector:
+    """Diagnoses generalization gaps between training and validation splits."""
+
+    @staticmethod
+    def evaluate(
+        train_metrics: dict[str, float],
+        val_metrics: dict[str, float],
+        threshold: float = 0.15,
+    ) -> dict[str, Any]:
+        gap_details: dict[str, float] = {}
+        overfitting_detected = False
+
+        common_keys = set(train_metrics.keys()).intersection(val_metrics.keys())
+        for k in common_keys:
+            tr_v = train_metrics[k]
+            val_v = val_metrics[k]
+            if (
+                isinstance(tr_v, (int, float))
+                and isinstance(val_v, (int, float))
+                and k in ("accuracy", "roc_auc", "pr_auc", "f1", "f1_macro", "r2")
+            ):
+                gap = tr_v - val_v
+                gap_details[k] = round(gap, 4)
+                if gap >= threshold:
+                    overfitting_detected = True
+
+        return {
+            "overfitting_detected": overfitting_detected,
+            "metric_gaps": gap_details,
+            "severity": "HIGH"
+            if any(g > 0.25 for g in gap_details.values())
+            else ("MEDIUM" if overfitting_detected else "LOW"),
+            "observation": (
+                f"Validation performance lags training metrics by {max(gap_details.values()):.2%}, indicating model memorization."
+                if overfitting_detected and gap_details
+                else "Generalization gap is within acceptable operational tolerance."
+            ),
+        }
+
+
+class DataLeakageDetector:
+    """Detects suspicious statistical patterns indicative of label or feature leakage."""
+
+    @staticmethod
+    def evaluate(
+        train_metrics: dict[str, float],
+        val_metrics: dict[str, float],
+    ) -> dict[str, Any]:
+        signals: list[str] = []
+        suspicious = False
+
+        # Suspiciously perfect validation performance
+        for k in ("roc_auc", "accuracy", "f1"):
+            if val_metrics.get(k) is not None and val_metrics[k] >= 0.999:
+                signals.append(
+                    f"Metric '{k}' is suspiciously perfect ({val_metrics[k]:.4f}) on validation split."
+                )
+                suspicious = True
+
+        # Validation strictly outperforms training by a large margin (often indicates wrong split or inverted leakage)
+        for k in ("accuracy", "f1", "roc_auc"):
+            if k in train_metrics and k in val_metrics and val_metrics[k] - train_metrics[k] > 0.20:
+                signals.append(
+                    f"Validation '{k}' ({val_metrics[k]:.2f}) significantly exceeds training ({train_metrics[k]:.2f})."
+                )
+                suspicious = True
+
+        return {
+            "leakage_risk_detected": suspicious,
+            "signals": signals,
+            "confidence": "HIGH" if len(signals) > 1 else ("MEDIUM" if suspicious else "NONE"),
+        }
+
+
+class ErrorPatternAnalyzer:
+    """Analyzes confusion matrices and residual error distributions."""
+
+    @staticmethod
+    def evaluate_confusion_matrix(cm: list[list[int]]) -> dict[str, Any]:
+        """Analyze false positives and false negatives in binary or multiclass matrix."""
+        cm_arr = np.array(cm)
+        total_samples = int(cm_arr.sum())
+        if total_samples == 0:
+            return {"status": "EMPTY"}
+
+        if cm_arr.shape == (2, 2):
+            tn, fp, fn, tp = cm_arr.ravel()
+            fp_rate = float(fp / (fp + tn)) if (fp + tn) > 0 else 0.0
+            fn_rate = float(fn / (fn + tp)) if (fn + tp) > 0 else 0.0
+
+            dominant_error = (
+                "FALSE_POSITIVES" if fp > fn else ("FALSE_NEGATIVES" if fn > fp else "BALANCED")
+            )
+
+            return {
+                "total_samples": total_samples,
+                "true_positives": int(tp),
+                "true_negatives": int(tn),
+                "false_positives": int(fp),
+                "false_negatives": int(fn),
+                "false_positive_rate": round(fp_rate, 4),
+                "false_negative_rate": round(fn_rate, 4),
+                "dominant_error_pattern": dominant_error,
+            }
+        else:
+            # Multiclass
+            diagonal = np.diag(cm_arr)
+            per_class_accuracy = diagonal / np.maximum(cm_arr.sum(axis=1), 1)
+            lowest_class = int(np.argmin(per_class_accuracy))
+            return {
+                "total_samples": total_samples,
+                "classes_count": int(cm_arr.shape[0]),
+                "lowest_performing_class_index": lowest_class,
+                "lowest_class_accuracy": round(float(per_class_accuracy[lowest_class]), 4),
+            }

ml_mcp/application/analysis/service.py

@@ -0,0 +1,155 @@
+"""Structured experiment analysis application service."""
+
+import logging
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ml_mcp.application.analysis.diagnostics import (
+    DataLeakageDetector,
+    OverfittingDetector,
+)
+from ml_mcp.domain.errors import InvalidInputError, ResourceNotFoundError
+from ml_mcp.infrastructure.postgres.base import generate_uuid7
+from ml_mcp.infrastructure.postgres.models import AnalysisRunOrm, ExperimentOrm, MetricOrm
+
+logger = logging.getLogger(__name__)
+
+
+class AnalysisService:
+    """Service providing structured, read-only analytical evaluation of ML experiment results."""
+
+    def __init__(self, session: AsyncSession) -> None:
+        self.session = session
+
+    async def analyze_experiment(self, tenant_id: str, experiment_id: str) -> dict[str, Any]:
+        """Perform comprehensive statistical analysis of an experiment's metrics and predictions."""
+        # 1. Fetch experiment
+        stmt = select(ExperimentOrm).where(
+            ExperimentOrm.tenant_id == tenant_id, ExperimentOrm.id == experiment_id
+        )
+        result = await self.session.execute(stmt)
+        exp = result.scalar_one_or_none()
+        if not exp:
+            raise ResourceNotFoundError("Experiment", experiment_id)
+
+        if exp.status != "SUCCEEDED":
+            raise InvalidInputError(
+                f"Experiment '{experiment_id}' has not completed successfully (current status: {exp.status}). "
+                "Only completed experiments with recorded metrics can be analyzed."
+            )
+
+        # 2. Fetch metrics
+        m_stmt = select(MetricOrm).where(MetricOrm.experiment_id == experiment_id)
+        m_result = await self.session.execute(m_stmt)
+        all_metrics = m_result.scalars().all()
+        if not all_metrics:
+            raise InvalidInputError(
+                f"Experiment '{experiment_id}' has no recorded evaluation metrics to analyze."
+            )
+
+        train_metrics: dict[str, float] = {}
+        val_metrics: dict[str, float] = {}
+
+        for m in all_metrics:
+            if m.split == "train":
+                train_metrics[m.metric_name] = m.metric_value
+            elif m.split == "validation":
+                val_metrics[m.metric_name] = m.metric_value
+
+        # Check for confusion matrix in artifacts or metrics
+        # 3. Run diagnostic engines
+        overfitting = OverfittingDetector.evaluate(train_metrics, val_metrics)
+        leakage = DataLeakageDetector.evaluate(train_metrics, val_metrics)
+
+        # 4. Synthesize recommendations
+        recommendations: list[str] = []
+        if overfitting["overfitting_detected"]:
+            recommendations.append(
+                "Increase regularization (e.g. increase alpha, reduce max_depth, or increase min_samples_split)."
+            )
+            recommendations.append(
+                "Apply cross-validation (K-Fold) to evaluate variance across folds."
+            )
+        if leakage["leakage_risk_detected"]:
+            recommendations.append(
+                "Audit dataset columns for target leakage or deterministic identifiers."
+            )
+        if not recommendations:
+            recommendations.append(
+                "Model demonstrates healthy generalization. Consider testing next candidate architecture or fine-tuning learning rate."
+            )
+
+        # 5. Format according to Architecture.md Section 21
+        structured_findings: dict[str, Any] = {
+            "summary": f"Experiment {experiment_id} evaluated with {len(val_metrics)} validation metrics.",
+            "metric_comparison": {
+                "train": train_metrics,
+                "validation": val_metrics,
+            },
+            "statistical_observations": {
+                "primary_metric": exp.spec_json.get("evaluation_config", {}).get("primary_metric"),
+                "primary_metric_validation_value": val_metrics.get(
+                    exp.spec_json.get("evaluation_config", {}).get("primary_metric", "")
+                ),
+            },
+            "error_patterns": {
+                "observed_anomalies": len(leakage["signals"]),
+            },
+            "data_quality_findings": {
+                "target_column": exp.spec_json.get("target_column"),
+                "task_type": exp.spec_json.get("task_type"),
+            },
+            "overfitting_signals": overfitting,
+            "leakage_signals": leakage,
+            "limitations": [
+                "Analysis performed strictly on tabular evaluation split.",
+                "Hyperparameter space explored only for single trial run.",
+            ],
+            "recommended_next_experiments": recommendations,
+            "confidence": "HIGH",
+        }
+
+        # 6. Upsert read-only analysis run record
+        analysis_run_stmt = select(AnalysisRunOrm).where(
+            AnalysisRunOrm.experiment_id == experiment_id
+        )
+        existing_run = (await self.session.execute(analysis_run_stmt)).scalar_one_or_none()
+
+        if not existing_run:
+            analysis_orm = AnalysisRunOrm(
+                id=generate_uuid7(),
+                experiment_id=experiment_id,
+                tenant_id=tenant_id,
+                summary_json={"summary": structured_findings["summary"]},
+                diagnostics_json=structured_findings,
+            )
+            self.session.add(analysis_orm)
+            await self.session.flush()
+
+        return structured_findings
+
+    async def analyze_model_errors(self, tenant_id: str, experiment_id: str) -> dict[str, Any]:
+        """Deep dive into false positives and false negatives."""
+        analysis = await self.analyze_experiment(tenant_id, experiment_id)
+        return {
+            "experiment_id": experiment_id,
+            "error_patterns": analysis["error_patterns"],
+            "overfitting_diagnosis": analysis["overfitting_signals"],
+        }
+
+    async def check_experiment_validity(self, tenant_id: str, experiment_id: str) -> dict[str, Any]:
+        """Validate whether an experiment's metrics can be trusted or should be flagged for review."""
+        analysis = await self.analyze_experiment(tenant_id, experiment_id)
+        is_valid = not (
+            analysis["leakage_signals"]["leakage_risk_detected"]
+            or analysis["overfitting_signals"]["severity"] == "HIGH"
+        )
+        return {
+            "experiment_id": experiment_id,
+            "is_valid": is_valid,
+            "leakage_risk": analysis["leakage_signals"]["leakage_risk_detected"],
+            "overfitting_severity": analysis["overfitting_signals"]["severity"],
+            "recommendations": analysis["recommended_next_experiments"],
+        }

ml_mcp/application/artifacts/__init__.py

@@ -0,0 +1,5 @@
+"""Artifacts application module."""
+
+from ml_mcp.application.artifacts.service import ArtifactService
+
+__all__ = ["ArtifactService"]

ml_mcp/application/artifacts/service.py

@@ -0,0 +1,149 @@
+"""Artifact and results application service managing metrics, predictions, and model artifacts."""
+
+import contextlib
+import io
+from typing import Any
+
+import pandas as pd
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ml_mcp.domain.errors import ResourceNotFoundError
+from ml_mcp.infrastructure.object_storage.s3 import S3StorageService, get_storage_service
+from ml_mcp.infrastructure.postgres.repositories.artifacts import ArtifactRepository
+from ml_mcp.infrastructure.postgres.repositories.experiments import ExperimentRepository
+from ml_mcp.infrastructure.postgres.repositories.metrics import MetricRepository
+
+
+class ArtifactService:
+    """Encapsulates querying of experiment results, metrics, prediction samples, and stored artifacts."""
+
+    def __init__(
+        self,
+        session: AsyncSession,
+        storage_service: S3StorageService | None = None,
+    ) -> None:
+        self.session = session
+        self.storage = storage_service or get_storage_service()
+        self.exp_repo = ExperimentRepository(session)
+        self.metric_repo = MetricRepository(session)
+        self.artifact_repo = ArtifactRepository(session)
+
+    async def get_metrics(
+        self,
+        tenant_id: str,
+        experiment_id: str,
+        split: str | None = None,
+    ) -> dict[str, Any]:
+        """Fetch metrics for an experiment with tenant isolation and anti-enumeration enforcement."""
+        exp = await self.exp_repo.get_experiment(tenant_id, experiment_id)
+        if not exp:
+            raise ResourceNotFoundError("Experiment", experiment_id)
+
+        metrics = await self.metric_repo.get_metrics(tenant_id, experiment_id, split)
+
+        formatted: dict[str, Any] = {}
+        for m in metrics:
+            key = f"{m.split}_{m.metric_name}" if not split else m.metric_name
+            formatted[key] = m.metric_value
+
+        return {
+            "experiment_id": experiment_id,
+            "split": split or "all",
+            "metrics": formatted,
+        }
+
+    async def get_predictions(
+        self,
+        tenant_id: str,
+        experiment_id: str,
+        limit: int = 5,
+    ) -> dict[str, Any]:
+        """Fetch sample predictions and generate presigned download URL with tenant verification."""
+        exp = await self.exp_repo.get_experiment(tenant_id, experiment_id)
+        if not exp:
+            raise ResourceNotFoundError("Experiment", experiment_id)
+
+        key = self.storage.get_experiment_artifact_key(
+            tenant_id, experiment_id, "predictions", "predictions.parquet"
+        )
+
+        download_url = self.storage.generate_presigned_get_url(key, expires_in=900)
+
+        sample_preview = []
+        with contextlib.suppress(Exception):
+            data = self.storage.get_object(key)
+            df = pd.read_parquet(io.BytesIO(data))
+            sample_preview = df.head(limit).to_dict(orient="records")
+
+        return {
+            "experiment_id": experiment_id,
+            "sample_preview": sample_preview,
+            "download_url": download_url,
+            "expires_in_seconds": 900,
+        }
+
+    async def list_artifacts(
+        self,
+        tenant_id: str,
+        experiment_id: str,
+    ) -> list[dict[str, Any]]:
+        """List stored artifacts for an experiment after tenant ownership check."""
+        exp = await self.exp_repo.get_experiment(tenant_id, experiment_id)
+        if not exp:
+            raise ResourceNotFoundError("Experiment", experiment_id)
+
+        artifacts = await self.artifact_repo.list_artifacts(tenant_id, experiment_id)
+        return [
+            {
+                "artifact_id": a.id,
+                "artifact_type": a.artifact_type,
+                "size_bytes": a.size_bytes,
+                "content_hash": a.content_hash,
+                "created_at": a.created_at.isoformat(),
+            }
+            for a in artifacts
+        ]
+
+    async def get_artifact(
+        self,
+        tenant_id: str,
+        artifact_id: str,
+    ) -> dict[str, Any]:
+        """Fetch artifact metadata and presigned URL by artifact ID."""
+        art = await self.artifact_repo.get_artifact(tenant_id, artifact_id)
+        if not art:
+            raise ResourceNotFoundError("Artifact", artifact_id)
+
+        prefix = f"s3://{self.storage.bucket_name}/"
+        key = (
+            art.storage_uri[len(prefix) :]
+            if art.storage_uri.startswith(prefix)
+            else art.storage_uri
+        )
+        download_url = self.storage.generate_presigned_get_url(key, expires_in=900)
+
+        return {
+            "artifact_id": art.id,
+            "artifact_type": art.artifact_type,
+            "size_bytes": art.size_bytes,
+            "content_hash": art.content_hash,
+            "download_url": download_url,
+            "metadata": art.metadata_json,
+        }
+
+    async def compare_experiments(
+        self,
+        tenant_id: str,
+        experiment_ids: list[str],
+    ) -> dict[str, Any]:
+        """Compare validation metrics across experiments ensuring caller owns every experiment."""
+        for exp_id in experiment_ids:
+            exp = await self.exp_repo.get_experiment(tenant_id, exp_id)
+            if not exp:
+                raise ResourceNotFoundError("Experiment", exp_id)
+
+        comparison = await self.metric_repo.compare_metrics(tenant_id, experiment_ids)
+        return {
+            "compared_experiment_ids": experiment_ids,
+            "validation_metrics_by_experiment": comparison,
+        }

ml_mcp/application/datasets/__init__.py

@@ -0,0 +1,6 @@
+"""Dataset management and ingestion package."""
+
+from ml_mcp.application.datasets.service import DatasetService
+from ml_mcp.application.datasets.validator import DatasetValidator
+
+__all__ = ["DatasetValidator", "DatasetService"]

ml_mcp/application/datasets/service.py

@@ -0,0 +1,178 @@
+"""Dataset application service managing registration, validation, and inspection."""
+
+import io
+from typing import Any
+
+import pandas as pd
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ml_mcp.application.datasets.validator import DatasetValidator
+from ml_mcp.domain.errors import ResourceNotFoundError
+from ml_mcp.domain.value_objects import DatasetFormat
+from ml_mcp.infrastructure.object_storage.s3 import S3StorageService, get_storage_service
+from ml_mcp.infrastructure.postgres.base import generate_uuid7
+from ml_mcp.infrastructure.postgres.models import DatasetOrm, DatasetVersionOrm
+from ml_mcp.infrastructure.postgres.repositories.datasets import DatasetRepository
+
+
+class DatasetService:
+    """Application service for dataset lifecycle and profiling."""
+
+    def __init__(
+        self,
+        session: AsyncSession,
+        storage_service: S3StorageService | None = None,
+        validator: DatasetValidator | None = None,
+    ) -> None:
+        self.session = session
+        self.repo = DatasetRepository(session)
+        self.storage = storage_service or get_storage_service()
+        self.validator = validator or DatasetValidator()
+
+    async def register_dataset(
+        self,
+        tenant_id: str,
+        project_id: str,
+        name: str,
+        description: str,
+        data_format: DatasetFormat | str,
+        raw_bytes: bytes,
+        version: str = "1.0",
+    ) -> dict[str, Any]:
+        """Validate, store to S3, and register a new dataset version."""
+        # 1. Validate & extract schema
+        df, parquet_bytes, schema = self.validator.process(raw_bytes, data_format)
+
+        # 2. Check or create parent Dataset record
+        dataset_id = generate_uuid7()
+        version_id = generate_uuid7()
+
+        dataset_key = self.storage.get_dataset_key(tenant_id, dataset_id, version, "data.parquet")
+        storage_uri, _ = self.storage.put_object(
+            key=dataset_key,
+            data=parquet_bytes,
+            content_type="application/vnd.apache.parquet",
+            metadata={"row_count": str(schema.row_count), "column_count": str(schema.column_count)},
+        )
+
+        # 3. Persist metadata to DB
+        fmt = (
+            data_format.value
+            if isinstance(data_format, DatasetFormat)
+            else str(data_format).lower()
+        )
+        dataset_orm = DatasetOrm(
+            id=dataset_id,
+            tenant_id=tenant_id,
+            project_id=project_id,
+            name=name,
+            description=description,
+            format=fmt,
+        )
+        await self.repo.create_dataset(dataset_orm)
+
+        version_orm = DatasetVersionOrm(
+            id=version_id,
+            dataset_id=dataset_id,
+            version=version,
+            schema_json=schema.model_dump(),
+            row_count=schema.row_count,
+            column_count=schema.column_count,
+            size_bytes=schema.size_bytes,
+            content_hash=schema.content_hash,
+            storage_uri=storage_uri,
+            validation_status="VALIDATED",
+        )
+        await self.repo.create_version(version_orm)
+
+        return {
+            "dataset_id": dataset_id,
+            "version_id": version_id,
+            "version": version,
+            "name": name,
+            "format": fmt,
+            "row_count": schema.row_count,
+            "column_count": schema.column_count,
+            "content_hash": schema.content_hash,
+            "storage_uri": storage_uri,
+        }
+
+    async def validate_only(
+        self,
+        raw_bytes: bytes,
+        data_format: DatasetFormat | str,
+    ) -> dict[str, Any]:
+        """Validate dataset and extract schema without persisting."""
+        _, _, schema = self.validator.process(raw_bytes, data_format)
+        return schema.model_dump()
+
+    async def get_dataset(self, tenant_id: str, dataset_id: str) -> dict[str, Any]:
+        ds = await self.repo.get_dataset(tenant_id, dataset_id)
+        if not ds:
+            raise ResourceNotFoundError("Dataset", dataset_id)
+
+        return {
+            "dataset_id": ds.id,
+            "name": ds.name,
+            "description": ds.description,
+            "format": ds.format,
+            "versions": [
+                {
+                    "version_id": v.id,
+                    "version": v.version,
+                    "row_count": v.row_count,
+                    "column_count": v.column_count,
+                    "content_hash": v.content_hash,
+                    "created_at": v.created_at.isoformat(),
+                }
+                for v in ds.versions
+            ],
+        }
+
+    async def list_datasets(self, tenant_id: str, project_id: str | None = None) -> list[dict[str, Any]]:
+        datasets = await self.repo.list_datasets(tenant_id, project_id)
+        return [
+            {
+                "dataset_id": ds.id,
+                "project_id": ds.project_id,
+                "name": ds.name,
+                "format": ds.format,
+                "latest_version": ds.versions[0].version if ds.versions else None,
+            }
+            for ds in datasets
+        ]
+
+    async def inspect_dataset(
+        self,
+        tenant_id: str,
+        dataset_id: str,
+        version: str = "1.0",
+        sample_rows: int = 5,
+    ) -> dict[str, Any]:
+        """Retrieve dataset schema, summary statistics, and head sample rows."""
+        dsv = await self.repo.get_version(tenant_id, dataset_id, version)
+        if not dsv:
+            raise ResourceNotFoundError("DatasetVersion", f"{dataset_id}:{version}")
+
+        # Download parquet from storage and load sample rows
+        key = self.storage.get_dataset_key(tenant_id, dataset_id, version, "data.parquet")
+        data = self.storage.get_object(key)
+        df = pd.read_parquet(io.BytesIO(data))
+
+        head_df = df.head(sample_rows)
+        # Convert to records format with string/native conversion
+        sample_data = head_df.to_dict(orient="records")
+
+        # Generate presigned download URL for full dataset access
+        download_url = self.storage.generate_presigned_get_url(key, expires_in=900)
+
+        return {
+            "dataset_id": dataset_id,
+            "version": version,
+            "schema": dsv.schema_json,
+            "row_count": dsv.row_count,
+            "column_count": dsv.column_count,
+            "size_bytes": dsv.size_bytes,
+            "sample_rows": sample_data,
+            "download_url": download_url,
+        }