mdmp-protocol 0.1.0__py3-none-any.whl

mdmp_ai/__init__.py

@@ -0,0 +1,3 @@
+from .lineage import DatasetRef, LineageCard, LineageTracker, load_lineage_card
+
+__all__ = ["DatasetRef", "LineageCard", "LineageTracker", "load_lineage_card"]

mdmp_ai/lineage.py

@@ -0,0 +1,198 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+import json
+
+import pandas as pd
+import yaml
+
+from mdmp_core.contracts import load_contract
+from mdmp_core.fingerprint import check_fingerprint, compute_fingerprint
+from mdmp_core.runner import ContractRunner
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+@dataclass
+class DatasetRef:
+    path: str
+    fingerprint: str
+    grade: str
+    rows_used: int
+    date_validated: str
+    consent: str
+    contract_fingerprint: str
+    expires: str
+    status: str = "valid"
+    stale_reason: Optional[str] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "path": self.path,
+            "fingerprint": self.fingerprint,
+            "grade": self.grade,
+            "rows_used": self.rows_used,
+            "date_validated": self.date_validated,
+            "consent": self.consent,
+            "contract_fingerprint": self.contract_fingerprint,
+            "expires": self.expires,
+            "status": self.status,
+            "stale_reason": self.stale_reason,
+        }
+
+    @classmethod
+    def from_dict(cls, payload: Dict[str, Any]) -> "DatasetRef":
+        return cls(
+            path=str(payload.get("path", "")),
+            fingerprint=str(payload.get("fingerprint", "")),
+            grade=str(payload.get("grade", "draft")),
+            rows_used=int(payload.get("rows_used", 0)),
+            date_validated=str(payload.get("date_validated", "")),
+            consent=str(payload.get("consent", "not_verified")),
+            contract_fingerprint=str(payload.get("contract_fingerprint", "")),
+            expires=str(payload.get("expires", "")),
+            status=str(payload.get("status", "valid")),
+            stale_reason=payload.get("stale_reason"),
+        )
+
+
+@dataclass
+class LineageCard:
+    model_name: str
+    training_date: str
+    datasets: List[DatasetRef] = field(default_factory=list)
+
+    @property
+    def stale_datasets(self) -> List[DatasetRef]:
+        return [row for row in self.datasets if row.status == "stale"]
+
+    @property
+    def lineage_status(self) -> str:
+        return "stale" if self.stale_datasets else "valid"
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "model": {
+                "name": self.model_name,
+                "training_date": self.training_date,
+                "lineage_status": self.lineage_status,
+                "stale_lineage": bool(self.stale_datasets),
+                "stale_datasets": [row.fingerprint for row in self.stale_datasets],
+                "trained_on": [row.to_dict() for row in self.datasets],
+            }
+        }
+
+    @classmethod
+    def from_dict(cls, payload: Dict[str, Any]) -> "LineageCard":
+        model = payload.get("model", {}) if isinstance(payload, dict) else {}
+        trained_on = model.get("trained_on", []) if isinstance(model, dict) else []
+        rows: List[DatasetRef] = []
+        if isinstance(trained_on, list):
+            for entry in trained_on:
+                if isinstance(entry, dict):
+                    rows.append(DatasetRef.from_dict(entry))
+        return cls(
+            model_name=str(model.get("name", "unknown_model")),
+            training_date=str(model.get("training_date", _now_iso())),
+            datasets=rows,
+        )
+
+    def export(self, output_path: str | Path) -> Dict[str, Any]:
+        payload = self.to_dict()
+        output = Path(output_path)
+        output.parent.mkdir(parents=True, exist_ok=True)
+        if output.suffix.lower() in {".yaml", ".yml"}:
+            output.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8")
+        else:
+            output.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+        return payload
+
+    def refresh(self, *, data_dir: Optional[Path] = None) -> "LineageCard":
+        for dataset in self.datasets:
+            resolved_path = Path(dataset.path)
+            if not resolved_path.is_file() and data_dir is not None:
+                candidate = data_dir / dataset.path
+                if candidate.is_file():
+                    resolved_path = candidate
+            if not resolved_path.is_file():
+                dataset.status = "stale"
+                dataset.stale_reason = "dataset_not_found"
+                continue
+
+            checked = check_fingerprint(
+                {
+                    "fingerprint": dataset.fingerprint,
+                    "expires": dataset.expires,
+                    "status": dataset.status,
+                    "stale_reason": dataset.stale_reason,
+                },
+                data_path=resolved_path,
+            )
+            dataset.status = str(checked.get("status", "stale"))
+            dataset.stale_reason = checked.get("stale_reason")
+        return self
+
+
+def load_lineage_card(path: str | Path) -> LineageCard:
+    card_path = Path(path)
+    raw = card_path.read_text(encoding="utf-8")
+    if card_path.suffix.lower() in {".yaml", ".yml"}:
+        payload = yaml.safe_load(raw) or {}
+    else:
+        payload = json.loads(raw)
+    if not isinstance(payload, dict):
+        raise ValueError("Lineage card root must be a mapping")
+    return LineageCard.from_dict(payload)
+
+
+@dataclass
+class LineageTracker:
+    datasets: list[DatasetRef] = field(default_factory=list)
+    model_name: str | None = None
+
+    def register_dataset(
+        self,
+        dataset_path: str | Path,
+        *,
+        contract: str | Path,
+        expires_days: int = 365,
+    ) -> Dict[str, Any]:
+        dataset_path = Path(dataset_path)
+        contract_path = Path(contract)
+        df = pd.read_csv(dataset_path)
+        contract_obj = load_contract(contract_path)
+        validation = ContractRunner(contract_obj).run(df)
+        fp_record = compute_fingerprint(dataset_path, expires_days=expires_days)
+
+        row = DatasetRef(
+            path=str(dataset_path),
+            fingerprint=str(fp_record["fingerprint"]),
+            grade=validation.grade,
+            rows_used=validation.row_count,
+            date_validated=validation.created_utc,
+            consent="verified" if validation.consent_verified else "not_verified",
+            contract_fingerprint=f"sha256:{validation.contract_fingerprint_sha256}",
+            expires=str(fp_record["expires"]),
+            status=str(fp_record["status"]),
+            stale_reason=fp_record.get("stale_reason"),
+        )
+        self.datasets.append(row)
+        return row.to_dict()
+
+    def attach_to_model(self, model_name: str) -> None:
+        self.model_name = model_name
+
+    def export_card(self, output_path: str | Path) -> Dict[str, Any]:
+        if not self.model_name:
+            raise ValueError("Call attach_to_model() before export_card()")
+        card = LineageCard(
+            model_name=self.model_name,
+            training_date=_now_iso(),
+            datasets=self.datasets,
+        )
+        return card.export(output_path)

mdmp_core/__init__.py

@@ -0,0 +1,48 @@
+from .contracts import DataContract, load_contract, parse_contract, save_contract
+from .runner import (
+    MDMP_PROTOCOL_VERSION,
+    MDMP_GRADE_ORDER,
+    ContractRunner,
+    ValidationResult,
+    dataframe_fingerprint,
+    grade_meets_minimum,
+)
+from .fingerprint import compute_fingerprint, check_fingerprint
+from .fingerprint_store import FingerprintStore
+from .registry import (
+    REGISTRY_VERSION,
+    init_registry,
+    load_registry,
+    save_registry,
+    upsert_record,
+    lookup_record,
+    list_records,
+)
+from .hf import build_hf_mdmp_section, load_report
+from .visualizer import build_dashboard_html
+
+__all__ = [
+    "DataContract",
+    "load_contract",
+    "parse_contract",
+    "save_contract",
+    "MDMP_PROTOCOL_VERSION",
+    "MDMP_GRADE_ORDER",
+    "ContractRunner",
+    "ValidationResult",
+    "dataframe_fingerprint",
+    "grade_meets_minimum",
+    "compute_fingerprint",
+    "check_fingerprint",
+    "FingerprintStore",
+    "REGISTRY_VERSION",
+    "init_registry",
+    "load_registry",
+    "save_registry",
+    "upsert_record",
+    "lookup_record",
+    "list_records",
+    "build_hf_mdmp_section",
+    "load_report",
+    "build_dashboard_html",
+]

mdmp_core/cli.py

@@ -0,0 +1,346 @@
+from __future__ import annotations
+
+from pathlib import Path
+import json
+
+import pandas as pd
+import typer
+import yaml
+
+from mdmp_ai import LineageTracker, load_lineage_card
+from mdmp_core.contracts import load_contract, parse_contract, save_contract
+from mdmp_core.fingerprint import check_fingerprint, compute_fingerprint
+from mdmp_core.fingerprint_store import FingerprintStore
+from mdmp_core.hf import build_hf_mdmp_section, load_report
+from mdmp_core.registry import (
+    init_registry,
+    list_records,
+    lookup_record,
+    upsert_record,
+)
+from mdmp_core.runner import ContractRunner, dataframe_fingerprint
+from mdmp_core.visualizer import build_dashboard_html
+from mdmp_flavors import TEMPLATES
+
+
+app = typer.Typer(help="MDMP CLI - contracts, grading, fingerprints, and lineage")
+registry_app = typer.Typer(help="MDMP registry scaffold commands")
+app.add_typer(registry_app, name="registry")
+
+
+@app.command("init")
+def init_contract(
+    output: Path = typer.Option(Path("mdmp_contract.yaml"), help="Output contract file"),
+    flavor: str = typer.Option("health", help="Template flavor: health|finance|industrial"),
+) -> None:
+    normalized = flavor.strip().lower()
+    if normalized not in TEMPLATES:
+        allowed = ", ".join(sorted(TEMPLATES))
+        raise typer.BadParameter(f"Unknown flavor '{flavor}'. Allowed: {allowed}")
+
+    payload = TEMPLATES[normalized]()
+    contract = parse_contract(payload)
+    save_contract(output, contract)
+    typer.echo(f"Created contract: {output}")
+
+
+@app.command("validate")
+def validate(
+    contract_path: Path,
+    dataset_csv: Path,
+    output_json: Path = typer.Option(Path("results/mdmp_report.json"), help="Output report JSON"),
+    fingerprint_store: Path | None = typer.Option(
+        None,
+        help="Optional fingerprint store JSON for stale/expiry checks",
+    ),
+) -> None:
+    contract = load_contract(contract_path)
+    df = pd.read_csv(dataset_csv)
+    result = ContractRunner(contract).run(df)
+    payload = result.to_dict()
+
+    if fingerprint_store is not None:
+        store = FingerprintStore(fingerprint_store)
+        stored = store.get_by_dataset(str(dataset_csv))
+        if stored is not None:
+            staleness = check_fingerprint(stored, data_path=dataset_csv)
+            payload["staleness"] = staleness
+            warnings = payload.setdefault("warnings", [])
+            if staleness.get("status") == "stale":
+                reason = staleness.get("stale_reason", "unknown")
+                warnings.append(
+                    f"Dataset fingerprint is stale ({reason}). Re-grade required before AI training."
+                )
+
+    output_json.parent.mkdir(parents=True, exist_ok=True)
+    output_json.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+    typer.echo(f"Saved report: {output_json}")
+    typer.echo(f"Grade: {result.grade} | Compliance: {result.compliance_score}%")
+    if payload.get("staleness", {}).get("status") == "stale":
+        typer.echo(f"Staleness: STALE ({payload['staleness'].get('stale_reason')})")
+
+
+@app.command("grade")
+def grade(contract_path: Path, dataset_csv: Path) -> None:
+    contract = load_contract(contract_path)
+    df = pd.read_csv(dataset_csv)
+    result = ContractRunner(contract).run(df)
+    typer.echo(result.grade)
+
+
+@app.command("fingerprint")
+def fingerprint(dataset_csv: Path) -> None:
+    df = pd.read_csv(dataset_csv)
+    typer.echo(f"sha256:{dataframe_fingerprint(df)}")
+
+
+@app.command("fingerprint-record")
+def fingerprint_record(
+    dataset_path: Path,
+    output_json: Path = typer.Option(Path("results/fingerprint.json"), help="Output fingerprint JSON"),
+    expires_days: int = typer.Option(365, help="Validity period in days"),
+    fingerprint_store: Path | None = typer.Option(
+        None,
+        help="Optional fingerprint store JSON to upsert this record",
+    ),
+) -> None:
+    record = compute_fingerprint(dataset_path, expires_days=expires_days)
+    output_json.parent.mkdir(parents=True, exist_ok=True)
+    output_json.write_text(json.dumps(record, indent=2), encoding="utf-8")
+    typer.echo(f"Saved fingerprint record: {output_json}")
+    typer.echo(f"Fingerprint: {record['fingerprint']}")
+    typer.echo(f"Expires: {record['expires']}")
+
+    if fingerprint_store is not None:
+        store = FingerprintStore(fingerprint_store)
+        store.upsert(dataset_path=str(dataset_path), record=record)
+        typer.echo(f"Updated fingerprint store: {fingerprint_store}")
+
+
+@app.command("fingerprint-check")
+def fingerprint_check(
+    fingerprint_json: Path,
+    dataset_path: Path,
+    output_json: Path | None = typer.Option(None, help="Optional output path for check result"),
+) -> None:
+    stored = json.loads(fingerprint_json.read_text(encoding="utf-8"))
+    checked = check_fingerprint(stored, data_path=dataset_path)
+    typer.echo(f"Fingerprint: {checked.get('fingerprint')}")
+    typer.echo(f"Status: {str(checked.get('status', '')).upper()}")
+    typer.echo(f"Reason: {checked.get('stale_reason')}")
+    typer.echo(f"Created: {checked.get('created')}")
+    typer.echo(f"Expires: {checked.get('expires')}")
+    typer.echo(f"Checked at: {checked.get('checked_at')}")
+
+    if output_json is not None:
+        output_json.parent.mkdir(parents=True, exist_ok=True)
+        output_json.write_text(json.dumps(checked, indent=2), encoding="utf-8")
+        typer.echo(f"Saved check result: {output_json}")
+
+
+@app.command("report")
+def report(
+    report_json: Path,
+    output_html: Path = typer.Option(Path("results/mdmp_dashboard.html"), help="Output dashboard HTML"),
+    title: str = typer.Option("MDMP Dashboard", help="HTML title"),
+) -> None:
+    payload = json.loads(report_json.read_text(encoding="utf-8"))
+    html = build_dashboard_html(payload, title=title)
+    output_html.parent.mkdir(parents=True, exist_ok=True)
+    output_html.write_text(html, encoding="utf-8")
+    typer.echo(f"Saved dashboard: {output_html}")
+
+
+@app.command("lineage-card")
+def lineage_card(
+    model: str = typer.Option(..., help="Model name"),
+    dataset: Path = typer.Option(..., help="Dataset CSV path"),
+    contract: Path = typer.Option(..., help="Contract YAML path"),
+    output: Path = typer.Option(Path("results/mdmp_model_card.yaml"), help="Output model card (.yaml/.json)"),
+    expires_days: int = typer.Option(365, help="Validity period for dataset fingerprint in days"),
+    fingerprint_store: Path | None = typer.Option(
+        None,
+        help="Optional fingerprint store JSON to persist lineage dataset fingerprints",
+    ),
+) -> None:
+    tracker = LineageTracker()
+    record = tracker.register_dataset(dataset, contract=contract, expires_days=expires_days)
+    tracker.attach_to_model(model)
+    card = tracker.export_card(output)
+
+    typer.echo(f"Dataset registered: {record['fingerprint']} ({record['grade']})")
+    typer.echo(f"Saved model card: {output}")
+    typer.echo(yaml.safe_dump(card, sort_keys=False))
+
+    if fingerprint_store is not None:
+        store = FingerprintStore(fingerprint_store)
+        store.upsert(
+            dataset_path=str(dataset),
+            record={
+                "fingerprint": record["fingerprint"],
+                "created": record["date_validated"],
+                "expires": record["expires"],
+                "source_path": str(dataset),
+                "status": record.get("status", "valid"),
+                "stale_reason": record.get("stale_reason"),
+            },
+        )
+        typer.echo(f"Updated fingerprint store: {fingerprint_store}")
+
+
+@app.command("lineage-card-refresh")
+def lineage_card_refresh(
+    card_path: Path,
+    output: Path | None = typer.Option(None, help="Optional output path (default: overwrite input card)"),
+    data_dir: Path | None = typer.Option(None, help="Optional data directory for resolving relative dataset paths"),
+) -> None:
+    card = load_lineage_card(card_path)
+    card.refresh(data_dir=data_dir)
+
+    destination = output or card_path
+    payload = card.export(destination)
+    stale = payload.get("model", {}).get("stale_datasets", [])
+    typer.echo(f"Model: {payload.get('model', {}).get('name')}")
+    typer.echo(f"Lineage status: {payload.get('model', {}).get('lineage_status')}")
+    typer.echo(f"Stale datasets: {len(stale)}")
+    typer.echo(f"Saved lineage card: {destination}")
+
+
+@registry_app.command("init")
+def registry_init(
+    registry: Path = typer.Option(Path("registry/mdmp_registry.json"), help="Registry JSON path"),
+) -> None:
+    payload = init_registry(registry)
+    typer.echo(f"Initialized registry: {registry}")
+    typer.echo(f"Version: {payload['version']}")
+
+
+@registry_app.command("push")
+def registry_push(
+    registry: Path = typer.Option(Path("registry/mdmp_registry.json"), help="Registry JSON path"),
+    fingerprint: str | None = typer.Option(None, help="Dataset fingerprint (sha256:...)"),
+    report: Path | None = typer.Option(None, help="Optional MDMP report JSON to ingest"),
+    grade: str | None = typer.Option(None, help="Optional grade override"),
+    source: str | None = typer.Option(None, help="Optional source label (dataset/report name)"),
+    visibility: str = typer.Option("private", help="Record visibility: private|public"),
+    model_id: list[str] = typer.Option([], help="Model id(s) using this dataset"),
+    expires: str | None = typer.Option(None, help="Optional expires timestamp"),
+    status: str | None = typer.Option(None, help="Optional status override: valid|stale"),
+    stale_reason: str | None = typer.Option(None, help="Optional stale reason"),
+    metadata_json: Path | None = typer.Option(None, help="Optional metadata JSON file"),
+) -> None:
+    report_payload = None
+    if report is not None:
+        report_payload = json.loads(report.read_text(encoding="utf-8"))
+    metadata = None
+    if metadata_json is not None:
+        metadata = json.loads(metadata_json.read_text(encoding="utf-8"))
+        if not isinstance(metadata, dict):
+            raise typer.BadParameter("metadata_json must contain a JSON object")
+
+    record = upsert_record(
+        registry,
+        fingerprint=fingerprint,
+        report=report_payload,
+        grade=grade,
+        source=source,
+        visibility=visibility,
+        used_in_models=model_id,
+        expires=expires,
+        status=status,
+        stale_reason=stale_reason,
+        metadata=metadata,
+    )
+    typer.echo(f"Registry updated: {registry}")
+    typer.echo(f"Fingerprint: {record['fingerprint']}")
+    typer.echo(f"Grade: {record['grade']} | Status: {record['status']} | Visibility: {record['visibility']}")
+
+
+@registry_app.command("lookup")
+def registry_lookup(
+    fingerprint: str,
+    registry: Path = typer.Option(Path("registry/mdmp_registry.json"), help="Registry JSON path"),
+    output_json: Path | None = typer.Option(None, help="Optional output JSON path"),
+) -> None:
+    record = lookup_record(registry, fingerprint)
+    if record is None:
+        typer.echo(f"Fingerprint not found: {fingerprint}")
+        raise typer.Exit(code=1)
+    text = json.dumps(record, indent=2)
+    typer.echo(text)
+    if output_json is not None:
+        output_json.parent.mkdir(parents=True, exist_ok=True)
+        output_json.write_text(text, encoding="utf-8")
+        typer.echo(f"Saved lookup: {output_json}")
+
+
+@registry_app.command("list")
+def registry_list(
+    registry: Path = typer.Option(Path("registry/mdmp_registry.json"), help="Registry JSON path"),
+    grade: str | None = typer.Option(None, help="Filter by grade"),
+    visibility: str | None = typer.Option(None, help="Filter by visibility"),
+    status: str | None = typer.Option(None, help="Filter by status"),
+    limit: int = typer.Option(20, help="Maximum rows"),
+    output_json: Path | None = typer.Option(None, help="Optional output JSON path"),
+) -> None:
+    records = list_records(
+        registry,
+        grade=grade,
+        visibility=visibility,
+        status=status,
+        limit=limit,
+    )
+    payload = {"count": len(records), "records": records}
+    text = json.dumps(payload, indent=2)
+    typer.echo(text)
+    if output_json is not None:
+        output_json.parent.mkdir(parents=True, exist_ok=True)
+        output_json.write_text(text, encoding="utf-8")
+        typer.echo(f"Saved list: {output_json}")
+
+
+@app.command("hf-export")
+def hf_export(
+    dataset_id: str = typer.Option(..., help="Hugging Face dataset id (e.g. owner/name)"),
+    report_json: Path | None = typer.Option(None, help="Optional MDMP report JSON"),
+    fingerprint: str | None = typer.Option(None, help="Optional dataset fingerprint (sha256:...)"),
+    grade: str | None = typer.Option(None, help="Optional grade override"),
+    output_md: Path = typer.Option(Path("results/mdmp_hf_section.md"), help="Output Markdown path"),
+    registry_url: str | None = typer.Option(None, help="Optional registry URL"),
+) -> None:
+    report = load_report(report_json) if report_json is not None else {}
+
+    resolved_grade = grade or str(report.get("grade", report.get("mdmp_grade", "draft")))
+    if fingerprint:
+        resolved_fp = fingerprint
+    else:
+        fp_raw = report.get("dataset_fingerprint_sha256")
+        if not fp_raw:
+            raise typer.BadParameter("Provide --fingerprint or --report-json with dataset_fingerprint_sha256")
+        resolved_fp = f"sha256:{fp_raw}" if not str(fp_raw).startswith("sha256:") else str(fp_raw)
+
+    contract_fp_raw = report.get("contract_fingerprint_sha256")
+    contract_fp = None
+    if contract_fp_raw:
+        contract_fp = (
+            f"sha256:{contract_fp_raw}" if not str(contract_fp_raw).startswith("sha256:") else str(contract_fp_raw)
+        )
+
+    score = report.get("compliance_score")
+    protocol = report.get("protocol_version", report.get("mdmp_protocol_version"))
+    section = build_hf_mdmp_section(
+        dataset_id=dataset_id,
+        grade=resolved_grade,
+        fingerprint=resolved_fp,
+        contract_fingerprint=contract_fp,
+        compliance_score=float(score) if score is not None else None,
+        protocol_version=str(protocol) if protocol is not None else None,
+        registry_url=registry_url,
+    )
+    output_md.parent.mkdir(parents=True, exist_ok=True)
+    output_md.write_text(section, encoding="utf-8")
+    typer.echo(f"Saved HF MDMP section: {output_md}")
+
+
+if __name__ == "__main__":
+    app()