multimodal-mllog 0.1.0__tar.gz

multimodal_mllog-0.1.0/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 2.4
+Name: multimodal-mllog
+Version: 0.1.0
+Summary: Experiment logbook pipeline: MLflow runs + git context -> structured logbook entries
+Requires-Python: >=3.11
+Requires-Dist: mlflow
+Requires-Dist: pydantic>=2
+Requires-Dist: click
+Provides-Extra: synthesis
+Requires-Dist: anthropic; extra == "synthesis"

multimodal_mllog-0.1.0/README.md

@@ -0,0 +1,96 @@
+# multimodal_mllogger
+
+Experiment Logbook Pipeline for a researcher finetuning an ML model using coding agents.
+
+A researcher finetunes a model inside Claude Code; the training script logs to MLflow. At
+session end, `/mllog` generates a structured, validated **logbook entry** — what was tried,
+what changed in the code, what the metrics were, what likely happened, next steps — and
+stores it back into MLflow as artifacts and tags.
+
+```
+researcher → Claude Code → MLflow → logbook entry   (triggered by /mllog at session end)
+```
+
+Full design spec: [BUILD.md](BUILD.md).
+
+## Quick start
+
+```bash
+python -m venv .venv
+.venv/Scripts/activate          # Windows; on Unix: source .venv/bin/activate
+pip install -e .
+pip install pytest              # for the test suite
+
+python examples/finetune_demo.py                     # demo run: 4 params, 2 metrics
+mlflow ui --backend-store-uri sqlite:///mlflow.db    # optional: inspect runs
+```
+
+Then, in a Claude Code session opened in this repo, run **`/mllog`**. It collects ground
+truth via the CLI, fills in the logbook from the live session, and ingests it.
+
+## Storage (v0.1)
+
+- Tracking backend: **local SQLite**, default `MLFLOW_TRACKING_URI=sqlite:///mlflow.db`
+  (serverless; override via the env var). Run artifacts live under `./mlruns/`.
+- `logbook.json` is canonical; `logbook.md` is a deterministic rendering of it — never
+  edited or parsed.
+
+## CLI
+
+```bash
+mllog get-mlflow-info [--run-id ID | --latest] [--experiment NAME] --json
+    # params, metrics, tags, artifacts, UTC start/end of one MLflow run; exits 1 if no run
+
+mllog get-git-info [--base REF] --json
+    # working-tree diff vs HEAD, recent commits (UTC ISO 8601), HEAD sha
+
+mllog create-logbook <logbook.json> [--run-id ID | --latest]
+    # validate against the schema, render logbook.md, attach both as artifacts,
+    # set logbook.* tags; exits 1 on validation failure or missing run
+```
+
+## Schema and guardrails
+
+`mllog/schema.py` defines the versioned `LogbookEntry` (pydantic v2). Validators enforce:
+
+1. **Grounding** — `hypothesis`, `likely_cause`, and `next_experiment` must each have a key
+   in `evidence`.
+2. **Metric references resolve** — every `metric:<name>` evidence value must name a metric
+   present in `metrics`.
+3. **Timestamps** — `date` must be a full ISO 8601 UTC timestamp.
+
+Evidence values are typed references: `metric:val_acc`, `diff:planner.py@@-120,6`,
+`session:user asked to lower lr` (`video:` / `telemetry:` reserved for later milestones).
+The pipeline never fabricates data: no MLflow run or no git repo means a loud non-zero exit.
+
+## Layout
+
+```
+mllog/
+├── cli.py            # `mllog` entrypoint
+├── schema.py         # versioned LogbookEntry + Metric (the contract)
+├── mlflow_io.py      # collector: read run; writer: artifacts + tags
+├── git_context.py    # collector: diff + recent commits
+├── render.py         # LogbookEntry -> markdown (deterministic projection)
+└── synthesis/        # empty in v0; M2 LLM layer (pip install mllog[synthesis])
+.claude/commands/mllog.md   # the /mllog command (thin agent wrapper)
+examples/finetune_demo.py   # tiny demo finetune that logs to MLflow
+tests/                      # schema, render (golden file), mlflow_io
+```
+
+## Tests
+
+```bash
+python -m pytest -q
+```
+
+The deterministic core is tested without any LLM. `tests/test_render.py` includes a
+golden-file test (`tests/golden/logbook.md`) locking the markdown output; if a render
+change is intentional, regenerate the golden file from the `valid_entry` fixture.
+
+## Roadmap
+
+- **M1 (done):** `run-info`, `git-context`, `ingest`, `/mllog` — full loop on the demo.
+- **M2:** `mllog.synthesis` (Anthropic API, optional extra), `mllog logbook`,
+  session-id run selection, multi-run sessions (`related_run_ids`).
+- **M3:** video/telemetry collectors, time alignment, object-storage artifacts.

multimodal_mllog-0.1.0/mllog/__init__.py

@@ -0,0 +1,6 @@
+"""mllog: deterministic experiment-logbook core (MLflow + git -> logbook)."""
+
+from mllog.schema import LogbookEntry, Metric
+
+__all__ = ["LogbookEntry", "Metric"]
+__version__ = "0.1.0"

multimodal_mllog-0.1.0/mllog/cli.py

@@ -0,0 +1,97 @@
+"""mllog CLI. Fails loudly: non-zero exit when a source is absent or invalid."""
+
+import json
+import sys
+from pathlib import Path
+
+import click
+from pydantic import ValidationError
+
+
+
+@click.group()
+def main() -> None:
+    """Experiment logbook tools: MLflow + git -> validated logbook entries."""
+
+
+@main.command("get-mlflow-info")
+@click.option("--run-id", "run_id", default=None, help="Specific MLflow run id.")
+@click.option("--latest", is_flag=True, help="Most recently started run in the experiment.")
+@click.option("--experiment", "experiment_name", default=None, help="Experiment name (default: Default).")
+@click.option("--json", "as_json", is_flag=True, help="Print JSON payload.")
+def get_mlflow_info(run_id: str | None, latest: bool, experiment_name: str | None, as_json: bool) -> None:
+    """Print params, metrics, tags, artifacts and UTC times for one MLflow run."""
+    from mllog.mlflow_io import RunNotFoundError, get_mlflow_info
+
+    if not run_id and not latest:
+        raise click.UsageError("missing parameters: provide --run-id or --latest")
+    try:
+        info = get_mlflow_info(run_id=run_id, latest=latest, experiment_name=experiment_name)
+    except RunNotFoundError as exc:
+        click.echo(f"error: {exc}", err=True)
+        sys.exit(1)
+
+    click.echo(
+        f"selected run: {info['run_id']} (experiment: {info['experiment_name']})",
+        err=True,
+    )
+    click.echo(json.dumps(info, indent=2))
+
+
+@main.command("get-git-info")
+@click.option("--base", default="HEAD", help="Diff base ref (default: HEAD).")
+@click.option("--json", "as_json", is_flag=True, help="Print JSON payload.")
+def get_git_info(base: str, as_json: bool) -> None:
+    """Print working-tree diff, recent commits (UTC timestamps), and HEAD sha."""
+    from mllog.git_context import GitContextError, get_git_context
+
+    try:
+        ctx = get_git_context(base=base)
+    except GitContextError as exc:
+        click.echo(f"error: {exc}", err=True)
+        sys.exit(1)
+    click.echo(json.dumps(ctx, indent=2))
+
+
+@main.command("create-logbook")
+@click.argument("logbook_path", type=click.Path(exists=True, dir_okay=False, path_type=Path))
+@click.option("--run-id", "run_id", default=None, help="Specific MLflow run id.")
+@click.option("--latest", is_flag=True, help="Most recently started run in the experiment.")
+@click.option("--experiment", "experiment_name", default=None, help="Experiment name (default: Default).")
+def verify_log(logbook_path: Path, run_id: str | None, latest: bool, experiment_name: str | None) -> None:
+    """Validate logbook JSON, render markdown, attach both to the MLflow run."""
+    from mllog.mlflow_io import RunNotFoundError, attach_to_mlflow
+    from mllog.schema import LogbookEntry
+
+    if not run_id and not latest:
+        raise click.UsageError("specify --run-id ID or --latest")
+
+    # Load logbook JSON
+    try:
+        data = json.loads(logbook_path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        click.echo(f"error: {logbook_path} is not valid JSON: {exc}", err=True)
+        sys.exit(1)
+
+    # Validate logbook schema
+    try:
+        entry = LogbookEntry.model_validate(data)
+    except ValidationError as exc:
+        click.echo(f"error: logbook failed schema validation:\n{exc}", err=True)
+        sys.exit(1)
+
+    # Attach the logbook to the MLflow run
+    try:
+        result = attach_to_mlflow(
+            entry, run_id=run_id, latest=latest, experiment_name=experiment_name
+        )
+    except RunNotFoundError as exc:
+        click.echo(f"error: {exc}", err=True)
+        sys.exit(1)
+
+    click.echo(f"logbook ingested into run: {result['run_id']}")
+    click.echo(f"artifacts: {', '.join(result['artifacts'])}")
+
+
+if __name__ == "__main__":
+    main()

multimodal_mllog-0.1.0/mllog/git_context.py

@@ -0,0 +1,70 @@
+"""Git collector: diff + recent commits as a timestamped, typed payload.
+
+Collector symmetry with mlflow_io: explicit inputs, typed payload with UTC
+ISO 8601 timestamps, fails loudly (raises GitContextError) when git/repo is absent.
+"""
+
+import subprocess
+from datetime import datetime, timezone
+
+
+class GitContextError(Exception):
+    """Raised when git context cannot be collected."""
+
+
+def _git(*args: str, cwd: str | None = None) -> str:
+    try:
+        proc = subprocess.run(
+            ["git", *args],
+            cwd=cwd,
+            capture_output=True,
+            text=True,
+            encoding="utf-8",
+            errors="replace",
+        )
+    except FileNotFoundError as exc:
+        raise GitContextError("git executable not found") from exc
+    if proc.returncode != 0:
+        raise GitContextError(
+            f"git {' '.join(args)} failed: {proc.stderr.strip() or proc.stdout.strip()}"
+        )
+    return proc.stdout
+
+
+def get_git_context(base: str = "HEAD", cwd: str | None = None, max_commits: int = 10) -> dict:
+    """Return {"collected_at", "diff", "recent_commits", "head"}."""
+
+    # Get the current HEAD commit
+    head = _git("rev-parse", "HEAD", cwd=cwd).strip()
+
+    # Get the diff
+    diff = _git("diff", base, cwd=cwd)
+
+    # Get recent commits
+    log_raw = _git(
+        "log",
+        f"-{max_commits}",
+        "--pretty=format:%H%x1f%cI%x1f%s",  # %cI = committer date, strict ISO 8601
+        cwd=cwd,
+    )
+    commits = []
+    for line in log_raw.splitlines():
+        if not line.strip():
+            continue
+        sha, raw_ts, subject = line.split("\x1f", 2)
+        utc_ts = (
+            datetime.fromisoformat(raw_ts)
+            .astimezone(timezone.utc)
+            .isoformat(timespec="seconds")
+            .replace("+00:00", "Z")
+        )
+        commits.append({"sha": sha, "timestamp": utc_ts, "subject": subject})
+
+    return {
+        "collected_at": datetime.now(timezone.utc)
+        .isoformat(timespec="seconds")
+        .replace("+00:00", "Z"),
+        "diff": diff,
+        "recent_commits": commits,
+        "head": head,
+    }

multimodal_mllog-0.1.0/mllog/mlflow_io.py

@@ -0,0 +1,126 @@
+"""MLflow collector (read run -> typed payload) and writer (artifacts + tags).
+
+Collector symmetry: explicit inputs, timestamped typed payload, fails loudly
+(raises RunNotFoundError) when the source is absent.
+"""
+
+import json
+import os
+import tempfile
+from datetime import datetime, timezone
+from pathlib import Path
+
+# v0.1 default: local serverless SQLite backend (MLflow's recommended local store).
+os.environ.setdefault("MLFLOW_TRACKING_URI", "sqlite:///mlflow.db")
+
+import mlflow
+from mlflow.tracking import MlflowClient
+
+from mllog.render import render_markdown
+from mllog.schema import LogbookEntry
+
+
+class RunNotFoundError(Exception):
+    """Raised when no MLflow run matches the request."""
+
+
+def _ms_to_utc_iso(ms: int | None) -> str | None:
+    if ms is None:
+        return None
+    return (
+        datetime.fromtimestamp(ms / 1000, tz=timezone.utc)
+        .isoformat(timespec="seconds")
+        .replace("+00:00", "Z")
+    )
+
+
+def _resolve_experiment(client: MlflowClient, experiment_name: str | None):
+    if experiment_name is None:
+        experiment_name = "Default"
+    experiment = client.get_experiment_by_name(experiment_name)
+    if experiment is None:
+        raise RunNotFoundError(f"MLflow experiment not found: {experiment_name!r}")
+    return experiment
+
+
+def get_mlflow_info(
+    run_id: str | None = None,
+    latest: bool = False,
+    experiment_name: str | None = None,
+) -> dict:
+    """Return a typed, timestamped payload describing one MLflow run."""
+    client = MlflowClient()
+
+    # Find the run
+    if run_id is not None:
+        try:
+            run = client.get_run(run_id)
+        except Exception as exc:
+            raise RunNotFoundError(f"MLflow run not found: {run_id!r} ({exc})") from exc
+    elif latest:
+        experiment = _resolve_experiment(client, experiment_name)
+        runs = client.search_runs(
+            [experiment.experiment_id],
+            order_by=["attributes.start_time DESC"],
+            max_results=1,
+        )
+        if not runs:
+            raise RunNotFoundError(f"no runs found in experiment {experiment.name!r}")
+        run = runs[0]
+    else:
+        raise RunNotFoundError("specify --run-id or --latest")
+
+    # Get run info (metrics+params)
+    experiment = client.get_experiment(run.info.experiment_id)
+    artifacts = [f.path for f in client.list_artifacts(run.info.run_id)]
+    return {
+        "collected_at": datetime.now(timezone.utc)
+        .isoformat(timespec="seconds")
+        .replace("+00:00", "Z"),
+        "run_id": run.info.run_id,
+        "experiment_id": run.info.experiment_id,
+        "experiment_name": experiment.name,
+        "status": run.info.status,
+        "start_time": _ms_to_utc_iso(run.info.start_time),
+        "end_time": _ms_to_utc_iso(run.info.end_time),
+        "params": dict(run.data.params),
+        "metrics": dict(run.data.metrics),
+        "tags": dict(run.data.tags),
+        "artifacts": artifacts,
+    }
+
+
+def attach_to_mlflow(
+    entry: LogbookEntry,
+    run_id: str | None = None,
+    latest: bool = False,
+    experiment_name: str | None = None,
+) -> dict:
+    """Render markdown, log logbook.json + logbook.md as artifacts, set tags.
+
+    Returns {"run_id": ..., "artifacts": [...]}.
+    """
+    info = get_mlflow_info(run_id=run_id, latest=latest, experiment_name=experiment_name)
+    target_run_id = info["run_id"]
+    client = MlflowClient()
+
+    # Create logbook artifacts
+    with tempfile.TemporaryDirectory() as tmp:
+        json_path = Path(tmp) / "logbook.json"
+        md_path = Path(tmp) / "logbook.md"
+        json_path.write_text(
+            json.dumps(entry.model_dump(), indent=2, ensure_ascii=False),
+            encoding="utf-8",
+        )
+        md_path.write_text(render_markdown(entry), encoding="utf-8")
+        # Attach to the MLflow run
+        client.log_artifact(target_run_id, str(json_path))
+        client.log_artifact(target_run_id, str(md_path))
+
+    # Set MLflow tags
+    client.set_tag(target_run_id, "logbook.generated", "true")
+    client.set_tag(target_run_id, "logbook.task", entry.task)
+    client.set_tag(target_run_id, "logbook.result", entry.result)
+    client.set_tag(target_run_id, "logbook.schema_version", entry.schema_version)
+
+    return {"run_id": target_run_id, "artifacts": ["logbook.json", "logbook.md"]}

multimodal_mllog-0.1.0/mllog/render.py

@@ -0,0 +1,78 @@
+"""Deterministic LogbookEntry -> markdown projection.
+
+logbook.md is never edited or parsed; to change it, amend the JSON and re-render.
+"""
+
+from mllog.schema import LogbookEntry
+
+
+def render_markdown(entry: LogbookEntry) -> str:
+    lines: list[str] = []
+    lines.append(f"# Logbook: {entry.task}")
+    lines.append("")
+    lines.append(f"- **Date:** {entry.date}")
+    lines.append(f"- **Experiment:** {entry.experiment_id}")
+    lines.append(f"- **MLflow run:** `{entry.mlflow_run_id}`")
+    if entry.related_run_ids:
+        lines.append(
+            "- **Related runs:** " + ", ".join(f"`{r}`" for r in entry.related_run_ids)
+        )
+    if entry.git_before or entry.git_after:
+        lines.append(
+            f"- **Git:** `{entry.git_before or '?'}` -> `{entry.git_after or '?'}`"
+        )
+    lines.append(f"- **Schema version:** {entry.schema_version}")
+    lines.append("")
+
+    lines.append("## Prompt")
+    lines.append("")
+    lines.append(entry.prompt)
+    lines.append("")
+
+    lines.append("## Hypothesis")
+    lines.append("")
+    lines.append(entry.hypothesis)
+    lines.append("")
+
+    lines.append("## Code changes")
+    lines.append("")
+    for change in entry.code_changes:
+        lines.append(f"- {change}")
+    lines.append("")
+
+    lines.append("## Metrics")
+    lines.append("")
+    lines.append("| Name | Value | Unit |")
+    lines.append("|------|-------|------|")
+    for m in entry.metrics:
+        lines.append(f"| {m.name} | {m.value} | {m.unit or ''} |")
+    lines.append("")
+
+    lines.append("## Result")
+    lines.append("")
+    lines.append(entry.result)
+    lines.append("")
+
+    lines.append("## Observations")
+    lines.append("")
+    for obs in entry.observations:
+        lines.append(f"- {obs}")
+    lines.append("")
+
+    lines.append("## Likely cause")
+    lines.append("")
+    lines.append(entry.likely_cause)
+    lines.append("")
+
+    lines.append("## Next experiment")
+    lines.append("")
+    lines.append(entry.next_experiment)
+    lines.append("")
+
+    lines.append("## Evidence")
+    lines.append("")
+    for claim in sorted(entry.evidence):
+        lines.append(f"- **{claim}** -> `{entry.evidence[claim]}`")
+    lines.append("")
+
+    return "\n".join(lines)

multimodal_mllog-0.1.0/mllog/schema.py

@@ -0,0 +1,69 @@
+"""Versioned logbook schema. This is the contract; validators enforce grounding."""
+
+from datetime import datetime
+
+from pydantic import BaseModel, Field, model_validator
+
+GROUNDED_FIELDS = ("hypothesis", "likely_cause", "next_experiment")
+
+
+class Metric(BaseModel):
+    name: str
+    value: float
+    unit: str | None = None
+
+
+class LogbookEntry(BaseModel):
+    schema_version: str = "1.0"
+    experiment_id: str
+    date: str  # full ISO 8601 UTC timestamp, e.g. "2026-06-09T18:45:01Z"
+    task: str
+    prompt: str
+    hypothesis: str
+    code_changes: list[str]
+    metrics: list[Metric]
+    result: str
+    observations: list[str]
+    likely_cause: str
+    next_experiment: str
+    evidence: dict[str, str]  # claim -> typed reference (metric:, diff:, session:, ...)
+    mlflow_run_id: str
+    related_run_ids: list[str] = Field(default_factory=list)  # RESERVED: multi-run sessions (M2+)
+    git_before: str | None = None
+    git_after: str | None = None
+    extra_inputs: dict = Field(default_factory=dict)  # RESERVED for video/telemetry; {} in v0
+
+    @model_validator(mode="after")
+    def _validate_date_iso8601(self) -> "LogbookEntry":
+        try:
+            datetime.fromisoformat(self.date.replace("Z", "+00:00"))
+        except ValueError as exc:
+            raise ValueError(
+                f"date must be a full ISO 8601 timestamp, got {self.date!r}: {exc}"
+            ) from exc
+        return self
+
+    @model_validator(mode="after")
+    def _validate_grounding(self) -> "LogbookEntry":
+        missing = [f for f in GROUNDED_FIELDS if f not in self.evidence]
+        if missing:
+            raise ValueError(
+                "evidence must contain a key for each grounded claim; missing: "
+                + ", ".join(missing)
+            )
+        return self
+
+    @model_validator(mode="after")
+    def _validate_metric_references(self) -> "LogbookEntry":
+        metric_names = {m.name for m in self.metrics}
+        bad = [
+            ref
+            for ref in self.evidence.values()
+            if ref.startswith("metric:") and ref.removeprefix("metric:") not in metric_names
+        ]
+        if bad:
+            raise ValueError(
+                "evidence metric references must name a metric present in `metrics`; "
+                "unresolved: " + ", ".join(bad)
+            )
+        return self

multimodal_mllog-0.1.0/multimodal_mllog.egg-info/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 2.4
+Name: multimodal-mllog
+Version: 0.1.0
+Summary: Experiment logbook pipeline: MLflow runs + git context -> structured logbook entries
+Requires-Python: >=3.11
+Requires-Dist: mlflow
+Requires-Dist: pydantic>=2
+Requires-Dist: click
+Provides-Extra: synthesis
+Requires-Dist: anthropic; extra == "synthesis"

multimodal_mllog-0.1.0/multimodal_mllog.egg-info/SOURCES.txt

@@ -0,0 +1,17 @@
+README.md
+pyproject.toml
+mllog/__init__.py
+mllog/cli.py
+mllog/git_context.py
+mllog/mlflow_io.py
+mllog/render.py
+mllog/schema.py
+multimodal_mllog.egg-info/PKG-INFO
+multimodal_mllog.egg-info/SOURCES.txt
+multimodal_mllog.egg-info/dependency_links.txt
+multimodal_mllog.egg-info/entry_points.txt
+multimodal_mllog.egg-info/requires.txt
+multimodal_mllog.egg-info/top_level.txt
+tests/test_mlflow_io.py
+tests/test_render.py
+tests/test_schema.py

multimodal_mllog-0.1.0/multimodal_mllog.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

multimodal_mllog-0.1.0/multimodal_mllog.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+mllog = mllog.cli:main

multimodal_mllog-0.1.0/multimodal_mllog.egg-info/requires.txt

@@ -0,0 +1,6 @@
+mlflow
+pydantic>=2
+click
+
+[synthesis]
+anthropic

multimodal_mllog-0.1.0/multimodal_mllog.egg-info/top_level.txt

@@ -0,0 +1 @@
+mllog

multimodal_mllog-0.1.0/pyproject.toml

@@ -0,0 +1,23 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "multimodal-mllog"
+version = "0.1.0"
+description = "Experiment logbook pipeline: MLflow runs + git context -> structured logbook entries"
+requires-python = ">=3.11"
+dependencies = [
+    "mlflow",
+    "pydantic>=2",
+    "click",
+]
+
+[project.optional-dependencies]
+synthesis = ["anthropic"]
+
+[project.scripts]
+mllog = "mllog.cli:main"
+
+[tool.setuptools.packages.find]
+include = ["mllog*"]

multimodal_mllog-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

multimodal_mllog-0.1.0/tests/test_mlflow_io.py

@@ -0,0 +1,77 @@
+import json
+import re
+
+import mlflow
+import pytest
+
+from mllog.mlflow_io import RunNotFoundError, attach_to_mlflow, get_mlflow_info
+
+ISO_UTC = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$")
+
+
+@pytest.fixture
+def tracking(tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)  # keep ./mlruns artifact root inside tmp
+    uri = f"sqlite:///{(tmp_path / 'mlflow.db').as_posix()}"
+    monkeypatch.setenv("MLFLOW_TRACKING_URI", uri)
+    mlflow.set_tracking_uri(uri)
+    yield uri
+
+
+@pytest.fixture
+def demo_run(tracking):
+    with mlflow.start_run(run_name="test-run") as run:
+        mlflow.log_params({"learning_rate": 3e-4, "epochs": 5})
+        mlflow.log_metric("train_loss", 0.271)
+        mlflow.log_metric("val_acc", 0.86)
+    return run.info.run_id
+
+
+def test_run_info_latest(demo_run):
+    info = get_mlflow_info(latest=True)
+    assert info["run_id"] == demo_run
+    assert info["params"] == {"learning_rate": "0.0003", "epochs": "5"}
+    assert info["metrics"] == {"train_loss": 0.271, "val_acc": 0.86}
+    assert ISO_UTC.match(info["start_time"])
+    assert ISO_UTC.match(info["collected_at"])
+
+
+def test_run_info_by_id(demo_run):
+    info = get_mlflow_info(run_id=demo_run)
+    assert info["run_id"] == demo_run
+    assert info["experiment_name"] == "Default"
+
+
+def test_run_info_no_runs_fails_loudly(tracking):
+    with pytest.raises(RunNotFoundError):
+        get_mlflow_info(latest=True)
+
+
+def test_run_info_bad_run_id_fails_loudly(tracking):
+    with pytest.raises(RunNotFoundError):
+        get_mlflow_info(run_id="nonexistent")
+
+
+def test_ingest_attaches_artifacts_and_tags(demo_run, valid_entry):
+    entry = valid_entry.model_copy(update={"mlflow_run_id": demo_run})
+    result = attach_to_mlflow(entry, run_id=demo_run)
+    assert result["run_id"] == demo_run
+
+    client = mlflow.tracking.MlflowClient()
+    artifacts = {f.path for f in client.list_artifacts(demo_run)}
+    assert {"logbook.json", "logbook.md"} <= artifacts
+
+    run = client.get_run(demo_run)
+    assert run.data.tags["logbook.generated"] == "true"
+    assert run.data.tags["logbook.task"] == entry.task
+    assert run.data.tags["logbook.result"] == entry.result
+    assert run.data.tags["logbook.schema_version"] == "1.0"
+
+    json_path = client.download_artifacts(demo_run, "logbook.json")
+    stored = json.loads(open(json_path, encoding="utf-8").read())
+    assert stored["mlflow_run_id"] == demo_run
+
+
+def test_ingest_missing_run_fails_loudly(tracking, valid_entry):
+    with pytest.raises(RunNotFoundError):
+        attach_to_mlflow(valid_entry, run_id="nonexistent")

multimodal_mllog-0.1.0/tests/test_render.py

@@ -0,0 +1,26 @@
+from pathlib import Path
+
+from mllog.render import render_markdown
+
+GOLDEN = Path(__file__).parent / "golden" / "logbook.md"
+
+
+def test_render_is_deterministic(valid_entry):
+    assert render_markdown(valid_entry) == render_markdown(valid_entry)
+
+
+def test_render_contains_metrics_and_evidence(valid_entry):
+    md = render_markdown(valid_entry)
+    assert "| train_loss | 0.271 |" in md
+    assert "| val_acc | 0.86 |" in md
+    assert "`metric:train_loss`" in md
+    assert "abc123" in md
+
+
+def test_golden_file(valid_entry):
+    """Locks the markdown projection of a fixed LogbookEntry.
+
+    If a render change is intentional, regenerate with:
+    python -c "from tests.conftest import *; ..." (see README) or update the file.
+    """
+    assert render_markdown(valid_entry) == GOLDEN.read_text(encoding="utf-8")

multimodal_mllog-0.1.0/tests/test_schema.py

@@ -0,0 +1,47 @@
+import pytest
+from pydantic import ValidationError
+
+from mllog.schema import LogbookEntry
+
+
+def test_valid_entry_passes(valid_entry):
+    assert valid_entry.schema_version == "1.0"
+    assert valid_entry.related_run_ids == []
+    assert valid_entry.extra_inputs == {}
+
+
+@pytest.mark.parametrize("missing", ["hypothesis", "likely_cause", "next_experiment"])
+def test_grounding_required_for_each_claim(valid_entry, missing):
+    data = valid_entry.model_dump()
+    del data["evidence"][missing]
+    with pytest.raises(ValidationError, match=missing):
+        LogbookEntry.model_validate(data)
+
+
+def test_metric_reference_must_resolve(valid_entry):
+    data = valid_entry.model_dump()
+    data["evidence"]["likely_cause"] = "metric:does_not_exist"
+    with pytest.raises(ValidationError, match="metric:does_not_exist"):
+        LogbookEntry.model_validate(data)
+
+
+def test_non_metric_references_not_resolved_at_schema_level(valid_entry):
+    data = valid_entry.model_dump()
+    data["evidence"]["likely_cause"] = "diff:anything.py@@-1,1"
+    data["evidence"]["next_experiment"] = "session:we discussed it"
+    LogbookEntry.model_validate(data)  # should not raise
+
+
+@pytest.mark.parametrize("bad_date", ["2026-13-40", "yesterday", "", "18:45"])
+def test_date_must_be_iso8601(valid_entry, bad_date):
+    data = valid_entry.model_dump()
+    data["date"] = bad_date
+    with pytest.raises(ValidationError):
+        LogbookEntry.model_validate(data)
+
+
+def test_date_accepts_z_suffix_and_offset(valid_entry):
+    data = valid_entry.model_dump()
+    for ok in ("2026-06-09T18:45:01Z", "2026-06-09T18:45:01+00:00"):
+        data["date"] = ok
+        LogbookEntry.model_validate(data)