diffeval 0.1.0__py3-none-any.whl

diffeval/runner/run.py

@@ -0,0 +1,24 @@
+"""Run one evaluation and write results.json."""
+
+from pathlib import Path
+
+from diffeval.aggregate.table import aggregate
+from diffeval.registry.lookup import get
+from diffeval.registry.tables import BACKENDS
+from diffeval.runner.metric_device import metric_device
+from diffeval.runner.options import chosen
+from diffeval.runner.payload import payload
+from diffeval.runner.seeds import score_seeds
+from diffeval.runner.write_reports import write_reports
+
+
+def run_evaluation(options):
+    picked = chosen(options)
+    backend = get(BACKENDS, options.backend, "backend").from_config(options.backend_config)
+    backend.prepare()
+    device = metric_device(options.backend_config.get("device"))
+    out = Path(options.output_dir)
+    per_seed = score_seeds(backend, options.suite, picked, out, device)
+    body = payload(options, backend, device, picked["seeds"], per_seed, aggregate(per_seed))
+    write_reports(body, out)
+    return body

diffeval/runner/safe_metric.py

@@ -0,0 +1,14 @@
+"""Call compute and record an error if it raises."""
+
+
+def _record(name, value, status, error, higher, protocol, details=None) -> dict:
+    return {"name": name, "value": value, "status": status, "error": error, "higher_is_better": higher, "details": details or {}, "protocol": protocol}
+
+
+def safe_metric(metric, samples, context) -> dict:
+    try:
+        result = metric.compute(samples, context)
+    except Exception as exc:
+        higher = getattr(metric, "higher_is_better", None)
+        return _record(metric.name, None, "error", str(exc), higher, getattr(metric, "protocol", {}))
+    return _record(result.name, result.value, result.status, result.error, result.higher_is_better, result.protocol, result.details)

diffeval/runner/save_png.py

@@ -0,0 +1,8 @@
+"""Write one uint8 RGB PNG."""
+
+from pathlib import Path
+
+
+def save_png(image, path: Path) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    image.convert("RGB").save(path, format="PNG")

diffeval/runner/seeds.py

@@ -0,0 +1,23 @@
+"""Score every seed."""
+
+from diffeval.metrics.skipped import skipped
+from diffeval.registry.lookup import get
+from diffeval.registry.tables import METRICS
+from diffeval.runner.one_seed import one_seed
+
+_PENDING = {"clip_score", "cmmd", "fid", "kid"}
+
+
+def _metric(name):
+    try:
+        return get(METRICS, name, "metric")()
+    except KeyError:
+        if name not in _PENDING:
+            raise
+        compute = lambda self, samples, context: skipped(name, "metric is not installed", None, {})
+        return type("Missing", (), {"name": name, "compute": compute, "higher_is_better": None, "protocol": {}})()
+
+
+def score_seeds(backend, suite, picked, output_dir, device):
+    metrics = {name: _metric(name) for name in picked["metrics"]}
+    return [one_seed(backend, suite, seed, picked, metrics, output_dir, device) for seed in picked["seeds"]]

diffeval/runner/write_json.py

@@ -0,0 +1,11 @@
+"""Write results.json."""
+
+import json
+from pathlib import Path
+
+
+def write_json(body: dict, output_dir: Path) -> Path:
+    path = Path(output_dir) / "results.json"
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(body, indent=2) + "\n", encoding="utf-8")
+    return path

diffeval/runner/write_reports.py

@@ -0,0 +1,14 @@
+"""Write report.md and report.html."""
+
+from pathlib import Path
+
+from diffeval.report.html_page import html_page
+from diffeval.report.markdown import markdown
+from diffeval.runner.write_json import write_json
+
+
+def write_reports(body: dict, output_dir) -> None:
+    write_json(body, output_dir)
+    folder = Path(output_dir)
+    (folder / "report.md").write_text(markdown(body), encoding="utf-8")
+    (folder / "report.html").write_text(html_page(body), encoding="utf-8")

diffeval/system/__init__.py

@@ -0,0 +1 @@
+"""Versions, GPU, and attention flags for a run."""

diffeval/system/describe.py

@@ -0,0 +1,10 @@
+"""Platform, GPU, and attention flags."""
+
+import platform
+
+from diffeval.system.gpu import gpu
+from diffeval.system.sdp_flags import sdp_flags
+
+
+def describe() -> dict:
+    return {"platform": platform.platform(), "gpu": gpu(), "torch_attention": sdp_flags()}

diffeval/system/gpu.py

@@ -0,0 +1,11 @@
+"""CUDA, otherwise MPS, otherwise null."""
+
+
+def gpu():
+    try:
+        import torch
+    except ImportError:
+        return None
+    from diffeval.system.gpu_cuda import gpu_cuda
+    from diffeval.system.gpu_mps import gpu_mps
+    return gpu_cuda(torch) or gpu_mps(torch)

diffeval/system/gpu_cuda.py

@@ -0,0 +1,9 @@
+"""CUDA device name, index, count, and memory."""
+
+
+def gpu_cuda(torch):
+    if not torch.cuda.is_available():
+        return None
+    index = torch.cuda.current_device()
+    props = torch.cuda.get_device_properties(index)
+    return {"name": props.name, "index": index, "count": torch.cuda.device_count(), "total_memory": props.total_memory}

diffeval/system/gpu_mps.py

@@ -0,0 +1,8 @@
+"""MPS entry when torch reports Apple GPU."""
+
+
+def gpu_mps(torch):
+    mps = getattr(torch.backends, "mps", None)
+    if mps is None or not mps.is_available():
+        return None
+    return {"name": "Apple MPS", "index": 0, "count": 1, "total_memory": None}

diffeval/system/sdp_flags.py

@@ -0,0 +1,15 @@
+"""PyTorch scaled-dot-product attention flags."""
+
+
+def sdp_flags():
+    try:
+        import torch
+    except ImportError:
+        return None
+    cuda = torch.backends.cuda
+    return {
+        "flash_sdp": cuda.flash_sdp_enabled(),
+        "mem_efficient_sdp": cuda.mem_efficient_sdp_enabled(),
+        "math_sdp": cuda.math_sdp_enabled(),
+        "cudnn_sdp": cuda.cudnn_sdp_enabled(),
+    }

diffeval/system/version_of.py

@@ -0,0 +1,10 @@
+"""importlib.metadata version, or null when missing."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+
+def version_of(name: str):
+    try:
+        return version(name)
+    except PackageNotFoundError:
+        return None

diffeval/system/versions.py

@@ -0,0 +1,20 @@
+"""Package versions recorded in results.json."""
+
+import platform
+
+from diffeval.system.version_of import version_of
+
+
+def versions() -> dict:
+    return {
+        "diffeval": version_of("diffeval"),
+        "python": platform.python_version(),
+        "numpy": version_of("numpy"),
+        "pillow": version_of("pillow"),
+        "pyyaml": version_of("pyyaml"),
+        "torch": version_of("torch"),
+        "diffusers": version_of("diffusers"),
+        "transformers": version_of("transformers"),
+        "accelerate": version_of("accelerate"),
+        "clean-fid": version_of("clean-fid"),
+    }

diffeval/tasks/__init__.py

@@ -0,0 +1,3 @@
+"""Load prompt and seed suites."""
+
+import diffeval.tasks.core

diffeval/tasks/build_suite.py

@@ -0,0 +1,7 @@
+"""Assemble a Suite from parsed fields."""
+
+from diffeval.types.suite import Suite
+
+
+def build_suite(fields: dict) -> Suite:
+    return Suite(**fields)

diffeval/tasks/core.py

@@ -0,0 +1,15 @@
+"""Register the bundled core suite."""
+
+from diffeval.registry.register import register
+from diffeval.registry.tables import TASKS
+from diffeval.tasks.load_bundled import load_bundled
+
+
+class Core:
+    description = "Four synthetic prompts for a CPU smoke run."
+
+    def load(self):
+        return load_bundled("core.yaml")
+
+
+register(TASKS, "core")(Core())

diffeval/tasks/errors.py

@@ -0,0 +1,5 @@
+"""Errors raised while loading a suite."""
+
+
+class SuiteError(ValueError):
+    """A suite file is missing a field or breaks a rule."""

diffeval/tasks/fields_metrics.py

@@ -0,0 +1,14 @@
+"""Metric name list."""
+
+from diffeval.tasks.errors import SuiteError
+
+_DEFAULT = ("clip_score", "cmmd", "fid", "kid")
+
+
+def metrics(data: dict) -> dict:
+    raw = data.get("metrics", list(_DEFAULT))
+    if not isinstance(raw, list) or not raw:
+        raise SuiteError("metrics must be a non-empty list")
+    if any(not isinstance(item, str) or not item for item in raw):
+        raise SuiteError("metric names must be non-empty strings")
+    return {"metrics": tuple(raw)}

diffeval/tasks/fields_paths.py

@@ -0,0 +1,25 @@
+"""reference_dir, cmmd_ref_embed, and clean_fid_stats."""
+
+from pathlib import Path
+
+from diffeval.tasks.errors import SuiteError
+
+
+def _path(base: Path, value):
+    if value is None:
+        return None
+    if not isinstance(value, str) or not value:
+        raise SuiteError("path fields must be strings")
+    path = Path(value)
+    return path if path.is_absolute() else base / path
+
+
+def paths(data: dict, base: Path) -> dict:
+    stats = data.get("clean_fid_stats")
+    if stats is not None and not isinstance(stats, dict):
+        raise SuiteError("clean_fid_stats must be a mapping")
+    return {
+        "reference_dir": _path(base, data.get("reference_dir")),
+        "cmmd_ref_embed": _path(base, data.get("cmmd_ref_embed")),
+        "clean_fid_stats": stats,
+    }

diffeval/tasks/fields_prompts.py

@@ -0,0 +1,15 @@
+"""Prompt list and duplicate ids, compared ignoring case."""
+
+from diffeval.tasks.errors import SuiteError
+from diffeval.tasks.one_prompt import one_prompt
+
+
+def prompts(data: dict) -> dict:
+    raw = data.get("prompts")
+    if not isinstance(raw, list) or not raw:
+        raise SuiteError("prompts must be a non-empty list")
+    built = tuple(one_prompt(item) for item in raw)
+    ids = [item.id.casefold() for item in built]
+    if len(ids) != len(set(ids)):
+        raise SuiteError("duplicate prompt id (ids are compared ignoring case)")
+    return {"prompts": built}

diffeval/tasks/fields_seeds.py

@@ -0,0 +1,14 @@
+"""Non-empty list of distinct seeds."""
+
+from diffeval.tasks.errors import SuiteError
+
+
+def seeds(data: dict) -> dict:
+    raw = data.get("seeds")
+    if not isinstance(raw, list) or not raw:
+        raise SuiteError("seeds must be a non-empty list of ints")
+    if any(type(item) is not int for item in raw):
+        raise SuiteError("seeds must be ints")
+    if len(set(raw)) != len(raw):
+        raise SuiteError("seeds must not repeat")
+    return {"seeds": tuple(raw)}

diffeval/tasks/fields_size.py

@@ -0,0 +1,12 @@
+"""Width and height, defaulting to 512."""
+
+from diffeval.tasks.errors import SuiteError
+
+
+def width_and_height(data: dict) -> dict:
+    width = data.get("width", 512)
+    height = data.get("height", 512)
+    bad = type(width) is not int or type(height) is not int
+    if bad or width < 1 or height < 1:
+        raise SuiteError("width and height must be positive ints")
+    return {"width": width, "height": height}

diffeval/tasks/fields_text.py

@@ -0,0 +1,13 @@
+"""Suite name and description."""
+
+from diffeval.tasks.errors import SuiteError
+
+
+def name_and_description(data: dict) -> dict:
+    name = data.get("name")
+    description = data.get("description", "")
+    if not isinstance(name, str) or not name:
+        raise SuiteError("name must be a non-empty string")
+    if not isinstance(description, str):
+        raise SuiteError("description must be a string")
+    return {"name": name, "description": description}

diffeval/tasks/from_mapping.py

@@ -0,0 +1,22 @@
+"""Build a Suite from a mapping."""
+
+from pathlib import Path
+
+from diffeval.tasks.build_suite import build_suite
+from diffeval.tasks.fields_metrics import metrics
+from diffeval.tasks.fields_paths import paths
+from diffeval.tasks.fields_prompts import prompts
+from diffeval.tasks.fields_seeds import seeds
+from diffeval.tasks.fields_size import width_and_height
+from diffeval.tasks.fields_text import name_and_description
+
+
+def from_mapping(data: dict, source: str, base: Path):
+    fields = {"source": source}
+    fields.update(name_and_description(data))
+    fields.update(width_and_height(data))
+    fields.update(seeds(data))
+    fields.update(prompts(data))
+    fields.update(metrics(data))
+    fields.update(paths(data, base))
+    return build_suite(fields)

diffeval/tasks/load_bundled.py

@@ -0,0 +1,9 @@
+"""Load a suite YAML packaged with the tasks."""
+
+from pathlib import Path
+
+from diffeval.tasks.load_path import load_path
+
+
+def load_bundled(name: str):
+    return load_path(Path(__file__).parent / "suites" / name)

diffeval/tasks/load_path.py

@@ -0,0 +1,12 @@
+"""Load a suite from a filesystem YAML path."""
+
+from pathlib import Path
+
+from diffeval.tasks.from_mapping import from_mapping
+from diffeval.tasks.read_yaml import read_yaml
+from diffeval.types.suite import Suite
+
+
+def load_path(path) -> Suite:
+    file = Path(path)
+    return from_mapping(read_yaml(file), str(file), file.parent)

diffeval/tasks/one_prompt.py

@@ -0,0 +1,17 @@
+"""One prompt mapping."""
+
+from diffeval.tasks.errors import SuiteError
+from diffeval.tasks.prompt_id import check_id
+from diffeval.types.prompt import Prompt
+
+
+def one_prompt(item: dict) -> Prompt:
+    if not isinstance(item, dict):
+        raise SuiteError("prompt must be a mapping")
+    text = item.get("prompt")
+    if not isinstance(text, str) or not text.strip():
+        raise SuiteError("prompt text must be a non-empty string")
+    negative = item.get("negative_prompt")
+    if negative is not None and not isinstance(negative, str):
+        raise SuiteError("negative_prompt must be a string")
+    return Prompt(check_id(item.get("id")), text, negative)

diffeval/tasks/prompt_id.py

@@ -0,0 +1,13 @@
+"""Prompt ids that are safe filenames."""
+
+import re
+
+from diffeval.tasks.errors import SuiteError
+
+_ID = re.compile(r"[A-Za-z0-9_-]+")
+
+
+def check_id(value: str) -> str:
+    if not isinstance(value, str) or _ID.fullmatch(value) is None:
+        raise SuiteError(f"bad prompt id {value!r}")
+    return value

diffeval/tasks/read_yaml.py

@@ -0,0 +1,14 @@
+"""Read a YAML file to a mapping."""
+
+from pathlib import Path
+
+import yaml
+
+from diffeval.tasks.errors import SuiteError
+
+
+def read_yaml(path: Path) -> dict:
+    data = yaml.safe_load(Path(path).read_text(encoding="utf-8"))
+    if not isinstance(data, dict):
+        raise SuiteError(f"{path} must be a mapping")
+    return data

diffeval/tasks/resolve.py

@@ -0,0 +1,18 @@
+"""Resolve a suite from a path or a registered name."""
+
+from pathlib import Path
+
+from diffeval.registry.lookup import get
+from diffeval.registry.tables import TASKS
+from diffeval.tasks.errors import SuiteError
+from diffeval.tasks.load_path import load_path
+
+
+def resolve(source: str):
+    path = Path(source)
+    if path.suffix in {".yaml", ".yml"}:
+        return load_path(path)
+    try:
+        return get(TASKS, source, "task").load()
+    except KeyError as exc:
+        raise SuiteError(str(exc)) from None

diffeval/tasks/suites/core.yaml

@@ -0,0 +1,12 @@
+name: core
+description: Synthetic smoke prompts. The drawings are not COCO.
+seeds: [0, 1]
+width: 512
+height: 512
+metrics: [clip_score, cmmd, fid, kid]
+reference_dir: references/core
+prompts:
+- {id: red_cube, prompt: a red cube}
+- {id: blue_sphere, prompt: a blue sphere}
+- {id: green_triangle, prompt: a green triangle}
+- {id: yellow_star, prompt: a yellow star}

diffeval/types/__init__.py

@@ -0,0 +1 @@
+"""Records shared by tasks, backends, and metrics."""

diffeval/types/context.py

@@ -0,0 +1,15 @@
+"""Inputs a metric sees for one seed."""
+
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class EvalContext:
+    generated_dir: Path
+    reference_dir: Path | None
+    device: str
+    seed: int
+    clean_fid_stats: dict | None
+    cmmd_ref_embed: Path | None
+    batch_size: int

diffeval/types/prompt.py

@@ -0,0 +1,10 @@
+"""One prompt in a suite."""
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Prompt:
+    id: str
+    prompt: str
+    negative_prompt: str | None = None

diffeval/types/request.py

@@ -0,0 +1,13 @@
+"""One image a backend is asked to generate."""
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class GenerationRequest:
+    prompt_id: str
+    prompt: str
+    seed: int
+    width: int
+    height: int
+    negative_prompt: str | None = None

diffeval/types/result.py

@@ -0,0 +1,14 @@
+"""One metric score for one seed."""
+
+from dataclasses import dataclass
+
+
+@dataclass
+class MetricResult:
+    name: str
+    value: float | None
+    status: str
+    error: str | None
+    higher_is_better: bool | None
+    details: dict
+    protocol: dict

diffeval/types/result_rules.py

@@ -0,0 +1,13 @@
+"""Finite value and status rules for a metric result."""
+
+import math
+
+
+def check(status: str, value: float | None, error: str | None) -> None:
+    if status == "ok":
+        finite = isinstance(value, (int, float)) and not isinstance(value, bool)
+        if finite and math.isfinite(value) and error is None:
+            return
+    elif status in ("skipped", "error") and value is None and error:
+        return
+    raise ValueError(f"bad metric result status {status!r}")

diffeval/types/sample.py

@@ -0,0 +1,15 @@
+"""One generated image and the request that produced it."""
+
+from dataclasses import dataclass
+
+
+@dataclass
+class Sample:
+    prompt_id: str
+    prompt: str
+    seed: int
+    width: int
+    height: int
+    image: object | None
+    path: str | None
+    negative_prompt: str | None = None

diffeval/types/suite.py

@@ -0,0 +1,21 @@
+"""A loaded prompt and seed suite."""
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from diffeval.types.prompt import Prompt
+
+
+@dataclass(frozen=True)
+class Suite:
+    name: str
+    description: str
+    prompts: tuple[Prompt, ...]
+    seeds: tuple[int, ...]
+    width: int
+    height: int
+    metrics: tuple[str, ...]
+    reference_dir: Path | None
+    clean_fid_stats: dict | None
+    cmmd_ref_embed: Path | None
+    source: str