kalfa 3.0.0.dev0__py3-none-any.whl

kalfa/std/objectives.py

@@ -0,0 +1,360 @@
+import copy
+import math
+
+import torch
+from torch import nn
+from torch.nn import functional
+
+from kalfa.core.aliases import register
+from kalfa.core.errors import RunError
+from kalfa.core.predict import batch_size, target_names
+from kalfa.core.protocols import Objective
+from kalfa.core.rng import Rng
+from kalfa.core.wrap import loss_scalar
+
+
+NOISE = "noise"
+
+
+def title(owner):
+    name = getattr(owner, "name", "")
+    return f"loss {name!r}" if name else type(owner).__name__
+
+
+def named(obj):
+    return getattr(obj, "name", "") or getattr(obj, "__name__", "") or type(obj).__name__
+
+
+def field(batch, name, owner):
+    if name not in batch:
+        raise RunError(f"{title(owner)}: the batch has no field {name!r}; its fields are {sorted(map(str, batch))}")
+    return batch[name]
+
+
+def first(result):
+    if isinstance(result, (tuple, list)):
+        return result[0]
+    if isinstance(result, dict):
+        return next(iter(result.values()))
+    return result
+
+
+def call(module, value, label):
+    return module(value) if label is None else module(value, label)
+
+
+def device_of(module, batch):
+    for parameter in module.parameters():
+        return parameter.device
+    for value in batch.values() if isinstance(batch, dict) else ():
+        if isinstance(value, torch.Tensor):
+            return value.device
+    return torch.device("cpu")
+
+
+def normal(ctx, shape, like=None, device=None):
+    rng = getattr(ctx, "rng", None)
+    if rng is not None:
+        return rng.normal(NOISE, shape, like=like, device=device)
+    if like is None:
+        return torch.randn(shape, device=device)
+    return torch.randn(shape, device=like.device, dtype=like.dtype if like.dtype.is_floating_point else None)
+
+
+def uniform(ctx, shape, like):
+    rng = getattr(ctx, "rng", None)
+    if rng is not None:
+        return rng.uniform(NOISE, shape, like=like)
+    return torch.rand(shape, device=like.device, dtype=like.dtype if like.dtype.is_floating_point else None)
+
+
+def integers(ctx, high, count, device):
+    rng = getattr(ctx, "rng", None)
+    if rng is not None:
+        return rng.integers(NOISE, 0, high, (count,), device=device)
+    return torch.randint(0, int(high), (count,), device=device)
+
+
+def owned_modules(ctx):
+    found = []
+    for entry in (getattr(ctx, "models", None) or {}).values():
+        for module in (getattr(entry, "module", None), getattr(entry, "ema", None)):
+            if isinstance(module, nn.Module):
+                found.extend(module.modules())
+    return found
+
+
+def release(objective, ctx):
+    owned = owned_modules(ctx)
+    for name, module in list(objective.named_children()):
+        if any(module is part for part in owned):
+            delattr(objective, name)
+            object.__setattr__(objective, name, module)
+
+
+def gaussian_kl(mu, logvar):
+    return -0.5 * torch.mean(torch.sum(1.0 + logvar - mu.pow(2) - logvar.exp(), dim=-1))
+
+
+class BetaSchedule:
+    def __init__(self, steps):
+        self.steps = int(steps)
+        if self.steps < 1:
+            raise ValueError(f"a beta schedule needs at least one step, got {steps}")
+        self.betas = self.compute()
+
+    def compute(self):
+        raise NotImplementedError
+
+
+class LinearBeta(BetaSchedule):
+    def __init__(self, steps, start=1.0e-4, end=0.02):
+        self.start = float(start)
+        self.end = float(end)
+        if not (0.0 < self.start < 1.0 and 0.0 < self.end < 1.0):
+            raise ValueError(f"linear_beta needs start and end inside (0, 1), got {start} and {end}")
+        super().__init__(steps)
+
+    def compute(self):
+        return torch.linspace(self.start, self.end, self.steps, dtype=torch.float64).to(torch.float32)
+
+
+class CosineBeta(BetaSchedule):
+    def __init__(self, steps, s=0.008):
+        self.s = float(s)
+        super().__init__(steps)
+
+    def compute(self):
+        grid = torch.arange(self.steps + 1, dtype=torch.float64) / self.steps
+        curve = torch.cos((grid + self.s) / (1.0 + self.s) * math.pi / 2.0) ** 2
+        cumulative = curve / curve[0]
+        betas = 1.0 - cumulative[1:] / cumulative[:-1]
+        return betas.clamp(1.0e-8, 0.999).to(torch.float32)
+
+
+class NoiseTables:
+    def __init__(self, schedule):
+        self.schedule = schedule
+        self.steps = schedule.steps
+        self.betas = schedule.betas
+        self.alphas = 1.0 - self.betas
+        self.cumulative = torch.cumprod(self.alphas, dim=0)
+        self.placed = {}
+
+    def on(self, device, dtype=torch.float32):
+        where = Rng.place(device)
+        key = (str(where), dtype)
+        found = self.placed.get(key)
+        if found is None:
+            found = copy.copy(self)
+            found.betas = self.betas.to(where, dtype)
+            found.alphas = self.alphas.to(where, dtype)
+            found.cumulative = self.cumulative.to(where, dtype)
+            self.placed[key] = found
+        return found
+
+    def __iter__(self):
+        return iter((self.betas, self.alphas, self.cumulative))
+
+
+class VaeElbo(Objective):
+    def __init__(self, recon, w_rec=1.0, w_kl=1.0, mu="mu", logvar="logvar"):
+        super().__init__()
+        self.recon = recon
+        self.w_rec = float(w_rec)
+        self.w_kl = w_kl
+        self.mu = mu
+        self.logvar = logvar
+
+    def weight(self, ctx):
+        if callable(self.w_kl):
+            return float(self.w_kl(getattr(ctx, "updates", 0)))
+        return float(self.w_kl)
+
+    def forward(self, outputs, batch, ctx):
+        outputs = {} if outputs is None else outputs
+        missing = [name for name in (self.mu, self.logvar) if name not in outputs]
+        if missing:
+            raise RunError(f"{title(self)}: the step outputs have no {missing}; they are {sorted(map(str, outputs))}; "
+                           "name the encoder outputs with mu and logvar")
+        kl = gaussian_kl(outputs[self.mu].float(), outputs[self.logvar].float())
+        recon = loss_scalar(self.recon(outputs, batch, ctx), named(self.recon))
+        w_kl = self.weight(ctx)
+        return {"loss": self.w_rec * recon + w_kl * kl, "recon": recon, "kl": kl, "w_kl": w_kl}
+
+
+class WganGpD(Objective):
+    on = ("train",)
+
+    def __init__(self, generator, critic, latent, real="image", label=None, gp_weight=10.0):
+        super().__init__()
+        self.generator = generator
+        self.critic = critic
+        self.latent = int(latent)
+        self.real = real
+        self.label = label
+        self.gp_weight = float(gp_weight)
+
+    def bind(self, ctx):
+        release(self, ctx)
+
+    def forward(self, outputs, batch, ctx):
+        real = field(batch, self.real, self).detach()
+        label = None if self.label is None else field(batch, self.label, self)
+        count = real.shape[0]
+        z = normal(ctx, (count, self.latent), device=real.device)
+        with torch.no_grad():
+            fake = first(call(self.generator, z, label)).detach()
+        if fake.shape != real.shape:
+            raise RunError(f"{title(self)}: the generator makes {tuple(fake.shape)} but the real field "
+                           f"{self.real!r} is {tuple(real.shape)}")
+        score_real = first(call(self.critic, real, label))
+        score_fake = first(call(self.critic, fake, label))
+        with torch.enable_grad():
+            mix = uniform(ctx, (count,) + (1,) * (real.ndim - 1), like=real)
+            between = (mix * real + (1.0 - mix) * fake).requires_grad_(True)
+            score = first(call(self.critic, between, label))
+            gradients = torch.autograd.grad(score.sum(), between, create_graph=True)[0]
+            penalty = ((gradients.flatten(1).norm(dim=1) - 1.0) ** 2).mean()
+        wasserstein = score_real.mean() - score_fake.mean()
+        return {"loss": -wasserstein + self.gp_weight * penalty, "wasserstein": wasserstein, "gp": penalty}
+
+
+class WganG(Objective):
+    def __init__(self, generator, critic, latent, label=None):
+        super().__init__()
+        self.generator = generator
+        self.critic = critic
+        self.latent = int(latent)
+        self.label = label
+
+    def bind(self, ctx):
+        release(self, ctx)
+
+    def forward(self, outputs, batch, ctx):
+        label = None if self.label is None else field(batch, self.label, self)
+        z = normal(ctx, (batch_size(batch), self.latent), device=device_of(self.generator, batch))
+        fake = first(call(self.generator, z, label))
+        return -first(call(self.critic, fake, label)).mean()
+
+
+class Ddpm(Objective):
+    def __init__(self, model, schedule, x="image"):
+        super().__init__()
+        self.model = model
+        self.schedule = schedule
+        self.x = x
+        self.tables = NoiseTables(schedule)
+
+    def bind(self, ctx):
+        release(self, ctx)
+
+    def forward(self, outputs, batch, ctx):
+        if self.tables.schedule is not self.schedule:
+            self.tables = NoiseTables(self.schedule)
+        x = field(batch, self.x, self)
+        tables = self.tables.on(x.device)
+        count = x.shape[0]
+        t = integers(ctx, tables.steps, count, x.device)
+        noise = normal(ctx, x.shape, like=x)
+        weight = tables.cumulative[t].to(x.dtype).reshape((count,) + (1,) * (x.ndim - 1))
+        noised = weight.sqrt() * x + (1.0 - weight).sqrt() * noise
+        predicted = first(self.model(noised, t))
+        return functional.mse_loss(predicted.float(), noise.float())
+
+
+class Distill(Objective):
+    def __init__(self, student, teacher, x=None, label=None, temperature=1.0, alpha=0.5):
+        super().__init__()
+        self.student = student
+        self.teacher = teacher
+        self.x = x
+        self.label = label
+        self.temperature = float(temperature)
+        self.alpha = float(alpha)
+
+    def bind(self, ctx):
+        release(self, ctx)
+
+    def input_name(self, ctx):
+        if self.x is not None:
+            return self.x
+        inputs = list(getattr(getattr(ctx, "data", None), "inputs", None) or ())
+        if not inputs:
+            raise RunError(f"{title(self)}: x is not written and the data declares no inputs; write x: <field>")
+        return inputs[0]
+
+    def label_name(self, ctx):
+        if self.label is not None:
+            return self.label
+        names = target_names(getattr(ctx, "data", None))
+        if len(names) != 1:
+            raise RunError(f"{title(self)}: label is not written and the data has {len(names)} targets {names}; "
+                           "write label: <field>")
+        return names[0]
+
+    def forward(self, outputs, batch, ctx):
+        x = field(batch, self.input_name(ctx), self)
+        labels = field(batch, self.label_name(ctx), self).reshape(-1).long()
+        logits = first(self.student(x)).float()
+        with torch.no_grad():
+            guide = first(self.teacher(x)).float()
+        scale = self.temperature
+        soft = functional.softmax(guide / scale, dim=-1)
+        guess = functional.log_softmax(logits / scale, dim=-1)
+        kl = functional.kl_div(guess, soft, reduction="batchmean") * scale * scale
+        ce = functional.cross_entropy(logits, labels)
+        return {"loss": self.alpha * kl + (1.0 - self.alpha) * ce, "ce": ce, "kl": kl}
+
+
+class Ntxent(Objective):
+    def __init__(self, encoder, projector=None, views="image", temperature=0.5):
+        super().__init__()
+        self.encoder = encoder
+        self.projector = projector
+        self.views = views
+        self.temperature = float(temperature)
+
+    def bind(self, ctx):
+        release(self, ctx)
+
+    def forward(self, outputs, batch, ctx):
+        views = field(batch, self.views, self)
+        if views.ndim < 3 or views.shape[1] != 2:
+            raise RunError(f"{title(self)}: ntxent needs two views per item, a (B, 2, ...) field, but {self.views!r} "
+                           f"is {tuple(views.shape)}; put two_views in the train transform")
+        count = views.shape[0]
+        embedded = first(self.encoder(views.flatten(0, 1)))
+        if self.projector is not None:
+            embedded = first(self.projector(embedded))
+        z = functional.normalize(embedded.flatten(1).float(), dim=1)
+        similarity = z @ z.t() / self.temperature
+        diagonal = torch.eye(2 * count, device=z.device, dtype=torch.bool)
+        similarity = similarity.masked_fill(diagonal, float("-inf"))
+        partner = torch.arange(2 * count, device=z.device) ^ 1
+        return functional.cross_entropy(similarity, partner)
+
+
+REGISTRATIONS = (
+    ("vae_elbo", "kalfa.std.objectives.VaeElbo", "loss",
+     "w_rec times the recon criterion on the outputs plus w_kl (a number or a schedule of the update count) times the "
+     "gaussian KL of the mu and logvar outputs; terms recon, kl, w_kl"),
+    ("wgan_gp_d", "kalfa.std.objectives.WganGpD", "loss",
+     "the critic loss of WGAN GP: fake minus real scores plus gp_weight times the gradient penalty on interpolates; "
+     "the latent noise comes from the noise stream; terms wasserstein, gp; train only by default"),
+    ("wgan_g", "kalfa.std.objectives.WganG", "loss",
+     "the generator loss of WGAN: minus the critic score of fresh fakes"),
+    ("ddpm", "kalfa.std.objectives.Ddpm", "loss",
+     "the denoising loss: noise the x field at a random step of the beta schedule and regress the noise with "
+     "model(x_t, t)"),
+    ("distill", "kalfa.std.objectives.Distill", "loss",
+     "alpha times the KL between student and teacher logits at temperature plus (1 - alpha) times cross entropy with "
+     "the label field (default the only target of the data); terms ce, kl"),
+    ("ntxent", "kalfa.std.objectives.Ntxent", "loss",
+     "NT-Xent over two views per image: the encoder (and projector) embeds both views, partners are positives"),
+    ("linear_beta", "kalfa.std.objectives.LinearBeta", "value", "betas rising linearly from start to end over steps"),
+    ("cosine_beta", "kalfa.std.objectives.CosineBeta", "value", "the cosine beta schedule of Nichol and Dhariwal"),
+)
+
+for name, target, slot, doc in REGISTRATIONS:
+    register(name, target, slot=slot, doc=doc)

kalfa/std/plots/__init__.py

@@ -0,0 +1,75 @@
+from kalfa.core.aliases import register
+from kalfa.std.plots.classification import (
+    MAX_CLASSES,
+    SKEW,
+    binary_curve,
+    binary_precision_recall_curve,
+    binary_roc,
+    class_histogram,
+    classified,
+    confusion_matrix,
+    floats,
+    label_set,
+    score_column,
+)
+from kalfa.std.plots.common import (
+    GRID,
+    INK,
+    LEGEND_LIMIT,
+    MUTED,
+    color_of,
+    directory_of,
+    dress,
+    label_text,
+    quiet,
+    spans_decades,
+)
+from kalfa.std.plots.curves import DEFAULT_PREFIXES, history_series, loss_curve, turn_word
+from kalfa.std.plots.images import (
+    TEXT_LINES,
+    TEXT_WIDTH,
+    first_output,
+    image_field,
+    image_pairs,
+    pixels,
+    sample_images,
+    samples_grid,
+    shaped,
+    show,
+    text_sheet,
+)
+from kalfa.std.plots.pairs import chosen_pair, pick_pair, pred_column, prediction_pairs, read_predictions
+from kalfa.std.plots.predictions import (
+    HORIZON,
+    forecast_samples,
+    forecast_series,
+    horizon_pair,
+    indexed,
+    numeric,
+    pred_vs_true,
+    r_squared,
+)
+
+
+__all__ = ["DEFAULT_PREFIXES", "GRID", "HORIZON", "INK", "LEGEND_LIMIT", "MAX_CLASSES", "MUTED", "SKEW", "TEXT_LINES",
+           "TEXT_WIDTH", "binary_curve", "binary_precision_recall_curve", "binary_roc", "chosen_pair",
+           "class_histogram", "classified", "color_of", "confusion_matrix", "directory_of", "dress", "first_output",
+           "floats", "forecast_samples", "forecast_series", "history_series", "horizon_pair", "image_field",
+           "image_pairs", "indexed", "label_set", "label_text", "loss_curve", "numeric", "pick_pair", "pixels",
+           "pred_column", "pred_vs_true", "prediction_pairs", "quiet", "r_squared", "read_predictions",
+           "sample_images", "samples_grid", "score_column", "shaped", "show", "spans_decades", "text_sheet",
+           "turn_word"]
+
+for _name, _doc in (
+    ("loss_curve", "history.jsonl series against the turn; series names the keys, default every train/ and val/ key"),
+    ("pred_vs_true", "scatter of each pred_<output> against its target with a y = x line; pairs maps output to target"),
+    ("confusion_matrix", "confusion matrix of a label output against its target with counts; pred and target name the "
+                         "pair"),
+    ("image_pairs", "n test images above their first model output (reconstructions); needs the run"),
+    ("samples_grid", "a grid of the first n images in samples/<file>.pt, or the text of samples/<file>.txt"),
+    ("forecast_samples", "n windows of a horizon target: the truth and the prediction over the horizon steps"),
+    ("class_histogram", "the histogram of the positive class score per true class"),
+    ("binary_roc", "the ROC curve of a binary classification with its AUC"),
+    ("binary_precision_recall_curve", "the precision recall curve of a binary classification with its AP"),
+):
+    register(_name, f"kalfa.std.plots.{_name}", slot="plot", doc=_doc)

kalfa/std/plots/classification.py

@@ -0,0 +1,207 @@
+import numpy
+
+from kalfa.core.errors import KalfaError
+from kalfa.records import reader, stats
+from kalfa.std.plots.common import INK, LEGEND_LIMIT, MUTED, color_of, directory_of, dress, label_text, quiet
+from kalfa.std.plots.pairs import chosen_pair, pick_pair, read_predictions
+
+
+MAX_CLASSES = 64
+SKEW = 20.0
+
+
+def label_set(truth, guess):
+    values = set(truth.tolist()) | set(guess.tolist())
+    try:
+        return sorted(values)
+    except TypeError:
+        return sorted(values, key=str)
+
+
+def confusion_matrix(record, figures, pred=None, target=None):
+    directory = directory_of(record)
+    frame = read_predictions(directory)
+    if frame is None:
+        return None
+    pair = pick_pair(frame, quiet(reader.read_plan, directory), pred, target)
+    if pair is None:
+        return None
+    column, truth_column = pair
+    keep = frame[column].notna() & frame[truth_column].notna()
+    guess = frame.loc[keep, column].to_numpy()
+    truth = frame.loc[keep, truth_column].to_numpy()
+    labels = label_set(truth, guess)
+    if not labels or len(labels) > MAX_CLASSES:
+        return None
+    index = {label: position for position, label in enumerate(labels)}
+    matrix = numpy.zeros((len(labels), len(labels)), dtype="int64")
+    for true_value, guess_value in zip(truth.tolist(), guess.tolist()):
+        matrix[index[true_value], index[guess_value]] += 1
+    shares = matrix / numpy.maximum(matrix.sum(axis=1, keepdims=True), 1)
+    side = min(16.0, 2.0 + 0.7 * len(labels))
+    figure = figures.new(side + 1.2, side)
+    axis = figure.subplots()
+    drawn = axis.imshow(shares, cmap="Blues", vmin=0.0, vmax=1.0)
+    for row in range(len(labels)):
+        for col in range(len(labels)):
+            axis.text(col, row, f"{matrix[row, col]:,}\n{shares[row, col] * 100:.0f}%", ha="center", va="center",
+                      fontsize=8, color="#ffffff" if shares[row, col] > 0.55 else INK)
+    names = [label_text(label) for label in labels]
+    tilt = len(names) > 6 or max(len(name) for name in names) > 6
+    axis.set_xticks(range(len(names)), names, rotation=45 if tilt else 0, ha="right" if tilt else "center")
+    axis.set_yticks(range(len(names)), names)
+    axis.grid(False)
+    figure.colorbar(drawn, ax=axis, fraction=0.046, pad=0.03, label="row share")
+    axis.set_title(f"Confusion matrix ({truth_column}, n = {int(matrix.sum()):,})", loc="left")
+    axis.set_xlabel(f"predicted ({column[len('pred_'):]})")
+    axis.set_ylabel(f"true {truth_column}")
+    figures.save(figure, "confusion_matrix")
+    return None
+
+
+def classified(directory, plot, pred, target):
+    frame = stats.read_predictions(directory)
+    if frame is None or not len(frame):
+        return None
+    plan = quiet(reader.read_plan, directory)
+    fitted = quiet(reader.fields_of, directory)
+    pair = chosen_pair(frame, plan, fitted, pred, target, prefer="classification")
+    if pair is None:
+        return None
+    if pair["kind"] != "classification":
+        raise KalfaError(f"{plot}: the predictions carry no probabilities for {pair['pred']}; its target "
+                         f"{pair['target']} is a regression target, not a label")
+    return frame, plan, fitted, pair
+
+
+def score_column(pair, classes):
+    scores = pair["scores"]
+    if scores:
+        named = stats.positive_column(scores, pair["output"], classes) if classes and len(classes) == 2 else None
+        return named or scores[-1]
+    return pair["raw"]
+
+
+def class_histogram(record, figures, pred=None, target=None, bins=40):
+    import pandas
+    from matplotlib.colors import to_rgba
+
+    directory = directory_of(record)
+    found = classified(directory, "class_histogram", pred, target)
+    if found is None:
+        return None
+    frame, plan, fitted, pair = found
+    output, truth_column = pair["output"], pair["target"]
+    classes = stats.classes_for(plan, fitted, truth_column)
+    column = score_column(pair, classes)
+    if column is None:
+        raise KalfaError(f"class_histogram: the predictions carry no probabilities (prob_{output}_<class>) and no "
+                         f"raw_{output} scores for {pair['pred']}")
+    scores = pandas.to_numeric(frame[column], errors="coerce").to_numpy(dtype="float64")
+    known = frame[truth_column].notna().to_numpy()
+    keep = numpy.isfinite(scores) & known
+    if not keep.any():
+        return None
+    scores = scores[keep]
+    truth = numpy.asarray([label_text(value) for value in frame.loc[keep, truth_column].tolist()], dtype=object)
+    seen = sorted({str(value) for value in truth.tolist()})
+    order = list(classes or [])
+    labels = [label for label in order if label in seen] + [label for label in seen if label not in order]
+    if len(labels) > MAX_CLASSES:
+        return None
+    if column.startswith("prob_") and scores.min() >= 0.0 and scores.max() <= 1.0:
+        edges = numpy.linspace(0.0, 1.0, int(bins) + 1)
+    else:
+        edges = numpy.histogram_bin_edges(scores, bins=int(bins))
+    counts = [numpy.histogram(scores[truth == label], bins=edges)[0] for label in labels]
+    peaks = [int(count.max()) for count in counts if count.max() > 0]
+    log = len(peaks) > 1 and max(peaks) / min(peaks) > SKEW
+    figure = figures.new(8.0, 5.0)
+    axis = figure.subplots()
+    for position, (label, count) in enumerate(zip(labels, counts)):
+        color = color_of(figures, position)
+        axis.stairs(count, edges, fill=True, baseline=0.5 if log else 0.0, facecolor=to_rgba(color, 0.3),
+                    edgecolor=color, linewidth=1.4, label=f"{label} (n = {int(count.sum()):,})")
+    if log:
+        axis.set_yscale("log")
+        axis.set_ylim(bottom=0.5)
+    if len(labels) <= LEGEND_LIMIT:
+        axis.legend(loc="upper center", frameon=False, title=f"true {truth_column}")
+    prefix = f"prob_{output}_"
+    axis.set_xlabel(f"P({column[len(prefix):]})" if column.startswith(prefix) else column)
+    axis.set_ylabel("rows")
+    axis.set_title(f"Score by true class ({truth_column}, n = {len(scores):,})", loc="left")
+    dress(axis)
+    figures.save(figure, "class_histogram")
+    return None
+
+
+def floats(values):
+    return numpy.asarray([numpy.nan if value is None else value for value in values], dtype="float64")
+
+
+def binary_curve(record, plot, pred, target):
+    found = classified(directory_of(record), plot, pred, target)
+    if found is None:
+        return None
+    frame, plan, fitted, pair = found
+    output, truth_column = pair["output"], pair["target"]
+    if not pair["scores"]:
+        raise KalfaError(f"{plot}: the predictions carry no probabilities for {pair['pred']}; a binary curve reads "
+                         f"the prob_{output}_<class> columns that a label target writes")
+    classes = stats.classes_for(plan, fitted, truth_column)
+    result = stats.classification_stats(frame, pair["pred"], truth_column, pair["raw"], pair["scores"], classes)
+    if len(result["classes"]) != 2:
+        raise KalfaError(f"{plot} needs two classes; {truth_column} has {len(result['classes'])}: "
+                         f"{', '.join(result['classes'])}")
+    if result["roc"] is None:
+        return None
+    return result, pair
+
+
+def binary_roc(record, figures, pred=None, target=None):
+    found = binary_curve(record, "binary_roc", pred, target)
+    if found is None:
+        return None
+    result, pair = found
+    roc = result["roc"]
+    figure = figures.new(5.6, 5.0)
+    axis = figure.subplots()
+    axis.plot(floats(roc["fpr"]), floats(roc["tpr"]), color=color_of(figures, 0), linewidth=2.0,
+              label=f"{pair['output']} (AUC {roc['auc']:.3f})")
+    axis.plot([0.0, 1.0], [0.0, 1.0], color=MUTED, linewidth=1.0, linestyle="--", label="chance")
+    axis.set_xlim(0.0, 1.0)
+    axis.set_ylim(0.0, 1.02)
+    axis.legend(loc="lower right", frameon=False)
+    positive = result["classes"][1]
+    axis.set_title(f"ROC ({pair['target']} = {positive}, AUC {roc['auc']:.3f}, n = {result['count']:,})", loc="left")
+    axis.set_xlabel("false positive rate")
+    axis.set_ylabel("true positive rate")
+    dress(axis)
+    figures.save(figure, "binary_roc")
+    return None
+
+
+def binary_precision_recall_curve(record, figures, pred=None, target=None):
+    found = binary_curve(record, "binary_precision_recall_curve", pred, target)
+    if found is None:
+        return None
+    result, pair = found
+    pr = result["pr"]
+    positive = result["classes"][1]
+    rate = result["per_class"][positive]["support"] / result["count"]
+    figure = figures.new(5.6, 5.0)
+    axis = figure.subplots()
+    axis.step(floats(pr["recall"]), floats(pr["precision"]), where="post", color=color_of(figures, 0), linewidth=2.0,
+              label=f"{pair['output']} (AP {pr['ap']:.3f})")
+    axis.axhline(rate, color=MUTED, linewidth=1.0, linestyle="--", label=f"base rate {rate:.3f}")
+    axis.set_xlim(0.0, 1.0)
+    axis.set_ylim(0.0, 1.02)
+    axis.legend(loc="lower left", frameon=False)
+    axis.set_title(f"Precision and recall ({pair['target']} = {positive}, AP {pr['ap']:.3f}, "
+                   f"n = {result['count']:,})", loc="left")
+    axis.set_xlabel("recall")
+    axis.set_ylabel("precision")
+    dress(axis)
+    figures.save(figure, "binary_precision_recall_curve")
+    return None

kalfa/std/plots/common.py

@@ -0,0 +1,44 @@
+import math
+from pathlib import Path
+
+
+LEGEND_LIMIT = 12
+GRID = "#e3e2dd"
+INK = "#0b0b0b"
+MUTED = "#8c8b86"
+
+
+def directory_of(record):
+    return Path(getattr(record, "directory", record))
+
+
+def quiet(read, directory):
+    try:
+        return read(directory)
+    except (OSError, ValueError):
+        return None
+
+
+def color_of(figures, position):
+    colors = getattr(figures, "colors", None) or tuple(f"C{index}" for index in range(10))
+    return colors[position % len(colors)]
+
+
+def dress(axis):
+    axis.grid(True, color=GRID, linewidth=0.8)
+    axis.set_axisbelow(True)
+    for side in ("top", "right"):
+        axis.spines[side].set_visible(False)
+
+
+def spans_decades(values):
+    finite = [value for value in values if math.isfinite(value)]
+    if not finite or any(value <= 0 for value in finite):
+        return False
+    return max(finite) / min(finite) > 100.0
+
+
+def label_text(value):
+    if isinstance(value, float) and value.is_integer():
+        return str(int(value))
+    return str(value)