kalfa 3.0.0.dev0__py3-none-any.whl

kalfa/records/stats/pairs.py

@@ -0,0 +1,225 @@
+import numpy
+import pandas
+
+from .tables import finite, label_text, labels, numbers_of, numeric, ordered_labels, texts
+
+
+CODES = (3, 20)
+
+
+def plan_nodes(plan):
+    nodes = (plan or {}).get("nodes")
+    return nodes if isinstance(nodes, dict) else {}
+
+
+def plan_targets(plan):
+    node = plan_nodes(plan).get("data")
+    params = node.get("params") if isinstance(node, dict) else None
+    targets = params.get("targets") if isinstance(params, dict) else None
+    if isinstance(targets, dict) and targets:
+        return [str(name) for name in targets]
+    if isinstance(targets, (list, tuple)) and targets:
+        return [str(name) for name in targets]
+    if isinstance(targets, str):
+        return [targets]
+    fields = ((plan or {}).get("data") or {}).get("fields")
+    if isinstance(fields, dict):
+        return [str(name) for name, field in fields.items() if isinstance(field, dict) and field.get("target")]
+    return []
+
+
+def plan_classes(plan):
+    return labels(((plan or {}).get("data") or {}).get("classes"))
+
+
+def fitted_classes(fields, target=None):
+    if not isinstance(fields, dict):
+        return None
+    if target is not None:
+        for item in fields.get("target_fields") or ():
+            if isinstance(item, dict) and item.get("name") == target:
+                return labels(item.get("classes"))
+    return labels(fields.get("classes"))
+
+
+def fitted_entries(fields):
+    if not isinstance(fields, dict):
+        return {}
+    return {str(item["name"]): labels(item.get("classes")) for item in fields.get("target_fields") or ()
+            if isinstance(item, dict) and item.get("name") is not None}
+
+
+def classes_for(plan, fitted, target=None):
+    entries = fitted_entries(fitted)
+    if target in entries and (entries[target] is None or sum(1 for found in entries.values() if found) > 1):
+        return entries[target]
+    return plan_classes(plan) or fitted_classes(fitted, target)
+
+
+def plan_pairings(plan):
+    pairs = {}
+    for path, node in plan_nodes(plan).items():
+        if not isinstance(node, dict) or node.get("slot") not in ("loss", "metric"):
+            continue
+        keys = node.get("keys") if isinstance(node.get("keys"), dict) else {}
+        pred, target = keys.get("pred"), keys.get("target")
+        if isinstance(pred, str) and isinstance(target, str):
+            pairs.setdefault(pred, target)
+    return pairs
+
+
+def frame_targets(frame):
+    return [column for column in frame.columns
+            if column != "row" and not str(column).startswith(("pred_", "raw_", "prob_"))]
+
+
+def kind_of(frame, target, classes, scores):
+    if classes or scores:
+        return "classification"
+    return "regression" if pandas.api.types.is_numeric_dtype(frame[target]) else "classification"
+
+
+def spread_member(column, columns):
+    stem, _, position = str(column).rpartition("_")
+    if stem and position.isdigit() and f"{stem}_0" in columns and f"{stem}_1" in columns:
+        return stem, int(position)
+    return None
+
+
+def declared_targets(plan, fitted, frame):
+    targets = plan_targets(plan)
+    if targets:
+        return targets
+    if isinstance(fitted, dict) and isinstance(fitted.get("targets"), list):
+        return [str(name) for name in fitted["targets"]]
+    found = frame_targets(frame)
+    return found if len(found) == 1 else []
+
+
+def matches(frame, plan, fitted=None):
+    columns = [str(column) for column in frame.columns]
+    pairings = plan_pairings(plan)
+    targets = declared_targets(plan, fitted, frame)
+    fallback = targets[0] if len(targets) == 1 else None
+    found = []
+    for column in columns:
+        if not column.startswith("pred_"):
+            continue
+        output = column[len("pred_"):]
+        target, base = pairings.get(output), None
+        member = None if target is not None else spread_member(column, columns)
+        if member is not None:
+            base = pairings.get(member[0][len("pred_"):], fallback)
+            target = None if base is None else f"{base}_{member[1]}"
+        elif target is None:
+            target = fallback
+        if target is None or target not in columns or target == column:
+            target = None
+        found.append((column, output, target, base))
+    return found
+
+
+def raw_of(columns, output):
+    return f"raw_{output}" if f"raw_{output}" in columns else None
+
+
+def scores_of(columns, output):
+    return [name for name in columns if name.startswith(f"prob_{output}_")]
+
+
+def suffix_label(suffix, known):
+    if suffix in known:
+        return suffix
+    try:
+        number = float(suffix)
+    except ValueError:
+        return suffix
+    return label_text(number) or suffix
+
+
+def score_labels(scores, output, known=()):
+    prefix = f"prob_{output}_"
+    known = set(known or ())
+    return {column: suffix_label(column[len(prefix):], known) for column in scores if column.startswith(prefix)}
+
+
+def label_columns(scores, output, known=()):
+    found = {}
+    for column, label in score_labels(scores, output, known).items():
+        found.setdefault(label, column)
+    return found
+
+
+def prediction_pairs(frame, plan, fitted=None):
+    if frame is None:
+        return []
+    columns = [str(column) for column in frame.columns]
+    pairs = []
+    for column, output, target, base in matches(frame, plan, fitted):
+        if target is None:
+            continue
+        raw, scores = raw_of(columns, output), scores_of(columns, output)
+        kind = kind_of(frame, target, classes_for(plan, fitted, base or target), scores)
+        pairs.append({"output": output, "pred": column, "target": target, "kind": kind, "raw": raw, "scores": scores})
+    return pairs
+
+
+def observed(frame, pred, target):
+    return ordered_labels(set(texts(frame[target]).tolist()) | set(texts(frame[pred]).tolist()))
+
+
+def detected(frame, pred, target, classes, scores):
+    truth = frame[target]
+    if classes or scores or not numeric(truth):
+        found = classes or observed(frame, pred, target)
+        if len(found) == 2:
+            return "binary", found, ["binary", "multiclass"]
+        return "multiclass", found, ["multiclass"]
+    distinct = numpy.unique(finite(numbers_of(truth)))
+    if len(distinct) == 2:
+        return "binary", [label_text(value) for value in distinct], ["binary", "regression"]
+    low, high = CODES
+    if numeric(frame[pred]) and low <= len(distinct) <= high and bool(numpy.all(distinct == numpy.round(distinct))):
+        guessed = numpy.unique(finite(numbers_of(frame[pred])))
+        if bool(numpy.isin(guessed, distinct).all()):
+            return "multiclass", [label_text(value) for value in distinct], ["multiclass", "regression"]
+    return "regression", None, ["regression"]
+
+
+def target_pair(frame, column, output, target, classes):
+    columns = [str(name) for name in frame.columns]
+    scores = scores_of(columns, output)
+    auto, found, tasks = detected(frame, column, target, classes, scores)
+    return {"output": output, "pred": column, "columns": [column], "target": target, "auto": auto, "tasks": tasks,
+            "raw": raw_of(columns, output), "scores": scores, "classes": found}
+
+
+def loose_pair(output, pred, task, columns, raw=None):
+    return {"output": output, "pred": pred, "columns": columns, "target": None, "auto": task, "tasks": [task],
+            "raw": raw, "scores": [], "classes": None}
+
+
+def task_pairs(frame, plan, fitted=None):
+    columns = [str(column) for column in frame.columns]
+    found, spreads = [], {}
+    for column, output, target, base in matches(frame, plan, fitted):
+        if target is not None:
+            found.append(target_pair(frame, column, output, target, classes_for(plan, fitted, base or target)))
+            continue
+        member = spread_member(column, columns)
+        if member is None:
+            found.append(loose_pair(output, column, "values", [column], raw_of(columns, output)))
+            continue
+        stem = member[0]
+        if stem not in spreads:
+            spreads[stem] = loose_pair(stem[len("pred_"):], stem, "embedding", [])
+            found.append(spreads[stem])
+        spreads[stem]["columns"].append(column)
+    return [single(pair, columns) if pair["auto"] == "embedding" and len(pair["columns"]) < 2 else pair
+            for pair in found]
+
+
+def single(pair, columns):
+    column = pair["columns"][0]
+    output = column[len("pred_"):]
+    return loose_pair(output, column, "values", [column], raw_of(columns, output))

kalfa/records/stats/tables.py

@@ -0,0 +1,153 @@
+import math
+import numbers
+from pathlib import Path
+
+import numpy
+import pandas
+
+from ...core.errors import KalfaError
+from ..reader import fields_of
+
+
+PREDICTIONS = "predictions.parquet"
+IMAGE_TYPES = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".gif": "image/gif",
+               ".bmp": "image/bmp", ".webp": "image/webp"}
+IMAGE_SUFFIXES = tuple(IMAGE_TYPES)
+
+
+def load_table(directory, name=PREDICTIONS):
+    path = Path(directory) / name
+    if not path.is_file():
+        return None
+    try:
+        return pandas.read_parquet(path)
+    except (OSError, ValueError):
+        return None
+
+
+def filtered(frame, where, name=PREDICTIONS):
+    if not where:
+        return frame
+    try:
+        return frame.query(where)
+    except Exception as exc:
+        raise KalfaError(f"bad where for {name}: {exc}") from exc
+
+
+def read_predictions(directory, name=PREDICTIONS, where=None):
+    frame = load_table(directory, name)
+    return None if frame is None else filtered(frame, where, name)
+
+
+def prediction_tables(directory):
+    root = Path(directory)
+    if not root.is_dir():
+        return []
+    return sorted(path.name for path in root.glob("predictions*.parquet") if path.is_file())
+
+
+def read_fitted_fields(directory):
+    return fields_of(directory)
+
+
+def image_column(frame):
+    if "path" not in [str(column) for column in frame.columns]:
+        return None
+    values = [value for value in frame["path"].tolist() if not missing(value)]
+    if values and all(isinstance(value, str) and value.lower().endswith(IMAGE_SUFFIXES) for value in values):
+        return "path"
+    return None
+
+
+def missing(value):
+    if value is None:
+        return True
+    try:
+        return bool(pandas.isna(value))
+    except (TypeError, ValueError):
+        return False
+
+
+def label_text(value):
+    if isinstance(value, numpy.generic):
+        value = value.item()
+    if missing(value):
+        return None
+    if isinstance(value, bool):
+        return str(value)
+    if isinstance(value, numbers.Integral):
+        return str(int(value))
+    if isinstance(value, numbers.Real):
+        number = float(value)
+        if math.isfinite(number) and number.is_integer():
+            return str(int(number))
+        return format(number, ".12g")
+    return str(value)
+
+
+def labels(classes):
+    if not isinstance(classes, (list, tuple)) or not classes:
+        return None
+    return [label_text(label) or str(label) for label in classes]
+
+
+def label_key(text):
+    try:
+        number = float(text)
+    except (TypeError, ValueError):
+        return 1, 0.0, str(text)
+    return (0, number, "") if math.isfinite(number) else (1, 0.0, str(text))
+
+
+def ordered_labels(values):
+    return sorted({value for value in values if value is not None}, key=label_key)
+
+
+def cached_text(known, value):
+    try:
+        return known[(type(value), value)]
+    except KeyError:
+        text = known[(type(value), value)] = label_text(value)
+        return text
+    except TypeError:
+        return label_text(value)
+
+
+def texts(series):
+    known = {}
+    return numpy.asarray([cached_text(known, value) for value in series.tolist()], dtype=object)
+
+
+def present(values):
+    return numpy.fromiter((value is not None for value in values), dtype=bool, count=len(values))
+
+
+def numeric(series):
+    return pandas.api.types.is_numeric_dtype(series) and not pandas.api.types.is_bool_dtype(series)
+
+
+def numbers_of(series):
+    return pandas.to_numeric(series, errors="coerce").to_numpy(dtype=numpy.float64)
+
+
+def finite(values):
+    array = numpy.asarray(values, dtype=numpy.float64).reshape(-1)
+    return array[numpy.isfinite(array)]
+
+
+def plain(values):
+    return [None if isinstance(value, float) and not math.isfinite(value) else value for value in values.tolist()]
+
+
+def plain_value(value):
+    if isinstance(value, numpy.generic):
+        value = value.item()
+    if isinstance(value, float) and not math.isfinite(value):
+        return None
+    return None if missing(value) else value
+
+
+def rows_in(frame):
+    if "row" in frame.columns:
+        return frame["row"].to_numpy()
+    return frame.index.to_numpy()

kalfa/records/sweep.py

@@ -0,0 +1,244 @@
+import json
+import math
+import re
+from pathlib import Path
+
+import numpy
+
+from .reader import (
+    MANIFEST,
+    SWEEP,
+    best_in,
+    is_finite,
+    is_number,
+    is_sweep_root,
+    normal_mode,
+    read_history,
+    read_json,
+    record_stamp,
+    safe_json,
+    summary_of,
+)
+
+
+POINT_NAME = re.compile(r"^\d{4}$")
+FAILED = "failed"
+
+
+is_sweep = is_sweep_root
+
+
+def read_sweep(root):
+    return read_json(root, SWEEP)
+
+
+def point_dirs(root):
+    root = Path(root)
+    try:
+        found = [entry for entry in root.iterdir() if entry.is_dir() and POINT_NAME.match(entry.name)]
+    except OSError:
+        return []
+    return sorted(found, key=lambda path: int(path.name))
+
+
+def objective_of(summary, objective):
+    if not isinstance(summary, dict) or not isinstance(objective, dict):
+        return None
+    watch = objective.get("watch")
+    if watch is None:
+        return None
+    if str(objective.get("at") or "best") == "last":
+        last = summary.get("last")
+        value = last.get(watch) if isinstance(last, dict) else None
+    else:
+        value = best_value(summary, watch, mode_of(objective))
+    return float(value) if is_finite(value) else None
+
+
+def best_value(summary, watch, mode):
+    best = summary.get("best")
+    if isinstance(best, dict) and best.get("series") == watch and normal_mode(best.get("mode")) == mode:
+        return best.get("value")
+    path = summary.get("path")
+    if not isinstance(path, (str, Path)):
+        return None
+    return best_in(read_history(path), watch, mode)[0]
+
+
+def point_id(summary, directory):
+    sweep = summary.get("sweep")
+    if isinstance(sweep, dict) and is_number(sweep.get("id")):
+        return int(sweep["id"])
+    return int(directory.name)
+
+
+def point_values(summary):
+    sweep = summary.get("sweep")
+    if isinstance(sweep, dict) and isinstance(sweep.get("values"), dict):
+        return dict(sweep["values"])
+    params = summary.get("params")
+    return dict(params) if isinstance(params, dict) else {}
+
+
+def point_of(directory, root, objective):
+    summary = summary_of(directory)
+    if not isinstance(summary, dict):
+        return None
+    status = summary.get("status")
+    return {
+        "id": point_id(summary, directory),
+        "path": directory.relative_to(root).as_posix(),
+        "values": point_values(summary),
+        "status": status,
+        "turn": summary.get("turn"),
+        "total": summary.get("total"),
+        "objective": None if status == FAILED else objective_of(summary, objective),
+        "summary": summary,
+    }
+
+
+def sweep_points(root, cache=None):
+    root = Path(root)
+    sweep = read_sweep(root)
+    objective = sweep.get("objective") if isinstance(sweep, dict) else None
+    asked = json.dumps(objective, sort_keys=True, default=str)
+    points = []
+    for directory in point_dirs(root):
+        if cache is None:
+            point = point_of(directory, root, objective)
+        else:
+            stamp = record_stamp(directory, safe_json(directory, MANIFEST))
+            point = cache.get(("point", str(directory), str(root), asked), stamp,
+                              lambda directory=directory: point_of(directory, root, objective))
+        if point is not None:
+            points.append(dict(point))
+    return points
+
+
+def is_valued(point):
+    return is_finite(point.get("objective")) and point.get("status") != FAILED
+
+
+def id_key(point):
+    identity = point.get("id")
+    return (0, identity, "") if is_number(identity) else (1, 0, str(identity))
+
+
+def mode_of(objective):
+    return normal_mode(objective.get("mode") if isinstance(objective, dict) else objective)
+
+
+def rank_points(points, objective):
+    maximize = mode_of(objective) == "max"
+    valued = [point for point in points if is_valued(point)]
+    unvalued = [point for point in points if not is_valued(point)]
+    valued.sort(key=lambda point: ((-point["objective"] if maximize else point["objective"]), id_key(point)))
+    unvalued.sort(key=id_key)
+    return valued + unvalued
+
+
+def best_point(points, objective):
+    ranked = rank_points(points, objective)
+    if ranked and is_valued(ranked[0]):
+        return ranked[0]
+    return None
+
+
+def running_best(points, objective):
+    maximize = mode_of(objective) == "max"
+    ordered = sorted(points, key=id_key)
+    best = None
+    found = {"ids": [], "objective": [], "best": []}
+    for point in ordered:
+        value = point["objective"] if is_valued(point) else None
+        if value is not None and (best is None or (value > best if maximize else value < best)):
+            best = value
+        found["ids"].append(point.get("id"))
+        found["objective"].append(value)
+        found["best"].append(best)
+    return found
+
+
+def is_log_space(spec):
+    return isinstance(spec, dict) and isinstance(spec.get("log"), (list, tuple))
+
+
+def label_of(value):
+    if isinstance(value, float):
+        return f"{value:.4g}"
+    return value if isinstance(value, str) else json.dumps(value, default=str)
+
+
+def axis_of(key, values, spec):
+    numeric = all(is_finite(value) for value in values)
+    if numeric and is_log_space(spec) and all(value > 0 for value in values):
+        shown = sorted(set(values))
+        return {"key": key, "label": f"{key} (log)", "values": [math.log10(value) for value in values],
+                "ticks": [math.log10(value) for value in shown] if len(shown) <= 12 else None,
+                "text": [label_of(float(value)) for value in shown] if len(shown) <= 12 else None}
+    if numeric:
+        return {"key": key, "label": key, "values": [float(value) for value in values], "ticks": None, "text": None}
+    names = []
+    for value in values:
+        name = label_of(value)
+        if name not in names:
+            names.append(name)
+    return {"key": key, "label": key, "values": [float(names.index(label_of(value))) for value in values],
+            "ticks": [float(position) for position in range(len(names))], "text": names}
+
+
+def axes_of(points, sweep):
+    sweep = sweep if isinstance(sweep, dict) else {}
+    objective = sweep.get("objective") if isinstance(sweep.get("objective"), dict) else {}
+    space = sweep.get("space") if isinstance(sweep.get("space"), dict) else {}
+    valued = sorted((point for point in points if is_valued(point)), key=id_key)
+    keys = []
+    for point in valued:
+        keys.extend(key for key in point["values"] if key not in keys)
+    keys = [key for key in keys if all(key in point["values"] for point in valued)]
+    axes = [axis_of(key, [point["values"][key] for point in valued], space.get(key)) for key in keys]
+    watch = str(objective.get("watch") or "objective")
+    axes.append({"key": "objective", "label": watch, "values": [float(point["objective"]) for point in valued],
+                 "ticks": None, "text": None})
+    return {"ids": [point.get("id") for point in valued], "axes": axes if valued else [],
+            "left_out": len(points) - len(valued), "mode": mode_of(objective)}
+
+
+def frozen(value):
+    if isinstance(value, dict):
+        return tuple(sorted((str(key), frozen(item)) for key, item in value.items()))
+    if isinstance(value, (list, tuple, set)):
+        return tuple(frozen(item) for item in value)
+    return value
+
+
+def numeric_last(summary):
+    last = summary.get("last")
+    if not isinstance(last, dict):
+        return {}
+    return {key: float(value) for key, value in last.items() if is_finite(value)}
+
+
+def group_folds(summaries, key="fold"):
+    groups = {}
+    for summary in summaries:
+        if not isinstance(summary, dict):
+            continue
+        params = summary.get("params")
+        if not isinstance(params, dict) or key not in params:
+            continue
+        rest = {name: value for name, value in params.items() if name != key}
+        group_key = frozen(rest)
+        if group_key not in groups:
+            groups[group_key] = {"group": rest, "members": [], "mean": {}, "std": {}}
+        groups[group_key]["members"].append(summary)
+    for group in groups.values():
+        lasts = [numeric_last(member) for member in group["members"]]
+        common = set(lasts[0])
+        for last in lasts[1:]:
+            common &= set(last)
+        for series in sorted(common):
+            values = numpy.asarray([last[series] for last in lasts], dtype=float)
+            group["mean"][series] = float(values.mean())
+            group["std"][series] = float(numpy.std(values))
+    return list(groups.values())

kalfa/std/__init__.py

@@ -0,0 +1,17 @@
+from . import (  # noqa: F401
+    actions,
+    data,
+    init,
+    loggers,
+    losses,
+    metrics,
+    models,
+    objectives,
+    plots,
+    samplers,
+    samples,
+    schedules,
+    steps,
+    triggers,
+)
+from . import notes  # noqa: E402,F401

kalfa/std/actions.py

@@ -0,0 +1,35 @@
+import torch
+
+from kalfa.core.aliases import register
+from kalfa.core.errors import BuildError, RunError
+from kalfa.core.protocols import Action
+from kalfa.std.samples import save_samples
+
+
+class WriteSamples(Action):
+    requires_generate = True
+
+    def __init__(self, sampler=None, n=16):
+        self.sampler = sampler
+        self.n = int(n)
+
+    def bind(self, ctx):
+        if self.sampler is None and getattr(ctx, "generate", None) is None:
+            raise BuildError(f"{self.path or 'write_samples'}: write_samples needs a sampler: write sampler: "
+                             "{use: gan_sampler, ...} or a generate section")
+
+    def __call__(self, ctx):
+        sampler = self.sampler if self.sampler is not None else getattr(ctx, "generate", None)
+        if sampler is None:
+            raise RunError(f"{self.path or 'write_samples'} needs a sampler: write sampler: <a sampler call> or a "
+                           "generate section")
+        with torch.no_grad():
+            samples = sampler.sample(ctx, self.n)
+        if getattr(ctx, "record", None) is None:
+            return []
+        return save_samples(samples, ctx.record.path("samples"), f"turn_{ctx.turn}")
+
+
+register("write_samples", "kalfa.std.actions.WriteSamples", slot="action",
+         doc="draws n samples from its sampler (or the run's generate) and writes samples/turn_<k>.pt, a png grid "
+             "for images or a txt for text")