supaschema 0.1.0-rc.1

package/benchmarks/plot-lib.js

@@ -0,0 +1,234 @@
+export const fixtureScale = {
+  additive: { order: 0, tables: "1 table" },
+  "functions-policies": { order: 1, tables: "1 table" },
+  realistic: { order: 2, tables: "50 tables" },
+  xl: { order: 3, tables: "1,000 tables" },
+  xxl: { order: 4, tables: "2,500 tables" },
+};
+
+export const theme = {
+  accent: "#34d399",
+  accentDeep: "#059669",
+  amber: "#fbbf24",
+  bg: "#0b1220",
+  fail: "#f87171",
+  failStroke: "#ef4444",
+  grid: "#1e293b",
+  muted: "#64748b",
+  pass: "#34d399",
+  passStroke: "#10b981",
+  slateBar: "#526079",
+  slateBarDeep: "#3b4757",
+  subtitle: "#94a3b8",
+  text: "#e2e8f0",
+  title: "#f8fafc",
+};
+
+const fontStack = "ui-sans-serif, system-ui, 'Segoe UI', Helvetica, Arial, sans-serif";
+
+export function formatSeconds(ms) {
+  const seconds = ms / 1000;
+  if (seconds < 10) {
+    return `${seconds.toFixed(2)}s`;
+  }
+  if (seconds < 100) {
+    return `${seconds.toFixed(1)}s`;
+  }
+  return `${Math.round(seconds)}s`;
+}
+
+export function isSupaschema(label) {
+  return label.startsWith("supaschema");
+}
+
+export function isWorkflow(label) {
+  return label.endsWith("-workflow");
+}
+
+export function logTicks(minMs, maxMs) {
+  const candidates = [
+    10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10_000, 20_000, 50_000, 100_000, 200_000, 500_000,
+    1_000_000,
+  ];
+  return candidates.filter((value) => value >= minMs && value <= maxMs * 1.05);
+}
+
+export function logDomain(values) {
+  const min = Math.min(...values);
+  const max = Math.max(...values);
+  const floor = 10 ** Math.floor(Math.log10(Math.max(1, min)));
+  const ceil = 10 ** Math.ceil(Math.log10(Math.max(floor * 10, max)));
+  return { ceil, floor };
+}
+
+export function logX(value, domain, x0, chartWidth) {
+  const span = Math.log10(domain.ceil) - Math.log10(domain.floor);
+  return x0 + ((Math.log10(Math.max(1, value)) - Math.log10(domain.floor)) / span) * chartWidth;
+}
+
+export function envFooter(environments, fixture) {
+  const env =
+    (fixture === undefined
+      ? environments[0]
+      : environments.find((item) => item.fixtures.includes(fixture))) ??
+    environments[0] ??
+    {};
+  const supabaseVersion = env.toolVersions?.supabase
+    ? `Supabase CLI ${env.toolVersions.supabase}`
+    : undefined;
+  const supaschemaVersion = env.toolVersions?.supaschema
+    ? `supaschema ${env.toolVersions.supaschema}`
+    : undefined;
+  return [
+    supaschemaVersion,
+    supabaseVersion,
+    env.node ? `Node ${env.node}` : undefined,
+    platformLabel(env.platform, env.arch),
+    iterationsLabel(fixture === undefined ? environments : [env]),
+  ]
+    .filter(Boolean)
+    .join("  ·  ");
+}
+
+function iterationsLabel(environments) {
+  const counts = [
+    ...new Set(environments.map((item) => item.iterations).filter((value) => value > 0)),
+  ].sort((left, right) => left - right);
+  if (counts.length === 0) {
+    return undefined;
+  }
+  if (counts.length === 1) {
+    return `${counts[0]} iteration${counts[0] === 1 ? "" : "s"}`;
+  }
+  return `${counts[0]}–${counts.at(-1)} iterations per fixture`;
+}
+
+function platformLabel(platform, arch) {
+  if (platform === "darwin") {
+    return arch === "arm64" ? "Apple Silicon" : "macOS";
+  }
+  return [platform, arch].filter(Boolean).join(" ") || undefined;
+}
+
+export function groupedStats(rows) {
+  const map = new Map();
+  for (const row of rows) {
+    const label = row.adapter;
+    const bucket = map.get(label) ?? [];
+    bucket.push(row.elapsedMs);
+    map.set(label, bucket);
+  }
+  return [...map.entries()]
+    .map(([label, values]) => ({
+      label,
+      median: percentile(values, 0.5),
+      p95: percentile(values, 0.95),
+    }))
+    .sort((left, right) => left.median - right.median || left.label.localeCompare(right.label));
+}
+
+export function groupedCorrectness(rows) {
+  const map = new Map();
+  for (const row of rows) {
+    const label = row.adapter;
+    const bucket = map.get(label) ?? [];
+    bucket.push(row);
+    map.set(label, bucket);
+  }
+  return [...map.entries()]
+    .map(([label, values]) => {
+      const measured = values.filter((item) => !item.skipped && !item.unsupported);
+      const scored = measured.filter((item) => typeof item.outputF1 === "number");
+      return {
+        f1:
+          scored.length > 0
+            ? scored.reduce((sum, item) => sum + item.outputF1, 0) / scored.length
+            : undefined,
+        label,
+        match: measured.filter((item) => item.matchesTargetAfterFirstApply).length,
+        once: measured.filter((item) => item.appliesOnce).length,
+        total: measured.length,
+        twice: measured.filter((item) => item.appliesTwice).length,
+      };
+    })
+    .sort(
+      (left, right) =>
+        Number(isSupaschema(right.label)) - Number(isSupaschema(left.label)) ||
+        left.label.localeCompare(right.label),
+    );
+}
+
+export function percentile(values, percentileValue) {
+  const sorted = [...values].sort((left, right) => left - right);
+  const index = Math.min(sorted.length - 1, Math.ceil(sorted.length * percentileValue) - 1);
+  return sorted[index] ?? 0;
+}
+
+export function svgHeader(width, height) {
+  return `<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}" viewBox="0 0 ${width} ${height}" role="img">`;
+}
+
+export function defs() {
+  return `<defs>
+<linearGradient id="supaGradient" x1="0" y1="0" x2="1" y2="0">
+<stop offset="0%" stop-color="${theme.accentDeep}" />
+<stop offset="100%" stop-color="${theme.accent}" />
+</linearGradient>
+<linearGradient id="slateGradient" x1="0" y1="0" x2="1" y2="0">
+<stop offset="0%" stop-color="${theme.slateBarDeep}" />
+<stop offset="100%" stop-color="${theme.slateBar}" />
+</linearGradient>
+</defs>`;
+}
+
+export function svgFooter() {
+  return "</svg>";
+}
+
+export function text(x, y, value, options = {}) {
+  const anchor = options.anchor ? ` text-anchor="${options.anchor}"` : "";
+  const weight = options.weight ?? "400";
+  const fill = options.fill ?? theme.text;
+  const size = options.size ?? 12;
+  return `<text x="${typeof x === "number" ? x.toFixed(1) : x}" y="${typeof y === "number" ? y.toFixed(1) : y}" fill="${fill}" font-family="${fontStack}" font-size="${size}" font-weight="${weight}"${anchor}>${escapeXml(value)}</text>`;
+}
+
+export function chip(x, y, label, kind) {
+  const palette = {
+    fail: { fill: "rgba(239,68,68,0.12)", stroke: theme.failStroke, text: theme.fail },
+    muted: { fill: "rgba(100,116,139,0.12)", stroke: theme.muted, text: theme.subtitle },
+    pass: { fill: "rgba(16,185,129,0.14)", stroke: theme.passStroke, text: theme.pass },
+    warn: { fill: "rgba(251,191,36,0.12)", stroke: theme.amber, text: theme.amber },
+  }[kind];
+  return [
+    `<rect x="${x}" y="${y}" width="120" height="23" rx="11.5" fill="${palette.fill}" stroke="${palette.stroke}" stroke-opacity="0.55" />`,
+    text(x + 60, y + 15.5, label, {
+      anchor: "middle",
+      fill: palette.text,
+      size: 11.5,
+      weight: "600",
+    }),
+  ].join("\n");
+}
+
+export function passFailChip(x, y, passed, total) {
+  if (passed === total) {
+    return chip(x, y, `✓ ${passed}/${total}`, "pass");
+  }
+  if (passed === 0) {
+    return chip(x, y, `✗ 0/${total}`, "fail");
+  }
+  return chip(x, y, `△ ${passed}/${total}`, "warn");
+}
+
+export function truncate(value, length) {
+  return value.length > length ? `${value.slice(0, length - 1)}...` : value;
+}
+
+export function escapeXml(value) {
+  return value
+    .replaceAll("&", "&amp;")
+    .replaceAll("<", "&lt;")
+    .replaceAll(">", "&gt;")
+    .replaceAll('"', "&quot;");
+}

package/benchmarks/plot-svg.js

@@ -0,0 +1,339 @@
+import {
+  chip,
+  defs,
+  envFooter,
+  fixtureScale,
+  formatSeconds,
+  groupedCorrectness,
+  groupedStats,
+  isSupaschema,
+  logDomain,
+  logTicks,
+  logX,
+  passFailChip,
+  percentile,
+  svgFooter,
+  svgHeader,
+  text,
+  theme,
+  truncate,
+} from "./plot-lib.js";
+
+export function renderLatencySvg(rows, fixture, environments) {
+  const groups = groupedStats(rows);
+  const width = 1200;
+  const rowHeight = 34;
+  const labelX = 36;
+  const x0 = 300;
+  const chartWidth = 660;
+  const valueX = width - 36;
+  const top = 108;
+  const height = top + groups.length * rowHeight + 64;
+  const domain = logDomain(groups.flatMap((item) => [item.median, item.p95]));
+  const ticks = logTicks(domain.floor, domain.ceil);
+  const tablesNote = fixtureScale[fixture]?.tables;
+  const parts = [
+    svgHeader(width, height),
+    defs(),
+    `<rect width="100%" height="100%" rx="14" fill="${theme.bg}" />`,
+    text(labelX, 46, `Diff latency — ${fixture} fixture${tablesNote ? ` (${tablesNote})` : ""}`, {
+      fill: theme.title,
+      size: 21,
+      weight: "700",
+    }),
+    text(labelX, 70, "Median seconds per diff, log scale · whisker marks p95 · lower is better", {
+      fill: theme.subtitle,
+      size: 12.5,
+    }),
+  ];
+  const plotBottom = top + groups.length * rowHeight - 10;
+  for (const tickValue of ticks) {
+    const tickX = logX(tickValue, domain, x0, chartWidth);
+    parts.push(
+      `<line x1="${tickX.toFixed(1)}" y1="${top - 14}" x2="${tickX.toFixed(1)}" y2="${plotBottom}" stroke="${theme.grid}" stroke-width="1" />`,
+    );
+    parts.push(
+      text(tickX, plotBottom + 22, formatSeconds(tickValue), {
+        anchor: "middle",
+        fill: theme.muted,
+        size: 11,
+      }),
+    );
+  }
+  for (const [index, group] of groups.entries()) {
+    const y = top + index * rowHeight;
+    const supa = isSupaschema(group.label);
+    const barFill = supa ? "url(#supaGradient)" : "url(#slateGradient)";
+    const barEnd = logX(group.median, domain, x0, chartWidth);
+    const p95X = logX(group.p95, domain, x0, chartWidth);
+    if (supa) {
+      parts.push(`<circle cx="${labelX + 5}" cy="${y + 9}" r="4" fill="${theme.accent}" />`);
+    }
+    parts.push(
+      text(labelX + 18, y + 13, truncate(group.label, 30), {
+        fill: supa ? theme.title : theme.text,
+        size: 13,
+        weight: supa ? "650" : "450",
+      }),
+    );
+    parts.push(
+      `<rect x="${x0}" y="${y}" width="${Math.max(2, barEnd - x0).toFixed(1)}" height="18" rx="4" fill="${barFill}" />`,
+    );
+    parts.push(
+      `<line x1="${p95X.toFixed(1)}" y1="${y - 3}" x2="${p95X.toFixed(1)}" y2="${y + 21}" stroke="${theme.title}" stroke-opacity="0.55" stroke-width="2" />`,
+    );
+    parts.push(
+      text(valueX, y + 13, `${formatSeconds(group.median)}  ·  p95 ${formatSeconds(group.p95)}`, {
+        anchor: "end",
+        fill: supa ? theme.accent : theme.subtitle,
+        size: 12,
+        weight: supa ? "650" : "450",
+      }),
+    );
+  }
+  parts.push(
+    text(labelX, height - 22, envFooter(environments, fixture), { fill: theme.muted, size: 11 }),
+    svgFooter(),
+  );
+  return parts.join("\n");
+}
+
+export function renderCorrectnessSvg(rows, fixture, environments) {
+  const groups = groupedCorrectness(rows);
+  const width = 1200;
+  const rowHeight = 38;
+  const labelX = 36;
+  const top = 132;
+  const height = top + groups.length * rowHeight + 56;
+  const columns = [
+    { key: "once", label: "applies once", x: 430 },
+    { key: "twice", label: "applies twice", x: 610 },
+    { key: "match", label: "matches target", x: 790 },
+    { key: "f1", label: "output F1", x: 985 },
+  ];
+  const tablesNote = fixtureScale[fixture]?.tables;
+  const parts = [
+    svgHeader(width, height),
+    defs(),
+    `<rect width="100%" height="100%" rx="14" fill="${theme.bg}" />`,
+    text(
+      labelX,
+      46,
+      `Verification & accuracy — ${fixture} fixture${tablesNote ? ` (${tablesNote})` : ""}`,
+      { fill: theme.title, size: 21, weight: "700" },
+    ),
+    text(
+      labelX,
+      70,
+      "Each generated migration is applied in one transaction, applied again, and the catalog is fingerprinted against the target.",
+      { fill: theme.subtitle, size: 12.5 },
+    ),
+    text(
+      labelX,
+      88,
+      "Output F1 scores generated SQL content against the fixture's ground-truth change manifest (1.000 = exact).",
+      { fill: theme.subtitle, size: 12.5 },
+    ),
+  ];
+  for (const column of columns) {
+    parts.push(
+      text(column.x + 60, top - 16, column.label, {
+        anchor: "middle",
+        fill: theme.muted,
+        size: 11.5,
+        weight: "600",
+      }),
+    );
+  }
+  for (const [index, group] of groups.entries()) {
+    const y = top + index * rowHeight;
+    const supa = isSupaschema(group.label);
+    if (supa) {
+      parts.push(`<circle cx="${labelX + 5}" cy="${y + 11}" r="4" fill="${theme.accent}" />`);
+    }
+    parts.push(
+      text(labelX + 18, y + 15, truncate(group.label, 30), {
+        fill: supa ? theme.title : theme.text,
+        size: 13,
+        weight: supa ? "650" : "450",
+      }),
+    );
+    if (group.total === 0) {
+      parts.push(chip(columns[0].x, y, "skipped", "muted"));
+      continue;
+    }
+    parts.push(passFailChip(columns[0].x, y, group.once, group.total));
+    parts.push(passFailChip(columns[1].x, y, group.twice, group.total));
+    parts.push(passFailChip(columns[2].x, y, group.match, group.total));
+    if (group.f1 === undefined) {
+      parts.push(chip(columns[3].x, y, "—", "muted"));
+    } else {
+      parts.push(chip(columns[3].x, y, group.f1.toFixed(3), group.f1 >= 0.9995 ? "pass" : "warn"));
+    }
+  }
+  parts.push(
+    text(labelX, height - 22, envFooter(environments, fixture), { fill: theme.muted, size: 11 }),
+    svgFooter(),
+  );
+  return parts.join("\n");
+}
+
+export function renderScalingSvg(allResults, fixtures, environments, options = {}) {
+  const title = options.title ?? "Diff latency vs schema size";
+  const subtitle =
+    options.subtitle ??
+    "Median seconds per diff at each fixture scale, log scale · lower is better";
+  const width = 1200;
+  const height = 560;
+  const left = 96;
+  const right = width - 290;
+  const top = 116;
+  const bottom = height - 86;
+  const adapters = [...new Set(allResults.map((item) => item.adapter))].sort(
+    (a, b) => Number(isSupaschema(b)) - Number(isSupaschema(a)) || a.localeCompare(b),
+  );
+  const series = adapters.map((adapter) => ({
+    adapter,
+    points: fixtures
+      .map((fixture, index) => {
+        const values = allResults
+          .filter(
+            (item) =>
+              item.adapter === adapter &&
+              item.fixture === fixture &&
+              !item.skipped &&
+              !item.unsupported,
+          )
+          .map((item) => item.elapsedMs);
+        return values.length > 0 ? { index, median: percentile(values, 0.5) } : undefined;
+      })
+      .filter(Boolean),
+  }));
+  const allMedians = series.flatMap((item) => item.points.map((point) => point.median));
+  const domain = logDomain(allMedians);
+  const ticks = logTicks(domain.floor, domain.ceil);
+  const xFor = (index) => left + (index / Math.max(1, fixtures.length - 1)) * (right - left);
+  const yFor = (value) => {
+    const span = Math.log10(domain.ceil) - Math.log10(domain.floor);
+    return (
+      bottom - ((Math.log10(Math.max(1, value)) - Math.log10(domain.floor)) / span) * (bottom - top)
+    );
+  };
+  const supaColors = [theme.accent, "#22d3ee"];
+  const engineColors = ["#94a3b8", "#7c8aa0", "#64748b", "#8896ab", "#566379"];
+  let supaIndex = 0;
+  let engineIndex = 0;
+  const parts = [
+    svgHeader(width, height),
+    defs(),
+    `<rect width="100%" height="100%" rx="14" fill="${theme.bg}" />`,
+    text(36, 46, title, { fill: theme.title, size: 21, weight: "700" }),
+    text(36, 70, subtitle, { fill: theme.subtitle, size: 12.5 }),
+  ];
+  for (const tickValue of ticks) {
+    const y = yFor(tickValue);
+    parts.push(
+      `<line x1="${left}" y1="${y.toFixed(1)}" x2="${right}" y2="${y.toFixed(1)}" stroke="${theme.grid}" stroke-width="1" />`,
+    );
+    parts.push(
+      text(left - 12, y + 4, formatSeconds(tickValue), {
+        anchor: "end",
+        fill: theme.muted,
+        size: 11,
+      }),
+    );
+  }
+  for (const [index, fixture] of fixtures.entries()) {
+    const x = xFor(index);
+    parts.push(
+      text(x, bottom + 26, fixture, {
+        anchor: "middle",
+        fill: theme.text,
+        size: 12,
+        weight: "550",
+      }),
+    );
+    const tables = fixtureScale[fixture]?.tables;
+    if (tables) {
+      parts.push(text(x, bottom + 44, tables, { anchor: "middle", fill: theme.muted, size: 11 }));
+    }
+  }
+  const endLabels = [];
+  for (const item of series) {
+    const supa = isSupaschema(item.adapter);
+    const color = supa
+      ? supaColors[supaIndex++ % supaColors.length]
+      : engineColors[engineIndex++ % engineColors.length];
+    const path = item.points
+      .map(
+        (point, order) =>
+          `${order === 0 ? "M" : "L"}${xFor(point.index).toFixed(1)},${yFor(point.median).toFixed(1)}`,
+      )
+      .join(" ");
+    parts.push(
+      `<path d="${path}" fill="none" stroke="${color}" stroke-width="${supa ? 3 : 2}" stroke-linecap="round" stroke-linejoin="round"${supa ? "" : ' stroke-opacity="0.75"'} />`,
+    );
+    for (const point of item.points) {
+      parts.push(
+        `<circle cx="${xFor(point.index).toFixed(1)}" cy="${yFor(point.median).toFixed(1)}" r="${supa ? 4.5 : 3.5}" fill="${color}" />`,
+      );
+    }
+    const last = item.points.at(-1);
+    if (last) {
+      endLabels.push({
+        anchorY: yFor(last.median),
+        color,
+        label: `${item.adapter} ${formatSeconds(last.median)}`,
+        supa,
+        y: yFor(last.median),
+      });
+    }
+  }
+  parts.push(...renderEndLabels(endLabels, { bottom, right, top }));
+  parts.push(
+    text(36, height - 22, envFooter(environments, undefined), {
+      fill: theme.muted,
+      size: 11,
+    }),
+    svgFooter(),
+  );
+  return parts.join("\n");
+}
+
+function renderEndLabels(endLabels, bounds) {
+  const minGap = 18;
+  endLabels.sort((left, right) => left.anchorY - right.anchorY);
+  for (let index = 1; index < endLabels.length; index += 1) {
+    if (endLabels[index].y - endLabels[index - 1].y < minGap) {
+      endLabels[index].y = endLabels[index - 1].y + minGap;
+    }
+  }
+  const lastLabel = endLabels.at(-1);
+  const overflow = lastLabel ? lastLabel.y - bounds.bottom : 0;
+  if (overflow > 0) {
+    for (const item of endLabels) {
+      item.y -= Math.min(overflow, Math.max(0, item.y - bounds.top));
+    }
+    for (let index = endLabels.length - 2; index >= 0; index -= 1) {
+      if (endLabels[index + 1].y - endLabels[index].y < minGap) {
+        endLabels[index].y = endLabels[index + 1].y - minGap;
+      }
+    }
+  }
+  const parts = [];
+  for (const item of endLabels) {
+    if (Math.abs(item.y - item.anchorY) > 4) {
+      parts.push(
+        `<line x1="${bounds.right + 6}" y1="${item.anchorY.toFixed(1)}" x2="${bounds.right + 24}" y2="${(item.y - 4).toFixed(1)}" stroke="${item.color}" stroke-opacity="0.45" stroke-width="1" />`,
+      );
+    }
+    parts.push(
+      text(bounds.right + 28, item.y, item.label, {
+        fill: item.supa ? item.color : theme.subtitle,
+        size: 12,
+        weight: item.supa ? "650" : "450",
+      }),
+    );
+  }
+  return parts;
+}