settld 0.1.2 → 0.2.0

package/scripts/ci/run-10x-throughput-drill.mjs

@@ -0,0 +1,318 @@
+#!/usr/bin/env node
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
+import { spawn } from "node:child_process";
+
+function parseIntEnv(name, fallback, { min = null, max = null } = {}) {
+  const raw = process.env[name];
+  if (raw === undefined || raw === null || String(raw).trim() === "") return fallback;
+  const value = Number(raw);
+  if (!Number.isSafeInteger(value)) throw new Error(`${name} must be an integer`);
+  if (min !== null && value < min) throw new Error(`${name} must be >= ${min}`);
+  if (max !== null && value > max) throw new Error(`${name} must be <= ${max}`);
+  return value;
+}
+
+function parseFloatEnv(name, fallback, { min = null, max = null } = {}) {
+  const raw = process.env[name];
+  if (raw === undefined || raw === null || String(raw).trim() === "") return fallback;
+  const value = Number(raw);
+  if (!Number.isFinite(value)) throw new Error(`${name} must be a number`);
+  if (min !== null && value < min) throw new Error(`${name} must be >= ${min}`);
+  if (max !== null && value > max) throw new Error(`${name} must be <= ${max}`);
+  return value;
+}
+
+function parseBoolEnv(name, fallback = false) {
+  const raw = process.env[name];
+  if (raw === undefined || raw === null || String(raw).trim() === "") return fallback;
+  const value = String(raw).trim().toLowerCase();
+  if (value === "1" || value === "true" || value === "yes" || value === "y") return true;
+  if (value === "0" || value === "false" || value === "no" || value === "n") return false;
+  throw new Error(`${name} must be boolean-like (true/false)`);
+}
+
+function runCommand(cmd, args, { env = process.env, stdio = "inherit" } = {}) {
+  return new Promise((resolvePromise, rejectPromise) => {
+    const child = spawn(cmd, args, { env, stdio });
+    child.on("error", rejectPromise);
+    child.on("exit", (code) => {
+      resolvePromise(code ?? 1);
+    });
+  });
+}
+
+async function ensureK6() {
+  try {
+    const code = await runCommand("k6", ["version"], { stdio: "ignore" });
+    return code === 0;
+  } catch {
+    return false;
+  }
+}
+
+async function ensureDocker() {
+  try {
+    const code = await runCommand("docker", ["--version"], { stdio: "ignore" });
+    return code === 0;
+  } catch {
+    return false;
+  }
+}
+
+function toWorkspacePath(hostPath, cwd) {
+  const target = resolve(hostPath);
+  const root = resolve(cwd);
+  const rel = relative(root, target);
+  if (
+    rel === "" ||
+    rel.startsWith(`..${sep}`) ||
+    rel === ".." ||
+    (isAbsolute(rel) && resolve(rel) !== target)
+  ) {
+    throw new Error(`docker k6 fallback requires paths under working directory: ${hostPath}`);
+  }
+  const normalized = rel.split(sep).join("/");
+  return normalized === "" ? "/workspace" : `/workspace/${normalized}`;
+}
+
+async function resolveK6Runner({ cwd, allowDockerFallback = true } = {}) {
+  if (await ensureK6()) {
+    return {
+      kind: "k6_binary",
+      cmd: "k6",
+      baseArgs: []
+    };
+  }
+  if (!allowDockerFallback) {
+    throw new Error("k6 is required on PATH (docker fallback disabled)");
+  }
+  if (!(await ensureDocker())) {
+    throw new Error("k6 is required on PATH or docker must be installed for fallback");
+  }
+  const networkMode = process.env.K6_DOCKER_NETWORK_MODE && process.env.K6_DOCKER_NETWORK_MODE.trim() !== "" ? process.env.K6_DOCKER_NETWORK_MODE.trim() : "host";
+  const image = process.env.K6_DOCKER_IMAGE && process.env.K6_DOCKER_IMAGE.trim() !== "" ? process.env.K6_DOCKER_IMAGE.trim() : "grafana/k6:0.48.0";
+  return {
+    kind: "docker_k6",
+    cmd: "docker",
+    baseArgs: ["run", "--rm", "--network", networkMode, "-v", `${cwd}:/workspace`, "-w", "/workspace", image],
+    toRunnerPath: (value) => toWorkspacePath(value, cwd)
+  };
+}
+
+function readMetric(summary, metricName, key) {
+  const metric = summary?.metrics?.[metricName];
+  if (!metric || typeof metric !== "object") return null;
+  const valueFromValues = metric?.values?.[key];
+  if (Number.isFinite(Number(valueFromValues))) return Number(valueFromValues);
+  const valueDirect = metric?.[key];
+  if (Number.isFinite(Number(valueDirect))) return Number(valueDirect);
+  return null;
+}
+
+function resolveThroughputReportPath(cwd = process.cwd(), env = process.env) {
+  return resolve(cwd, env.THROUGHPUT_REPORT_PATH || "artifacts/throughput/10x-drill-summary.json");
+}
+
+function toFailureSummary(err) {
+  return {
+    message: err?.message ?? String(err),
+    code: typeof err?.code === "string" ? err.code : null
+  };
+}
+
+async function writeFailureReport({ reportPath, runConfig = null, error }) {
+  await mkdir(dirname(reportPath), { recursive: true });
+  const report = {
+    schemaVersion: "ThroughputDrill10xReport.v1",
+    generatedAt: new Date().toISOString(),
+    mode: "error",
+    runConfig,
+    metrics: null,
+    checks: {
+      scriptCompleted: false
+    },
+    failure: toFailureSummary(error),
+    verdict: {
+      ok: false,
+      reason: "execution_error"
+    }
+  };
+  await writeFile(reportPath, JSON.stringify(report, null, 2) + "\n", "utf8");
+  process.stdout.write(`wrote throughput report: ${reportPath}\n`);
+}
+
+async function main() {
+  const baseJobsPerMinPerTenant = parseIntEnv("BASELINE_JOBS_PER_MIN_PER_TENANT", 10, { min: 1 });
+  const multiplier = parseIntEnv("THROUGHPUT_MULTIPLIER", 10, { min: 1 });
+  const jobsPerMinPerTenant = parseIntEnv(
+    "JOBS_PER_MIN_PER_TENANT",
+    baseJobsPerMinPerTenant * multiplier,
+    { min: 1 }
+  );
+  const tenants = parseIntEnv("TENANTS", 3, { min: 1 });
+  const robotsPerTenant = parseIntEnv("ROBOTS_PER_TENANT", 3, { min: 1 });
+  const duration = process.env.DURATION && process.env.DURATION.trim() !== "" ? process.env.DURATION.trim() : "120s";
+  const targetP95Ms = parseFloatEnv("TARGET_P95_MS", 5000, { min: 1 });
+  const maxFailureRate = parseFloatEnv("MAX_FAILURE_RATE", 0.05, { min: 0, max: 1 });
+  const allowRejectedRate = parseFloatEnv("MAX_INGEST_REJECTED_PER_MIN", 50, { min: 0 });
+  const dryRun = parseBoolEnv("DRY_RUN", false);
+  const allowDockerFallback = parseBoolEnv("ALLOW_DOCKER_K6_FALLBACK", true);
+
+  const reportPath = resolveThroughputReportPath(process.cwd(), process.env);
+  const summaryPath = resolve(process.cwd(), process.env.THROUGHPUT_K6_SUMMARY_PATH || "artifacts/throughput/10x-drill-k6-summary.json");
+  await mkdir(dirname(reportPath), { recursive: true });
+  await mkdir(dirname(summaryPath), { recursive: true });
+
+  const runConfig = {
+    baseUrl: process.env.BASE_URL || "http://127.0.0.1:3000",
+    opsTokenPresent: Boolean(process.env.OPS_TOKEN && process.env.OPS_TOKEN.trim() !== ""),
+    tenants,
+    robotsPerTenant,
+    baseJobsPerMinPerTenant,
+    throughputMultiplier: multiplier,
+    jobsPerMinPerTenant,
+    expectedJobsPerMinTotal: tenants * jobsPerMinPerTenant,
+    duration,
+    targetP95Ms,
+    maxFailureRate,
+    maxIngestRejectedPerMin: allowRejectedRate,
+    script: "scripts/load/ingest-burst.k6.js",
+    runner: "unresolved"
+  };
+
+  if (dryRun) {
+    const dryReport = {
+      schemaVersion: "ThroughputDrill10xReport.v1",
+      generatedAt: new Date().toISOString(),
+      mode: "dry_run",
+      runConfig,
+      verdict: {
+        ok: false,
+        reason: "dry_run"
+      }
+    };
+    await writeFile(reportPath, JSON.stringify(dryReport, null, 2) + "\n", "utf8");
+    process.stdout.write(`wrote dry-run report: ${reportPath}\n`);
+    return;
+  }
+
+  const k6Runner = await resolveK6Runner({
+    cwd: process.cwd(),
+    allowDockerFallback
+  });
+  runConfig.runner = k6Runner.kind;
+
+  const k6Env = {
+    ...process.env,
+    BASE_URL: runConfig.baseUrl,
+    OPS_TOKEN: process.env.OPS_TOKEN || "",
+    TENANTS: String(tenants),
+    ROBOTS_PER_TENANT: String(robotsPerTenant),
+    JOBS_PER_MIN_PER_TENANT: String(jobsPerMinPerTenant),
+    DURATION: duration,
+    TARGET_P95_MS: String(targetP95Ms),
+    MAX_FAILURE_RATE: String(maxFailureRate)
+  };
+
+  const resolvedScriptPath = resolve(process.cwd(), runConfig.script);
+  const scriptPathForRunner = k6Runner.kind === "docker_k6" ? k6Runner.toRunnerPath(resolvedScriptPath) : runConfig.script;
+  const summaryPathForRunner = k6Runner.kind === "docker_k6" ? k6Runner.toRunnerPath(summaryPath) : summaryPath;
+  const runnerEnvArgs =
+    k6Runner.kind === "docker_k6"
+      ? Object.entries(k6Env).flatMap(([name, value]) => ["-e", `${name}=${String(value)}`])
+      : [];
+
+  const startedAt = Date.now();
+  const exitCode = await runCommand(
+    k6Runner.cmd,
+    [...k6Runner.baseArgs, ...runnerEnvArgs, "run", "--summary-export", summaryPathForRunner, scriptPathForRunner],
+    { env: k6Env }
+  );
+  const completedAt = Date.now();
+
+  let summary;
+  try {
+    summary = JSON.parse(await readFile(summaryPath, "utf8"));
+  } catch (err) {
+    const report = {
+      schemaVersion: "ThroughputDrill10xReport.v1",
+      generatedAt: new Date().toISOString(),
+      mode: "error",
+      runConfig,
+      metrics: null,
+      checks: {
+        k6ExitCodeZero: exitCode === 0,
+        k6SummaryPresent: false
+      },
+      failure: {
+        message: `k6 runner exited with code ${exitCode} before writing summary export`,
+        code: typeof err?.code === "string" ? err.code : null
+      },
+      verdict: {
+        ok: false,
+        reason: "k6_summary_missing",
+        k6ExitCode: exitCode,
+        summaryPath
+      }
+    };
+    await writeFile(reportPath, JSON.stringify(report, null, 2) + "\n", "utf8");
+    process.stdout.write(`wrote throughput report: ${reportPath}\n`);
+    process.exitCode = 1;
+    return;
+  }
+  const p95 = readMetric(summary, "http_req_duration", "p(95)");
+  const failureRate = readMetric(summary, "http_req_failed", "rate") ?? readMetric(summary, "http_req_failed", "value");
+  const rejectedCount = readMetric(summary, "ingest_rejected_requests_total", "count") ?? 0;
+  const durationSeconds = Math.max(1, Math.round((completedAt - startedAt) / 1000));
+  const rejectedPerMin = (rejectedCount / durationSeconds) * 60;
+
+  const checks = {
+    k6ExitCodeZero: exitCode === 0,
+    p95WithinTarget: p95 !== null ? p95 <= targetP95Ms : false,
+    failureRateWithinTarget: failureRate !== null ? failureRate <= maxFailureRate : false,
+    ingestRejectedWithinTarget: rejectedPerMin <= allowRejectedRate
+  };
+  const ok = Object.values(checks).every((value) => value === true);
+
+  const report = {
+    schemaVersion: "ThroughputDrill10xReport.v1",
+    generatedAt: new Date().toISOString(),
+    runConfig,
+    metrics: {
+      httpReqDurationP95Ms: p95,
+      httpReqFailedRate: failureRate,
+      ingestRejectedCount: rejectedCount,
+      ingestRejectedPerMin: rejectedPerMin,
+      runDurationSeconds: durationSeconds
+    },
+    checks,
+    verdict: {
+      ok,
+      k6ExitCode: exitCode,
+      summaryPath
+    }
+  };
+
+  await writeFile(reportPath, JSON.stringify(report, null, 2) + "\n", "utf8");
+  process.stdout.write(`wrote throughput report: ${reportPath}\n`);
+  if (!ok) {
+    process.exitCode = 1;
+  }
+}
+
+main().catch((err) => {
+  const reportPath = resolveThroughputReportPath(process.cwd(), process.env);
+  writeFailureReport({
+    reportPath,
+    runConfig: null,
+    error: err
+  })
+    .catch((writeErr) => {
+      process.stderr.write(`${writeErr?.stack || writeErr?.message || String(writeErr)}\n`);
+    })
+    .finally(() => {
+      process.stderr.write(`${err?.stack || err?.message || String(err)}\n`);
+      process.exit(1);
+    });
+});

package/scripts/ci/run-10x-throughput-incident-rehearsal.mjs

@@ -0,0 +1,368 @@
+#!/usr/bin/env node
+import { mkdir, writeFile } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+
+function parseIntEnv(name, fallback, { min = null, max = null } = {}) {
+  const raw = process.env[name];
+  if (raw === undefined || raw === null || String(raw).trim() === "") return fallback;
+  const value = Number(raw);
+  if (!Number.isSafeInteger(value)) throw new Error(`${name} must be an integer`);
+  if (min !== null && value < min) throw new Error(`${name} must be >= ${min}`);
+  if (max !== null && value > max) throw new Error(`${name} must be <= ${max}`);
+  return value;
+}
+
+function normalizeBaseUrl(raw) {
+  const value = typeof raw === "string" && raw.trim() !== "" ? raw.trim() : "http://127.0.0.1:3000";
+  return value.endsWith("/") ? value.slice(0, -1) : value;
+}
+
+function makePolicyPayload({ policyId, policyVersion, amberReleaseRatePct, description }) {
+  return {
+    policyId,
+    policyVersion,
+    verificationMethod: {
+      mode: "deterministic",
+      source: "verifier://settld-verify"
+    },
+    policy: {
+      mode: "automatic",
+      rules: {
+        requireDeterministicVerification: true,
+        autoReleaseOnGreen: true,
+        autoReleaseOnAmber: false,
+        autoReleaseOnRed: false,
+        greenReleaseRatePct: 100,
+        amberReleaseRatePct,
+        redReleaseRatePct: 0
+      }
+    },
+    description
+  };
+}
+
+function resolveIncidentReportPath(cwd = process.cwd(), env = process.env) {
+  return resolve(
+    cwd,
+    env.THROUGHPUT_INCIDENT_REHEARSAL_REPORT_PATH || "artifacts/throughput/10x-incident-rehearsal-summary.json"
+  );
+}
+
+function toFailureSummary(err) {
+  return {
+    message: err?.message ?? String(err),
+    statusCode: Number.isSafeInteger(err?.statusCode) ? err.statusCode : null,
+    details: err?.details ?? null
+  };
+}
+
+async function writeFailureReport({ reportPath, runConfig = null, error }) {
+  await mkdir(dirname(reportPath), { recursive: true });
+  const report = {
+    schemaVersion: "ThroughputIncidentRehearsalReport.v1",
+    generatedAt: new Date().toISOString(),
+    runConfig,
+    durationMs: 0,
+    checks: [],
+    snapshots: {},
+    failure: toFailureSummary(error),
+    verdict: {
+      ok: false,
+      requiredChecks: 0,
+      passedChecks: 0
+    }
+  };
+  await writeFile(reportPath, JSON.stringify(report, null, 2) + "\n", "utf8");
+  process.stdout.write(`wrote throughput incident rehearsal report: ${reportPath}\n`);
+}
+
+async function main() {
+  const startedAt = Date.now();
+  const baseUrl = normalizeBaseUrl(process.env.BASE_URL);
+  const opsToken = typeof process.env.OPS_TOKEN === "string" ? process.env.OPS_TOKEN.trim() : "";
+  if (!opsToken) throw new Error("OPS_TOKEN is required");
+  const tenantId =
+    typeof process.env.TENANT_ID === "string" && process.env.TENANT_ID.trim() !== ""
+      ? process.env.TENANT_ID.trim()
+      : "tenant_default";
+  const timeoutMs = parseIntEnv("HTTP_TIMEOUT_MS", 10_000, { min: 1000, max: 120_000 });
+  const protocolVersion =
+    typeof process.env.SETTLD_PROTOCOL === "string" && process.env.SETTLD_PROTOCOL.trim() !== ""
+      ? process.env.SETTLD_PROTOCOL.trim()
+      : "1.0";
+  const reportPath = resolveIncidentReportPath(process.cwd(), process.env);
+  const policyId =
+    typeof process.env.INCIDENT_REHEARSAL_POLICY_ID === "string" && process.env.INCIDENT_REHEARSAL_POLICY_ID.trim() !== ""
+      ? process.env.INCIDENT_REHEARSAL_POLICY_ID.trim()
+      : "market.incident.rehearsal.v1";
+
+  const rehearsalId = `inc_rh_${Date.now()}`;
+  const noteDegraded = `${rehearsalId}:comms:announce_degraded_mode`;
+  const noteRollback = `${rehearsalId}:comms:announce_rollback_complete`;
+
+  await mkdir(dirname(reportPath), { recursive: true });
+
+  const defaultHeaders = {
+    "x-proxy-tenant-id": tenantId,
+    "x-proxy-ops-token": opsToken,
+    accept: "application/json"
+  };
+
+  async function requestJson({ method, path, body = null, write = false, idempotencyKey = null }) {
+    const headers = { ...defaultHeaders };
+    if (write) {
+      headers["content-type"] = "application/json";
+      headers["x-settld-protocol"] = protocolVersion;
+      if (idempotencyKey) headers["x-idempotency-key"] = idempotencyKey;
+    }
+    const response = await fetch(`${baseUrl}${path}`, {
+      method,
+      headers,
+      body: body === null ? undefined : JSON.stringify(body),
+      signal: AbortSignal.timeout(timeoutMs)
+    });
+    const text = await response.text();
+    let json = null;
+    try {
+      json = text ? JSON.parse(text) : null;
+    } catch {}
+    if (!response.ok) {
+      const details =
+        json && typeof json === "object"
+          ? json
+          : {
+              body: text
+            };
+      const err = new Error(`${method} ${path} failed (${response.status})`);
+      err.statusCode = response.status;
+      err.details = details;
+      throw err;
+    }
+    return {
+      statusCode: response.status,
+      json
+    };
+  }
+
+  async function getPolicyState() {
+    const res = await requestJson({
+      method: "GET",
+      path: `/ops/settlement-policies/state?policyId=${encodeURIComponent(policyId)}`
+    });
+    return res.json ?? {};
+  }
+
+  async function ensurePolicyVersion({ policyVersion, amberReleaseRatePct, description }) {
+    const state = await getPolicyState();
+    const policies = Array.isArray(state?.policies) ? state.policies : [];
+    const existing = policies.find(
+      (row) =>
+        String(row?.policyId ?? "") === policyId &&
+        Number(row?.policyVersion ?? -1) === Number(policyVersion)
+    );
+    if (existing) return existing;
+    const created = await requestJson({
+      method: "POST",
+      path: "/marketplace/settlement-policies",
+      write: true,
+      idempotencyKey: `${rehearsalId}_policy_${policyVersion}`,
+      body: makePolicyPayload({
+        policyId,
+        policyVersion,
+        amberReleaseRatePct,
+        description
+      })
+    });
+    return created?.json?.policy ?? null;
+  }
+
+  function activeVersionFromState(state) {
+    return Number(state?.rollout?.stages?.active?.policyVersion ?? NaN);
+  }
+
+  const checks = [];
+  const snapshots = {};
+  let failure = null;
+
+  try {
+    const health = await requestJson({ method: "GET", path: "/healthz" });
+    checks.push({
+      id: "api_healthy",
+      ok: health?.json?.ok === true,
+      details: health?.json ?? null
+    });
+
+    await ensurePolicyVersion({
+      policyVersion: 1,
+      amberReleaseRatePct: 10,
+      description: "incident rehearsal degraded policy"
+    });
+    await ensurePolicyVersion({
+      policyVersion: 2,
+      amberReleaseRatePct: 30,
+      description: "incident rehearsal stable policy"
+    });
+
+    const stateBefore = await getPolicyState();
+    snapshots.stateBefore = stateBefore;
+    const stateBeforeActive = activeVersionFromState(stateBefore);
+    if (stateBeforeActive !== 2) {
+      await requestJson({
+        method: "POST",
+        path: "/ops/settlement-policies/rollout",
+        write: true,
+        idempotencyKey: `${rehearsalId}_baseline_rollout`,
+        body: {
+          stage: "active",
+          policyRef: { policyId, policyVersion: 2 },
+          note: `${rehearsalId}:baseline_stable`
+        }
+      });
+    }
+
+    const degradedSignal = await requestJson({
+      method: "GET",
+      path:
+        "/ops/network/command-center?windowHours=24&disputeSlaHours=1&emitAlerts=true&persistAlerts=true" +
+        "&httpClientErrorRateThresholdPct=0&httpServerErrorRateThresholdPct=0&deliveryDlqThreshold=0" +
+        "&disputeOverSlaThreshold=0&determinismRejectThreshold=0&kernelVerificationErrorThreshold=0"
+    });
+    snapshots.degradedSignal = degradedSignal?.json ?? null;
+    const degradedBreachCount = Number(degradedSignal?.json?.alerts?.breachCount ?? 0);
+    checks.push({
+      id: "degraded_mode_alert_signal_emitted",
+      ok: degradedBreachCount > 0,
+      details: {
+        breachCount: degradedBreachCount,
+        emittedCount: Number(degradedSignal?.json?.alerts?.emittedCount ?? 0)
+      }
+    });
+
+    const degradedRollout = await requestJson({
+      method: "POST",
+      path: "/ops/settlement-policies/rollout",
+      write: true,
+      idempotencyKey: `${rehearsalId}_degraded_rollout`,
+      body: {
+        stage: "active",
+        policyRef: { policyId, policyVersion: 1 },
+        note: noteDegraded
+      }
+    });
+    snapshots.degradedRollout = degradedRollout?.json ?? null;
+
+    const stateDegraded = await getPolicyState();
+    snapshots.stateDegraded = stateDegraded;
+    const degradedActiveVersion = activeVersionFromState(stateDegraded);
+    checks.push({
+      id: "degraded_policy_activated",
+      ok: degradedActiveVersion === 1,
+      details: {
+        activePolicyVersion: Number.isFinite(degradedActiveVersion) ? degradedActiveVersion : null
+      }
+    });
+
+    const rollback = await requestJson({
+      method: "POST",
+      path: "/ops/settlement-policies/rollback",
+      write: true,
+      idempotencyKey: `${rehearsalId}_rollback`,
+      body: {
+        note: noteRollback
+      }
+    });
+    snapshots.rollback = rollback?.json ?? null;
+
+    const stateRecovered = await getPolicyState();
+    snapshots.stateRecovered = stateRecovered;
+    const recoveredActiveVersion = activeVersionFromState(stateRecovered);
+    checks.push({
+      id: "rollback_restores_stable_policy",
+      ok: recoveredActiveVersion === 2,
+      details: {
+        activePolicyVersion: Number.isFinite(recoveredActiveVersion) ? recoveredActiveVersion : null
+      }
+    });
+
+    const postRunCommandCenter = await requestJson({
+      method: "GET",
+      path: "/ops/network/command-center?windowHours=24&disputeSlaHours=1&emitAlerts=true"
+    });
+    snapshots.postRunCommandCenter = postRunCommandCenter?.json ?? null;
+    const postRunBreachCount = Number(postRunCommandCenter?.json?.alerts?.breachCount ?? 0);
+    checks.push({
+      id: "post_rollback_command_center_clear",
+      ok: postRunBreachCount === 0,
+      details: {
+        breachCount: postRunBreachCount
+      }
+    });
+
+    const audits = await requestJson({ method: "GET", path: "/ops/audit?limit=200" });
+    const rows = Array.isArray(audits?.json?.audit) ? audits.json.audit : [];
+    const noteSet = new Set(
+      rows
+        .map((row) => (typeof row?.details?.note === "string" ? row.details.note : ""))
+        .filter(Boolean)
+    );
+    const hasDegradedCommsAudit = noteSet.has(noteDegraded);
+    const hasRollbackCommsAudit = noteSet.has(noteRollback);
+    checks.push({
+      id: "communications_drill_audited",
+      ok: hasDegradedCommsAudit && hasRollbackCommsAudit,
+      details: {
+        hasDegradedCommsAudit,
+        hasRollbackCommsAudit
+      }
+    });
+  } catch (err) {
+    failure = {
+      message: err?.message ?? String(err),
+      statusCode: Number.isSafeInteger(err?.statusCode) ? err.statusCode : null,
+      details: err?.details ?? null
+    };
+  }
+
+  const ok = failure === null && checks.every((check) => check.ok === true);
+  const report = {
+    schemaVersion: "ThroughputIncidentRehearsalReport.v1",
+    generatedAt: new Date().toISOString(),
+    runConfig: {
+      rehearsalId,
+      baseUrl,
+      tenantId,
+      policyId,
+      protocolVersion,
+      timeoutMs
+    },
+    durationMs: Date.now() - startedAt,
+    checks,
+    snapshots,
+    failure,
+    verdict: {
+      ok,
+      requiredChecks: checks.length,
+      passedChecks: checks.filter((check) => check.ok === true).length
+    }
+  };
+
+  await writeFile(reportPath, JSON.stringify(report, null, 2) + "\n", "utf8");
+  process.stdout.write(`wrote throughput incident rehearsal report: ${reportPath}\n`);
+  if (!ok) process.exitCode = 1;
+}
+
+main().catch((err) => {
+  const reportPath = resolveIncidentReportPath(process.cwd(), process.env);
+  writeFailureReport({
+    reportPath,
+    runConfig: null,
+    error: err
+  })
+    .catch((writeErr) => {
+      process.stderr.write(`${writeErr?.stack || writeErr?.message || String(writeErr)}\n`);
+    })
+    .finally(() => {
+      process.stderr.write(`${err?.stack || err?.message || String(err)}\n`);
+      process.exit(1);
+    });
+});

package/scripts/ci/run-arbitration-workspace-browser-e2e.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+TEST_FILE="test/api-e2e-ops-arbitration-workspace-browser.test.js"
+
+export SETTLD_RUN_BROWSER_E2E="${SETTLD_RUN_BROWSER_E2E:-1}"
+
+# Optional host library shim for environments where Playwright browser deps are staged manually.
+if [[ -z "${SETTLD_PW_DEPS_LIB_DIR:-}" && -d "/tmp/pw-deps/root/usr/lib/x86_64-linux-gnu" ]]; then
+  export SETTLD_PW_DEPS_LIB_DIR="/tmp/pw-deps/root/usr/lib/x86_64-linux-gnu"
+fi
+if [[ -n "${SETTLD_PW_DEPS_LIB_DIR:-}" && -d "${SETTLD_PW_DEPS_LIB_DIR}" ]]; then
+  if [[ -n "${LD_LIBRARY_PATH:-}" ]]; then
+    export LD_LIBRARY_PATH="${SETTLD_PW_DEPS_LIB_DIR}:${LD_LIBRARY_PATH}"
+  else
+    export LD_LIBRARY_PATH="${SETTLD_PW_DEPS_LIB_DIR}"
+  fi
+fi
+
+cd "${ROOT_DIR}"
+node --test "${TEST_FILE}"