settld 0.1.2 → 0.2.0

package/scripts/governance/export.mjs

@@ -0,0 +1,169 @@
+import path from "node:path";
+
+import { createPgStore } from "../../src/db/store-pg.js";
+import { canonicalJsonStringify } from "../../src/core/canonical-json.js";
+import { sha256Hex, signHashHexEd25519 } from "../../src/core/crypto.js";
+import { DEFAULT_TENANT_ID, normalizeTenantId } from "../../src/core/tenancy.js";
+import { GOVERNANCE_STREAM_ID } from "../../src/core/governance.js";
+import { canonicalJsonlLines, computeProofBundleManifestV1 } from "../../src/core/proof-bundle.js";
+
+import { ensureDir, writeFilesToDir, writeZipFromDir } from "../proof-bundle/lib.mjs";
+
+function readArg(name) {
+  const idx = process.argv.indexOf(name);
+  if (idx === -1) return null;
+  return process.argv[idx + 1] ?? null;
+}
+
+function usageAndExit() {
+  // eslint-disable-next-line no-console
+  console.error(
+    [
+      "usage:",
+      "  DATABASE_URL=... node scripts/governance/export.mjs [--tenant <tenantId>] [--out <dir>] [--zip]",
+      "",
+      "notes:",
+      "  - exports both governance scopes: tenant + global",
+      "  - output is self-contained (events + snapshots + keys + manifest + head attestation)"
+    ].join("\n")
+  );
+  process.exit(2);
+}
+
+const DATABASE_URL = process.env.DATABASE_URL ?? null;
+if (!DATABASE_URL) usageAndExit();
+
+const tenantId = normalizeTenantId(process.env.TENANT_ID ?? readArg("--tenant") ?? DEFAULT_TENANT_ID);
+const outBase = readArg("--out") ?? path.join("demo", "governance-export");
+const zipFlag = process.argv.includes("--zip");
+
+const store = await createPgStore({ databaseUrl: DATABASE_URL, schema: process.env.PROXY_PG_SCHEMA ?? "public", migrateOnStartup: true });
+try {
+  const generatedAt = store.nowIso ? store.nowIso() : new Date().toISOString();
+
+  const tenantEvents = await store.listAggregateEvents({ tenantId, aggregateType: "month", aggregateId: GOVERNANCE_STREAM_ID });
+  const globalEvents = await store.listAggregateEvents({ tenantId: DEFAULT_TENANT_ID, aggregateType: "month", aggregateId: GOVERNANCE_STREAM_ID });
+
+  const tenantSnapshot = {
+    streamId: GOVERNANCE_STREAM_ID,
+    lastChainHash: tenantEvents.length ? tenantEvents[tenantEvents.length - 1]?.chainHash ?? null : null,
+    lastEventId: tenantEvents.length ? tenantEvents[tenantEvents.length - 1]?.id ?? null : null
+  };
+  const globalSnapshot = {
+    streamId: GOVERNANCE_STREAM_ID,
+    lastChainHash: globalEvents.length ? globalEvents[globalEvents.length - 1]?.chainHash ?? null : null,
+    lastEventId: globalEvents.length ? globalEvents[globalEvents.length - 1]?.id ?? null : null
+  };
+
+  const files = new Map();
+
+  function writeGovernanceScope(scope, events, snapshot) {
+    const payloadMaterial = events.map((e) => ({
+      v: e?.v ?? null,
+      id: e?.id ?? null,
+      at: e?.at ?? null,
+      streamId: e?.streamId ?? null,
+      type: e?.type ?? null,
+      actor: e?.actor ?? null,
+      payload: e?.payload ?? null
+    }));
+    files.set(`governance/${scope}/events/events.jsonl`, new TextEncoder().encode(canonicalJsonlLines(events)));
+    files.set(`governance/${scope}/events/payload_material.jsonl`, new TextEncoder().encode(canonicalJsonlLines(payloadMaterial)));
+    files.set(`governance/${scope}/snapshot.json`, new TextEncoder().encode(`${canonicalJsonStringify(snapshot)}\n`));
+  }
+
+  writeGovernanceScope("tenant", tenantEvents, tenantSnapshot);
+  writeGovernanceScope("global", globalEvents, globalSnapshot);
+
+  // keys/public_keys.json: include keys used to sign any governance event.
+  const signerKeyIds = Array.from(
+    new Set([...tenantEvents, ...globalEvents].map((e) => (typeof e?.signerKeyId === "string" ? e.signerKeyId : null)).filter(Boolean))
+  ).sort();
+  const publicKeyByKeyId = store.publicKeyByKeyId instanceof Map ? store.publicKeyByKeyId : new Map();
+
+  let signerKeys = [];
+  if (typeof store.listSignerKeys === "function") {
+    const tenantKeys = await store.listSignerKeys({ tenantId });
+    const defaultKeys = await store.listSignerKeys({ tenantId: DEFAULT_TENANT_ID });
+    const all = [...(tenantKeys ?? []), ...(defaultKeys ?? [])];
+    const byKeyId = new Map();
+    for (const r of all) {
+      const keyId = r?.keyId ? String(r.keyId) : null;
+      if (!keyId) continue;
+      byKeyId.set(keyId, r);
+    }
+    signerKeys = Array.from(byKeyId.values());
+  }
+
+  const metaByKeyId = new Map();
+  for (const r of signerKeys) {
+    if (!r || typeof r !== "object") continue;
+    const kid = typeof r.keyId === "string" && r.keyId.trim() ? r.keyId : null;
+    if (!kid) continue;
+    metaByKeyId.set(kid, r);
+  }
+
+  const keys = [];
+  for (const keyId of signerKeyIds) {
+    const publicKeyPem = publicKeyByKeyId.get(keyId) ?? null;
+    if (!publicKeyPem) continue;
+    const meta = metaByKeyId.get(keyId) ?? null;
+    keys.push({
+      keyId,
+      publicKeyPem: String(publicKeyPem),
+      tenantId: meta?.tenantId ?? tenantId,
+      purpose: meta?.purpose ?? null,
+      status: meta?.status ?? null,
+      createdAt: meta?.createdAt ?? null,
+      rotatedAt: meta?.rotatedAt ?? null,
+      revokedAt: meta?.revokedAt ?? null
+    });
+  }
+  keys.sort((a, b) => String(a.keyId).localeCompare(String(b.keyId)));
+  const publicKeysFile = { schemaVersion: "PublicKeys.v1", tenantId, generatedAt, order: "keyId_asc", keys };
+  files.set("keys/public_keys.json", new TextEncoder().encode(`${canonicalJsonStringify(publicKeysFile)}\n`));
+
+  const kind = "GovernanceExport.v1";
+  const scope = { tenantId };
+  const { manifest, manifestHash } = computeProofBundleManifestV1({ files, generatedAt, kind, tenantId, scope });
+  files.set("manifest.json", new TextEncoder().encode(`${canonicalJsonStringify({ ...manifest, manifestHash })}\n`));
+
+  // Head attestation (server-signed).
+  if (store?.serverSigner?.privateKeyPem) {
+    const attCore = {
+      schemaVersion: "BundleHeadAttestation.v1",
+      kind,
+      tenantId,
+      scope,
+      generatedAt,
+      manifestHash,
+      heads: {
+        governance: {
+          tenant: { lastChainHash: tenantSnapshot.lastChainHash, lastEventId: tenantSnapshot.lastEventId },
+          global: { lastChainHash: globalSnapshot.lastChainHash, lastEventId: globalSnapshot.lastEventId }
+        }
+      },
+      signedAt: generatedAt,
+      signerKeyId: store.serverSigner.keyId
+    };
+    const attestationHash = sha256Hex(canonicalJsonStringify(attCore));
+    const signature = signHashHexEd25519(attestationHash, store.serverSigner.privateKeyPem);
+    const att = { ...attCore, attestationHash, signature };
+    files.set("attestation/bundle_head_attestation.json", new TextEncoder().encode(`${canonicalJsonStringify(att)}\n`));
+  }
+
+  const outDir = path.join(outBase, `gov_${tenantId}_${manifestHash.slice(0, 12)}`);
+  ensureDir(outDir);
+  writeFilesToDir({ files, outDir });
+
+  if (zipFlag) {
+    const zipPath = `${outDir}.zip`;
+    await writeZipFromDir({ dir: outDir, outPath: zipPath });
+    process.stdout.write(`${zipPath}\n`);
+  } else {
+    process.stdout.write(`${outDir}\n`);
+  }
+} finally {
+  await store.close?.();
+}
+

package/scripts/load/delivery-stress.k6.js

@@ -0,0 +1,183 @@
+import http from "k6/http";
+import { check, sleep } from "k6";
+
+const BASE_URL = __ENV.BASE_URL || "http://localhost:3000";
+const OPS_TOKEN = __ENV.OPS_TOKEN || "";
+
+const TENANTS = Number.parseInt(__ENV.TENANTS || "3", 10);
+const ROBOTS_PER_TENANT = Number.parseInt(__ENV.ROBOTS_PER_TENANT || "3", 10);
+const JOBS_PER_MIN_PER_TENANT = Number.parseInt(__ENV.JOBS_PER_MIN_PER_TENANT || "100", 10);
+const DURATION = __ENV.DURATION || "2m";
+
+const jobsTotalRate = Math.max(1, TENANTS * JOBS_PER_MIN_PER_TENANT);
+
+export const options = {
+  scenarios: {
+    jobs: {
+      executor: "constant-arrival-rate",
+      rate: jobsTotalRate,
+      timeUnit: "1m",
+      duration: DURATION,
+      preAllocatedVUs: Math.min(300, Math.max(30, jobsTotalRate)),
+      maxVUs: 800
+    },
+    poll: {
+      executor: "constant-arrival-rate",
+      rate: Math.max(30, TENANTS * 30),
+      timeUnit: "1m",
+      duration: DURATION,
+      preAllocatedVUs: 5,
+      maxVUs: 20,
+      exec: "pollOps"
+    }
+  },
+  thresholds: {
+    http_req_failed: ["rate<0.1"]
+  }
+};
+
+function jsonHeaders({ tenantId, token, extra = {} } = {}) {
+  const headers = {
+    "content-type": "application/json",
+    "x-proxy-tenant-id": tenantId
+  };
+  if (token) headers.authorization = `Bearer ${token}`;
+  for (const [k, v] of Object.entries(extra)) headers[k] = v;
+  return { headers };
+}
+
+function opsTokenHeaders({ tenantId, extra = {} } = {}) {
+  const headers = {
+    "content-type": "application/json",
+    "x-proxy-tenant-id": tenantId,
+    "x-proxy-ops-token": OPS_TOKEN
+  };
+  for (const [k, v] of Object.entries(extra)) headers[k] = v;
+  return { headers };
+}
+
+function randomId(prefix) {
+  const n = Math.floor(Math.random() * 1e9);
+  return `${prefix}_${Date.now()}_${n}`;
+}
+
+function isoNowPlusMs(deltaMs) {
+  return new Date(Date.now() + deltaMs).toISOString();
+}
+
+export function setup() {
+  if (!OPS_TOKEN) throw new Error("OPS_TOKEN is required");
+  const runId = randomId("run");
+  const tenants = [];
+
+  const availStartAt = isoNowPlusMs(-60 * 60_000);
+  const availEndAt = isoNowPlusMs(24 * 60 * 60_000);
+
+  for (let i = 0; i < TENANTS; i += 1) {
+    const tenantId = i === 0 ? "tenant_default" : `tenant_${i}`;
+
+    const keyRes = http.post(
+      `${BASE_URL}/ops/api-keys`,
+      JSON.stringify({
+        scopes: ["ops_read", "ops_write", "audit_read", "finance_read", "finance_write"],
+        description: `load:${runId}:${tenantId}`
+      }),
+      opsTokenHeaders({ tenantId })
+    );
+    check(keyRes, { "setup: created api key": (r) => r.status === 201 });
+    const keyJson = keyRes.json();
+    const token = `${keyJson.keyId}.${keyJson.secret}`;
+
+    for (let j = 0; j < ROBOTS_PER_TENANT; j += 1) {
+      const robotId = `rob_${runId}_${tenantId}_${j}`;
+      const reg = http.post(
+        `${BASE_URL}/robots/register`,
+        JSON.stringify({ robotId, trustScore: 0.8, homeZoneId: "zone_a" }),
+        jsonHeaders({ tenantId, token, extra: { "x-idempotency-key": `reg_${robotId}` } })
+      );
+      check(reg, { "setup: robot registered": (r) => r.status === 201 });
+      const lastChainHash = reg.json()?.robot?.lastChainHash;
+      const avail = http.post(
+        `${BASE_URL}/robots/${robotId}/availability`,
+        JSON.stringify({ availability: [{ startAt: availStartAt, endAt: availEndAt }] }),
+        jsonHeaders({
+          tenantId,
+          token,
+          extra: { "x-idempotency-key": `avail_${robotId}`, "x-proxy-expected-prev-chain-hash": String(lastChainHash || "") }
+        })
+      );
+      check(avail, { "setup: robot availability set": (r) => r.status === 201 });
+    }
+
+    tenants.push({ tenantId, token, runId });
+  }
+
+  return { runId, tenants };
+}
+
+export default function (data) {
+  const tenants = data?.tenants || [];
+  const t = tenants[Math.floor(Math.random() * tenants.length)];
+  const tenantId = t.tenantId;
+  const token = t.token;
+
+  const startAt = isoNowPlusMs(10 * 60_000);
+  const endAt = isoNowPlusMs(70 * 60_000);
+
+  const created = http.post(
+    `${BASE_URL}/jobs`,
+    JSON.stringify({ templateId: "reset_lite", constraints: { zoneId: "zone_a" } }),
+    jsonHeaders({ tenantId, token, extra: { "x-idempotency-key": randomId("job") } })
+  );
+  if (!check(created, { "job: created": (r) => r.status === 201 })) return;
+
+  const jobId = created.json()?.job?.id;
+  let prev = created.json()?.job?.lastChainHash;
+  if (!jobId || !prev) return;
+
+  const quote = http.post(
+    `${BASE_URL}/jobs/${jobId}/quote`,
+    JSON.stringify({ startAt, endAt, environmentTier: "ENV_MANAGED_BUILDING" }),
+    jsonHeaders({ tenantId, token, extra: { "x-idempotency-key": randomId("quote"), "x-proxy-expected-prev-chain-hash": String(prev) } })
+  );
+  if (!check(quote, { "job: quoted": (r) => r.status === 201 })) return;
+  prev = quote.json()?.job?.lastChainHash;
+
+  const book = http.post(
+    `${BASE_URL}/jobs/${jobId}/book`,
+    JSON.stringify({ paymentHoldId: randomId("hold"), startAt, endAt, environmentTier: "ENV_MANAGED_BUILDING" }),
+    jsonHeaders({ tenantId, token, extra: { "x-idempotency-key": randomId("book"), "x-proxy-expected-prev-chain-hash": String(prev) } })
+  );
+  if (!check(book, { "job: booked": (r) => r.status === 201 })) return;
+
+  const cancel = http.post(
+    `${BASE_URL}/ops/jobs/${jobId}/cancel`,
+    JSON.stringify({ reason: "OPS" }),
+    jsonHeaders({ tenantId, token, extra: { "x-idempotency-key": randomId("cancel") } })
+  );
+  if (!check(cancel, { "job: cancelled": (r) => r.status === 201 })) return;
+  prev = cancel.json()?.job?.lastChainHash;
+  if (!prev) return;
+
+  const settled = http.post(
+    `${BASE_URL}/jobs/${jobId}/events`,
+    JSON.stringify({ type: "SETTLED", actor: { type: "ops", id: "load" }, payload: null }),
+    jsonHeaders({ tenantId, token, extra: { "x-idempotency-key": randomId("settle"), "x-proxy-expected-prev-chain-hash": String(prev) } })
+  );
+  check(settled, { "job: settled": (r) => r.status === 201 });
+
+  sleep(0.05);
+}
+
+export function pollOps(data) {
+  const tenants = data?.tenants || [];
+  const t = tenants[Math.floor(Math.random() * tenants.length)];
+  const tenantId = t.tenantId;
+  const token = t.token;
+
+  http.get(`${BASE_URL}/healthz`, jsonHeaders({ tenantId, token }));
+  http.get(`${BASE_URL}/ops/deliveries?limit=100`, jsonHeaders({ tenantId, token }));
+  http.get(`${BASE_URL}/ops/deliveries?state=failed&limit=100`, jsonHeaders({ tenantId, token }));
+  sleep(0.2);
+}
+

package/scripts/load/ingest-burst.k6.js

@@ -0,0 +1,236 @@
+import http from "k6/http";
+import { check, sleep } from "k6";
+import { Counter } from "k6/metrics";
+
+const BASE_URL = __ENV.BASE_URL || "http://localhost:3000";
+const OPS_TOKEN = __ENV.OPS_TOKEN || "";
+
+const TENANTS = Number.parseInt(__ENV.TENANTS || "10", 10);
+const ROBOTS_PER_TENANT = Number.parseInt(__ENV.ROBOTS_PER_TENANT || "3", 10);
+const JOBS_PER_MIN_PER_TENANT = Number.parseInt(__ENV.JOBS_PER_MIN_PER_TENANT || "50", 10);
+const DURATION = __ENV.DURATION || "2m";
+
+const INJECT_REJECTS_PCT = Number.parseFloat(__ENV.INJECT_REJECTS_PCT || "0.02"); // 2%
+
+const jobsTotalRate = Math.max(1, TENANTS * JOBS_PER_MIN_PER_TENANT);
+
+const ingestRejected = new Counter("ingest_rejected_requests_total");
+
+export const options = {
+  scenarios: {
+    jobs: {
+      executor: "constant-arrival-rate",
+      rate: jobsTotalRate,
+      timeUnit: "1m",
+      duration: DURATION,
+      preAllocatedVUs: Math.min(200, Math.max(20, jobsTotalRate)),
+      maxVUs: 500
+    },
+    ops_read: {
+      executor: "constant-arrival-rate",
+      rate: Math.max(10, TENANTS * 20),
+      timeUnit: "1m",
+      duration: DURATION,
+      preAllocatedVUs: 10,
+      maxVUs: 50,
+      exec: "opsRead"
+    }
+  },
+  thresholds: {
+    http_req_failed: ["rate<0.05"]
+  }
+};
+
+function jsonHeaders({ tenantId, token, extra = {} } = {}) {
+  const headers = {
+    "content-type": "application/json",
+    "x-proxy-tenant-id": tenantId,
+    "x-settld-protocol": "1.0"
+  };
+  if (token) headers.authorization = `Bearer ${token}`;
+  for (const [k, v] of Object.entries(extra)) headers[k] = v;
+  return { headers };
+}
+
+function opsTokenHeaders({ tenantId, extra = {} } = {}) {
+  const headers = {
+    "content-type": "application/json",
+    "x-proxy-tenant-id": tenantId,
+    "x-settld-protocol": "1.0",
+    "x-proxy-ops-token": OPS_TOKEN
+  };
+  for (const [k, v] of Object.entries(extra)) headers[k] = v;
+  return { headers };
+}
+
+function randomId(prefix) {
+  const n = Math.floor(Math.random() * 1e9);
+  return `${prefix}_${Date.now()}_${n}`;
+}
+
+function isoNowPlusMs(deltaMs) {
+  return new Date(Date.now() + deltaMs).toISOString();
+}
+
+export function setup() {
+  if (!OPS_TOKEN) throw new Error("OPS_TOKEN is required (configure PROXY_OPS_TOKENS on the server and pass OPS_TOKEN here)");
+  if (!Number.isFinite(TENANTS) || TENANTS <= 0) throw new Error("TENANTS must be a positive integer");
+
+  const runId = randomId("run");
+  const tenants = [];
+
+  const availStartAt = isoNowPlusMs(-60 * 60000);
+  const availEndAt = isoNowPlusMs(24 * 60 * 60000);
+
+  for (let i = 0; i < TENANTS; i += 1) {
+    const tenantId = i === 0 ? "tenant_default" : `tenant_${i}`;
+
+    // Create an API key for this tenant (exercise auth_keys path).
+    const keyRes = http.post(
+      `${BASE_URL}/ops/api-keys`,
+      JSON.stringify({
+        scopes: ["ops_read", "ops_write", "audit_read", "finance_read", "finance_write"],
+        description: `load:${runId}:${tenantId}`
+      }),
+      opsTokenHeaders({ tenantId })
+    );
+    check(keyRes, { "setup: created api key": (r) => r.status === 201 });
+    const keyJson = keyRes.json();
+    const token = `${keyJson.keyId}.${keyJson.secret}`;
+
+    // Seed robots to make /quote succeed.
+    const robots = [];
+    for (let j = 0; j < ROBOTS_PER_TENANT; j += 1) {
+      const robotId = `rob_${runId}_${tenantId}_${j}`;
+      const reg = http.post(
+        `${BASE_URL}/robots/register`,
+        JSON.stringify({ robotId, trustScore: 0.8, homeZoneId: "zone_a" }),
+        jsonHeaders({ tenantId, token, extra: { "x-idempotency-key": `reg_${robotId}` } })
+      );
+      check(reg, { "setup: robot registered": (r) => r.status === 201 });
+      const regJson = reg.json();
+      const lastChainHash = regJson && regJson.robot ? regJson.robot.lastChainHash : null;
+      const avail = http.post(
+        `${BASE_URL}/robots/${robotId}/availability`,
+        JSON.stringify({ availability: [{ startAt: availStartAt, endAt: availEndAt }] }),
+        jsonHeaders({
+          tenantId,
+          token,
+          extra: {
+            "x-idempotency-key": `avail_${robotId}`,
+            "x-proxy-expected-prev-chain-hash": String(lastChainHash || "")
+          }
+        })
+      );
+      check(avail, { "setup: robot availability set": (r) => r.status === 201 });
+      robots.push({ robotId });
+    }
+
+    tenants.push({ tenantId, token, runId, robots });
+  }
+
+  return { runId, tenants };
+}
+
+export default function (data) {
+  const tenants = data && data.tenants ? data.tenants : [];
+  const t = tenants[Math.floor(Math.random() * tenants.length)];
+  const tenantId = t.tenantId;
+  const token = t.token;
+
+  const startAt = isoNowPlusMs(10 * 60000);
+  const endAt = isoNowPlusMs(70 * 60000);
+
+  const created = http.post(
+    `${BASE_URL}/jobs`,
+    JSON.stringify({ templateId: "reset_lite", constraints: { zoneId: "zone_a" } }),
+    jsonHeaders({ tenantId, token, extra: { "x-idempotency-key": randomId("job") } })
+  );
+  if (!check(created, { "job: created": (r) => r.status === 201 })) return;
+
+  const createdJson = created.json();
+  const jobId = createdJson && createdJson.job ? createdJson.job.id : null;
+  let prev = createdJson && createdJson.job ? createdJson.job.lastChainHash : null;
+  if (!jobId || !prev) return;
+
+  const quote = http.post(
+    `${BASE_URL}/jobs/${jobId}/quote`,
+    JSON.stringify({ startAt, endAt, environmentTier: "ENV_MANAGED_BUILDING" }),
+    jsonHeaders({
+      tenantId,
+      token,
+      extra: { "x-idempotency-key": randomId("quote"), "x-proxy-expected-prev-chain-hash": String(prev) }
+    })
+  );
+  if (!check(quote, { "job: quoted": (r) => r.status === 201 })) return;
+  const quoteJson = quote.json();
+  prev = quoteJson && quoteJson.job ? quoteJson.job.lastChainHash : null;
+
+  const book = http.post(
+    `${BASE_URL}/jobs/${jobId}/book`,
+    JSON.stringify({ paymentHoldId: randomId("hold"), startAt, endAt, environmentTier: "ENV_MANAGED_BUILDING" }),
+    jsonHeaders({
+      tenantId,
+      token,
+      extra: { "x-idempotency-key": randomId("book"), "x-proxy-expected-prev-chain-hash": String(prev) }
+    })
+  );
+  if (!check(book, { "job: booked": (r) => r.status === 201 })) return;
+
+  // Abort via ops endpoint (server sets cancelledAt==event.at).
+  const cancel = http.post(
+    `${BASE_URL}/ops/jobs/${jobId}/cancel`,
+    JSON.stringify({ reason: "OPS" }),
+    jsonHeaders({ tenantId, token, extra: { "x-idempotency-key": randomId("cancel") } })
+  );
+  if (!check(cancel, { "job: cancelled": (r) => r.status === 201 })) return;
+  const cancelJson = cancel.json();
+  prev = cancelJson && cancelJson.job ? cancelJson.job.lastChainHash : null;
+  if (!prev) return;
+
+  // Settle (server-signed, payload not required).
+  const settled = http.post(
+    `${BASE_URL}/jobs/${jobId}/events`,
+    JSON.stringify({ type: "SETTLED", actor: { type: "ops", id: "load" }, payload: null }),
+    jsonHeaders({
+      tenantId,
+      token,
+      extra: { "x-idempotency-key": randomId("settle"), "x-proxy-expected-prev-chain-hash": String(prev) }
+    })
+  );
+  check(settled, { "job: settled": (r) => r.status === 201 });
+
+  // Inject a small percentage of deterministic ingest rejects (future timestamp or unsupported signer kind).
+  if (Math.random() < INJECT_REJECTS_PCT) {
+    const farFuture = isoNowPlusMs(10 * 60 * 60000); // ~10 hours ahead
+    const externalEventId = randomId("ext");
+    const reject = http.post(
+      `${BASE_URL}/ingest/proxy`,
+      JSON.stringify({
+        source: "load",
+        jobId,
+        events: [{ externalEventId, type: "DISPATCH_REQUESTED", at: farFuture, payload: null }]
+      }),
+      jsonHeaders({ tenantId, token, extra: { "x-idempotency-key": `ing_${externalEventId}` } })
+    );
+    try {
+      const rejectJson = reject.json();
+      const results = rejectJson && Array.isArray(rejectJson.results) ? rejectJson.results : [];
+      if (results.length && results[0] && results[0].status === "rejected") ingestRejected.add(1);
+    } catch (err) {}
+  }
+
+  sleep(0.1);
+}
+
+export function opsRead(data) {
+  const tenants = data && data.tenants ? data.tenants : [];
+  const t = tenants[Math.floor(Math.random() * tenants.length)];
+  const tenantId = t.tenantId;
+  const token = t.token;
+
+  http.get(`${BASE_URL}/healthz`, jsonHeaders({ tenantId, token }));
+  http.get(`${BASE_URL}/ops/deliveries?limit=50`, jsonHeaders({ tenantId, token }));
+  http.get(`${BASE_URL}/ops/dlq?type=delivery&limit=50`, jsonHeaders({ tenantId, token }));
+  sleep(0.2);
+}

package/scripts/load/run-delivery-load.js

@@ -0,0 +1,66 @@
+import { spawn } from "node:child_process";
+
+function assertCmdExists(cmd) {
+  return new Promise((resolve) => {
+    const p = spawn(cmd, ["version"], { stdio: "ignore" });
+    p.on("error", () => resolve(false));
+    p.on("exit", (code) => resolve(code === 0));
+  });
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function waitForReceiverReady({ baseUrl, timeoutMs = 10_000 } = {}) {
+  const started = Date.now();
+  return new Promise((resolve, reject) => {
+    async function tick() {
+      try {
+        const res = await fetch(`${baseUrl}/health`);
+        if (res.status === 200) return resolve();
+      } catch {}
+      if (Date.now() - started > timeoutMs) return reject(new Error("webhook receiver did not become ready"));
+      setTimeout(tick, 250);
+    }
+    tick();
+  });
+}
+
+async function main() {
+  const ok = await assertCmdExists("k6");
+  if (!ok) {
+    process.stderr.write("k6 is required on PATH (https://k6.io/docs/get-started/installation/)\n");
+    process.exit(2);
+  }
+
+  const receiverPort = process.env.RECEIVER_PORT ?? "4010";
+  const receiverUrl = process.env.RECEIVER_URL ?? `http://127.0.0.1:${receiverPort}`;
+
+  const receiver = spawn(process.execPath, ["scripts/load/webhook-receiver.js"], {
+    stdio: ["ignore", "inherit", "inherit"],
+    env: { ...process.env, PORT: receiverPort }
+  });
+
+  try {
+    await waitForReceiverReady({ baseUrl: receiverUrl, timeoutMs: 15_000 });
+
+    // Give the API a moment to settle if it was just started.
+    await sleep(250);
+
+    const k6Args = ["run", "scripts/load/delivery-stress.k6.js"];
+    const k6 = spawn("k6", k6Args, { stdio: "inherit", env: process.env });
+    const exitCode = await new Promise((resolve) => k6.on("exit", (code) => resolve(code ?? 1)));
+    process.exitCode = exitCode;
+  } finally {
+    try {
+      receiver.kill("SIGTERM");
+    } catch {}
+  }
+}
+
+main().catch((err) => {
+  process.stderr.write(`${err?.stack || err?.message || String(err)}\n`);
+  process.exit(1);
+});
+