settld 0.1.2 → 0.2.0

package/scripts/slo/check.mjs

@@ -0,0 +1,239 @@
+import assert from "node:assert/strict";
+import fs from "node:fs/promises";
+
+const SLO_CHECK_SCHEMA_VERSION = "OperationalSloCheck.v1";
+
+function normalizeOptionalString(value) {
+  if (typeof value !== "string") return null;
+  const trimmed = value.trim();
+  return trimmed === "" ? null : trimmed;
+}
+
+export function assertFiniteNumber(n, name) {
+  if (!Number.isFinite(n)) throw new TypeError(`${name} must be finite`);
+}
+
+export function parseSloThresholds(env = process.env) {
+  return {
+    maxHttp5xxTotal: Number(env.SLO_MAX_HTTP_5XX_TOTAL ?? "0"),
+    maxOutboxPending: Number(env.SLO_MAX_OUTBOX_PENDING ?? "200"),
+    maxDeliveryDlq: Number(env.SLO_MAX_DELIVERY_DLQ ?? "0"),
+    maxDeliveriesPending: Number(env.SLO_MAX_DELIVERIES_PENDING ?? "0"),
+    maxDeliveriesFailed: Number(env.SLO_MAX_DELIVERIES_FAILED ?? "0")
+  };
+}
+
+export function validateSloThresholds(thresholds) {
+  for (const [k, v] of [
+    ["SLO_MAX_HTTP_5XX_TOTAL", thresholds.maxHttp5xxTotal],
+    ["SLO_MAX_OUTBOX_PENDING", thresholds.maxOutboxPending],
+    ["SLO_MAX_DELIVERY_DLQ", thresholds.maxDeliveryDlq],
+    ["SLO_MAX_DELIVERIES_PENDING", thresholds.maxDeliveriesPending],
+    ["SLO_MAX_DELIVERIES_FAILED", thresholds.maxDeliveriesFailed]
+  ]) {
+    assertFiniteNumber(v, k);
+    if (v < 0) throw new TypeError(`${k} must be >= 0`);
+  }
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function fetchTextWithTimeout(url, timeoutMs = 5000) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(new Error("timeout")), timeoutMs);
+  try {
+    const res = await fetch(url, { signal: controller.signal });
+    const text = await res.text();
+    return { status: res.status, text };
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+function unescapeLabelValue(value) {
+  return String(value).replaceAll("\\\\", "\\").replaceAll("\\n", "\n").replaceAll('\\"', '"');
+}
+
+function parseLabels(src) {
+  const labels = {};
+  let i = 0;
+  while (i < src.length) {
+    while (i < src.length && (src[i] === " " || src[i] === ",")) i += 1;
+    if (i >= src.length) break;
+    let key = "";
+    while (i < src.length && /[a-zA-Z0-9_]/.test(src[i])) {
+      key += src[i];
+      i += 1;
+    }
+    while (i < src.length && src[i] === " ") i += 1;
+    if (src[i] !== "=") break;
+    i += 1;
+    while (i < src.length && src[i] === " ") i += 1;
+    if (src[i] !== '"') break;
+    i += 1;
+    let value = "";
+    while (i < src.length) {
+      const ch = src[i];
+      if (ch === '"') {
+        i += 1;
+        break;
+      }
+      if (ch === "\\") {
+        const next = src[i + 1];
+        if (next === "n") {
+          value += "\n";
+          i += 2;
+          continue;
+        }
+        if (next === "\\" || next === '"') {
+          value += next;
+          i += 2;
+          continue;
+        }
+      }
+      value += ch;
+      i += 1;
+    }
+    labels[key] = unescapeLabelValue(value);
+    while (i < src.length && src[i] !== ",") i += 1;
+    if (src[i] === ",") i += 1;
+  }
+  return labels;
+}
+
+export function parsePrometheusText(text) {
+  const series = [];
+  const lines = String(text ?? "").split("\n");
+  for (const lineRaw of lines) {
+    const line = lineRaw.trim();
+    if (!line || line.startsWith("#")) continue;
+    const m = /^([a-zA-Z_:][a-zA-Z0-9_:]*)(\{[^}]*\})?\s+([-+]?(\d+(\.\d*)?|\.\d+)([eE][-+]?\d+)?|NaN|Inf|-Inf)\s*$/.exec(line);
+    if (!m) continue;
+    const name = m[1];
+    const labelsRaw = m[2] ?? "";
+    const value = Number(m[3]);
+    const labels = labelsRaw.startsWith("{") ? parseLabels(labelsRaw.slice(1, -1)) : {};
+    series.push({ name, labels, value });
+  }
+  return series;
+}
+
+export function sumWhere(series, { name, where = () => true } = {}) {
+  let sum = 0;
+  for (const sample of series) {
+    if (sample.name !== name) continue;
+    if (!where(sample.labels, sample.value)) continue;
+    const value = Number(sample.value);
+    if (!Number.isFinite(value)) continue;
+    sum += value;
+  }
+  return sum;
+}
+
+export function getOne(series, { name, where = () => true } = {}) {
+  for (const sample of series) {
+    if (sample.name !== name) continue;
+    if (!where(sample.labels, sample.value)) continue;
+    return Number(sample.value);
+  }
+  return null;
+}
+
+export function collectOperationalSloSummary(series) {
+  const http5xxTotal = sumWhere(series, {
+    name: "http_requests_total",
+    where: (labels) => typeof labels.status === "string" && labels.status.startsWith("5")
+  });
+  const outboxPending = sumWhere(series, { name: "outbox_pending_gauge" });
+  const deliveryDlq = sumWhere(series, { name: "delivery_dlq_pending_total_gauge" });
+  const deliveriesPending = sumWhere(series, {
+    name: "deliveries_pending_gauge",
+    where: (labels) => labels.state === "pending"
+  });
+  const deliveriesFailed = sumWhere(series, {
+    name: "deliveries_pending_gauge",
+    where: (labels) => labels.state === "failed"
+  });
+  return {
+    http5xxTotal,
+    outboxPending,
+    deliveryDlq,
+    deliveriesPending,
+    deliveriesFailed
+  };
+}
+
+export function assertOperationalSlo(summary, thresholds) {
+  assert.ok(
+    summary.http5xxTotal <= thresholds.maxHttp5xxTotal,
+    `SLO breach: http 5xx total ${summary.http5xxTotal} > ${thresholds.maxHttp5xxTotal}`
+  );
+  assert.ok(
+    summary.outboxPending <= thresholds.maxOutboxPending,
+    `SLO breach: outbox pending ${summary.outboxPending} > ${thresholds.maxOutboxPending}`
+  );
+  assert.ok(
+    summary.deliveryDlq <= thresholds.maxDeliveryDlq,
+    `SLO breach: delivery DLQ ${summary.deliveryDlq} > ${thresholds.maxDeliveryDlq}`
+  );
+  assert.ok(
+    summary.deliveriesPending <= thresholds.maxDeliveriesPending,
+    `SLO breach: deliveries pending ${summary.deliveriesPending} > ${thresholds.maxDeliveriesPending}`
+  );
+  assert.ok(
+    summary.deliveriesFailed <= thresholds.maxDeliveriesFailed,
+    `SLO breach: deliveries failed ${summary.deliveriesFailed} > ${thresholds.maxDeliveriesFailed}`
+  );
+}
+
+export async function loadMetricsText({
+  metricsFile = null,
+  apiBaseUrl = "http://127.0.0.1:3000",
+  metricsPath = "/metrics",
+  flushDelayMs = 250
+} = {}) {
+  if (metricsFile) {
+    return await fs.readFile(metricsFile, "utf8");
+  }
+  await sleep(flushDelayMs);
+  const response = await fetchTextWithTimeout(`${apiBaseUrl}${metricsPath}`, 10_000);
+  assert.equal(response.status, 200, `GET ${metricsPath} failed: http ${response.status}`);
+  return response.text;
+}
+
+export async function runSloCheck({ env = process.env } = {}) {
+  const thresholds = parseSloThresholds(env);
+  validateSloThresholds(thresholds);
+  const metricsFile = normalizeOptionalString(env.SLO_METRICS_FILE);
+  const metricsText = await loadMetricsText({
+    metricsFile,
+    apiBaseUrl: env.SLO_API_BASE_URL ?? "http://127.0.0.1:3000",
+    metricsPath: env.SLO_METRICS_PATH ?? "/metrics"
+  });
+  const series = parsePrometheusText(metricsText);
+  const summary = collectOperationalSloSummary(series);
+  assertOperationalSlo(summary, thresholds);
+  return summary;
+}
+
+async function main() {
+  const summary = await runSloCheck({ env: process.env });
+  console.log(JSON.stringify({ schemaVersion: SLO_CHECK_SCHEMA_VERSION, slo: summary }));
+}
+
+const isDirectExecution = (() => {
+  try {
+    return import.meta.url === new URL(`file://${process.argv[1]}`).href;
+  } catch {
+    return false;
+  }
+})();
+
+if (isDirectExecution) {
+  main().catch((err) => {
+    process.stderr.write(`${err?.stack ?? err?.message ?? String(err)}\n`);
+    process.exit(1);
+  });
+}

package/scripts/smoke/k8s-smoke.mjs

@@ -0,0 +1,214 @@
+import assert from "node:assert/strict";
+
+import { createEd25519Keypair, keyIdFromPublicKeyPem } from "../../src/core/crypto.js";
+import { createChainedEvent, finalizeChainedEvent } from "../../src/core/event-chain.js";
+import { computeSlaPolicy } from "../../src/core/sla.js";
+import { buildPolicySnapshot, computePolicyHash } from "../../src/core/policy.js";
+
+const API_BASE_URL = process.env.SMOKE_API_BASE_URL ?? "http://127.0.0.1:3000";
+const TENANT_ID = process.env.SMOKE_TENANT_ID ?? "tenant_default";
+const OPS_TOKEN = process.env.SMOKE_OPS_TOKEN ?? "dev";
+const PROTOCOL = process.env.SMOKE_PROTOCOL ?? "1.0";
+
+function sleep(ms) {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+async function waitFor(fn, { timeoutMs = 60_000, intervalMs = 250 } = {}) {
+  const deadline = Date.now() + timeoutMs;
+  let lastErr = null;
+  while (Date.now() < deadline) {
+    try {
+      const ok = await fn();
+      if (ok) return ok;
+    } catch (err) {
+      lastErr = err;
+    }
+    await sleep(intervalMs);
+  }
+  if (lastErr) throw lastErr;
+  throw new Error("timeout");
+}
+
+async function httpJson({ method, path, headers, body }) {
+  const res = await fetch(`${API_BASE_URL}${path}`, {
+    method,
+    headers: {
+      ...(headers ?? {}),
+      ...(body !== undefined ? { "content-type": "application/json; charset=utf-8" } : {})
+    },
+    body: body === undefined ? undefined : JSON.stringify(body)
+  });
+  const text = await res.text();
+  const isJson = res.headers.get("content-type")?.includes("json");
+  const json = isJson && text ? JSON.parse(text) : null;
+  return { status: res.status, json, text, headers: res.headers };
+}
+
+async function main() {
+  await waitFor(async () => {
+    const r = await httpJson({ method: "GET", path: "/healthz" }).catch(() => null);
+    return r && r.status === 200;
+  });
+  await waitFor(async () => {
+    const r = await httpJson({ method: "GET", path: "/capabilities" }).catch(() => null);
+    if (!r || r.status !== 200) return false;
+    if (r.headers.get("x-settld-protocol") !== PROTOCOL) return false;
+    return true;
+  });
+
+  const createKey = await httpJson({
+    method: "POST",
+    path: "/ops/api-keys",
+    headers: { "x-proxy-tenant-id": TENANT_ID, "x-proxy-ops-token": OPS_TOKEN, "x-settld-protocol": PROTOCOL },
+    body: { scopes: ["ops_read", "ops_write", "finance_read", "finance_write", "audit_read"] }
+  });
+  assert.equal(createKey.status, 201, createKey.text);
+  const bearer = `Bearer ${createKey.json.keyId}.${createKey.json.secret}`;
+  const headersBase = { "x-proxy-tenant-id": TENANT_ID, authorization: bearer, "x-settld-protocol": PROTOCOL };
+
+  // Robot registration + availability.
+  const { publicKeyPem: robotPublicKeyPem, privateKeyPem: robotPrivateKeyPem } = createEd25519Keypair();
+  const robotKeyId = keyIdFromPublicKeyPem(robotPublicKeyPem);
+  const robotId = `rob_smoke_${Date.now()}`;
+
+  const regRobot = await httpJson({
+    method: "POST",
+    path: "/robots/register",
+    headers: { ...headersBase, "x-idempotency-key": `robot_reg_${robotId}` },
+    body: { robotId, publicKeyPem: robotPublicKeyPem }
+  });
+  assert.equal(regRobot.status, 201, regRobot.text);
+
+  const now = Date.now();
+  const bookingStartAt = new Date(now + 5 * 60_000).toISOString();
+  const bookingEndAt = new Date(now + 65 * 60_000).toISOString();
+
+  const setAvail = await httpJson({
+    method: "POST",
+    path: `/robots/${robotId}/availability`,
+    headers: {
+      ...headersBase,
+      "x-idempotency-key": `robot_avail_${robotId}`,
+      "x-proxy-expected-prev-chain-hash": regRobot.json.robot.lastChainHash
+    },
+    body: {
+      availability: [{ startAt: new Date(now - 60 * 60_000).toISOString(), endAt: new Date(now + 24 * 60 * 60_000).toISOString() }]
+    }
+  });
+  assert.equal(setAvail.status, 201, setAvail.text);
+
+  // Job lifecycle (minimal but complete enough to trigger artifacts).
+  const createJob = await httpJson({
+    method: "POST",
+    path: "/jobs",
+    headers: { ...headersBase, "x-idempotency-key": `job_create_${robotId}` },
+    body: { templateId: "reset_lite", constraints: { privacyMode: "minimal" } }
+  });
+  assert.equal(createJob.status, 201, createJob.text);
+  const jobId = createJob.json.job.id;
+  let lastChainHash = createJob.json.job.lastChainHash;
+
+  const quote = await httpJson({
+    method: "POST",
+    path: `/jobs/${jobId}/quote`,
+    headers: { ...headersBase, "x-idempotency-key": `job_quote_${jobId}`, "x-proxy-expected-prev-chain-hash": lastChainHash },
+    body: { startAt: bookingStartAt, endAt: bookingEndAt, environmentTier: "ENV_MANAGED_BUILDING", requiresOperatorCoverage: false }
+  });
+  assert.equal(quote.status, 201, quote.text);
+  lastChainHash = quote.json.job.lastChainHash;
+
+  const book = await httpJson({
+    method: "POST",
+    path: `/jobs/${jobId}/book`,
+    headers: { ...headersBase, "x-idempotency-key": `job_book_${jobId}`, "x-proxy-expected-prev-chain-hash": lastChainHash },
+    body: {
+      paymentHoldId: `hold_${jobId}`,
+      startAt: bookingStartAt,
+      endAt: bookingEndAt,
+      environmentTier: "ENV_MANAGED_BUILDING",
+      requiresOperatorCoverage: false
+    }
+  });
+  assert.equal(book.status, 201, book.text);
+  lastChainHash = book.json.job.lastChainHash;
+
+  async function postServerEvent(type, payload, idempotencyKey) {
+    const r = await httpJson({
+      method: "POST",
+      path: `/jobs/${jobId}/events`,
+      headers: { ...headersBase, "x-idempotency-key": idempotencyKey, "x-proxy-expected-prev-chain-hash": lastChainHash },
+      body: { type, actor: { type: "system", id: "proxy" }, payload }
+    });
+    assert.equal(r.status, 201, r.text);
+    lastChainHash = r.json.job.lastChainHash;
+    return r.json.event;
+  }
+
+  async function postRobotEvent(type, payload) {
+    const draft = createChainedEvent({ streamId: jobId, type, actor: { type: "robot", id: robotId }, payload });
+    const finalized = finalizeChainedEvent({ event: draft, prevChainHash: lastChainHash, signer: { keyId: robotKeyId, privateKeyPem: robotPrivateKeyPem } });
+    const r = await httpJson({ method: "POST", path: `/jobs/${jobId}/events`, headers: { ...headersBase }, body: finalized });
+    assert.equal(r.status, 201, r.text);
+    lastChainHash = r.json.job.lastChainHash;
+    return r.json.event;
+  }
+
+  // Move the job through the state machine.
+  await postServerEvent("MATCHED", { robotId }, `m_${jobId}`);
+  await postServerEvent("RESERVED", { robotId, startAt: bookingStartAt, endAt: bookingEndAt, reservationId: `rsv_${jobId}` }, `r_${jobId}`);
+
+  const accessPlanId = `ap_${jobId}`;
+  await postServerEvent(
+    "ACCESS_PLAN_ISSUED",
+    {
+      jobId,
+      accessPlanId,
+      method: "BUILDING_CONCIERGE",
+      credentialRef: `vault://access/${accessPlanId}/v1`,
+      scope: { areas: ["ENTRYWAY"], noGo: [] },
+      validFrom: new Date(now - 60_000).toISOString(),
+      validTo: new Date(now + 60 * 60_000).toISOString(),
+      revocable: true,
+      requestedBy: "system"
+    },
+    `ap_${jobId}`
+  );
+
+  await postRobotEvent("EN_ROUTE", { etaSeconds: 60 });
+  await postRobotEvent("ACCESS_GRANTED", { jobId, accessPlanId, method: "BUILDING_CONCIERGE" });
+  await postRobotEvent("EXECUTION_STARTED", { plan: ["navigate"] });
+  await postRobotEvent("EXECUTION_COMPLETED", { report: { durationSeconds: 1 } });
+
+  await postServerEvent("SETTLED", { settlement: "smoke" }, `s_${jobId}`);
+
+  // Assert that artifacts are being produced (outbox + artifact worker + pg durability).
+  await waitFor(async () => {
+    const r = await httpJson({ method: "GET", path: `/jobs/${jobId}/artifacts`, headers: headersBase }).catch(() => null);
+    if (!r || r.status !== 200) return false;
+    const artifacts = Array.isArray(r.json?.artifacts) ? r.json.artifacts : [];
+    return artifacts.length > 0;
+  }, { timeoutMs: 90_000, intervalMs: 500 });
+
+  // Sanity: protocol headers present.
+  const status = await httpJson({ method: "GET", path: "/ops/status", headers: headersBase });
+  assert.equal(status.status, 200, status.text);
+  assert.equal(status.headers.get("x-settld-protocol"), PROTOCOL);
+
+  // Extra assertion: booking pinned policy hash is deterministic (exercise policy functions quickly).
+  const sla = computeSlaPolicy({ environmentTier: "ENV_MANAGED_BUILDING" });
+  const policySnapshot = buildPolicySnapshot({
+    contractId: "contract_default",
+    contractVersion: 1,
+    environmentTier: "ENV_MANAGED_BUILDING",
+    requiresOperatorCoverage: false,
+    sla,
+    creditPolicy: { enabled: false, defaultAmountCents: 0, maxAmountCents: 0, currency: "USD" },
+    evidencePolicy: { retentionDays: 0 },
+    claimPolicy: { currency: "USD", autoApproveThresholdCents: 0, maxPayoutCents: 0, reservePercent: 0 },
+    coveragePolicy: { required: false, responseSlaSeconds: 0, includedAssistSeconds: 0, overageRateCentsPerMinute: 0 }
+  });
+  assert.ok(computePolicyHash(policySnapshot));
+}
+
+await main();