sentinelayer-cli 0.1.0

package/src/swarm/scenario-dsl.js

@@ -0,0 +1,272 @@
+import fsp from "node:fs/promises";
+import path from "node:path";
+
+const SUPPORTED_ACTIONS = new Set(["goto", "click", "fill", "wait", "screenshot"]);
+
+function normalizeString(value) {
+  return String(value || "").trim();
+}
+
+function decodeQuoted(value) {
+  return String(value || "")
+    .replace(/^"/, "")
+    .replace(/"$/, "")
+    .replace(/\\"/g, '"')
+    .replace(/\\n/g, "\n")
+    .replace(/\\t/g, "\t");
+}
+
+function tokenizeLine(line = "") {
+  const tokens = [];
+  let cursor = 0;
+  while (cursor < line.length) {
+    while (cursor < line.length && /\s/.test(line[cursor])) {
+      cursor += 1;
+    }
+    if (cursor >= line.length) {
+      break;
+    }
+    if (line[cursor] === "#") {
+      break;
+    }
+
+    if (line[cursor] === '"') {
+      let token = '"';
+      cursor += 1;
+      let escaped = false;
+      while (cursor < line.length) {
+        const char = line[cursor];
+        token += char;
+        cursor += 1;
+        if (escaped) {
+          escaped = false;
+          continue;
+        }
+        if (char === "\\") {
+          escaped = true;
+          continue;
+        }
+        if (char === '"') {
+          break;
+        }
+      }
+      if (!token.endsWith('"')) {
+        throw new Error("Unterminated quoted string.");
+      }
+      tokens.push(decodeQuoted(token));
+      continue;
+    }
+
+    const start = cursor;
+    while (cursor < line.length && !/\s/.test(line[cursor])) {
+      cursor += 1;
+    }
+    tokens.push(line.slice(start, cursor));
+  }
+  return tokens;
+}
+
+function parseAction(tokens = [], lineNumber = 0) {
+  const type = normalizeString(tokens[0]).toLowerCase();
+  if (!SUPPORTED_ACTIONS.has(type)) {
+    throw new Error(
+      `Line ${lineNumber}: unsupported action '${type}'. Supported: ${[...SUPPORTED_ACTIONS].join(
+        ", "
+      )}.`
+    );
+  }
+
+  if (type === "goto") {
+    if (tokens.length < 2) {
+      throw new Error(`Line ${lineNumber}: action goto requires <url>.`);
+    }
+    return {
+      type,
+      url: normalizeString(tokens[1]),
+    };
+  }
+
+  if (type === "click") {
+    if (tokens.length < 2) {
+      throw new Error(`Line ${lineNumber}: action click requires <selector>.`);
+    }
+    return {
+      type,
+      selector: normalizeString(tokens[1]),
+    };
+  }
+
+  if (type === "fill") {
+    if (tokens.length < 3) {
+      throw new Error(`Line ${lineNumber}: action fill requires <selector> <text>.`);
+    }
+    return {
+      type,
+      selector: normalizeString(tokens[1]),
+      text: normalizeString(tokens[2]),
+    };
+  }
+
+  if (type === "wait") {
+    if (tokens.length < 2) {
+      throw new Error(`Line ${lineNumber}: action wait requires <ms>.`);
+    }
+    const ms = Number(tokens[1]);
+    if (!Number.isFinite(ms) || ms < 0) {
+      throw new Error(`Line ${lineNumber}: action wait requires non-negative milliseconds.`);
+    }
+    return {
+      type,
+      ms: Math.floor(ms),
+    };
+  }
+
+  if (type === "screenshot") {
+    return {
+      type,
+      path: normalizeString(tokens[1] || ""),
+    };
+  }
+
+  throw new Error(`Line ${lineNumber}: unsupported action '${type}'.`);
+}
+
+export function parseScenarioDsl(content = "", { source = "" } = {}) {
+  const raw = String(content || "");
+  const lines = raw.split(/\r?\n/);
+  const spec = {
+    id: "",
+    startUrl: "",
+    tags: [],
+    actions: [],
+    source: normalizeString(source),
+  };
+
+  for (let index = 0; index < lines.length; index += 1) {
+    const lineNumber = index + 1;
+    const line = lines[index];
+    const trimmed = normalizeString(line);
+    if (!trimmed || trimmed.startsWith("#")) {
+      continue;
+    }
+
+    let tokens;
+    try {
+      tokens = tokenizeLine(line);
+    } catch (error) {
+      throw new Error(`Line ${lineNumber}: ${error.message}`);
+    }
+    if (tokens.length === 0) {
+      continue;
+    }
+    const command = normalizeString(tokens[0]).toLowerCase();
+
+    if (command === "scenario") {
+      if (tokens.length < 2) {
+        throw new Error(`Line ${lineNumber}: scenario requires <id>.`);
+      }
+      spec.id = normalizeString(tokens[1]);
+      continue;
+    }
+
+    if (command === "start_url") {
+      if (tokens.length < 2) {
+        throw new Error(`Line ${lineNumber}: start_url requires <url>.`);
+      }
+      spec.startUrl = normalizeString(tokens[1]);
+      continue;
+    }
+
+    if (command === "tag") {
+      if (tokens.length < 2) {
+        throw new Error(`Line ${lineNumber}: tag requires <value>.`);
+      }
+      spec.tags.push(normalizeString(tokens[1]));
+      continue;
+    }
+
+    if (command === "action") {
+      const action = parseAction(tokens.slice(1), lineNumber);
+      spec.actions.push(action);
+      continue;
+    }
+
+    throw new Error(`Line ${lineNumber}: unknown command '${command}'.`);
+  }
+
+  return spec;
+}
+
+export function validateScenarioSpec(spec = {}) {
+  const errors = [];
+  const id = normalizeString(spec.id);
+  if (!id) {
+    errors.push("scenario id is required.");
+  }
+  if (!Array.isArray(spec.actions) || spec.actions.length === 0) {
+    errors.push("at least one action is required.");
+  }
+  for (const action of spec.actions || []) {
+    if (!SUPPORTED_ACTIONS.has(normalizeString(action.type).toLowerCase())) {
+      errors.push(`unsupported action type: ${action.type}`);
+    }
+  }
+  return {
+    valid: errors.length === 0,
+    errors,
+  };
+}
+
+export async function parseScenarioFile(filePath = "") {
+  const normalizedPath = normalizeString(filePath);
+  if (!normalizedPath) {
+    throw new Error("scenario file path is required.");
+  }
+  const resolved = path.resolve(process.cwd(), normalizedPath);
+  const content = await fsp.readFile(resolved, "utf-8");
+  const spec = parseScenarioDsl(content, { source: resolved });
+  return {
+    spec,
+    filePath: resolved,
+  };
+}
+
+export function renderScenarioTemplate({
+  scenarioId = "nightly_smoke",
+  startUrl = "https://example.com",
+} = {}) {
+  const id = normalizeString(scenarioId) || "nightly_smoke";
+  const url = normalizeString(startUrl) || "https://example.com";
+  return `# Sentinelayer swarm scenario DSL
+scenario "${id}"
+start_url "${url}"
+tag "smoke"
+tag "runtime"
+
+action goto "${url}"
+action wait 500
+action screenshot "${id}-home.png"
+`;
+}
+
+export async function writeScenarioTemplate({
+  scenarioId,
+  targetPath = ".",
+  outputFile = "",
+  startUrl = "https://example.com",
+} = {}) {
+  const normalizedTargetPath = path.resolve(String(targetPath || "."));
+  const resolvedOutputFile = normalizeString(outputFile)
+    ? path.resolve(normalizedTargetPath, outputFile)
+    : path.join(normalizedTargetPath, ".sentinelayer", "scenarios", `${scenarioId}.sls`);
+  const content = renderScenarioTemplate({
+    scenarioId,
+    startUrl,
+  });
+  await fsp.mkdir(path.dirname(resolvedOutputFile), { recursive: true });
+  await fsp.writeFile(resolvedOutputFile, `${content.trim()}\n`, "utf-8");
+  return {
+    filePath: resolvedOutputFile,
+    content,
+  };
+}

package/src/telemetry/ledger.js

@@ -0,0 +1,302 @@
+import fsp from "node:fs/promises";
+import path from "node:path";
+import { randomUUID } from "node:crypto";
+
+import { resolveOutputRoot } from "../config/service.js";
+
+const OBSERVABILITY_DIR_NAME = "observability";
+const RUN_EVENTS_FILE_NAME = "run-events.jsonl";
+const SCHEMA_VERSION = 1;
+
+export const RUN_EVENT_TYPES = Object.freeze([
+  "run_start",
+  "run_step",
+  "tool_call",
+  "usage",
+  "budget_check",
+  "run_stop",
+]);
+
+export const STOP_CLASSES = Object.freeze([
+  "NONE",
+  "MAX_COST_EXCEEDED",
+  "MAX_OUTPUT_TOKENS_EXCEEDED",
+  "DIMINISHING_RETURNS",
+  "MAX_RUNTIME_MS_EXCEEDED",
+  "MAX_TOOL_CALLS_EXCEEDED",
+  "MANUAL_STOP",
+  "ERROR",
+  "UNKNOWN",
+]);
+
+const RUN_EVENT_TYPE_SET = new Set(RUN_EVENT_TYPES);
+const STOP_CLASS_SET = new Set(STOP_CLASSES);
+
+function normalizeNonNegativeNumber(value, field) {
+  const normalized = Number(value || 0);
+  if (!Number.isFinite(normalized) || normalized < 0) {
+    throw new Error(`${field} must be a non-negative number.`);
+  }
+  return normalized;
+}
+
+function normalizeReasonCodes(reasonCodes = []) {
+  if (reasonCodes === undefined || reasonCodes === null) {
+    return [];
+  }
+  if (!Array.isArray(reasonCodes)) {
+    throw new Error("stop.reasonCodes must be an array of strings.");
+  }
+
+  const unique = new Set();
+  for (const reasonCode of reasonCodes) {
+    const normalized = String(reasonCode || "").trim().toUpperCase();
+    if (normalized) {
+      unique.add(normalized);
+    }
+  }
+
+  return [...unique];
+}
+
+function normalizeStop(stop = null) {
+  if (!stop) {
+    return null;
+  }
+  if (typeof stop !== "object" || Array.isArray(stop)) {
+    throw new Error("stop must be an object when provided.");
+  }
+
+  const stopClass = String(stop.stopClass || "NONE").trim().toUpperCase() || "NONE";
+  if (!STOP_CLASS_SET.has(stopClass)) {
+    throw new Error(`Unsupported stop class '${stopClass}'.`);
+  }
+
+  return {
+    stopClass,
+    blocking: Boolean(stop.blocking),
+    reasonCodes: normalizeReasonCodes(stop.reasonCodes),
+  };
+}
+
+function normalizeMetadata(metadata = {}) {
+  if (metadata === undefined || metadata === null) {
+    return {};
+  }
+  if (typeof metadata !== "object" || Array.isArray(metadata)) {
+    throw new Error("metadata must be an object when provided.");
+  }
+  return { ...metadata };
+}
+
+function normalizeUsage(usage = {}) {
+  if (usage === undefined || usage === null) {
+    return {
+      inputTokens: 0,
+      outputTokens: 0,
+      cacheReadTokens: 0,
+      cacheWriteTokens: 0,
+      costUsd: 0,
+      durationMs: 0,
+      toolCalls: 0,
+    };
+  }
+
+  if (typeof usage !== "object" || Array.isArray(usage)) {
+    throw new Error("usage must be an object when provided.");
+  }
+
+  return {
+    inputTokens: normalizeNonNegativeNumber(usage.inputTokens, "usage.inputTokens"),
+    outputTokens: normalizeNonNegativeNumber(usage.outputTokens, "usage.outputTokens"),
+    cacheReadTokens: normalizeNonNegativeNumber(usage.cacheReadTokens, "usage.cacheReadTokens"),
+    cacheWriteTokens: normalizeNonNegativeNumber(usage.cacheWriteTokens, "usage.cacheWriteTokens"),
+    costUsd: normalizeNonNegativeNumber(usage.costUsd, "usage.costUsd"),
+    durationMs: normalizeNonNegativeNumber(usage.durationMs, "usage.durationMs"),
+    toolCalls: normalizeNonNegativeNumber(usage.toolCalls, "usage.toolCalls"),
+  };
+}
+
+export function mapBudgetReasonToStopClass(reasonCode = "") {
+  const normalized = String(reasonCode || "").trim().toUpperCase();
+  switch (normalized) {
+    case "MAX_COST_EXCEEDED":
+      return "MAX_COST_EXCEEDED";
+    case "MAX_OUTPUT_TOKENS_EXCEEDED":
+      return "MAX_OUTPUT_TOKENS_EXCEEDED";
+    case "DIMINISHING_RETURNS":
+      return "DIMINISHING_RETURNS";
+    case "MAX_RUNTIME_MS_EXCEEDED":
+      return "MAX_RUNTIME_MS_EXCEEDED";
+    case "MAX_TOOL_CALLS_EXCEEDED":
+      return "MAX_TOOL_CALLS_EXCEEDED";
+    default:
+      return "UNKNOWN";
+  }
+}
+
+export function deriveStopClassFromBudget(budget = {}) {
+  if (!budget || !Array.isArray(budget.reasons) || budget.reasons.length === 0) {
+    return "NONE";
+  }
+  const firstReasonCode = String(budget.reasons[0]?.code || "").trim().toUpperCase();
+  return mapBudgetReasonToStopClass(firstReasonCode);
+}
+
+export function normalizeRunEvent(event = {}) {
+  if (!event || typeof event !== "object" || Array.isArray(event)) {
+    throw new Error("Run event must be an object.");
+  }
+
+  const eventType = String(event.eventType || "").trim();
+  if (!RUN_EVENT_TYPE_SET.has(eventType)) {
+    throw new Error(
+      `Unsupported event type '${eventType}'. Allowed: ${RUN_EVENT_TYPES.join(", ")}.`
+    );
+  }
+
+  return {
+    schemaVersion: SCHEMA_VERSION,
+    eventId: String(event.eventId || "").trim() || randomUUID(),
+    timestamp: String(event.timestamp || "").trim() || new Date().toISOString(),
+    sessionId: String(event.sessionId || "").trim() || "default",
+    runId: String(event.runId || "").trim() || "default",
+    eventType,
+    usage: normalizeUsage(event.usage),
+    stop: normalizeStop(event.stop),
+    metadata: normalizeMetadata(event.metadata),
+  };
+}
+
+function getInitialSummary() {
+  return {
+    eventCount: 0,
+    runCount: 0,
+    sessionCount: 0,
+    earliestTimestamp: null,
+    latestTimestamp: null,
+    eventTypeCounts: {},
+    stopClassCounts: {},
+    reasonCodeCounts: {},
+    usageTotals: {
+      inputTokens: 0,
+      outputTokens: 0,
+      cacheReadTokens: 0,
+      cacheWriteTokens: 0,
+      costUsd: 0,
+      durationMs: 0,
+      toolCalls: 0,
+    },
+  };
+}
+
+function incrementCounter(container, key) {
+  const normalizedKey = String(key || "").trim();
+  if (!normalizedKey) {
+    return;
+  }
+  container[normalizedKey] = Number(container[normalizedKey] || 0) + 1;
+}
+
+function mergeUsageTotals(totals, usage = {}) {
+  totals.inputTokens += normalizeNonNegativeNumber(usage.inputTokens, "usage.inputTokens");
+  totals.outputTokens += normalizeNonNegativeNumber(usage.outputTokens, "usage.outputTokens");
+  totals.cacheReadTokens += normalizeNonNegativeNumber(usage.cacheReadTokens, "usage.cacheReadTokens");
+  totals.cacheWriteTokens += normalizeNonNegativeNumber(
+    usage.cacheWriteTokens,
+    "usage.cacheWriteTokens"
+  );
+  totals.costUsd += normalizeNonNegativeNumber(usage.costUsd, "usage.costUsd");
+  totals.durationMs += normalizeNonNegativeNumber(usage.durationMs, "usage.durationMs");
+  totals.toolCalls += normalizeNonNegativeNumber(usage.toolCalls, "usage.toolCalls");
+}
+
+export function summarizeRunEvents(events = []) {
+  if (!Array.isArray(events)) {
+    throw new Error("events must be an array.");
+  }
+
+  const summary = getInitialSummary();
+  const runIds = new Set();
+  const sessionIds = new Set();
+
+  for (const rawEvent of events) {
+    const event = normalizeRunEvent(rawEvent);
+    summary.eventCount += 1;
+    runIds.add(event.runId);
+    sessionIds.add(event.sessionId);
+    incrementCounter(summary.eventTypeCounts, event.eventType);
+    mergeUsageTotals(summary.usageTotals, event.usage);
+
+    if (!summary.earliestTimestamp || event.timestamp < summary.earliestTimestamp) {
+      summary.earliestTimestamp = event.timestamp;
+    }
+    if (!summary.latestTimestamp || event.timestamp > summary.latestTimestamp) {
+      summary.latestTimestamp = event.timestamp;
+    }
+
+    if (event.stop) {
+      incrementCounter(summary.stopClassCounts, event.stop.stopClass);
+      for (const reasonCode of event.stop.reasonCodes || []) {
+        incrementCounter(summary.reasonCodeCounts, reasonCode);
+      }
+    }
+  }
+
+  summary.runCount = runIds.size;
+  summary.sessionCount = sessionIds.size;
+  return summary;
+}
+
+export async function resolveRunEventsPath({
+  targetPath = ".",
+  outputDirOverride = "",
+  env,
+  homeDir,
+} = {}) {
+  const outputRoot = await resolveOutputRoot({
+    cwd: path.resolve(targetPath),
+    outputDirOverride,
+    env,
+    homeDir,
+  });
+  return path.join(outputRoot, OBSERVABILITY_DIR_NAME, RUN_EVENTS_FILE_NAME);
+}
+
+export async function appendRunEvent(options = {}, event = {}) {
+  const normalizedEvent = normalizeRunEvent(event);
+  const filePath = await resolveRunEventsPath(options);
+  await fsp.mkdir(path.dirname(filePath), { recursive: true });
+  await fsp.appendFile(filePath, `${JSON.stringify(normalizedEvent)}\n`, "utf-8");
+  return {
+    filePath,
+    event: normalizedEvent,
+  };
+}
+
+export async function loadRunEvents(options = {}) {
+  const filePath = await resolveRunEventsPath(options);
+  try {
+    const raw = await fsp.readFile(filePath, "utf-8");
+    const lines = raw
+      .split(/\r?\n/)
+      .map((line) => line.trim())
+      .filter(Boolean);
+    const events = lines.map((line, index) => {
+      try {
+        return JSON.parse(line);
+      } catch (error) {
+        throw new Error(
+          `Invalid run event JSON at ${filePath}:${index + 1} (${error instanceof Error ? error.message : String(error)}).`
+        );
+      }
+    });
+    return { filePath, events };
+  } catch (error) {
+    if (error && typeof error === "object" && error.code === "ENOENT") {
+      return { filePath, events: [] };
+    }
+    throw error;
+  }
+}
+