site-agent-pro 1.0.0

package/dist/backend/dashboardData.js

@@ -0,0 +1,194 @@
+import { z } from "zod";
+import { renderHtmlReport } from "../reporting/html.js";
+import { AccessibilityResultSchema, FinalReportSchema, SiteChecksSchema, TaskRunResultSchema } from "../schemas/types.js";
+import { DashboardRunDetailSchema, DashboardRunSummarySchema, RunInputsSchema } from "../dashboard/contracts.js";
+const TaskRunResultsSchema = z.array(TaskRunResultSchema);
+const RawEventsSchema = z.array(z.unknown());
+function readHost(baseUrl, fallback) {
+    if (!baseUrl) {
+        return fallback;
+    }
+    try {
+        return new URL(baseUrl).hostname.replace(/^www\./, "") || fallback;
+    }
+    catch {
+        return fallback;
+    }
+}
+function isVisibleDashboardRun(run) {
+    return run.batchRole !== "child";
+}
+function collectSiteCheckWarnings(runId, siteChecks) {
+    if (!siteChecks) {
+        return [
+            `This run is missing supplemental site checks. Run \`npm run backfill:site-checks -- --run ${runId}\` or rerun the audit to verify performance, SEO, security, content, and mobile responsiveness.`
+        ];
+    }
+    const blockers = Object.values(siteChecks.coverage).flatMap((coverage) => coverage.blockers);
+    if (blockers.some((blocker) => /__name is not defined/i.test(blocker))) {
+        return [
+            `This run used an older broken probe build. Run \`npm run backfill:site-checks -- --run ${runId} --force\` or rerun the audit to refresh the blocked metrics, including the mobile responsiveness check.`
+        ];
+    }
+    return [];
+}
+async function findVideoArtifact(repository, runId) {
+    try {
+        const files = await repository.listArtifacts(runId);
+        const video = files.find((f) => f.endsWith(".webm") || f.endsWith(".mp4"));
+        return video ?? null;
+    }
+    catch {
+        return null;
+    }
+}
+async function enrichAgentRunsWithReplay(repository, agentRuns) {
+    return Promise.all(agentRuns.map(async (agentRun) => {
+        const childRunId = agentRun.runId?.trim();
+        if (!childRunId) {
+            return agentRun;
+        }
+        const clickReplayAvailable = (await repository.readBinaryArtifact(childRunId, "click-replay.webp")) !== null;
+        const videoArtifact = await findVideoArtifact(repository, childRunId);
+        return {
+            ...agentRun,
+            clickReplayAvailable,
+            clickReplayArtifact: clickReplayAvailable ? "click-replay.webp" : null,
+            ...(videoArtifact ? { videoArtifact } : {})
+        };
+    }));
+}
+export async function buildRunSummary(repository, runId) {
+    const inputs = await repository.readJsonArtifact(runId, "inputs.json", RunInputsSchema);
+    const report = await repository.readJsonArtifact(runId, "report.json", FinalReportSchema);
+    const accessibility = await repository.readJsonArtifact(runId, "accessibility.json", AccessibilityResultSchema);
+    return DashboardRunSummarySchema.parse({
+        id: runId,
+        baseUrl: inputs?.baseUrl ?? "",
+        host: readHost(inputs?.baseUrl, runId),
+        startedAt: inputs?.startedAt ?? null,
+        headed: inputs?.headed ?? false,
+        mobile: inputs?.mobile ?? false,
+        llmProvider: inputs?.llmProvider ?? null,
+        model: inputs?.model ?? null,
+        persona: inputs?.persona ?? null,
+        overallScore: report?.overall_score ?? null,
+        summary: report?.summary ?? null,
+        taskCount: report?.task_results.length ?? 0,
+        accessibilityViolationCount: accessibility?.violations.length ?? null,
+        batchRole: inputs?.batchRole ?? "single",
+        agentCount: inputs?.agentCount ?? 1,
+        completedAgentCount: inputs?.completedAgentCount ?? 0,
+        failedAgentCount: inputs?.failedAgentCount ?? 0,
+        agentLabel: inputs?.agentLabel ?? null,
+        agentProfileLabel: inputs?.agentProfileLabel ?? null
+    });
+}
+export async function listVisibleRunSummaries(repository) {
+    const runIds = await repository.listRunIds();
+    const allRuns = await Promise.all(runIds.map((runId) => buildRunSummary(repository, runId)));
+    return allRuns.filter(isVisibleDashboardRun);
+}
+export async function buildRunDetail(repository, runId) {
+    const inputs = await repository.readJsonArtifact(runId, "inputs.json", RunInputsSchema);
+    const inferredClickReplayArtifact = (await repository.readBinaryArtifact(runId, "click-replay.webp")) !== null ? "click-replay.webp" : null;
+    const inferredVideoArtifact = await findVideoArtifact(repository, runId);
+    const inputsWithReplay = inputs
+        ? {
+            ...inputs,
+            ...(inferredClickReplayArtifact && !inputs.clickReplayArtifact ? { clickReplayArtifact: inferredClickReplayArtifact } : {}),
+            ...(inferredVideoArtifact ? { videoArtifact: inferredVideoArtifact } : {}),
+            agentRuns: await enrichAgentRunsWithReplay(repository, inputs.agentRuns)
+        }
+        : inputs;
+    const report = await repository.readJsonArtifact(runId, "report.json", FinalReportSchema);
+    const accessibility = await repository.readJsonArtifact(runId, "accessibility.json", AccessibilityResultSchema);
+    const siteChecks = await repository.readJsonArtifact(runId, "site-checks.json", SiteChecksSchema);
+    const taskRuns = (await repository.readJsonArtifact(runId, "task-results.json", TaskRunResultsSchema)) ?? [];
+    const rawEvents = (await repository.readJsonArtifact(runId, "raw-events.json", RawEventsSchema)) ?? [];
+    if (!inputs && !report && !accessibility && !siteChecks && taskRuns.length === 0 && rawEvents.length === 0) {
+        return null;
+    }
+    const warnings = [];
+    if (!inputs) {
+        warnings.push("inputs.json is missing or invalid for this run.");
+    }
+    if (!report) {
+        warnings.push("report.json is missing or invalid for this run.");
+    }
+    if (!accessibility) {
+        warnings.push("accessibility.json is missing or invalid for this run.");
+    }
+    warnings.push(...collectSiteCheckWarnings(runId, siteChecks));
+    if (taskRuns.length === 0) {
+        warnings.push("task-results.json is missing or empty for this run.");
+    }
+    const reviewedTasks = new Map((report?.task_results ?? []).map((task) => [task.name, task]));
+    const taskRunsByName = new Map(taskRuns.map((task) => [task.name, task]));
+    const taskNames = Array.from(new Set([...taskRuns.map((task) => task.name), ...(report?.task_results.map((task) => task.name) ?? [])]));
+    const tasks = taskNames.map((taskName) => {
+        const taskRun = taskRunsByName.get(taskName);
+        const reviewedTask = reviewedTasks.get(taskName);
+        return {
+            name: taskName,
+            status: reviewedTask?.status ?? taskRun?.status ?? "failed",
+            reason: reviewedTask?.reason ?? taskRun?.reason ?? "No task reasoning was captured for this task.",
+            evidence: reviewedTask?.evidence ?? [],
+            finalUrl: taskRun?.finalUrl ?? "",
+            finalTitle: taskRun?.finalTitle ?? "",
+            history: taskRun?.history ?? []
+        };
+    });
+    return DashboardRunDetailSchema.parse({
+        id: runId,
+        host: readHost(inputsWithReplay?.baseUrl, runId),
+        inputs: inputsWithReplay,
+        report,
+        accessibility,
+        siteChecks,
+        tasks,
+        rawEventCount: rawEvents.length,
+        warnings
+    });
+}
+export async function buildStandaloneReportHtml(repository, runId) {
+    const inputs = await repository.readJsonArtifact(runId, "inputs.json", RunInputsSchema);
+    const report = await repository.readJsonArtifact(runId, "report.json", FinalReportSchema);
+    const accessibility = await repository.readJsonArtifact(runId, "accessibility.json", AccessibilityResultSchema);
+    const siteChecks = await repository.readJsonArtifact(runId, "site-checks.json", SiteChecksSchema);
+    const taskRuns = (await repository.readJsonArtifact(runId, "task-results.json", TaskRunResultsSchema)) ?? [];
+    const rawEvents = (await repository.readJsonArtifact(runId, "raw-events.json", RawEventsSchema)) ?? [];
+    if (!report) {
+        return repository.readTextArtifact(runId, "report.html");
+    }
+    return renderHtmlReport({
+        website: inputs?.baseUrl ?? runId,
+        persona: inputs?.persona ?? "first-time visitor",
+        acceptedTasks: inputs?.customTasks ?? [],
+        instructionText: inputs?.instructionText,
+        report,
+        taskResults: taskRuns,
+        accessibility: accessibility ?? undefined,
+        siteChecks: siteChecks ?? undefined,
+        siteBrief: inputs?.siteBrief,
+        rawEvents,
+        runId,
+        startedAt: inputs?.startedAt,
+        mobile: inputs?.mobile,
+        timeZone: inputs?.synchronizedTimezone ?? inputs?.browserTimezone ?? inputs?.deviceTimezone
+    });
+}
+export async function loadDashboardData(repository, selectedRunId) {
+    const runs = await listVisibleRunSummaries(repository);
+    let resolvedRunId = selectedRunId;
+    let detail = selectedRunId ? await buildRunDetail(repository, selectedRunId) : null;
+    if (!detail) {
+        resolvedRunId = runs.find((run) => run.id === selectedRunId)?.id ?? runs[0]?.id ?? null;
+        detail = resolvedRunId ? await buildRunDetail(repository, resolvedRunId) : null;
+    }
+    return {
+        runs,
+        detail,
+        selectedRunId: resolvedRunId
+    };
+}

package/dist/backend/runArtifacts.js

@@ -0,0 +1,48 @@
+import path from "node:path";
+const SAFE_RUN_NAME_PATTERN = /^[A-Za-z0-9._-]+$/;
+const IMAGE_ARTIFACT_PATTERN = /^[A-Za-z0-9._-]+\.(?:png|jpg|jpeg|webp)$/i;
+const VIDEO_ARTIFACT_PATTERN = /^[A-Za-z0-9._-]+\.(?:webm|mp4)$/i;
+const STATIC_REPORT_ARTIFACTS = new Set(["report.html", "report.json", "report.md"]);
+export function isSafeRunFileName(value) {
+    const trimmed = value.trim();
+    return trimmed.length > 0 && path.basename(trimmed) === trimmed && SAFE_RUN_NAME_PATTERN.test(trimmed);
+}
+export function isImageArtifact(fileName) {
+    return IMAGE_ARTIFACT_PATTERN.test(fileName.trim());
+}
+export function isVideoArtifact(fileName) {
+    return VIDEO_ARTIFACT_PATTERN.test(fileName.trim());
+}
+export function isStaticReportArtifact(fileName) {
+    return STATIC_REPORT_ARTIFACTS.has(fileName.trim());
+}
+export function isAllowedDashboardArtifact(fileName) {
+    return isStaticReportArtifact(fileName) || isImageArtifact(fileName) || isVideoArtifact(fileName);
+}
+export function artifactContentType(fileName) {
+    if (fileName.endsWith(".html")) {
+        return "text/html; charset=utf-8";
+    }
+    if (fileName.endsWith(".json")) {
+        return "application/json; charset=utf-8";
+    }
+    if (fileName.endsWith(".md")) {
+        return "text/markdown; charset=utf-8";
+    }
+    if (fileName.endsWith(".png")) {
+        return "image/png";
+    }
+    if (fileName.endsWith(".jpg") || fileName.endsWith(".jpeg")) {
+        return "image/jpeg";
+    }
+    if (fileName.endsWith(".webp")) {
+        return "image/webp";
+    }
+    if (fileName.endsWith(".webm")) {
+        return "video/webm";
+    }
+    if (fileName.endsWith(".mp4")) {
+        return "video/mp4";
+    }
+    return "application/octet-stream";
+}

package/dist/backend/runRepository.js

@@ -0,0 +1,93 @@
+import fs from "node:fs";
+import path from "node:path";
+import { readUtf8, resolveRunsDir } from "../utils/files.js";
+import { isSafeRunFileName } from "./runArtifacts.js";
+function normalizeRunId(runId) {
+    const trimmed = runId.trim();
+    return isSafeRunFileName(trimmed) ? trimmed : null;
+}
+function resolveLocalRunDir(runId) {
+    const normalizedRunId = normalizeRunId(runId);
+    if (!normalizedRunId) {
+        return null;
+    }
+    const baseDir = resolveRunsDir();
+    const runDir = path.resolve(baseDir, normalizedRunId);
+    if (!runDir.startsWith(`${baseDir}${path.sep}`)) {
+        return null;
+    }
+    if (!fs.existsSync(runDir) || !fs.statSync(runDir).isDirectory()) {
+        return null;
+    }
+    return runDir;
+}
+function resolveLocalArtifactPath(runId, fileName) {
+    const runDir = resolveLocalRunDir(runId);
+    const normalizedFileName = isSafeRunFileName(fileName) ? fileName.trim() : null;
+    if (!runDir || !normalizedFileName) {
+        return null;
+    }
+    const artifactPath = path.resolve(runDir, normalizedFileName);
+    if (!artifactPath.startsWith(`${runDir}${path.sep}`)) {
+        return null;
+    }
+    if (!fs.existsSync(artifactPath) || !fs.statSync(artifactPath).isFile()) {
+        return null;
+    }
+    return artifactPath;
+}
+export function createRunRepository(adapter) {
+    return {
+        listRunIds: adapter.listRunIds,
+        listArtifacts: adapter.listArtifacts ??
+            (async (runId) => {
+                const runDir = resolveLocalRunDir(runId);
+                if (!runDir)
+                    return [];
+                return fs.readdirSync(runDir).filter((f) => isSafeRunFileName(f));
+            }),
+        hasRun: adapter.hasRun ??
+            (async (runId) => {
+                const runIds = await adapter.listRunIds();
+                return runIds.includes(runId);
+            }),
+        readTextArtifact: adapter.readTextArtifact,
+        readBinaryArtifact: adapter.readBinaryArtifact,
+        readJsonArtifact: adapter.readJsonArtifact,
+        resolveArtifactPath: adapter.resolveArtifactPath ??
+            (async (runId, fileName) => {
+                return resolveLocalArtifactPath(runId, fileName);
+            })
+    };
+}
+export function createLocalRunRepository() {
+    return createRunRepository({
+        listRunIds: async () => {
+            const runsDir = resolveRunsDir();
+            if (!fs.existsSync(runsDir)) {
+                return [];
+            }
+            return fs
+                .readdirSync(runsDir, { withFileTypes: true })
+                .filter((entry) => entry.isDirectory())
+                .map((entry) => entry.name)
+                .sort((left, right) => right.localeCompare(left));
+        },
+        hasRun: async (runId) => resolveLocalRunDir(runId) !== null,
+        readTextArtifact: async (runId, fileName) => {
+            const artifactPath = resolveLocalArtifactPath(runId, fileName);
+            return artifactPath ? readUtf8(artifactPath) : null;
+        },
+        readBinaryArtifact: async (runId, fileName) => {
+            const artifactPath = resolveLocalArtifactPath(runId, fileName);
+            return artifactPath ? fs.readFileSync(artifactPath) : null;
+        },
+        readJsonArtifact: async (runId, fileName, schema) => {
+            const artifactPath = resolveLocalArtifactPath(runId, fileName);
+            if (!artifactPath) {
+                return null;
+            }
+            return schema.parse(JSON.parse(readUtf8(artifactPath)));
+        }
+    });
+}

package/dist/bin.js

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+import "./cli/run.js";

package/dist/cli/backfillSiteChecks.js

@@ -0,0 +1,143 @@
+import fs from "node:fs";
+import path from "node:path";
+import { Command } from "commander";
+import { chromium } from "playwright";
+import { z } from "zod";
+import { renderHtmlReport } from "../reporting/html.js";
+import { renderMarkdownReport } from "../reporting/markdown.js";
+import { AccessibilityResultSchema, FinalReportSchema, SiteChecksSchema, TaskRunResultSchema } from "../schemas/types.js";
+import { RunInputsSchema } from "../dashboard/contracts.js";
+import { runSiteChecks } from "../core/siteChecks.js";
+import { readUtf8, resolveRunsDir, writeJson, writeText } from "../utils/files.js";
+const TaskRunResultsSchema = TaskRunResultSchema.array();
+const RawEventsSchema = z.array(z.unknown());
+function loadJson(filePath, schema) {
+    if (!fs.existsSync(filePath) || !fs.statSync(filePath).isFile()) {
+        return null;
+    }
+    return schema.parse(JSON.parse(readUtf8(filePath)));
+}
+function needsProbeRefresh(siteChecks) {
+    if (!siteChecks) {
+        return true;
+    }
+    return Object.values(siteChecks.coverage).some((coverage) => coverage.blockers.some((blocker) => /__name is not defined|scrollWidth|Timeout \d+ms exceeded|ERR_CONNECTION_CLOSED|ERR_SOCKET_NOT_CONNECTED/i.test(blocker)));
+}
+async function backfillRun(runId, force, budgetMs) {
+    const runDir = path.join(resolveRunsDir(), runId);
+    if (!fs.existsSync(runDir) || !fs.statSync(runDir).isDirectory()) {
+        return { runId, status: "failed", note: "Run directory was not found." };
+    }
+    const siteChecksPath = path.join(runDir, "site-checks.json");
+    const existingSiteChecks = fs.existsSync(siteChecksPath) ? loadJson(siteChecksPath, SiteChecksSchema) : null;
+    if (!force && existingSiteChecks && !needsProbeRefresh(existingSiteChecks)) {
+        return { runId, status: "skipped", note: "site-checks.json already exists." };
+    }
+    if (!force && existingSiteChecks && needsProbeRefresh(existingSiteChecks)) {
+        process.stdout.write(`[refresh] ${runId}: existing site checks were created by an older broken probe build.\n`);
+    }
+    if (!force && fs.existsSync(siteChecksPath) && !existingSiteChecks) {
+        process.stdout.write(`[refresh] ${runId}: existing site-checks.json was invalid and will be regenerated.\n`);
+    }
+    const inputs = loadJson(path.join(runDir, "inputs.json"), RunInputsSchema);
+    const report = loadJson(path.join(runDir, "report.json"), FinalReportSchema);
+    const accessibility = loadJson(path.join(runDir, "accessibility.json"), AccessibilityResultSchema);
+    const taskResults = loadJson(path.join(runDir, "task-results.json"), TaskRunResultsSchema) ?? [];
+    const rawEvents = loadJson(path.join(runDir, "raw-events.json"), RawEventsSchema) ?? [];
+    if (!inputs || !report) {
+        return { runId, status: "failed", note: "inputs.json or report.json is missing." };
+    }
+    const browser = await chromium.launch({ headless: true });
+    try {
+        const siteChecks = await runSiteChecks({
+            browser,
+            baseUrl: inputs.baseUrl,
+            ignoreHttpsErrors: Boolean(inputs.ignoreHttpsErrors),
+            browserTimezone: inputs.synchronizedTimezone ?? inputs.browserTimezone ?? inputs.deviceTimezone ?? "UTC",
+            storageState: undefined,
+            rawEvents: rawEvents ?? [],
+            taskResults,
+            budgetMs
+        });
+        writeJson(siteChecksPath, siteChecks);
+        writeText(path.join(runDir, "report.html"), renderHtmlReport({
+            website: inputs.baseUrl,
+            persona: inputs.persona ?? "first-time visitor",
+            report,
+            taskResults,
+            accessibility: accessibility ?? undefined,
+            siteChecks,
+            rawEvents: rawEvents ?? [],
+            runId,
+            startedAt: inputs.startedAt,
+            mobile: inputs.mobile,
+            timeZone: inputs.synchronizedTimezone ?? inputs.browserTimezone ?? inputs.deviceTimezone
+        }));
+        writeText(path.join(runDir, "report.md"), renderMarkdownReport({
+            website: inputs.baseUrl,
+            persona: inputs.persona ?? "first-time visitor",
+            report,
+            taskResults,
+            accessibility: accessibility ?? undefined,
+            siteChecks,
+            rawEvents: rawEvents ?? [],
+            startedAt: inputs.startedAt,
+            mobile: inputs.mobile,
+            timeZone: inputs.synchronizedTimezone ?? inputs.browserTimezone ?? inputs.deviceTimezone
+        }));
+        return { runId, status: "updated", note: "site-checks.json and rendered reports were refreshed." };
+    }
+    catch (error) {
+        return {
+            runId,
+            status: "failed",
+            note: error instanceof Error ? error.message : "Unknown backfill failure"
+        };
+    }
+    finally {
+        await browser.close().catch(() => undefined);
+    }
+}
+async function main() {
+    const program = new Command();
+    program
+        .option("--run <runId>", "Backfill a single run by run ID")
+        .option("--all", "Backfill every run under the runs directory")
+        .option("--visible-only", "When used with --all, skip child agent runs that are hidden from the dashboard", false)
+        .option("--budget-ms <ms>", "Override the supplemental site-check budget used during backfill", "45000")
+        .option("--force", "Recompute site checks even if they already exist", false);
+    program.parse(process.argv);
+    const options = program.opts();
+    const parsedBudgetMs = Number.parseInt(options.budgetMs ?? "45000", 10);
+    const budgetMs = Number.isFinite(parsedBudgetMs) ? Math.max(12000, parsedBudgetMs) : 45000;
+    const runIds = options.all
+        ? fs
+            .readdirSync(resolveRunsDir(), { withFileTypes: true })
+            .filter((entry) => entry.isDirectory())
+            .filter((entry) => {
+            if (!options.visibleOnly) {
+                return true;
+            }
+            const inputs = loadJson(path.join(resolveRunsDir(), entry.name, "inputs.json"), RunInputsSchema);
+            if (!inputs) {
+                return false;
+            }
+            return (inputs.batchRole ?? "single") !== "child";
+        })
+            .map((entry) => entry.name)
+            .sort((left, right) => right.localeCompare(left))
+        : options.run
+            ? [options.run]
+            : [];
+    if (runIds.length === 0) {
+        throw new Error("Provide either --run <runId> or --all.");
+    }
+    for (const runId of runIds) {
+        const result = await backfillRun(runId, Boolean(options.force), budgetMs);
+        process.stdout.write(`[${result.status}] ${result.runId}: ${result.note}\n`);
+    }
+}
+main().catch((error) => {
+    process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
+    process.exitCode = 1;
+});