site-agent-pro 1.0.0

package/dist/core/processSubmissionBatch.js

@@ -0,0 +1,204 @@
+import path from "node:path";
+import { buildAgentVariants } from "./agentProfiles.js";
+import { createAggregateRun } from "./aggregateReport.js";
+import { buildCustomTaskSuite } from "./customTaskSuite.js";
+import { runAuditJob } from "./runAuditJob.js";
+import { SUBMISSION_TASKS_REQUIRED_MESSAGE } from "../submissions/customTasks.js";
+import { SubmissionSchema } from "../submissions/types.js";
+export async function processSubmissionBatch(args) {
+    if (args.submission.customTasks.length === 0) {
+        const failedSubmission = SubmissionSchema.parse({
+            ...args.submission,
+            status: "failed",
+            startedAt: args.submission.startedAt ?? new Date().toISOString(),
+            completedAt: new Date().toISOString(),
+            error: SUBMISSION_TASKS_REQUIRED_MESSAGE,
+            runId: null,
+            runDir: null,
+            reportSummary: null,
+            overallScore: null
+        });
+        await args.writeSubmission(failedSubmission);
+        return failedSubmission;
+    }
+    const baseSuite = buildCustomTaskSuite(args.submission.customTasks);
+    const variants = buildAgentVariants(args.submission.agentCount, baseSuite);
+    let currentSubmission = SubmissionSchema.parse({
+        ...args.submission,
+        agentCount: args.submission.agentCount || variants.length,
+        agentRuns: args.submission.agentRuns.length > 0 ? args.submission.agentRuns : variants.map((variant) => ({
+            id: variant.id,
+            index: variant.index,
+            label: variant.label,
+            profileLabel: variant.profileLabel,
+            personaName: variant.personaName,
+            personaVariantKey: variant.personaVariantKey,
+            status: "queued",
+            startedAt: null,
+            completedAt: null,
+            runId: null,
+            runDir: null,
+            error: null,
+            reportSummary: null,
+            overallScore: null
+        }))
+    });
+    let mutationChain = Promise.resolve();
+    const syncCounts = (submission) => SubmissionSchema.parse({
+        ...submission,
+        completedAgentCount: submission.agentRuns.filter((agentRun) => agentRun.status === "completed").length,
+        failedAgentCount: submission.agentRuns.filter((agentRun) => agentRun.status === "failed").length
+    });
+    const commit = async (mutator) => {
+        mutationChain = mutationChain.then(async () => {
+            const nextSubmission = structuredClone(currentSubmission);
+            mutator(nextSubmission);
+            currentSubmission = syncCounts(nextSubmission);
+            await args.writeSubmission(currentSubmission);
+        });
+        await mutationChain;
+        return currentSubmission;
+    };
+    const batchStartedAt = new Date().toISOString();
+    await commit((draft) => {
+        draft.status = "running";
+        draft.startedAt = batchStartedAt;
+        draft.completedAt = null;
+        draft.error = null;
+        draft.runId = null;
+        draft.runDir = null;
+        draft.reportSummary = null;
+        draft.overallScore = null;
+        draft.agentRuns = draft.agentRuns.map((agentRun) => ({
+            ...agentRun,
+            status: "queued",
+            startedAt: null,
+            completedAt: null,
+            runId: null,
+            runDir: null,
+            error: null,
+            reportSummary: null,
+            overallScore: null
+        }));
+    });
+    const agentOutcomes = await Promise.all(variants.map(async (variant) => {
+        const agentStartedAt = new Date().toISOString();
+        await commit((draft) => {
+            draft.agentRuns = draft.agentRuns.map((agentRun) => agentRun.id === variant.id
+                ? {
+                    ...agentRun,
+                    status: "running",
+                    startedAt: agentStartedAt,
+                    completedAt: null,
+                    error: null
+                }
+                : agentRun);
+        });
+        try {
+            const result = await runAuditJob({
+                baseUrl: currentSubmission.url,
+                suiteOverride: variant.taskSuite,
+                headed: currentSubmission.headed,
+                mobile: currentSubmission.mobile,
+                ignoreHttpsErrors: currentSubmission.ignoreHttpsErrors,
+                tradeOptions: currentSubmission.tradeOptions,
+                extraInputs: {
+                    source: args.source,
+                    submissionId: currentSubmission.id,
+                    reportToken: currentSubmission.reportToken,
+                    expiresAt: currentSubmission.expiresAt,
+                    batchRole: "child",
+                    parentSubmissionId: currentSubmission.id,
+                    agentCount: currentSubmission.agentCount,
+                    instructionText: currentSubmission.instructionText,
+                    instructionFileName: currentSubmission.instructionFileName,
+                    agentIndex: variant.index,
+                    agentLabel: variant.label,
+                    agentProfileLabel: variant.profileLabel,
+                    personaVariantKey: variant.personaVariantKey,
+                    customTasks: currentSubmission.customTasks,
+                    tradeOptions: currentSubmission.tradeOptions
+                }
+            });
+            const runId = path.basename(result.runDir);
+            await args.uploadRunArtifacts(runId, result.runDir);
+            const completedAt = new Date().toISOString();
+            await commit((draft) => {
+                draft.agentRuns = draft.agentRuns.map((agentRun) => agentRun.id === variant.id
+                    ? {
+                        ...agentRun,
+                        status: "completed",
+                        startedAt: agentStartedAt,
+                        completedAt,
+                        runId,
+                        runDir: null,
+                        error: null,
+                        reportSummary: result.report.summary,
+                        overallScore: result.report.overall_score
+                    }
+                    : agentRun);
+            });
+            return {
+                agentRun: {
+                    id: variant.id,
+                    index: variant.index,
+                    label: variant.label,
+                    profileLabel: variant.profileLabel,
+                    personaName: variant.personaName,
+                    personaVariantKey: variant.personaVariantKey,
+                    status: "completed",
+                    startedAt: agentStartedAt,
+                    completedAt,
+                    runId,
+                    runDir: null,
+                    error: null,
+                    reportSummary: result.report.summary,
+                    overallScore: result.report.overall_score
+                },
+                report: result.report,
+                taskResults: result.execution.taskResults,
+                accessibility: result.execution.accessibility,
+                siteChecks: result.execution.siteChecks,
+                runId
+            };
+        }
+        catch (error) {
+            const completedAt = new Date().toISOString();
+            const message = error instanceof Error ? error.message : "Unknown submission failure";
+            await commit((draft) => {
+                draft.agentRuns = draft.agentRuns.map((agentRun) => agentRun.id === variant.id
+                    ? {
+                        ...agentRun,
+                        status: "failed",
+                        startedAt: agentStartedAt,
+                        completedAt,
+                        error: message
+                    }
+                    : agentRun);
+            });
+            return null;
+        }
+    }));
+    const completedOutcomes = agentOutcomes.filter((outcome) => outcome !== null);
+    if (completedOutcomes.length === 0) {
+        await commit((draft) => {
+            draft.status = "failed";
+            draft.completedAt = new Date().toISOString();
+            draft.error = `All ${draft.agentCount} agent runs failed before an aggregate report could be produced.`;
+        });
+        return currentSubmission;
+    }
+    const aggregateSubmissionSnapshot = structuredClone(currentSubmission);
+    const aggregateRun = createAggregateRun(aggregateSubmissionSnapshot, completedOutcomes);
+    await args.uploadRunArtifacts(aggregateRun.runId, aggregateRun.runDir);
+    await commit((draft) => {
+        draft.status = "completed";
+        draft.completedAt = new Date().toISOString();
+        draft.runId = aggregateRun.runId;
+        draft.runDir = null;
+        draft.error = null;
+        draft.reportSummary = aggregateRun.report.summary;
+        draft.overallScore = aggregateRun.report.overall_score;
+    });
+    return currentSubmission;
+}

package/dist/core/runAuditJob.js

@@ -0,0 +1,170 @@
+import path from "node:path";
+import { getAccessIdentityLabel, runWithAccessIdentityContext } from "../auth/profile.js";
+import { clampRunDurationMs, config, deriveBrowserExecutionBudgetMs, deriveReportingReserveMs, resolveLlmRuntime } from "../config.js";
+import { evaluateRun } from "./evaluator.js";
+import { runTaskSuite } from "./runner.js";
+import { generateClickReplay } from "../reporting/clickReplay.js";
+import { renderHtmlReport } from "../reporting/html.js";
+import { renderMarkdownReport } from "../reporting/markdown.js";
+import { ensureDir, resolveRunDir, writeJson, writeText } from "../utils/files.js";
+function summarizeSessionPath(filePath) {
+    const resolvedPath = path.isAbsolute(filePath) ? filePath : path.resolve(process.cwd(), filePath);
+    const relativePath = path.relative(process.cwd(), resolvedPath);
+    return relativePath && relativePath !== "" && !relativePath.startsWith("..")
+        ? relativePath
+        : path.basename(resolvedPath);
+}
+export async function runAuditJob(options) {
+    const accessIdentityContext = {
+        ...(typeof options.extraInputs?.agentIndex === "number" ? { agentIndex: options.extraInputs.agentIndex } : {}),
+        ...(typeof options.extraInputs?.agentLabel === "string" ? { agentLabel: options.extraInputs.agentLabel } : {}),
+        ...(typeof options.extraInputs?.agentProfileLabel === "string"
+            ? { agentProfileLabel: options.extraInputs.agentProfileLabel }
+            : {})
+    };
+    return await runWithAccessIdentityContext(accessIdentityContext, async () => {
+        const suite = options.suiteOverride;
+        const runDir = options.runDir ?? resolveRunDir(options.baseUrl);
+        ensureDir(runDir);
+        const inputsPath = path.join(runDir, "inputs.json");
+        const startedAt = new Date().toISOString();
+        const startedAtMs = Date.now();
+        const requestedMaxSessionDurationMs = options.maxSessionDurationMs ?? config.maxSessionDurationMs;
+        const maxRunDurationMs = clampRunDurationMs(requestedMaxSessionDurationMs);
+        const browserExecutionBudgetMs = deriveBrowserExecutionBudgetMs(maxRunDurationMs);
+        const reportingReserveMs = deriveReportingReserveMs(maxRunDurationMs);
+        const llmRuntime = resolveLlmRuntime({
+            ...(options.llmProvider ? { provider: options.llmProvider } : {}),
+            ...(options.model ? { model: options.model } : {}),
+            ...(options.ollamaBaseUrl ? { ollamaBaseUrl: options.ollamaBaseUrl } : {})
+        });
+        const storageStatePath = options.storageStatePath ?? config.playwrightStorageStatePath;
+        const saveStorageStatePath = options.saveStorageStatePath;
+        const instructionText = typeof options.extraInputs?.instructionText === "string"
+            ? options.extraInputs.instructionText
+            : suite.tasks.map((task) => task.goal).join("\n");
+        const accessIdentityName = getAccessIdentityLabel(accessIdentityContext);
+        const baseInputs = {
+            baseUrl: options.baseUrl,
+            persona: suite.persona.name,
+            headed: Boolean(options.headed),
+            mobile: Boolean(options.mobile),
+            ignoreHttpsErrors: Boolean(options.ignoreHttpsErrors),
+            llmProvider: llmRuntime.provider,
+            storageStateLoaded: Boolean(storageStatePath),
+            storageStateSource: storageStatePath ? summarizeSessionPath(storageStatePath) : null,
+            saveStorageStateRequested: Boolean(saveStorageStatePath),
+            saveStorageStateTarget: saveStorageStatePath ? summarizeSessionPath(saveStorageStatePath) : null,
+            model: llmRuntime.model,
+            startedAt,
+            maxRunDurationMs,
+            maxRunDurationSeconds: Math.round(maxRunDurationMs / 1000),
+            browserExecutionBudgetMs,
+            reportingReserveMs,
+            maxRunDurationClamped: maxRunDurationMs !== requestedMaxSessionDurationMs,
+            deviceTimezone: config.deviceTimezone,
+            synchronizedTimezone: config.deviceTimezone,
+            customTasks: suite.tasks.map((task) => task.goal),
+            accessIdentityName,
+            recordVideo: config.recordVideo,
+            ...(options.tradeOptions ? { tradeOptions: options.tradeOptions } : {}),
+            ...(options.extraInputs ?? {})
+        };
+        writeJson(inputsPath, baseInputs);
+        const execution = await runTaskSuite({
+            baseUrl: options.baseUrl,
+            suite,
+            runDir,
+            headed: Boolean(options.headed),
+            mobile: Boolean(options.mobile),
+            ignoreHttpsErrors: Boolean(options.ignoreHttpsErrors),
+            storageStatePath,
+            saveStorageStatePath,
+            maxSessionDurationMs: browserExecutionBudgetMs,
+            ...(options.tradeOptions ? { tradeOptions: options.tradeOptions } : {}),
+            provider: llmRuntime.provider,
+            model: llmRuntime.model,
+            ollamaBaseUrl: llmRuntime.ollamaBaseUrl
+        });
+        let clickReplayArtifact = null;
+        let clickReplayFrameCount = null;
+        let clickReplayDurationMs = null;
+        try {
+            const clickReplay = await generateClickReplay({
+                runDir,
+                taskResults: execution.taskResults
+            });
+            clickReplayArtifact = clickReplay?.artifactName ?? null;
+            clickReplayFrameCount = clickReplay?.frameCount ?? null;
+            clickReplayDurationMs = clickReplay?.durationMs ?? null;
+        }
+        catch {
+            // Replay generation is optional and should never block the main report.
+        }
+        writeJson(inputsPath, {
+            ...baseInputs,
+            ...(execution.siteBrief ? { siteBrief: execution.siteBrief } : {}),
+            browserTimezone: execution.browserTimezone,
+            synchronizedTimezone: execution.browserTimezone || execution.deviceTimezone,
+            ...(clickReplayArtifact ? { clickReplayArtifact } : {}),
+            ...(clickReplayFrameCount !== null ? { clickReplayFrameCount } : {}),
+            ...(clickReplayDurationMs !== null ? { clickReplayDurationMs } : {})
+        });
+        const remainingEvaluationBudgetMs = Math.max(0, maxRunDurationMs - (Date.now() - startedAtMs));
+        const report = await evaluateRun({
+            baseUrl: options.baseUrl,
+            suite,
+            siteBrief: execution.siteBrief,
+            taskResults: execution.taskResults,
+            rawEvents: execution.rawEvents,
+            accessibility: execution.accessibility,
+            mobile: Boolean(options.mobile),
+            timeoutMs: remainingEvaluationBudgetMs,
+            totalRunDurationMs: maxRunDurationMs,
+            llm: {
+                provider: llmRuntime.provider,
+                model: llmRuntime.model,
+                ollamaBaseUrl: llmRuntime.ollamaBaseUrl
+            }
+        });
+        writeJson(path.join(runDir, "report.json"), report);
+        writeText(path.join(runDir, "report.html"), renderHtmlReport({
+            website: options.baseUrl,
+            persona: suite.persona.name,
+            acceptedTasks: suite.tasks.map((task) => task.goal),
+            instructionText,
+            report,
+            taskResults: execution.taskResults,
+            accessibility: execution.accessibility,
+            siteChecks: execution.siteChecks,
+            siteBrief: execution.siteBrief,
+            rawEvents: execution.rawEvents,
+            runId: path.basename(runDir),
+            startedAt,
+            mobile: Boolean(options.mobile),
+            timeZone: execution.browserTimezone || execution.deviceTimezone,
+            clickReplayArtifact
+        }));
+        writeText(path.join(runDir, "report.md"), renderMarkdownReport({
+            website: options.baseUrl,
+            persona: suite.persona.name,
+            acceptedTasks: suite.tasks.map((task) => task.goal),
+            instructionText,
+            report,
+            taskResults: execution.taskResults,
+            accessibility: execution.accessibility,
+            siteChecks: execution.siteChecks,
+            siteBrief: execution.siteBrief,
+            rawEvents: execution.rawEvents,
+            startedAt,
+            mobile: Boolean(options.mobile),
+            timeZone: execution.browserTimezone || execution.deviceTimezone
+        }));
+        return {
+            startedAt,
+            runDir,
+            report,
+            execution
+        };
+    });
+}