prose-qa 0.1.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -502
- package/dist/agent/bash.d.ts +1 -1
- package/dist/agent/bash.d.ts.map +1 -1
- package/dist/agent/bash.js +23 -4
- package/dist/agent/bash.js.map +1 -1
- package/dist/agent/bash.test.js +27 -3
- package/dist/agent/bash.test.js.map +1 -1
- package/dist/agent/llm-model.d.ts.map +1 -1
- package/dist/agent/llm-model.js +19 -10
- package/dist/agent/llm-model.js.map +1 -1
- package/dist/agent/prompt.d.ts.map +1 -1
- package/dist/agent/prompt.js +10 -2
- package/dist/agent/prompt.js.map +1 -1
- package/dist/agent/prompt.test.js +0 -1
- package/dist/agent/prompt.test.js.map +1 -1
- package/dist/agent/runner.d.ts.map +1 -1
- package/dist/agent/runner.js +80 -12
- package/dist/agent/runner.js.map +1 -1
- package/dist/agent/runner.test.d.ts +2 -0
- package/dist/agent/runner.test.d.ts.map +1 -0
- package/dist/agent/runner.test.js +83 -0
- package/dist/agent/runner.test.js.map +1 -0
- package/dist/agent/transcript-persist.d.ts +9 -0
- package/dist/agent/transcript-persist.d.ts.map +1 -0
- package/dist/agent/transcript-persist.js +6 -0
- package/dist/agent/transcript-persist.js.map +1 -0
- package/dist/agent/transcript-persist.test.d.ts +2 -0
- package/dist/agent/transcript-persist.test.d.ts.map +1 -0
- package/dist/agent/transcript-persist.test.js +56 -0
- package/dist/agent/transcript-persist.test.js.map +1 -0
- package/dist/agent/verdict.d.ts +8 -2
- package/dist/agent/verdict.d.ts.map +1 -1
- package/dist/agent/verdict.js +49 -0
- package/dist/agent/verdict.js.map +1 -1
- package/dist/agent/verdict.test.js +85 -1
- package/dist/agent/verdict.test.js.map +1 -1
- package/dist/cli/analyze.js +7 -7
- package/dist/cli/analyze.js.map +1 -1
- package/dist/cli/help.d.ts.map +1 -1
- package/dist/cli/help.js +53 -17
- package/dist/cli/help.js.map +1 -1
- package/dist/cli/help.test.js +1 -0
- package/dist/cli/help.test.js.map +1 -1
- package/dist/cli/index.js +18 -1
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/install-browser.d.ts +3 -0
- package/dist/cli/install-browser.d.ts.map +1 -0
- package/dist/cli/install-browser.js +42 -0
- package/dist/cli/install-browser.js.map +1 -0
- package/dist/cli/install-browser.test.d.ts +2 -0
- package/dist/cli/install-browser.test.d.ts.map +1 -0
- package/dist/cli/install-browser.test.js +19 -0
- package/dist/cli/install-browser.test.js.map +1 -0
- package/dist/cli/record.js +3 -3
- package/dist/cli/record.js.map +1 -1
- package/dist/cli/run.d.ts.map +1 -1
- package/dist/cli/run.js +32 -27
- package/dist/cli/run.js.map +1 -1
- package/dist/config/lightpanda.d.ts +8 -0
- package/dist/config/lightpanda.d.ts.map +1 -1
- package/dist/config/lightpanda.js +41 -4
- package/dist/config/lightpanda.js.map +1 -1
- package/dist/config/lightpanda.test.js +30 -1
- package/dist/config/lightpanda.test.js.map +1 -1
- package/dist/config/load.d.ts +4 -1
- package/dist/config/load.d.ts.map +1 -1
- package/dist/config/load.js +52 -28
- package/dist/config/load.js.map +1 -1
- package/dist/config/load.test.js +101 -13
- package/dist/config/load.test.js.map +1 -1
- package/dist/mcp/server.d.ts.map +1 -1
- package/dist/mcp/server.js +2 -1
- package/dist/mcp/server.js.map +1 -1
- package/dist/prompt/load.d.ts +2 -2
- package/dist/prompt/load.d.ts.map +1 -1
- package/dist/prompt/load.js +4 -9
- package/dist/prompt/load.js.map +1 -1
- package/dist/redact/env-secrets.d.ts.map +1 -1
- package/dist/redact/env-secrets.js +1 -0
- package/dist/redact/env-secrets.js.map +1 -1
- package/dist/redact/env-secrets.test.js +3 -3
- package/dist/redact/env-secrets.test.js.map +1 -1
- package/dist/reporter/index.d.ts.map +1 -1
- package/dist/reporter/index.js +6 -2
- package/dist/reporter/index.js.map +1 -1
- package/dist/reporter/index.test.js +59 -1
- package/dist/reporter/index.test.js.map +1 -1
- package/dist/scenarios/globs.d.ts +3 -4
- package/dist/scenarios/globs.d.ts.map +1 -1
- package/dist/scenarios/globs.js +5 -27
- package/dist/scenarios/globs.js.map +1 -1
- package/dist/scenarios/globs.test.js +11 -29
- package/dist/scenarios/globs.test.js.map +1 -1
- package/dist/skills/loader.d.ts +8 -1
- package/dist/skills/loader.d.ts.map +1 -1
- package/dist/skills/loader.js +31 -5
- package/dist/skills/loader.js.map +1 -1
- package/dist/skills/loader.test.d.ts +2 -0
- package/dist/skills/loader.test.d.ts.map +1 -0
- package/dist/skills/loader.test.js +40 -0
- package/dist/skills/loader.test.js.map +1 -0
- package/dist/skills/on-demand.d.ts +57 -0
- package/dist/skills/on-demand.d.ts.map +1 -0
- package/dist/skills/on-demand.js +225 -0
- package/dist/skills/on-demand.js.map +1 -0
- package/dist/skills/on-demand.test.d.ts +2 -0
- package/dist/skills/on-demand.test.d.ts.map +1 -0
- package/dist/skills/on-demand.test.js +111 -0
- package/dist/skills/on-demand.test.js.map +1 -0
- package/dist/types/config.d.ts +17 -7
- package/dist/types/config.d.ts.map +1 -1
- package/dist/types/verdict.d.ts +21 -1
- package/dist/types/verdict.d.ts.map +1 -1
- package/dist/types/verdict.js +16 -0
- package/dist/types/verdict.js.map +1 -1
- package/dist/version.d.ts +3 -0
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +6 -0
- package/dist/version.js.map +1 -0
- package/package.json +9 -7
- package/pqa.config.ts +8 -13
- package/prompt/SYSTEM.md +2 -1
- package/scripts/install-lightpanda.mjs +161 -0
- package/scripts/sync-skills.mjs +110 -0
- package/skills/agent-browser/SKILL.md +0 -1960
- package/skills/agent-browser/bundled/agentcore/SKILL.md +115 -0
- package/skills/agent-browser/bundled/dogfood/SKILL.md +220 -0
- package/skills/agent-browser/bundled/dogfood/references/issue-taxonomy.md +109 -0
- package/skills/agent-browser/bundled/dogfood/templates/dogfood-report-template.md +53 -0
- package/skills/agent-browser/bundled/electron/SKILL.md +236 -0
- package/skills/agent-browser/bundled/slack/SKILL.md +285 -0
- package/skills/agent-browser/bundled/slack/references/slack-tasks.md +348 -0
- package/skills/agent-browser/bundled/slack/templates/slack-report-template.md +163 -0
- package/skills/agent-browser/bundled/vercel-sandbox/SKILL.md +280 -0
- package/skills/agent-browser/manifest.json +42 -0
- package/skills/agent-browser/references/authentication.md +303 -0
- package/skills/agent-browser/references/commands.md +398 -0
- package/skills/agent-browser/references/profiling.md +120 -0
- package/skills/agent-browser/references/proxy-support.md +194 -0
- package/skills/agent-browser/references/session-management.md +193 -0
- package/skills/agent-browser/references/snapshot-refs.md +219 -0
- package/skills/agent-browser/references/trust-boundaries.md +89 -0
- package/skills/agent-browser/references/video-recording.md +175 -0
- package/skills/agent-browser/templates/authenticated-session.sh +105 -0
- package/skills/agent-browser/templates/capture-workflow.sh +69 -0
- package/skills/agent-browser/templates/form-automation.sh +62 -0
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import type { Scenario } from "../types/scenario.js";
|
|
2
|
+
import type { Skill } from "../types/skill.js";
|
|
3
|
+
export type SkillLoadKind = "reference" | "template" | "skill" | "custom";
|
|
4
|
+
export interface SkillManifest {
|
|
5
|
+
version: number;
|
|
6
|
+
core: {
|
|
7
|
+
references: string[];
|
|
8
|
+
templates: string[];
|
|
9
|
+
};
|
|
10
|
+
bundled: Array<{
|
|
11
|
+
name: string;
|
|
12
|
+
description: string;
|
|
13
|
+
}>;
|
|
14
|
+
}
|
|
15
|
+
export declare function resolveAgentBrowserSkillRoot(cwd: string): string;
|
|
16
|
+
export declare function readSkillManifest(cwd: string): SkillManifest | null;
|
|
17
|
+
export declare function listLoadableCustomSkills(cwd: string, skillDirs: string[], preloadedNames: Set<string>): Skill[];
|
|
18
|
+
export interface SkillLoadResult {
|
|
19
|
+
kind: SkillLoadKind;
|
|
20
|
+
name: string;
|
|
21
|
+
content: string;
|
|
22
|
+
truncated: boolean;
|
|
23
|
+
alreadyLoaded: boolean;
|
|
24
|
+
}
|
|
25
|
+
export interface SkillLoadRegistryOptions {
|
|
26
|
+
maxChars?: number;
|
|
27
|
+
/** Directories scanned for user SKILL.md files (pqa.config skills.dirs). */
|
|
28
|
+
skillDirs?: string[];
|
|
29
|
+
/** Skill names already injected in the system prompt — load_skill returns alreadyLoaded. */
|
|
30
|
+
preloadedNames?: Iterable<string>;
|
|
31
|
+
}
|
|
32
|
+
export declare class SkillLoadRegistry {
|
|
33
|
+
private readonly loaded;
|
|
34
|
+
private readonly maxChars;
|
|
35
|
+
private readonly skillDirs;
|
|
36
|
+
private readonly preloadedNames;
|
|
37
|
+
constructor(options?: SkillLoadRegistryOptions | number);
|
|
38
|
+
private key;
|
|
39
|
+
has(kind: SkillLoadKind, name: string): boolean;
|
|
40
|
+
mark(kind: SkillLoadKind, name: string): void;
|
|
41
|
+
private alreadyPreloaded;
|
|
42
|
+
load(cwd: string, kind: SkillLoadKind, name: string): SkillLoadResult;
|
|
43
|
+
}
|
|
44
|
+
export declare function formatOnDemandCatalog(cwd: string, skills: Skill[], options?: {
|
|
45
|
+
skillDirs?: string[];
|
|
46
|
+
}): string;
|
|
47
|
+
/** References the harness auto-injects before the agent runs (context-aware). */
|
|
48
|
+
export declare function inferAutoSkillLoads(options: {
|
|
49
|
+
scenario: Scenario;
|
|
50
|
+
authProfile?: string;
|
|
51
|
+
}): Array<{
|
|
52
|
+
kind: SkillLoadKind;
|
|
53
|
+
name: string;
|
|
54
|
+
reason: string;
|
|
55
|
+
}>;
|
|
56
|
+
export declare function formatAutoLoadedMessage(loads: SkillLoadResult[]): string | null;
|
|
57
|
+
//# sourceMappingURL=on-demand.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"on-demand.d.ts","sourceRoot":"","sources":["../../src/skills/on-demand.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAG/C,MAAM,MAAM,aAAa,GAAG,WAAW,GAAG,UAAU,GAAG,OAAO,GAAG,QAAQ,CAAC;AAE1E,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE;QACJ,UAAU,EAAE,MAAM,EAAE,CAAC;QACrB,SAAS,EAAE,MAAM,EAAE,CAAC;KACrB,CAAC;IACF,OAAO,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACvD;AAID,wBAAgB,4BAA4B,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAEhE;AAED,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CAQnE;AAkDD,wBAAgB,wBAAwB,CACtC,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,EAAE,EACnB,cAAc,EAAE,GAAG,CAAC,MAAM,CAAC,GAC1B,KAAK,EAAE,CAIT;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,aAAa,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,OAAO,CAAC;IACnB,aAAa,EAAE,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,4EAA4E;IAC5E,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,4FAA4F;IAC5F,cAAc,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;CACnC;AAED,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;IAC5C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAW;IACrC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAc;gBAEjC,OAAO,GAAE,wBAAwB,GAAG,MAAW;IAQ3D,OAAO,CAAC,GAAG;IAIX,GAAG,CAAC,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO;IAI/C,IAAI,CAAC,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI;IAI7C,OAAO,CAAC,gBAAgB;IAKxB,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,GAAG,eAAe;CA2DtE;AAED,wBAAgB,qBAAqB,CACnC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,KAAK,EAAE,EACf,OAAO,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,GACjC,MAAM,CAqDR;AAED,iFAAiF;AACjF,wBAAgB,mBAAmB,CAAC,OAAO,EAAE;IAC3C,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GAAG,KAAK,CAAC;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAkB/D;AAED,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,eAAe,EAAE,GACvB,MAAM,GAAG,IAAI,CAmBf"}
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { resolveBundledPath } from "../paths.js";
|
|
4
|
+
import { discoverSkills, getSkill } from "./loader.js";
|
|
5
|
+
const DEFAULT_MAX_CHARS = 50_000;
|
|
6
|
+
export function resolveAgentBrowserSkillRoot(cwd) {
|
|
7
|
+
return resolveBundledPath(cwd, path.join("skills", "agent-browser"));
|
|
8
|
+
}
|
|
9
|
+
export function readSkillManifest(cwd) {
|
|
10
|
+
const manifestPath = path.join(resolveAgentBrowserSkillRoot(cwd), "manifest.json");
|
|
11
|
+
if (!existsSync(manifestPath))
|
|
12
|
+
return null;
|
|
13
|
+
try {
|
|
14
|
+
return JSON.parse(readFileSync(manifestPath, "utf8"));
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
return null;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
function truncate(content, maxChars) {
|
|
21
|
+
if (content.length <= maxChars) {
|
|
22
|
+
return { content, truncated: false };
|
|
23
|
+
}
|
|
24
|
+
return {
|
|
25
|
+
content: `${content.slice(0, maxChars)}\n\n[… truncated at ${maxChars} characters — use a narrower reference]`,
|
|
26
|
+
truncated: true,
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
function readReference(root, name) {
|
|
30
|
+
const filePath = path.join(root, "references", `${name}.md`);
|
|
31
|
+
if (!existsSync(filePath)) {
|
|
32
|
+
throw new Error(`Reference not found: ${name}`);
|
|
33
|
+
}
|
|
34
|
+
return readFileSync(filePath, "utf8").trim();
|
|
35
|
+
}
|
|
36
|
+
function readTemplate(root, name) {
|
|
37
|
+
const filePath = path.join(root, "templates", `${name}.sh`);
|
|
38
|
+
if (!existsSync(filePath)) {
|
|
39
|
+
throw new Error(`Template not found: ${name}`);
|
|
40
|
+
}
|
|
41
|
+
return readFileSync(filePath, "utf8").trim();
|
|
42
|
+
}
|
|
43
|
+
function readBundledSkill(root, name) {
|
|
44
|
+
const filePath = path.join(root, "bundled", name, "SKILL.md");
|
|
45
|
+
if (!existsSync(filePath))
|
|
46
|
+
return null;
|
|
47
|
+
const raw = readFileSync(filePath, "utf8");
|
|
48
|
+
return raw.replace(/^---[\s\S]*?---\r?\n?/, "").trim();
|
|
49
|
+
}
|
|
50
|
+
function readCustomSkill(cwd, skillDirs, name) {
|
|
51
|
+
const skills = discoverSkills(skillDirs, cwd);
|
|
52
|
+
const skill = getSkill(skills, name);
|
|
53
|
+
if (!skill) {
|
|
54
|
+
throw new Error(`Custom skill not found: ${name}. Add a SKILL.md under skills.dirs (configured in pqa.config) with matching frontmatter name.`);
|
|
55
|
+
}
|
|
56
|
+
return skill.body;
|
|
57
|
+
}
|
|
58
|
+
export function listLoadableCustomSkills(cwd, skillDirs, preloadedNames) {
|
|
59
|
+
return discoverSkills(skillDirs, cwd).filter((s) => !preloadedNames.has(s.name));
|
|
60
|
+
}
|
|
61
|
+
export class SkillLoadRegistry {
|
|
62
|
+
loaded = new Set();
|
|
63
|
+
maxChars;
|
|
64
|
+
skillDirs;
|
|
65
|
+
preloadedNames;
|
|
66
|
+
constructor(options = {}) {
|
|
67
|
+
const resolved = typeof options === "number" ? { maxChars: options } : options;
|
|
68
|
+
this.maxChars = resolved.maxChars ?? DEFAULT_MAX_CHARS;
|
|
69
|
+
this.skillDirs = resolved.skillDirs ?? [];
|
|
70
|
+
this.preloadedNames = new Set(resolved.preloadedNames ?? []);
|
|
71
|
+
}
|
|
72
|
+
key(kind, name) {
|
|
73
|
+
return `${kind}:${name}`;
|
|
74
|
+
}
|
|
75
|
+
has(kind, name) {
|
|
76
|
+
return this.loaded.has(this.key(kind, name));
|
|
77
|
+
}
|
|
78
|
+
mark(kind, name) {
|
|
79
|
+
this.loaded.add(this.key(kind, name));
|
|
80
|
+
}
|
|
81
|
+
alreadyPreloaded(kind, name) {
|
|
82
|
+
if (kind !== "skill" && kind !== "custom")
|
|
83
|
+
return false;
|
|
84
|
+
return this.preloadedNames.has(name);
|
|
85
|
+
}
|
|
86
|
+
load(cwd, kind, name) {
|
|
87
|
+
const normalized = name.trim();
|
|
88
|
+
const key = this.key(kind, normalized);
|
|
89
|
+
if (this.loaded.has(key)) {
|
|
90
|
+
return {
|
|
91
|
+
kind,
|
|
92
|
+
name: normalized,
|
|
93
|
+
content: `Already loaded: ${kind} "${normalized}". Use the content from the earlier load_skill result.`,
|
|
94
|
+
truncated: false,
|
|
95
|
+
alreadyLoaded: true,
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
if (this.alreadyPreloaded(kind, normalized)) {
|
|
99
|
+
return {
|
|
100
|
+
kind,
|
|
101
|
+
name: normalized,
|
|
102
|
+
content: `Skill "${normalized}" is already preloaded in the system prompt.`,
|
|
103
|
+
truncated: false,
|
|
104
|
+
alreadyLoaded: true,
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
const root = resolveAgentBrowserSkillRoot(cwd);
|
|
108
|
+
let raw;
|
|
109
|
+
switch (kind) {
|
|
110
|
+
case "reference":
|
|
111
|
+
raw = readReference(root, normalized);
|
|
112
|
+
break;
|
|
113
|
+
case "template":
|
|
114
|
+
raw = readTemplate(root, normalized);
|
|
115
|
+
break;
|
|
116
|
+
case "custom":
|
|
117
|
+
raw = readCustomSkill(cwd, this.skillDirs, normalized);
|
|
118
|
+
break;
|
|
119
|
+
case "skill": {
|
|
120
|
+
const bundled = readBundledSkill(root, normalized);
|
|
121
|
+
if (bundled) {
|
|
122
|
+
raw = bundled;
|
|
123
|
+
break;
|
|
124
|
+
}
|
|
125
|
+
raw = readCustomSkill(cwd, this.skillDirs, normalized);
|
|
126
|
+
break;
|
|
127
|
+
}
|
|
128
|
+
default:
|
|
129
|
+
throw new Error(`Unknown skill load kind: ${kind}`);
|
|
130
|
+
}
|
|
131
|
+
const { content, truncated } = truncate(raw, this.maxChars);
|
|
132
|
+
this.loaded.add(key);
|
|
133
|
+
return {
|
|
134
|
+
kind,
|
|
135
|
+
name: normalized,
|
|
136
|
+
content,
|
|
137
|
+
truncated,
|
|
138
|
+
alreadyLoaded: false,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
export function formatOnDemandCatalog(cwd, skills, options) {
|
|
143
|
+
const core = skills.find((s) => s.name === "core");
|
|
144
|
+
if (!core)
|
|
145
|
+
return "";
|
|
146
|
+
const skillDirs = options?.skillDirs ?? [];
|
|
147
|
+
const preloaded = new Set(skills.map((s) => s.name));
|
|
148
|
+
const customSkills = listLoadableCustomSkills(cwd, skillDirs, preloaded);
|
|
149
|
+
const manifest = readSkillManifest(cwd);
|
|
150
|
+
if (!manifest) {
|
|
151
|
+
const customLines = customSkills
|
|
152
|
+
.map((s) => `- custom \`${s.name}\` — ${s.description}`)
|
|
153
|
+
.join("\n");
|
|
154
|
+
return [
|
|
155
|
+
"## On-demand skill loading",
|
|
156
|
+
"",
|
|
157
|
+
"Use the `load_skill` tool when you need detailed agent-browser docs beyond this core summary.",
|
|
158
|
+
"Run `pqa skills sync` (or `npm install`) if references are missing.",
|
|
159
|
+
"",
|
|
160
|
+
"**Custom skills** (load_skill kind=custom or kind=skill):",
|
|
161
|
+
customLines || "- (none — add SKILL.md under skills.dirs)",
|
|
162
|
+
].join("\n");
|
|
163
|
+
}
|
|
164
|
+
const refLines = manifest.core.references.map((r) => `- reference \`${r}\``).join("\n");
|
|
165
|
+
const tplLines = manifest.core.templates.map((t) => `- template \`${t}\``).join("\n");
|
|
166
|
+
const bundledLines = manifest.bundled
|
|
167
|
+
.map((b) => `- skill \`${b.name}\` — ${b.description || "bundled skill"}`)
|
|
168
|
+
.join("\n");
|
|
169
|
+
const customLines = customSkills
|
|
170
|
+
.map((s) => `- custom \`${s.name}\` — ${s.description}`)
|
|
171
|
+
.join("\n");
|
|
172
|
+
return [
|
|
173
|
+
"## On-demand skill loading",
|
|
174
|
+
"",
|
|
175
|
+
"Only preloaded skills appear in the system prompt above. Load everything else with `load_skill`",
|
|
176
|
+
"— do **not** run `agent-browser skills get` in bash (wastes turns).",
|
|
177
|
+
"",
|
|
178
|
+
"**Available references** (load_skill kind=reference):",
|
|
179
|
+
refLines || "- (none)",
|
|
180
|
+
"",
|
|
181
|
+
"**Available templates** (load_skill kind=template):",
|
|
182
|
+
tplLines || "- (none)",
|
|
183
|
+
"",
|
|
184
|
+
"**Bundled agent-browser skills** (load_skill kind=skill):",
|
|
185
|
+
bundledLines || "- (none)",
|
|
186
|
+
"",
|
|
187
|
+
"**Custom project skills** (load_skill kind=custom, or kind=skill as fallback):",
|
|
188
|
+
customLines || "- (none — add SKILL.md under skills.dirs in pqa.config)",
|
|
189
|
+
"",
|
|
190
|
+
"Load sparingly: one item at a time, only when the current step requires it.",
|
|
191
|
+
].join("\n");
|
|
192
|
+
}
|
|
193
|
+
/** References the harness auto-injects before the agent runs (context-aware). */
|
|
194
|
+
export function inferAutoSkillLoads(options) {
|
|
195
|
+
const loads = [];
|
|
196
|
+
// Consumer scenarios with auth: use preloaded state — no auth reference needed.
|
|
197
|
+
const needsAuth = Boolean(options.authProfile) ||
|
|
198
|
+
options.scenario.frontmatter.tags?.includes("auth") ||
|
|
199
|
+
options.scenario.frontmatter.name?.startsWith("login-");
|
|
200
|
+
if (needsAuth) {
|
|
201
|
+
loads.push({
|
|
202
|
+
kind: "reference",
|
|
203
|
+
name: "authentication",
|
|
204
|
+
reason: "scenario uses authentication",
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
return loads;
|
|
208
|
+
}
|
|
209
|
+
export function formatAutoLoadedMessage(loads) {
|
|
210
|
+
if (loads.length === 0)
|
|
211
|
+
return null;
|
|
212
|
+
const blocks = loads
|
|
213
|
+
.filter((l) => !l.alreadyLoaded)
|
|
214
|
+
.map((l) => `### Auto-loaded ${l.kind}: ${l.name}\n\n${l.content}`);
|
|
215
|
+
if (blocks.length === 0)
|
|
216
|
+
return null;
|
|
217
|
+
return [
|
|
218
|
+
"## Harness auto-loaded skill references",
|
|
219
|
+
"",
|
|
220
|
+
"The harness preloaded these references for this scenario. Do not reload them.",
|
|
221
|
+
"",
|
|
222
|
+
blocks.join("\n\n---\n\n"),
|
|
223
|
+
].join("\n");
|
|
224
|
+
}
|
|
225
|
+
//# sourceMappingURL=on-demand.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"on-demand.js","sourceRoot":"","sources":["../../src/skills/on-demand.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAGjD,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAavD,MAAM,iBAAiB,GAAG,MAAM,CAAC;AAEjC,MAAM,UAAU,4BAA4B,CAAC,GAAW;IACtD,OAAO,kBAAkB,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC,CAAC;AACvE,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,GAAW;IAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,EAAE,eAAe,CAAC,CAAC;IACnF,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3C,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAkB,CAAC;IACzE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,OAAe,EAAE,QAAgB;IACjD,IAAI,OAAO,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC/B,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IACvC,CAAC;IACD,OAAO;QACL,OAAO,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,uBAAuB,QAAQ,yCAAyC;QAC9G,SAAS,EAAE,IAAI;KAChB,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,IAAY,EAAE,IAAY;IAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,EAAE,GAAG,IAAI,KAAK,CAAC,CAAC;IAC7D,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,EAAE,CAAC,CAAC;IAClD,CAAC;IACD,OAAO,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;AAC/C,CAAC;AAED,SAAS,YAAY,CAAC,IAAY,EAAE,IAAY;IAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,EAAE,GAAG,IAAI,KAAK,CAAC,CAAC;IAC5D,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,EAAE,CAAC,CAAC;IACjD,CAAC;IACD,OAAO,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;AAC/C,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY,EAAE,IAAY;IAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IAC9D,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IACvC,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC3C,OAAO,GAAG,CAAC,OAAO,CAAC,uBAAuB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AACzD,CAAC;AAED,SAAS,eAAe,CACtB,GAAW,EACX,SAAmB,EACnB,IAAY;IAEZ,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACrC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CACb,2BAA2B,IAAI,+FAA+F,CAC/H,CAAC;IACJ,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC;AACpB,CAAC;AAED,MAAM,UAAU,wBAAwB,CACtC,GAAW,EACX,SAAmB,EACnB,cAA2B;IAE3B,OAAO,cAAc,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,MAAM,CAC1C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CACnC,CAAC;AACJ,CAAC;AAkBD,MAAM,OAAO,iBAAiB;IACX,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IAC3B,QAAQ,CAAS;IACjB,SAAS,CAAW;IACpB,cAAc,CAAc;IAE7C,YAAY,UAA6C,EAAE;QACzD,MAAM,QAAQ,GACZ,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;QAChE,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,IAAI,iBAAiB,CAAC;QACvD,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC,SAAS,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,cAAc,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC;IAC/D,CAAC;IAEO,GAAG,CAAC,IAAmB,EAAE,IAAY;QAC3C,OAAO,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC;IAC3B,CAAC;IAED,GAAG,CAAC,IAAmB,EAAE,IAAY;QACnC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED,IAAI,CAAC,IAAmB,EAAE,IAAY;QACpC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IACxC,CAAC;IAEO,gBAAgB,CAAC,IAAmB,EAAE,IAAY;QACxD,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QACxD,OAAO,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,CAAC,GAAW,EAAE,IAAmB,EAAE,IAAY;QACjD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QACvC,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,OAAO;gBACL,IAAI;gBACJ,IAAI,EAAE,UAAU;gBAChB,OAAO,EAAE,mBAAmB,IAAI,KAAK,UAAU,wDAAwD;gBACvG,SAAS,EAAE,KAAK;gBAChB,aAAa,EAAE,IAAI;aACpB,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,CAAC;YAC5C,OAAO;gBACL,IAAI;gBACJ,IAAI,EAAE,UAAU;gBAChB,OAAO,EAAE,UAAU,UAAU,8CAA8C;gBAC3E,SAAS,EAAE,KAAK;gBAChB,aAAa,EAAE,IAAI;aACpB,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,4BAA4B,CAAC,GAAG,CAAC,CAAC;QAC/C,IAAI,GAAW,CAAC;QAChB,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,WAAW;gBACd,GAAG,GAAG,aAAa,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;gBACtC,MAAM;YACR,KAAK,UAAU;gBACb,GAAG,GAAG,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;gBACrC,MAAM;YACR,KAAK,QAAQ;gBACX,GAAG,GAAG,eAAe,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;gBACvD,MAAM;YACR,KAAK,OAAO,CAAC,CAAC,CAAC;gBACb,MAAM,OAAO,GAAG,gBAAgB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;gBACnD,IAAI,OAAO,EAAE,CAAC;oBACZ,GAAG,GAAG,OAAO,CAAC;oBACd,MAAM;gBACR,CAAC;gBACD,GAAG,GAAG,eAAe,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;gBACvD,MAAM;YACR,CAAC;YACD;gBACE,MAAM,IAAI,KAAK,CAAC,4BAA4B,IAAoB,EAAE,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5D,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAErB,OAAO;YACL,IAAI;YACJ,IAAI,EAAE,UAAU;YAChB,OAAO;YACP,SAAS;YACT,aAAa,EAAE,KAAK;SACrB,CAAC;IACJ,CAAC;CACF;AAED,MAAM,UAAU,qBAAqB,CACnC,GAAW,EACX,MAAe,EACf,OAAkC;IAElC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;IACnD,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IAErB,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,EAAE,CAAC;IAC3C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACrD,MAAM,YAAY,GAAG,wBAAwB,CAAC,GAAG,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAEzE,MAAM,QAAQ,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC;IACxC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,WAAW,GAAG,YAAY;aAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;aACvD,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,OAAO;YACL,4BAA4B;YAC5B,EAAE;YACF,+FAA+F;YAC/F,qEAAqE;YACrE,EAAE;YACF,2DAA2D;YAC3D,WAAW,IAAI,2CAA2C;SAC3D,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC;IAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxF,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtF,MAAM,YAAY,GAAG,QAAQ,CAAC,OAAO;SAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC,WAAW,IAAI,eAAe,EAAE,CAAC;SACzE,IAAI,CAAC,IAAI,CAAC,CAAC;IACd,MAAM,WAAW,GAAG,YAAY;SAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;SACvD,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,OAAO;QACL,4BAA4B;QAC5B,EAAE;QACF,iGAAiG;QACjG,qEAAqE;QACrE,EAAE;QACF,uDAAuD;QACvD,QAAQ,IAAI,UAAU;QACtB,EAAE;QACF,qDAAqD;QACrD,QAAQ,IAAI,UAAU;QACtB,EAAE;QACF,2DAA2D;QAC3D,YAAY,IAAI,UAAU;QAC1B,EAAE;QACF,gFAAgF;QAChF,WAAW,IAAI,yDAAyD;QACxE,EAAE;QACF,6EAA6E;KAC9E,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,mBAAmB,CAAC,OAGnC;IACC,MAAM,KAAK,GAAiE,EAAE,CAAC;IAE/E,gFAAgF;IAChF,MAAM,SAAS,GACb,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC;QAC5B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC;QACnD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC;IAE1D,IAAI,SAAS,EAAE,CAAC;QACd,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,WAAW;YACjB,IAAI,EAAE,gBAAgB;YACtB,MAAM,EAAE,8BAA8B;SACvC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,KAAwB;IAExB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEpC,MAAM,MAAM,GAAG,KAAK;SACjB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;SAC/B,GAAG,CACF,CAAC,CAAC,EAAE,EAAE,CACJ,mBAAmB,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,CACzD,CAAC;IAEJ,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAErC,OAAO;QACL,yCAAyC;QACzC,EAAE;QACF,+EAA+E;QAC/E,EAAE;QACF,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC;KAC3B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"on-demand.test.d.ts","sourceRoot":"","sources":["../../src/skills/on-demand.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
4
|
+
import { describe, it } from "node:test";
|
|
5
|
+
import { SkillLoadRegistry, formatOnDemandCatalog, inferAutoSkillLoads, readSkillManifest, } from "./on-demand.js";
|
|
6
|
+
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
|
7
|
+
function makeScenario(overrides = {}) {
|
|
8
|
+
return {
|
|
9
|
+
filePath: "/tmp/example.md",
|
|
10
|
+
frontmatter: { name: "example" },
|
|
11
|
+
skills: ["core"],
|
|
12
|
+
goal: "Goal",
|
|
13
|
+
steps: "1. Step",
|
|
14
|
+
then: ['page shows "ok"'],
|
|
15
|
+
rawCheckpoints: ['page shows "ok"'],
|
|
16
|
+
checkpoints: [{ raw: 'page shows "ok"', kind: "page_shows", value: "ok" }],
|
|
17
|
+
...overrides,
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
describe("readSkillManifest", () => {
|
|
21
|
+
it("reads manifest after sync-skills", () => {
|
|
22
|
+
const manifest = readSkillManifest(repoRoot);
|
|
23
|
+
assert.ok(manifest);
|
|
24
|
+
assert.ok(manifest.core.references.includes("authentication"));
|
|
25
|
+
assert.ok(manifest.bundled.length > 0);
|
|
26
|
+
});
|
|
27
|
+
});
|
|
28
|
+
describe("SkillLoadRegistry", () => {
|
|
29
|
+
it("loads a reference and deduplicates subsequent loads", () => {
|
|
30
|
+
const registry = new SkillLoadRegistry();
|
|
31
|
+
const first = registry.load(repoRoot, "reference", "authentication");
|
|
32
|
+
assert.equal(first.alreadyLoaded, false);
|
|
33
|
+
assert.match(first.content, /auth/i);
|
|
34
|
+
const second = registry.load(repoRoot, "reference", "authentication");
|
|
35
|
+
assert.equal(second.alreadyLoaded, true);
|
|
36
|
+
});
|
|
37
|
+
it("throws for unknown reference via error message in load path", () => {
|
|
38
|
+
const registry = new SkillLoadRegistry();
|
|
39
|
+
assert.throws(() => registry.load(repoRoot, "reference", "does-not-exist-xyz"), /not found/i);
|
|
40
|
+
});
|
|
41
|
+
it("loads a custom skill from skills.dirs", () => {
|
|
42
|
+
const registry = new SkillLoadRegistry({
|
|
43
|
+
skillDirs: ["skills"],
|
|
44
|
+
preloadedNames: ["core"],
|
|
45
|
+
});
|
|
46
|
+
const result = registry.load(repoRoot, "custom", "prose-qa");
|
|
47
|
+
assert.equal(result.alreadyLoaded, false);
|
|
48
|
+
assert.match(result.content, /Prose-QA scenario authoring/i);
|
|
49
|
+
});
|
|
50
|
+
it("falls back to custom skill when bundled skill is missing", () => {
|
|
51
|
+
const registry = new SkillLoadRegistry({
|
|
52
|
+
skillDirs: ["skills"],
|
|
53
|
+
preloadedNames: ["core"],
|
|
54
|
+
});
|
|
55
|
+
const result = registry.load(repoRoot, "skill", "prose-qa");
|
|
56
|
+
assert.equal(result.alreadyLoaded, false);
|
|
57
|
+
assert.match(result.content, /scenario authoring/i);
|
|
58
|
+
});
|
|
59
|
+
it("returns alreadyLoaded for skills preloaded in the system prompt", () => {
|
|
60
|
+
const registry = new SkillLoadRegistry({
|
|
61
|
+
skillDirs: ["skills"],
|
|
62
|
+
preloadedNames: ["core", "prose-qa"],
|
|
63
|
+
});
|
|
64
|
+
const result = registry.load(repoRoot, "custom", "prose-qa");
|
|
65
|
+
assert.equal(result.alreadyLoaded, true);
|
|
66
|
+
assert.match(result.content, /already preloaded/i);
|
|
67
|
+
});
|
|
68
|
+
});
|
|
69
|
+
describe("inferAutoSkillLoads", () => {
|
|
70
|
+
it("auto-loads authentication for auth scenarios", () => {
|
|
71
|
+
const loads = inferAutoSkillLoads({
|
|
72
|
+
scenario: makeScenario({
|
|
73
|
+
frontmatter: { name: "login-admin", tags: ["auth"] },
|
|
74
|
+
}),
|
|
75
|
+
authProfile: "admin",
|
|
76
|
+
});
|
|
77
|
+
assert.deepEqual(loads, [
|
|
78
|
+
{ kind: "reference", name: "authentication", reason: "scenario uses authentication" },
|
|
79
|
+
]);
|
|
80
|
+
});
|
|
81
|
+
it("does not auto-load authentication for consumer scenarios with auth profile", () => {
|
|
82
|
+
const loads = inferAutoSkillLoads({
|
|
83
|
+
scenario: makeScenario({
|
|
84
|
+
frontmatter: { name: "checkout", auth: "admin" },
|
|
85
|
+
}),
|
|
86
|
+
});
|
|
87
|
+
assert.deepEqual(loads, []);
|
|
88
|
+
});
|
|
89
|
+
});
|
|
90
|
+
describe("formatOnDemandCatalog", () => {
|
|
91
|
+
it("lists references and bundled skills for core", () => {
|
|
92
|
+
const skills = [
|
|
93
|
+
{
|
|
94
|
+
name: "core",
|
|
95
|
+
description: "core skill",
|
|
96
|
+
dir: path.join(repoRoot, "skills/agent-browser"),
|
|
97
|
+
frontmatter: { name: "core", description: "core skill" },
|
|
98
|
+
body: "minimal core body",
|
|
99
|
+
},
|
|
100
|
+
];
|
|
101
|
+
const catalog = formatOnDemandCatalog(repoRoot, skills, {
|
|
102
|
+
skillDirs: ["skills"],
|
|
103
|
+
});
|
|
104
|
+
assert.match(catalog, /load_skill/);
|
|
105
|
+
assert.match(catalog, /authentication/);
|
|
106
|
+
assert.match(catalog, /Custom project skills/);
|
|
107
|
+
assert.match(catalog, /prose-qa/);
|
|
108
|
+
assert.match(catalog, /do \*\*not\*\* run `agent-browser skills get`/);
|
|
109
|
+
});
|
|
110
|
+
});
|
|
111
|
+
//# sourceMappingURL=on-demand.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"on-demand.test.js","sourceRoot":"","sources":["../../src/skills/on-demand.test.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AAGzC,OAAO,EACL,iBAAiB,EACjB,qBAAqB,EACrB,mBAAmB,EACnB,iBAAiB,GAClB,MAAM,gBAAgB,CAAC;AAExB,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAC3B,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAC5C,OAAO,CACR,CAAC;AAEF,SAAS,YAAY,CAAC,YAA+B,EAAE;IACrD,OAAO;QACL,QAAQ,EAAE,iBAAiB;QAC3B,WAAW,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;QAChC,MAAM,EAAE,CAAC,MAAM,CAAC;QAChB,IAAI,EAAE,MAAM;QACZ,KAAK,EAAE,SAAS;QAChB,IAAI,EAAE,CAAC,iBAAiB,CAAC;QACzB,cAAc,EAAE,CAAC,iBAAiB,CAAC;QACnC,WAAW,EAAE,CAAC,EAAE,GAAG,EAAE,iBAAiB,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;QAC1E,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;IACjC,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;QAC1C,MAAM,QAAQ,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAC7C,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;QACpB,MAAM,CAAC,EAAE,CAAC,QAAS,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC;QAChE,MAAM,CAAC,EAAE,CAAC,QAAS,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;IACjC,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;QAC7D,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,gBAAgB,CAAC,CAAC;QACrE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QACzC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAErC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,gBAAgB,CAAC,CAAC;QACtE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6DAA6D,EAAE,GAAG,EAAE;QACrE,MAAM,QAAQ,GAAG,IAAI,iBAAiB,EAAE,CAAC;QACzC,MAAM,CAAC,MAAM,CACX,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,oBAAoB,CAAC,EAChE,YAAY,CACb,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC;YACrC,SAAS,EAAE,CAAC,QAAQ,CAAC;YACrB,cAAc,EAAE,CAAC,MAAM,CAAC;SACzB,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QAC7D,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QAC1C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,8BAA8B,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,GAAG,EAAE;QAClE,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC;YACrC,SAAS,EAAE,CAAC,QAAQ,CAAC;YACrB,cAAc,EAAE,CAAC,MAAM,CAAC;SACzB,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;QAC5D,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QAC1C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iEAAiE,EAAE,GAAG,EAAE;QACzE,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC;YACrC,SAAS,EAAE,CAAC,QAAQ,CAAC;YACrB,cAAc,EAAE,CAAC,MAAM,EAAE,UAAU,CAAC;SACrC,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QAC7D,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;QACzC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,oBAAoB,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,KAAK,GAAG,mBAAmB,CAAC;YAChC,QAAQ,EAAE,YAAY,CAAC;gBACrB,WAAW,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,MAAM,CAAC,EAAE;aACrD,CAAC;YACF,WAAW,EAAE,OAAO;SACrB,CAAC,CAAC;QACH,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE;YACtB,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,8BAA8B,EAAE;SACtF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4EAA4E,EAAE,GAAG,EAAE;QACpF,MAAM,KAAK,GAAG,mBAAmB,CAAC;YAChC,QAAQ,EAAE,YAAY,CAAC;gBACrB,WAAW,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE;aACjD,CAAC;SACH,CAAC,CAAC;QACH,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;IACrC,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,MAAM,GAAY;YACtB;gBACE,IAAI,EAAE,MAAM;gBACZ,WAAW,EAAE,YAAY;gBACzB,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,sBAAsB,CAAC;gBAChD,WAAW,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE;gBACxD,IAAI,EAAE,mBAAmB;aAC1B;SACF,CAAC;QACF,MAAM,OAAO,GAAG,qBAAqB,CAAC,QAAQ,EAAE,MAAM,EAAE;YACtD,SAAS,EAAE,CAAC,QAAQ,CAAC;SACtB,CAAC,CAAC;QACH,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QACpC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;QACxC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,uBAAuB,CAAC,CAAC;QAC/C,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QAClC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,+CAA+C,CAAC,CAAC;IACzE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/dist/types/config.d.ts
CHANGED
|
@@ -35,7 +35,7 @@ export type BrowserEngine = "chrome" | "lightpanda";
|
|
|
35
35
|
/** Options when browser.engine is lightpanda (AGENT_BROWSER_EXECUTABLE_PATH, telemetry). */
|
|
36
36
|
export interface LightpandaBrowserConfig {
|
|
37
37
|
/**
|
|
38
|
-
* Path to the lightpanda binary, or a directory containing it (e.g. "
|
|
38
|
+
* Path to the lightpanda binary, or a directory containing it (e.g. "./.pqa/engine").
|
|
39
39
|
* Relative paths resolve from the project cwd. Maps to AGENT_BROWSER_EXECUTABLE_PATH.
|
|
40
40
|
*/
|
|
41
41
|
executablePath?: string;
|
|
@@ -45,16 +45,15 @@ export interface LightpandaBrowserConfig {
|
|
|
45
45
|
/** Reasoning intensity; applied only on providers that support it (e.g. OpenAI, Anthropic, Google). */
|
|
46
46
|
export type LlmReasoningEffort = "none" | "minimal" | "low" | "medium" | "high" | "xhigh";
|
|
47
47
|
export interface PqaConfig {
|
|
48
|
-
/** Root directory for scenario markdown files. Default: scenarios
|
|
48
|
+
/** Root directory for scenario markdown files. Default: scenarios */
|
|
49
49
|
scenariosDir?: string;
|
|
50
|
-
systemPromptPath?: string;
|
|
51
50
|
/** Env var names injected into the system prompt and expected at runtime */
|
|
52
51
|
envVars?: string[];
|
|
53
52
|
/** Env var names whose values are redacted from transcripts, verdicts, reports and verbose logs */
|
|
54
53
|
sensitiveEnvVars?: string[];
|
|
55
54
|
llm: {
|
|
56
|
-
provider
|
|
57
|
-
model
|
|
55
|
+
provider?: "anthropic" | "openai" | "fireworks" | "ollama" | "google" | "openrouter";
|
|
56
|
+
model?: string;
|
|
58
57
|
/**
|
|
59
58
|
* Extended thinking / reasoning (opt-in).
|
|
60
59
|
* `budgetTokens`: Anthropic, Fireworks, Google & OpenRouter reasoning budget.
|
|
@@ -76,8 +75,19 @@ export interface PqaConfig {
|
|
|
76
75
|
lightpanda?: LightpandaBrowserConfig;
|
|
77
76
|
};
|
|
78
77
|
skills: {
|
|
79
|
-
|
|
80
|
-
|
|
78
|
+
/** Directories scanned for custom SKILL.md files. Default: [] */
|
|
79
|
+
dirs?: string[];
|
|
80
|
+
/** Extra skill names always appended to the system prompt (core is always loaded). Default: [] */
|
|
81
|
+
preloads?: string[];
|
|
82
|
+
/** On-demand loading of agent-browser references and bundled skills. */
|
|
83
|
+
onDemand?: {
|
|
84
|
+
/** Master switch for load_skill tool and auto-load. Default: true */
|
|
85
|
+
enabled?: boolean;
|
|
86
|
+
/** Max characters returned per load_skill call. Default: 50000 */
|
|
87
|
+
maxChars?: number;
|
|
88
|
+
/** Harness auto-injects context-aware references before the run. Default: true */
|
|
89
|
+
autoLoad?: boolean;
|
|
90
|
+
};
|
|
81
91
|
};
|
|
82
92
|
agent: {
|
|
83
93
|
maxTurns: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/types/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,eAAe,CAAC;AAEjE,MAAM,WAAW,aAAa;IAC5B,mCAAmC;IACnC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,0EAA0E;IAC1E,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,qEAAqE;IACrE,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,6FAA6F;IAC7F,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,cAAc;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,WAAW;IAC1B,qEAAqE;IACrE,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,mCAAmC;IACnC,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,8EAA8E;IAC9E,GAAG,CAAC,EAAE,OAAO,CAAC;CACf;AAED,qEAAqE;AACrE,MAAM,MAAM,aAAa,GAAG,QAAQ,GAAG,YAAY,CAAC;AAEpD,4FAA4F;AAC5F,MAAM,WAAW,uBAAuB;IACtC;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,+FAA+F;IAC/F,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,uGAAuG;AACvG,MAAM,MAAM,kBAAkB,GAC1B,MAAM,GACN,SAAS,GACT,KAAK,GACL,QAAQ,GACR,MAAM,GACN,OAAO,CAAC;AAEZ,MAAM,WAAW,SAAS;IACxB,
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/types/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,eAAe,CAAC;AAEjE,MAAM,WAAW,aAAa;IAC5B,mCAAmC;IACnC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,0EAA0E;IAC1E,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,qEAAqE;IACrE,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,6FAA6F;IAC7F,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,cAAc;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,WAAW;IAC1B,qEAAqE;IACrE,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,mCAAmC;IACnC,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,8EAA8E;IAC9E,GAAG,CAAC,EAAE,OAAO,CAAC;CACf;AAED,qEAAqE;AACrE,MAAM,MAAM,aAAa,GAAG,QAAQ,GAAG,YAAY,CAAC;AAEpD,4FAA4F;AAC5F,MAAM,WAAW,uBAAuB;IACtC;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,+FAA+F;IAC/F,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,uGAAuG;AACvG,MAAM,MAAM,kBAAkB,GAC1B,MAAM,GACN,SAAS,GACT,KAAK,GACL,QAAQ,GACR,MAAM,GACN,OAAO,CAAC;AAEZ,MAAM,WAAW,SAAS;IACxB,qEAAqE;IACrE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,4EAA4E;IAC5E,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,mGAAmG;IACnG,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,GAAG,EAAE;QACH,QAAQ,CAAC,EACL,WAAW,GACX,QAAQ,GACR,WAAW,GACX,QAAQ,GACR,QAAQ,GACR,YAAY,CAAC;QACjB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;;;WAKG;QACH,QAAQ,CAAC,EAAE;YACT,OAAO,EAAE,OAAO,CAAC;YACjB,YAAY,CAAC,EAAE,MAAM,CAAC;YACtB,eAAe,CAAC,EAAE,kBAAkB,CAAC;SACtC,CAAC;KACH,CAAC;IACF,OAAO,EAAE;QACP,MAAM,EAAE,OAAO,CAAC;QAChB,WAAW,EAAE,MAAM,CAAC;QACpB,cAAc,EAAE,MAAM,CAAC;QACvB,2DAA2D;QAC3D,MAAM,EAAE,aAAa,CAAC;QACtB,UAAU,CAAC,EAAE,uBAAuB,CAAC;KACtC,CAAC;IACF,MAAM,EAAE;QACN,iEAAiE;QACjE,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;QAChB,kGAAkG;QAClG,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;QACpB,wEAAwE;QACxE,QAAQ,CAAC,EAAE;YACT,qEAAqE;YACrE,OAAO,CAAC,EAAE,OAAO,CAAC;YAClB,kEAAkE;YAClE,QAAQ,CAAC,EAAE,MAAM,CAAC;YAClB,kFAAkF;YAClF,QAAQ,CAAC,EAAE,OAAO,CAAC;SACpB,CAAC;KACH,CAAC;IACF,KAAK,EAAE;QACL,QAAQ,EAAE,MAAM,CAAC;QACjB,aAAa,EAAE,MAAM,CAAC;QACtB;;;;WAIG;QACH,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;IACxC,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB;AAED,MAAM,WAAW,iBAAiB;IAChC,sDAAsD;IACtD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,qEAAqE;IACrE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,MAAM,aAAa,GAAG,YAAY,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE9D,MAAM,WAAW,UAAU;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,2BAA2B,CAAC;IACnC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,aAAa,CAAC;IACzB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iEAAiE;IACjE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,8FAA8F;IAC9F,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,8DAA8D;IAC9D,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,6EAA6E;IAC7E,aAAa,CAAC,EAAE,WAAW,GAAG,QAAQ,CAAC;IACvC,2DAA2D;IAC3D,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,mFAAmF;IACnF,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,sEAAsE;IACtE,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB"}
|
package/dist/types/verdict.d.ts
CHANGED
|
@@ -18,8 +18,28 @@ export declare const VerdictSchema: z.ZodObject<{
|
|
|
18
18
|
}, z.core.$strip>>;
|
|
19
19
|
summary: z.ZodString;
|
|
20
20
|
}, z.core.$strip>;
|
|
21
|
+
export declare const VerdictStatsSchema: z.ZodObject<{
|
|
22
|
+
durationMs: z.ZodNumber;
|
|
23
|
+
llmTurns: z.ZodNumber;
|
|
24
|
+
userTurns: z.ZodNumber;
|
|
25
|
+
toolCalls: z.ZodNumber;
|
|
26
|
+
failedToolCalls: z.ZodNumber;
|
|
27
|
+
llmDurationMs: z.ZodNumber;
|
|
28
|
+
bashDurationMs: z.ZodNumber;
|
|
29
|
+
healing: z.ZodOptional<z.ZodObject<{
|
|
30
|
+
used: z.ZodBoolean;
|
|
31
|
+
recoveryTurns: z.ZodNumber;
|
|
32
|
+
scenarioRetries: z.ZodNumber;
|
|
33
|
+
}, z.core.$strip>>;
|
|
34
|
+
}, z.core.$strip>;
|
|
21
35
|
export type CheckpointResult = z.infer<typeof CheckpointResultSchema>;
|
|
22
|
-
|
|
36
|
+
/** Parsed agent verdict (LLM output). */
|
|
37
|
+
export type ParsedVerdict = z.infer<typeof VerdictSchema>;
|
|
38
|
+
export type VerdictStats = z.infer<typeof VerdictStatsSchema>;
|
|
39
|
+
/** Harness-enriched verdict written to verdict.json and report.json. */
|
|
40
|
+
export type Verdict = ParsedVerdict & {
|
|
41
|
+
stats?: VerdictStats;
|
|
42
|
+
};
|
|
23
43
|
export type FailureKind = "transient" | "scenario_issue" | "product" | "unknown";
|
|
24
44
|
export interface HealingMeta {
|
|
25
45
|
used: boolean;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"verdict.d.ts","sourceRoot":"","sources":["../../src/types/verdict.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,sBAAsB;;;;;iBAKjC,CAAC;AAEH,eAAO,MAAM,aAAa;;;;;;;;;;;;iBAIxB,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AACtE,MAAM,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,aAAa,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"verdict.d.ts","sourceRoot":"","sources":["../../src/types/verdict.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,sBAAsB;;;;;iBAKjC,CAAC;AAEH,eAAO,MAAM,aAAa;;;;;;;;;;;;iBAIxB,CAAC;AAEH,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;iBAe7B,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AACtE,yCAAyC;AACzC,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,aAAa,CAAC,CAAC;AAC1D,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAC9D,wEAAwE;AACxE,MAAM,MAAM,OAAO,GAAG,aAAa,GAAG;IAAE,KAAK,CAAC,EAAE,YAAY,CAAA;CAAE,CAAC;AAE/D,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,gBAAgB,GAAG,SAAS,GAAG,SAAS,CAAC;AAEjF,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,OAAO,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,SAAS,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,kEAAkE;IAClE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,uDAAuD;IACvD,EAAE,EAAE,MAAM,CAAC;IACX,6EAA6E;IAC7E,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,mBAAoB,SAAQ,SAAS;IACpD,IAAI,EAAE,MAAM,CAAC;IACb,uDAAuD;IACvD,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,MAAM,eAAe,GAAG,sBAAsB,GAAG,mBAAmB,CAAC;AAE3E,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,eAAe,EAAE,CAAC;CAC5B;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IAC9C,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACxB,UAAU,EAAE,eAAe,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,WAAW,CAAC;CACvB;AAED,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,OAAO,EAAE;QACP,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CACH"}
|
package/dist/types/verdict.js
CHANGED
|
@@ -10,4 +10,20 @@ export const VerdictSchema = z.object({
|
|
|
10
10
|
checkpoints: z.array(CheckpointResultSchema),
|
|
11
11
|
summary: z.string(),
|
|
12
12
|
});
|
|
13
|
+
export const VerdictStatsSchema = z.object({
|
|
14
|
+
durationMs: z.number(),
|
|
15
|
+
llmTurns: z.number(),
|
|
16
|
+
userTurns: z.number(),
|
|
17
|
+
toolCalls: z.number(),
|
|
18
|
+
failedToolCalls: z.number(),
|
|
19
|
+
llmDurationMs: z.number(),
|
|
20
|
+
bashDurationMs: z.number(),
|
|
21
|
+
healing: z
|
|
22
|
+
.object({
|
|
23
|
+
used: z.boolean(),
|
|
24
|
+
recoveryTurns: z.number(),
|
|
25
|
+
scenarioRetries: z.number(),
|
|
26
|
+
})
|
|
27
|
+
.optional(),
|
|
28
|
+
});
|
|
13
29
|
//# sourceMappingURL=verdict.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"verdict.js","sourceRoot":"","sources":["../../src/types/verdict.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7C,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE;IACjB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;CACzC,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC;IAC5C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;CACpB,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"verdict.js","sourceRoot":"","sources":["../../src/types/verdict.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7C,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE;IACjB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;CACzC,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC;IAC5C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;CACpB,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;IACtB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;IACpB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE;IAC3B,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE;IACzB,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE;IAC1B,OAAO,EAAE,CAAC;SACP,MAAM,CAAC;QACN,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE;QACjB,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE;QACzB,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE;KAC5B,CAAC;SACD,QAAQ,EAAE;CACd,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"version.d.ts","sourceRoot":"","sources":["../src/version.ts"],"names":[],"mappings":"AAIA,iFAAiF;AACjF,eAAO,MAAM,eAAe,EAAE,MAEX,CAAC"}
|
package/dist/version.js
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { getPackageRoot } from "./paths.js";
|
|
4
|
+
/** Package version from package.json (resolved at runtime from package root). */
|
|
5
|
+
export const PACKAGE_VERSION = JSON.parse(readFileSync(join(getPackageRoot(), "package.json"), "utf8")).version;
|
|
6
|
+
//# sourceMappingURL=version.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"version.js","sourceRoot":"","sources":["../src/version.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE5C,iFAAiF;AACjF,MAAM,CAAC,MAAM,eAAe,GAAW,IAAI,CAAC,KAAK,CAC/C,YAAY,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,CAC7D,CAAC,OAAiB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,15 +1,17 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "prose-qa",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.2",
|
|
4
4
|
"description": "ProseQA — agent harness for NL E2E regression testing via agent-browser",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
7
|
-
"pqa": "
|
|
7
|
+
"pqa": "dist/cli/index.js"
|
|
8
8
|
},
|
|
9
9
|
"files": [
|
|
10
10
|
"dist",
|
|
11
11
|
"prompt",
|
|
12
12
|
"skills",
|
|
13
|
+
"scripts/sync-skills.mjs",
|
|
14
|
+
"scripts/install-lightpanda.mjs",
|
|
13
15
|
"pqa.config.ts"
|
|
14
16
|
],
|
|
15
17
|
"scripts": {
|
|
@@ -17,7 +19,7 @@
|
|
|
17
19
|
"test": "node --import tsx --test src/**/*.test.ts",
|
|
18
20
|
"dev": "tsx src/cli/index.ts",
|
|
19
21
|
"mcp": "tsx src/cli/index.ts mcp",
|
|
20
|
-
"postinstall": "
|
|
22
|
+
"postinstall": "node scripts/sync-skills.mjs",
|
|
21
23
|
"install-chrome": "agent-browser install --with-deps",
|
|
22
24
|
"install-lightpanda": "node scripts/install-lightpanda.mjs",
|
|
23
25
|
"demo:server": "node scripts/demo-server.mjs",
|
|
@@ -29,7 +31,7 @@
|
|
|
29
31
|
},
|
|
30
32
|
"repository": {
|
|
31
33
|
"type": "git",
|
|
32
|
-
"url": "https://github.com/FreakDev/Prose-QA.git"
|
|
34
|
+
"url": "git+https://github.com/FreakDev/Prose-QA.git"
|
|
33
35
|
},
|
|
34
36
|
"homepage": "https://github.com/FreakDev/Prose-QA#readme",
|
|
35
37
|
"bugs": {
|
|
@@ -47,10 +49,10 @@
|
|
|
47
49
|
"access": "public"
|
|
48
50
|
},
|
|
49
51
|
"dependencies": {
|
|
50
|
-
"@ai-sdk/anthropic": "^
|
|
52
|
+
"@ai-sdk/anthropic": "^3.0.81",
|
|
51
53
|
"@ai-sdk/fireworks": "^2.0.53",
|
|
52
|
-
"@ai-sdk/google": "^
|
|
53
|
-
"@ai-sdk/openai": "^
|
|
54
|
+
"@ai-sdk/google": "^3.0.80",
|
|
55
|
+
"@ai-sdk/openai": "^3.0.67",
|
|
54
56
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
55
57
|
"@openrouter/ai-sdk-provider": "^2.9.0",
|
|
56
58
|
"agent-browser": "^0.27.1",
|