prose-qa 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +570 -0
- package/dist/agent/bash.d.ts +52 -0
- package/dist/agent/bash.d.ts.map +1 -0
- package/dist/agent/bash.js +186 -0
- package/dist/agent/bash.js.map +1 -0
- package/dist/agent/bash.test.d.ts +2 -0
- package/dist/agent/bash.test.d.ts.map +1 -0
- package/dist/agent/bash.test.js +70 -0
- package/dist/agent/bash.test.js.map +1 -0
- package/dist/agent/llm-model.d.ts +5 -0
- package/dist/agent/llm-model.d.ts.map +1 -0
- package/dist/agent/llm-model.js +29 -0
- package/dist/agent/llm-model.js.map +1 -0
- package/dist/agent/llm-model.test.d.ts +2 -0
- package/dist/agent/llm-model.test.d.ts.map +1 -0
- package/dist/agent/llm-model.test.js +28 -0
- package/dist/agent/llm-model.test.js.map +1 -0
- package/dist/agent/prompt.d.ts +17 -0
- package/dist/agent/prompt.d.ts.map +1 -0
- package/dist/agent/prompt.js +97 -0
- package/dist/agent/prompt.js.map +1 -0
- package/dist/agent/prompt.test.d.ts +2 -0
- package/dist/agent/prompt.test.d.ts.map +1 -0
- package/dist/agent/prompt.test.js +124 -0
- package/dist/agent/prompt.test.js.map +1 -0
- package/dist/agent/provider-options.d.ts +8 -0
- package/dist/agent/provider-options.d.ts.map +1 -0
- package/dist/agent/provider-options.js +115 -0
- package/dist/agent/provider-options.js.map +1 -0
- package/dist/agent/provider-options.test.d.ts +2 -0
- package/dist/agent/provider-options.test.d.ts.map +1 -0
- package/dist/agent/provider-options.test.js +114 -0
- package/dist/agent/provider-options.test.js.map +1 -0
- package/dist/agent/runner.d.ts +27 -0
- package/dist/agent/runner.d.ts.map +1 -0
- package/dist/agent/runner.js +291 -0
- package/dist/agent/runner.js.map +1 -0
- package/dist/agent/verdict-retry-prompt.d.ts +3 -0
- package/dist/agent/verdict-retry-prompt.d.ts.map +1 -0
- package/dist/agent/verdict-retry-prompt.js +18 -0
- package/dist/agent/verdict-retry-prompt.js.map +1 -0
- package/dist/agent/verdict-retry-prompt.test.d.ts +2 -0
- package/dist/agent/verdict-retry-prompt.test.d.ts.map +1 -0
- package/dist/agent/verdict-retry-prompt.test.js +25 -0
- package/dist/agent/verdict-retry-prompt.test.js.map +1 -0
- package/dist/agent/verdict.d.ts +31 -0
- package/dist/agent/verdict.d.ts.map +1 -0
- package/dist/agent/verdict.js +123 -0
- package/dist/agent/verdict.js.map +1 -0
- package/dist/agent/verdict.test.d.ts +2 -0
- package/dist/agent/verdict.test.d.ts.map +1 -0
- package/dist/agent/verdict.test.js +156 -0
- package/dist/agent/verdict.test.js.map +1 -0
- package/dist/analyze/build-context.d.ts +58 -0
- package/dist/analyze/build-context.d.ts.map +1 -0
- package/dist/analyze/build-context.js +141 -0
- package/dist/analyze/build-context.js.map +1 -0
- package/dist/analyze/build-context.test.d.ts +2 -0
- package/dist/analyze/build-context.test.d.ts.map +1 -0
- package/dist/analyze/build-context.test.js +118 -0
- package/dist/analyze/build-context.test.js.map +1 -0
- package/dist/analyze/compare-runs.d.ts +49 -0
- package/dist/analyze/compare-runs.d.ts.map +1 -0
- package/dist/analyze/compare-runs.js +214 -0
- package/dist/analyze/compare-runs.js.map +1 -0
- package/dist/analyze/compare-runs.test.d.ts +2 -0
- package/dist/analyze/compare-runs.test.d.ts.map +1 -0
- package/dist/analyze/compare-runs.test.js +139 -0
- package/dist/analyze/compare-runs.test.js.map +1 -0
- package/dist/analyze/diff-hunks.d.ts +16 -0
- package/dist/analyze/diff-hunks.d.ts.map +1 -0
- package/dist/analyze/diff-hunks.js +287 -0
- package/dist/analyze/diff-hunks.js.map +1 -0
- package/dist/analyze/diff-hunks.test.d.ts +2 -0
- package/dist/analyze/diff-hunks.test.d.ts.map +1 -0
- package/dist/analyze/diff-hunks.test.js +54 -0
- package/dist/analyze/diff-hunks.test.js.map +1 -0
- package/dist/analyze/hunk-editor.d.ts +8 -0
- package/dist/analyze/hunk-editor.d.ts.map +1 -0
- package/dist/analyze/hunk-editor.js +129 -0
- package/dist/analyze/hunk-editor.js.map +1 -0
- package/dist/analyze/hunk-editor.test.d.ts +2 -0
- package/dist/analyze/hunk-editor.test.d.ts.map +1 -0
- package/dist/analyze/hunk-editor.test.js +48 -0
- package/dist/analyze/hunk-editor.test.js.map +1 -0
- package/dist/analyze/index.d.ts +23 -0
- package/dist/analyze/index.d.ts.map +1 -0
- package/dist/analyze/index.js +122 -0
- package/dist/analyze/index.js.map +1 -0
- package/dist/analyze/llm-fix.d.ts +11 -0
- package/dist/analyze/llm-fix.d.ts.map +1 -0
- package/dist/analyze/llm-fix.js +76 -0
- package/dist/analyze/llm-fix.js.map +1 -0
- package/dist/analyze/parse-proposal.d.ts +41 -0
- package/dist/analyze/parse-proposal.d.ts.map +1 -0
- package/dist/analyze/parse-proposal.js +53 -0
- package/dist/analyze/parse-proposal.js.map +1 -0
- package/dist/analyze/parse-proposal.test.d.ts +2 -0
- package/dist/analyze/parse-proposal.test.d.ts.map +1 -0
- package/dist/analyze/parse-proposal.test.js +40 -0
- package/dist/analyze/parse-proposal.test.js.map +1 -0
- package/dist/analyze/repl.d.ts +28 -0
- package/dist/analyze/repl.d.ts.map +1 -0
- package/dist/analyze/repl.js +284 -0
- package/dist/analyze/repl.js.map +1 -0
- package/dist/analyze/repl.test.d.ts +2 -0
- package/dist/analyze/repl.test.d.ts.map +1 -0
- package/dist/analyze/repl.test.js +101 -0
- package/dist/analyze/repl.test.js.map +1 -0
- package/dist/analyze/suggest.d.ts +5 -0
- package/dist/analyze/suggest.d.ts.map +1 -0
- package/dist/analyze/suggest.js +75 -0
- package/dist/analyze/suggest.js.map +1 -0
- package/dist/analyze/suggest.test.d.ts +2 -0
- package/dist/analyze/suggest.test.d.ts.map +1 -0
- package/dist/analyze/suggest.test.js +53 -0
- package/dist/analyze/suggest.test.js.map +1 -0
- package/dist/analyze/validate-markdown.d.ts +3 -0
- package/dist/analyze/validate-markdown.d.ts.map +1 -0
- package/dist/analyze/validate-markdown.js +25 -0
- package/dist/analyze/validate-markdown.js.map +1 -0
- package/dist/artifacts/policy.d.ts +9 -0
- package/dist/artifacts/policy.d.ts.map +1 -0
- package/dist/artifacts/policy.js +46 -0
- package/dist/artifacts/policy.js.map +1 -0
- package/dist/artifacts/policy.test.d.ts +2 -0
- package/dist/artifacts/policy.test.d.ts.map +1 -0
- package/dist/artifacts/policy.test.js +73 -0
- package/dist/artifacts/policy.test.js.map +1 -0
- package/dist/auth/resolve.d.ts +22 -0
- package/dist/auth/resolve.d.ts.map +1 -0
- package/dist/auth/resolve.js +148 -0
- package/dist/auth/resolve.js.map +1 -0
- package/dist/auth/store.d.ts +23 -0
- package/dist/auth/store.d.ts.map +1 -0
- package/dist/auth/store.js +103 -0
- package/dist/auth/store.js.map +1 -0
- package/dist/cache/generate.d.ts +8 -0
- package/dist/cache/generate.d.ts.map +1 -0
- package/dist/cache/generate.js +61 -0
- package/dist/cache/generate.js.map +1 -0
- package/dist/cache/hash.d.ts +5 -0
- package/dist/cache/hash.d.ts.map +1 -0
- package/dist/cache/hash.js +21 -0
- package/dist/cache/hash.js.map +1 -0
- package/dist/cache/hash.test.d.ts +2 -0
- package/dist/cache/hash.test.d.ts.map +1 -0
- package/dist/cache/hash.test.js +42 -0
- package/dist/cache/hash.test.js.map +1 -0
- package/dist/cache/resolve.d.ts +5 -0
- package/dist/cache/resolve.d.ts.map +1 -0
- package/dist/cache/resolve.js +8 -0
- package/dist/cache/resolve.js.map +1 -0
- package/dist/cache/store.d.ts +20 -0
- package/dist/cache/store.d.ts.map +1 -0
- package/dist/cache/store.js +90 -0
- package/dist/cache/store.js.map +1 -0
- package/dist/cache/store.test.d.ts +2 -0
- package/dist/cache/store.test.d.ts.map +1 -0
- package/dist/cache/store.test.js +101 -0
- package/dist/cache/store.test.js.map +1 -0
- package/dist/cli/analyze.d.ts +21 -0
- package/dist/cli/analyze.d.ts.map +1 -0
- package/dist/cli/analyze.js +148 -0
- package/dist/cli/analyze.js.map +1 -0
- package/dist/cli/concurrency.d.ts +17 -0
- package/dist/cli/concurrency.d.ts.map +1 -0
- package/dist/cli/concurrency.js +56 -0
- package/dist/cli/concurrency.js.map +1 -0
- package/dist/cli/concurrency.test.d.ts +2 -0
- package/dist/cli/concurrency.test.d.ts.map +1 -0
- package/dist/cli/concurrency.test.js +74 -0
- package/dist/cli/concurrency.test.js.map +1 -0
- package/dist/cli/config.d.ts +2 -0
- package/dist/cli/config.d.ts.map +1 -0
- package/dist/cli/config.js +14 -0
- package/dist/cli/config.js.map +1 -0
- package/dist/cli/help.d.ts +23 -0
- package/dist/cli/help.d.ts.map +1 -0
- package/dist/cli/help.js +458 -0
- package/dist/cli/help.js.map +1 -0
- package/dist/cli/help.test.d.ts +2 -0
- package/dist/cli/help.test.d.ts.map +1 -0
- package/dist/cli/help.test.js +41 -0
- package/dist/cli/help.test.js.map +1 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +300 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/mcp.d.ts +6 -0
- package/dist/cli/mcp.d.ts.map +1 -0
- package/dist/cli/mcp.js +17 -0
- package/dist/cli/mcp.js.map +1 -0
- package/dist/cli/record.d.ts +27 -0
- package/dist/cli/record.d.ts.map +1 -0
- package/dist/cli/record.js +244 -0
- package/dist/cli/record.js.map +1 -0
- package/dist/cli/run.d.ts +11 -0
- package/dist/cli/run.d.ts.map +1 -0
- package/dist/cli/run.js +676 -0
- package/dist/cli/run.js.map +1 -0
- package/dist/cli/subprocess.d.ts +19 -0
- package/dist/cli/subprocess.d.ts.map +1 -0
- package/dist/cli/subprocess.js +142 -0
- package/dist/cli/subprocess.js.map +1 -0
- package/dist/cli/subprocess.test.d.ts +2 -0
- package/dist/cli/subprocess.test.d.ts.map +1 -0
- package/dist/cli/subprocess.test.js +76 -0
- package/dist/cli/subprocess.test.js.map +1 -0
- package/dist/cli/tags.d.ts +5 -0
- package/dist/cli/tags.d.ts.map +1 -0
- package/dist/cli/tags.js +33 -0
- package/dist/cli/tags.js.map +1 -0
- package/dist/cli/tags.test.d.ts +2 -0
- package/dist/cli/tags.test.d.ts.map +1 -0
- package/dist/cli/tags.test.js +31 -0
- package/dist/cli/tags.test.js.map +1 -0
- package/dist/config/env-vars.d.ts +2 -0
- package/dist/config/env-vars.d.ts.map +1 -0
- package/dist/config/env-vars.js +14 -0
- package/dist/config/env-vars.js.map +1 -0
- package/dist/config/env.d.ts +2 -0
- package/dist/config/env.d.ts.map +1 -0
- package/dist/config/env.js +9 -0
- package/dist/config/env.js.map +1 -0
- package/dist/config/lightpanda.d.ts +6 -0
- package/dist/config/lightpanda.d.ts.map +1 -0
- package/dist/config/lightpanda.js +38 -0
- package/dist/config/lightpanda.js.map +1 -0
- package/dist/config/lightpanda.test.d.ts +2 -0
- package/dist/config/lightpanda.test.d.ts.map +1 -0
- package/dist/config/lightpanda.test.js +46 -0
- package/dist/config/lightpanda.test.js.map +1 -0
- package/dist/config/load.d.ts +22 -0
- package/dist/config/load.d.ts.map +1 -0
- package/dist/config/load.js +242 -0
- package/dist/config/load.js.map +1 -0
- package/dist/config/load.test.d.ts +2 -0
- package/dist/config/load.test.d.ts.map +1 -0
- package/dist/config/load.test.js +86 -0
- package/dist/config/load.test.js.map +1 -0
- package/dist/config/set.d.ts +8 -0
- package/dist/config/set.d.ts.map +1 -0
- package/dist/config/set.js +93 -0
- package/dist/config/set.js.map +1 -0
- package/dist/config/set.test.d.ts +2 -0
- package/dist/config/set.test.d.ts.map +1 -0
- package/dist/config/set.test.js +98 -0
- package/dist/config/set.test.js.map +1 -0
- package/dist/healing/classify.d.ts +15 -0
- package/dist/healing/classify.d.ts.map +1 -0
- package/dist/healing/classify.js +209 -0
- package/dist/healing/classify.js.map +1 -0
- package/dist/healing/classify.test.d.ts +2 -0
- package/dist/healing/classify.test.d.ts.map +1 -0
- package/dist/healing/classify.test.js +167 -0
- package/dist/healing/classify.test.js.map +1 -0
- package/dist/healing/recovery-prompt.d.ts +3 -0
- package/dist/healing/recovery-prompt.d.ts.map +1 -0
- package/dist/healing/recovery-prompt.js +22 -0
- package/dist/healing/recovery-prompt.js.map +1 -0
- package/dist/mcp/inline-scenario.d.ts +13 -0
- package/dist/mcp/inline-scenario.d.ts.map +1 -0
- package/dist/mcp/inline-scenario.js +23 -0
- package/dist/mcp/inline-scenario.js.map +1 -0
- package/dist/mcp/server.d.ts +4 -0
- package/dist/mcp/server.d.ts.map +1 -0
- package/dist/mcp/server.js +186 -0
- package/dist/mcp/server.js.map +1 -0
- package/dist/mcp/skill.d.ts +5 -0
- package/dist/mcp/skill.d.ts.map +1 -0
- package/dist/mcp/skill.js +38 -0
- package/dist/mcp/skill.js.map +1 -0
- package/dist/mcp/skill.test.d.ts +2 -0
- package/dist/mcp/skill.test.d.ts.map +1 -0
- package/dist/mcp/skill.test.js +18 -0
- package/dist/mcp/skill.test.js.map +1 -0
- package/dist/paths.d.ts +12 -0
- package/dist/paths.d.ts.map +1 -0
- package/dist/paths.js +61 -0
- package/dist/paths.js.map +1 -0
- package/dist/prompt/load.d.ts +4 -0
- package/dist/prompt/load.d.ts.map +1 -0
- package/dist/prompt/load.js +19 -0
- package/dist/prompt/load.js.map +1 -0
- package/dist/recorder/bridge-process.d.ts +14 -0
- package/dist/recorder/bridge-process.d.ts.map +1 -0
- package/dist/recorder/bridge-process.js +133 -0
- package/dist/recorder/bridge-process.js.map +1 -0
- package/dist/recorder/bridge-process.test.d.ts +2 -0
- package/dist/recorder/bridge-process.test.d.ts.map +1 -0
- package/dist/recorder/bridge-process.test.js +36 -0
- package/dist/recorder/bridge-process.test.js.map +1 -0
- package/dist/recorder/bridge-worker.d.ts +2 -0
- package/dist/recorder/bridge-worker.d.ts.map +1 -0
- package/dist/recorder/bridge-worker.js +76 -0
- package/dist/recorder/bridge-worker.js.map +1 -0
- package/dist/recorder/bridge.d.ts +12 -0
- package/dist/recorder/bridge.d.ts.map +1 -0
- package/dist/recorder/bridge.js +61 -0
- package/dist/recorder/bridge.js.map +1 -0
- package/dist/recorder/bridge.test.d.ts +2 -0
- package/dist/recorder/bridge.test.d.ts.map +1 -0
- package/dist/recorder/bridge.test.js +21 -0
- package/dist/recorder/bridge.test.js.map +1 -0
- package/dist/recorder/enrich-event.d.ts +31 -0
- package/dist/recorder/enrich-event.d.ts.map +1 -0
- package/dist/recorder/enrich-event.js +91 -0
- package/dist/recorder/enrich-event.js.map +1 -0
- package/dist/recorder/events.d.ts +11 -0
- package/dist/recorder/events.d.ts.map +1 -0
- package/dist/recorder/events.js +42 -0
- package/dist/recorder/events.js.map +1 -0
- package/dist/recorder/events.test.d.ts +2 -0
- package/dist/recorder/events.test.d.ts.map +1 -0
- package/dist/recorder/events.test.js +40 -0
- package/dist/recorder/events.test.js.map +1 -0
- package/dist/recorder/generate-scenario.d.ts +16 -0
- package/dist/recorder/generate-scenario.d.ts.map +1 -0
- package/dist/recorder/generate-scenario.js +78 -0
- package/dist/recorder/generate-scenario.js.map +1 -0
- package/dist/recorder/in-page-helpers.d.ts +6 -0
- package/dist/recorder/in-page-helpers.d.ts.map +1 -0
- package/dist/recorder/in-page-helpers.js +238 -0
- package/dist/recorder/in-page-helpers.js.map +1 -0
- package/dist/recorder/in-page-helpers.test.d.ts +2 -0
- package/dist/recorder/in-page-helpers.test.d.ts.map +1 -0
- package/dist/recorder/in-page-helpers.test.js +186 -0
- package/dist/recorder/in-page-helpers.test.js.map +1 -0
- package/dist/recorder/page-script.d.ts +7 -0
- package/dist/recorder/page-script.d.ts.map +1 -0
- package/dist/recorder/page-script.js +132 -0
- package/dist/recorder/page-script.js.map +1 -0
- package/dist/recorder/redact.d.ts +8 -0
- package/dist/recorder/redact.d.ts.map +1 -0
- package/dist/recorder/redact.js +26 -0
- package/dist/recorder/redact.js.map +1 -0
- package/dist/recorder/redact.test.d.ts +2 -0
- package/dist/recorder/redact.test.d.ts.map +1 -0
- package/dist/recorder/redact.test.js +27 -0
- package/dist/recorder/redact.test.js.map +1 -0
- package/dist/recorder/session.d.ts +8 -0
- package/dist/recorder/session.d.ts.map +1 -0
- package/dist/recorder/session.js +28 -0
- package/dist/recorder/session.js.map +1 -0
- package/dist/recorder/snapshot-match.d.ts +22 -0
- package/dist/recorder/snapshot-match.d.ts.map +1 -0
- package/dist/recorder/snapshot-match.js +102 -0
- package/dist/recorder/snapshot-match.js.map +1 -0
- package/dist/recorder/snapshot-match.test.d.ts +2 -0
- package/dist/recorder/snapshot-match.test.d.ts.map +1 -0
- package/dist/recorder/snapshot-match.test.js +34 -0
- package/dist/recorder/snapshot-match.test.js.map +1 -0
- package/dist/redact/env-secrets.d.ts +14 -0
- package/dist/redact/env-secrets.d.ts.map +1 -0
- package/dist/redact/env-secrets.js +86 -0
- package/dist/redact/env-secrets.js.map +1 -0
- package/dist/redact/env-secrets.test.d.ts +2 -0
- package/dist/redact/env-secrets.test.d.ts.map +1 -0
- package/dist/redact/env-secrets.test.js +103 -0
- package/dist/redact/env-secrets.test.js.map +1 -0
- package/dist/reporter/export.d.ts +14 -0
- package/dist/reporter/export.d.ts.map +1 -0
- package/dist/reporter/export.js +53 -0
- package/dist/reporter/export.js.map +1 -0
- package/dist/reporter/export.test.d.ts +2 -0
- package/dist/reporter/export.test.d.ts.map +1 -0
- package/dist/reporter/export.test.js +100 -0
- package/dist/reporter/export.test.js.map +1 -0
- package/dist/reporter/index.d.ts +11 -0
- package/dist/reporter/index.d.ts.map +1 -0
- package/dist/reporter/index.js +161 -0
- package/dist/reporter/index.js.map +1 -0
- package/dist/reporter/index.test.d.ts +2 -0
- package/dist/reporter/index.test.d.ts.map +1 -0
- package/dist/reporter/index.test.js +61 -0
- package/dist/reporter/index.test.js.map +1 -0
- package/dist/scenarios/globs.d.ts +15 -0
- package/dist/scenarios/globs.d.ts.map +1 -0
- package/dist/scenarios/globs.js +48 -0
- package/dist/scenarios/globs.js.map +1 -0
- package/dist/scenarios/globs.test.d.ts +2 -0
- package/dist/scenarios/globs.test.d.ts.map +1 -0
- package/dist/scenarios/globs.test.js +53 -0
- package/dist/scenarios/globs.test.js.map +1 -0
- package/dist/scenarios/parser.d.ts +15 -0
- package/dist/scenarios/parser.d.ts.map +1 -0
- package/dist/scenarios/parser.js +278 -0
- package/dist/scenarios/parser.js.map +1 -0
- package/dist/scenarios/parser.test.d.ts +2 -0
- package/dist/scenarios/parser.test.d.ts.map +1 -0
- package/dist/scenarios/parser.test.js +373 -0
- package/dist/scenarios/parser.test.js.map +1 -0
- package/dist/skills/loader.d.ts +10 -0
- package/dist/skills/loader.d.ts.map +1 -0
- package/dist/skills/loader.js +98 -0
- package/dist/skills/loader.js.map +1 -0
- package/dist/types/config.d.ts +131 -0
- package/dist/types/config.d.ts.map +1 -0
- package/dist/types/config.js +2 -0
- package/dist/types/config.js.map +1 -0
- package/dist/types/recorder.d.ts +71 -0
- package/dist/types/recorder.d.ts.map +1 -0
- package/dist/types/recorder.js +2 -0
- package/dist/types/recorder.js.map +1 -0
- package/dist/types/scenario.d.ts +41 -0
- package/dist/types/scenario.d.ts.map +1 -0
- package/dist/types/scenario.js +2 -0
- package/dist/types/scenario.js.map +1 -0
- package/dist/types/skill.d.ts +20 -0
- package/dist/types/skill.d.ts.map +1 -0
- package/dist/types/skill.js +13 -0
- package/dist/types/skill.js.map +1 -0
- package/dist/types/verdict.d.ts +82 -0
- package/dist/types/verdict.d.ts.map +1 -0
- package/dist/types/verdict.js +13 -0
- package/dist/types/verdict.js.map +1 -0
- package/package.json +75 -0
- package/pqa.config.ts +82 -0
- package/prompt/ANALYZE-FLAKY.md +62 -0
- package/prompt/ANALYZE.md +110 -0
- package/prompt/CACHE-HINTS.md +49 -0
- package/prompt/RECORD.md +114 -0
- package/prompt/SYSTEM.md +118 -0
- package/skills/agent-browser/SKILL.md +2438 -0
- package/skills/create-pqa-scenario/SKILL.md +273 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 FreakDev
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,570 @@
|
|
|
1
|
+
# Prose-QA
|
|
2
|
+
|
|
3
|
+
Agent harness for **end-to-end regression testing** of web apps. Scenarios are written in natural language with explicit verification checkpoints. An LLM agent executes them using [Vercel `agent-browser`](https://github.com/vercel-labs/agent-browser) via bash — no browser wrapper in TypeScript.
|
|
4
|
+
|
|
5
|
+
## How-to (step by step)
|
|
6
|
+
|
|
7
|
+
See **[docs/HOWTO.md](docs/HOWTO.md)** for a progressive guide (scenario format → agent-browser → debug/run → CI → auth → MCP → record → cache → healing → analyze).
|
|
8
|
+
|
|
9
|
+
## Features
|
|
10
|
+
|
|
11
|
+
- **Natural language scenarios** with `# Goal`, `# Steps`, and `# Then` checkpoints
|
|
12
|
+
- **Agent Skills** ([agentskills.io](https://agentskills.io/)) — Anthropic-compatible `SKILL.md` format
|
|
13
|
+
- **Pinned agent-browser skill** vendored at `skills/agent-browser/` (installed via `postinstall` on `npm ci` / `npm install`)
|
|
14
|
+
- **CI + local debug** modes with HTML/JSON reports
|
|
15
|
+
|
|
16
|
+
## Install
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
npm install -g prose-qa
|
|
20
|
+
# or in a project:
|
|
21
|
+
npm install prose-qa
|
|
22
|
+
npx pqa --help
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Requires Node.js 20+ and an LLM API key (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `FIREWORKS_API_KEY`, `OPENROUTER_API_KEY`, etc. depending on config).
|
|
26
|
+
|
|
27
|
+
On first install, `agent-browser` downloads its browser binary via `postinstall`. In CI, run:
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
npx agent-browser install --with-deps
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## New project setup
|
|
34
|
+
|
|
35
|
+
1. Install the package in your app repo (or globally).
|
|
36
|
+
2. Create `pqa.config.json` in your project root (or use `pqa config <key> <value>` to set values incrementally):
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pqa config llm.provider anthropic
|
|
40
|
+
pqa config llm.model claude-sonnet-4-20250514
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Supported config filenames (first match wins): `pqa.config.json`, `pqa.config.mjs`, `pqa.config.js`, `pqa.config.ts`.
|
|
44
|
+
|
|
45
|
+
3. Create scenarios under `scenarios/` (see [0_hello-world.md](scenarios/0_hello-world.md)).
|
|
46
|
+
4. Copy [`.env.example`](.env.example) to `.env.development.local` (or set env vars in CI) and fill in secrets.
|
|
47
|
+
5. Run:
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
export ANTHROPIC_API_KEY=...
|
|
51
|
+
pqa run scenarios/**/*.md --tags smoke
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Bundled harness assets (`prompt/`, `skills/`) ship inside the npm package. Your project only needs `pqa.config.*`, `scenarios/`, and optional `.agents/skills/` overrides.
|
|
55
|
+
|
|
56
|
+
## Development (this repo)
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
git clone https://github.com/FreakDev/Prose-QA.git
|
|
60
|
+
cd Prose-QA
|
|
61
|
+
npm ci
|
|
62
|
+
npm run build
|
|
63
|
+
|
|
64
|
+
export ANTHROPIC_API_KEY=...
|
|
65
|
+
|
|
66
|
+
# Bundled scenarios target http://127.0.0.1:8080/ — start the demo server first (separate terminal or background)
|
|
67
|
+
npm run demo:server
|
|
68
|
+
|
|
69
|
+
# CI mode
|
|
70
|
+
npm run dev -- run scenarios/**/*.md --tags example
|
|
71
|
+
|
|
72
|
+
# Debug single scenario
|
|
73
|
+
npm run dev -- debug scenarios/0_hello-world.md --verbose
|
|
74
|
+
|
|
75
|
+
# Auth demo (demo server with hardcoded credentials)
|
|
76
|
+
export PQA_TEST_EMAIL=demo@pqa.local PQA_TEST_PASSWORD=demo-password
|
|
77
|
+
npm run dev -- debug scenarios/1_example-authenticated.md --verbose
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
The demo server (`npm run demo:server` → `scripts/demo-server.mjs`) serves `/` (Hello World), `/login`, and protected `/projects`. Credentials match `.env.example`.
|
|
81
|
+
|
|
82
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for pull request guidelines.
|
|
83
|
+
|
|
84
|
+
## MCP server (Cursor, Claude Desktop, …)
|
|
85
|
+
|
|
86
|
+
Start the Prose-QA MCP server over stdio so clients can read the **create-pqa-scenario** skill and run scenarios from inline markdown (same format as `scenarios/*.md`):
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
pqa mcp
|
|
90
|
+
# or from this repo:
|
|
91
|
+
npm run mcp
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
**Cursor** (`.cursor/mcp.json` in your app repo — `cwd` must be the project with `pqa.config` and env vars):
|
|
95
|
+
|
|
96
|
+
```json
|
|
97
|
+
{
|
|
98
|
+
"mcpServers": {
|
|
99
|
+
"prose-qa": {
|
|
100
|
+
"command": "npx",
|
|
101
|
+
"args": ["-y", "prose-qa", "mcp"]
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
After `npm run build` in this repo, use `"command": "node"` and `"args": ["dist/cli/index.js", "mcp"]` with `cwd` set to the Prose-QA repo root.
|
|
108
|
+
|
|
109
|
+
| Surface | Purpose |
|
|
110
|
+
| -------- | -------- |
|
|
111
|
+
| Resource `pqa://skill/create-pqa-scenario` | Full create-pqa-scenario `SKILL.md` |
|
|
112
|
+
| Tool `get_create_pqa_scenario_skill` | Same skill as text |
|
|
113
|
+
| Tool `validate_scenario` | Parse `content` without running the browser |
|
|
114
|
+
| Tool `run_scenario` | Execute `content` (requires LLM + browser env) |
|
|
115
|
+
| Prompt `author_pqa_scenario` | Template that includes the skill |
|
|
116
|
+
|
|
117
|
+
## Scenario format
|
|
118
|
+
|
|
119
|
+
See [prompt/references/scenario-format.md](prompt/references/scenario-format.md).
|
|
120
|
+
|
|
121
|
+
```markdown
|
|
122
|
+
---
|
|
123
|
+
name: checkout-happy-path
|
|
124
|
+
tags: [smoke]
|
|
125
|
+
auth: admin
|
|
126
|
+
url: https://app.example.com
|
|
127
|
+
---
|
|
128
|
+
|
|
129
|
+
# Goal
|
|
130
|
+
As a user, complete checkout.
|
|
131
|
+
|
|
132
|
+
# Steps
|
|
133
|
+
1. Add item to cart and proceed to checkout.
|
|
134
|
+
2. Complete payment with test card.
|
|
135
|
+
|
|
136
|
+
# Then
|
|
137
|
+
- url contains "/order-confirmation"
|
|
138
|
+
- page shows "Thank you"
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
## Configuration
|
|
142
|
+
|
|
143
|
+
Prose-QA loads configuration from the bundled defaults ([`pqa.config.ts`](pqa.config.ts) in the npm package), then merges your local overrides. Only keys you set need to appear in your project file.
|
|
144
|
+
|
|
145
|
+
**Local config files** (first match in the project root wins): `pqa.config.json`, `pqa.config.mjs`, `pqa.config.js`, `pqa.config.ts`.
|
|
146
|
+
|
|
147
|
+
**CLI helper** — create or update `pqa.config.json` without editing by hand (dot notation for nested keys):
|
|
148
|
+
|
|
149
|
+
```bash
|
|
150
|
+
pqa config llm.provider anthropic
|
|
151
|
+
pqa config browser.headed true
|
|
152
|
+
pqa config envVars '["PQA_TEST_EMAIL","PQA_TEST_PASSWORD"]'
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
Unknown keys are rejected; only properties that exist in the bundled reference config are allowed.
|
|
156
|
+
|
|
157
|
+
### Minimal example
|
|
158
|
+
|
|
159
|
+
```json
|
|
160
|
+
{
|
|
161
|
+
"envVars": ["PQA_TEST_EMAIL", "PQA_TEST_PASSWORD"],
|
|
162
|
+
"sensitiveEnvVars": ["PQA_TEST_EMAIL", "PQA_TEST_PASSWORD"],
|
|
163
|
+
"llm": {
|
|
164
|
+
"provider": "anthropic",
|
|
165
|
+
"model": "claude-sonnet-4-20250514"
|
|
166
|
+
},
|
|
167
|
+
"auth": {
|
|
168
|
+
"admin": {
|
|
169
|
+
"scenario": "login-admin",
|
|
170
|
+
"statePath": ".pqa/auth/admin.json"
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
### Environment variables
|
|
177
|
+
|
|
178
|
+
| Variable | Description |
|
|
179
|
+
| --- | --- |
|
|
180
|
+
| `ANTHROPIC_API_KEY` | Required when `llm.provider` is `anthropic` |
|
|
181
|
+
| `OPENAI_API_KEY` | Required when `llm.provider` is `openai` |
|
|
182
|
+
| `FIREWORKS_API_KEY` | Required when `llm.provider` is `fireworks` |
|
|
183
|
+
| `GOOGLE_GENERATIVE_AI_API_KEY` | Required when `llm.provider` is `google` |
|
|
184
|
+
| `OPENROUTER_API_KEY` | Required when `llm.provider` is `openrouter` |
|
|
185
|
+
| `PQA_LLM_PROVIDER` | Overrides bundled default `llm.provider` (dev / CI shortcut) |
|
|
186
|
+
| `PQA_LLM_MODEL` | Overrides bundled default `llm.model` |
|
|
187
|
+
|
|
188
|
+
Ollama does not require an API key env var. Any name listed in `envVars` must be set before a run starts.
|
|
189
|
+
|
|
190
|
+
### All options
|
|
191
|
+
|
|
192
|
+
#### `scenariosDir` (string)
|
|
193
|
+
|
|
194
|
+
Root directory for scenario markdown files. Set directly in `pqa.config.json`.
|
|
195
|
+
|
|
196
|
+
| | |
|
|
197
|
+
| --- | --- |
|
|
198
|
+
| **Default** | `scenarios`, or `pqa/` when that directory exists and `scenarios/` does not |
|
|
199
|
+
|
|
200
|
+
#### `systemPromptPath` (string)
|
|
201
|
+
|
|
202
|
+
Path to the agent system prompt markdown file. Relative paths resolve against the project cwd first, then bundled package assets.
|
|
203
|
+
|
|
204
|
+
| | |
|
|
205
|
+
| --- | --- |
|
|
206
|
+
| **Default** | `prompt/SYSTEM.md` (bundled) |
|
|
207
|
+
|
|
208
|
+
#### `envVars` (string[])
|
|
209
|
+
|
|
210
|
+
Environment variable **names** the agent should know about. Injected into the system prompt at runtime (set / not-set status only — never values). Validated before each run.
|
|
211
|
+
|
|
212
|
+
| | |
|
|
213
|
+
| --- | --- |
|
|
214
|
+
| **Default** | `[]` |
|
|
215
|
+
|
|
216
|
+
#### `sensitiveEnvVars` (string[])
|
|
217
|
+
|
|
218
|
+
Env var names whose **values** are redacted from transcripts, verdicts, reports, and verbose logs (replaced with `${VAR_NAME}`). If omitted, defaults to `envVars`. The LLM API key for the configured provider is always redacted.
|
|
219
|
+
|
|
220
|
+
| | |
|
|
221
|
+
| --- | --- |
|
|
222
|
+
| **Default** | same as `envVars` |
|
|
223
|
+
|
|
224
|
+
---
|
|
225
|
+
|
|
226
|
+
#### `llm` (object)
|
|
227
|
+
|
|
228
|
+
LLM provider and model used for test runs, recording generation, and analysis.
|
|
229
|
+
|
|
230
|
+
| Key | Type | Default | Description |
|
|
231
|
+
| --- | --- | --- | --- |
|
|
232
|
+
| `provider` | `"anthropic"` \| `"openai"` \| `"fireworks"` \| `"ollama"` \| `"google"` \| `"openrouter"` | `"anthropic"` | LLM backend |
|
|
233
|
+
| `model` | string | `"claude-sonnet-4-20250514"` | Model identifier for the chosen provider |
|
|
234
|
+
|
|
235
|
+
##### `llm.thinking` (object, optional)
|
|
236
|
+
|
|
237
|
+
Extended thinking / reasoning. Provider support varies.
|
|
238
|
+
|
|
239
|
+
| Key | Type | Default | Description |
|
|
240
|
+
| --- | --- | --- | --- |
|
|
241
|
+
| `enabled` | boolean | `true` | Enable extended thinking |
|
|
242
|
+
| `budgetTokens` | number | `10000` | Thinking token budget (Anthropic, Fireworks, Google, OpenRouter) |
|
|
243
|
+
| `reasoningEffort` | `"none"` \| `"minimal"` \| `"low"` \| `"medium"` \| `"high"` \| `"xhigh"` | — | OpenAI reasoning effort; mapped to Anthropic effort, Google thinking level, and OpenRouter reasoning effort. Ollama uses `think` mode only (other fields ignored) |
|
|
244
|
+
|
|
245
|
+
---
|
|
246
|
+
|
|
247
|
+
#### `browser` (object)
|
|
248
|
+
|
|
249
|
+
Default browser behavior for scenario runs (overridable per run with `--headed` / `--no-headed`).
|
|
250
|
+
|
|
251
|
+
| Key | Type | Default | Description |
|
|
252
|
+
| --- | --- | --- | --- |
|
|
253
|
+
| `headed` | boolean | `false` | Run browser in visible (headed) mode |
|
|
254
|
+
| `sessionName` | string | `"pqa"` | agent-browser session name |
|
|
255
|
+
| `defaultTimeout` | number | `25000` | Default timeout in milliseconds |
|
|
256
|
+
|
|
257
|
+
---
|
|
258
|
+
|
|
259
|
+
#### `skills` (object)
|
|
260
|
+
|
|
261
|
+
Agent skill discovery and preloading ([agentskills.io](https://agentskills.io/) `SKILL.md` format).
|
|
262
|
+
|
|
263
|
+
| Key | Type | Default | Description |
|
|
264
|
+
| --- | --- | --- | --- |
|
|
265
|
+
| `dirs` | string[] | `["skills", ".agents/skills"]` | Directories scanned for skills. Relative paths resolve like bundled assets |
|
|
266
|
+
| `preloads` | string[] | `["core"]` | Skill names always appended to the system prompt (`core` = vendored agent-browser skill) |
|
|
267
|
+
|
|
268
|
+
---
|
|
269
|
+
|
|
270
|
+
#### `agent` (object)
|
|
271
|
+
|
|
272
|
+
Agent loop limits.
|
|
273
|
+
|
|
274
|
+
| Key | Type | Default | Description |
|
|
275
|
+
| --- | --- | --- | --- |
|
|
276
|
+
| `maxTurns` | number | `200` | Maximum agent turns per scenario |
|
|
277
|
+
| `bashTimeoutMs` | number | `120000` | Timeout for each bash (agent-browser) command in milliseconds |
|
|
278
|
+
|
|
279
|
+
---
|
|
280
|
+
|
|
281
|
+
#### `auth` (object)
|
|
282
|
+
|
|
283
|
+
Map of auth profile names to login scenario configuration. Consumer scenarios reference a profile via frontmatter `auth: <name>`.
|
|
284
|
+
|
|
285
|
+
Each profile key (e.g. `admin`) supports:
|
|
286
|
+
|
|
287
|
+
| Key | Type | Default | Description |
|
|
288
|
+
| --- | --- | --- | --- |
|
|
289
|
+
| `scenario` | string | — | `frontmatter.name` of the on-demand auth scenario (e.g. `"login-admin"`) |
|
|
290
|
+
| `statePath` | string | `.pqa/auth/<profile>.json` | agent-browser state file path |
|
|
291
|
+
|
|
292
|
+
When a scenario uses `auth: admin`, the harness loads cached state from `statePath` or runs the auth scenario once, saves browser state, then continues. See [Auth (hybrid authStore)](#auth-hybrid-authstore).
|
|
293
|
+
|
|
294
|
+
---
|
|
295
|
+
|
|
296
|
+
#### `healing` (object, optional)
|
|
297
|
+
|
|
298
|
+
Conservative self-healing: in-run recovery and transient-only scenario retries. See [Self-healing](#self-healing-conservative).
|
|
299
|
+
|
|
300
|
+
| Key | Type | Default | Description |
|
|
301
|
+
| --- | --- | --- | --- |
|
|
302
|
+
| `enabled` | boolean | `true` | Master switch for in-run recovery and transient retry gating |
|
|
303
|
+
| `maxRecoveryTurns` | number | `2` | Extra agent turns after a failed verdict (same browser session) |
|
|
304
|
+
| `recoverOnUnknown` | boolean | `false` | Allow recovery when failure class is unknown but bash output looks transient |
|
|
305
|
+
| `transientPatterns` | string[] | see below | Substrings matched against bash output and checkpoint reasons to classify transient failures |
|
|
306
|
+
|
|
307
|
+
Default `transientPatterns`: `timeout`, `timed out`, `not found`, `waiting for`, `navigation`, `net::`, `target closed`, `detached`, `stale`, `interrupted`.
|
|
308
|
+
|
|
309
|
+
CLI equivalents: `--no-healing`, `--retries N`, `--retries-policy transient|always`, `--no-cache`.
|
|
310
|
+
|
|
311
|
+
---
|
|
312
|
+
|
|
313
|
+
#### `cache` (object, optional)
|
|
314
|
+
|
|
315
|
+
Scenario replay cache settings. See [Scenario replay cache](#scenario-replay-cache).
|
|
316
|
+
|
|
317
|
+
| Key | Type | Default | Description |
|
|
318
|
+
| --- | --- | --- | --- |
|
|
319
|
+
| `dir` | string | `".pqa/cache"` | Directory for per-scenario replay hints |
|
|
320
|
+
| `enabled` | boolean | `true` | Master switch (opt-out via `--no-cache`) |
|
|
321
|
+
|
|
322
|
+
---
|
|
323
|
+
|
|
324
|
+
#### `recorder` (object, optional)
|
|
325
|
+
|
|
326
|
+
Settings for `pqa record`. See [Recording scenarios](#recording-scenarios).
|
|
327
|
+
|
|
328
|
+
| Key | Type | Default | Description |
|
|
329
|
+
| --- | --- | --- | --- |
|
|
330
|
+
| `bridgePort` | number | `17321` | Local HTTP port for the recording event bridge |
|
|
331
|
+
| `outputDir` | string | `".pqa/recordings"` | Directory for saved recording sessions |
|
|
332
|
+
| `defaultTags` | string[] | `["recorded"]` | Tags added to generated scenario frontmatter |
|
|
333
|
+
|
|
334
|
+
---
|
|
335
|
+
|
|
336
|
+
### Full reference example
|
|
337
|
+
|
|
338
|
+
```json
|
|
339
|
+
{
|
|
340
|
+
"scenariosDir": "pqa",
|
|
341
|
+
"systemPromptPath": "prompt/SYSTEM.md",
|
|
342
|
+
"envVars": ["PQA_TEST_EMAIL", "PQA_TEST_PASSWORD"],
|
|
343
|
+
"sensitiveEnvVars": ["PQA_TEST_EMAIL", "PQA_TEST_PASSWORD"],
|
|
344
|
+
"llm": {
|
|
345
|
+
"provider": "anthropic",
|
|
346
|
+
"model": "claude-sonnet-4-20250514",
|
|
347
|
+
"thinking": {
|
|
348
|
+
"enabled": true,
|
|
349
|
+
"budgetTokens": 10000,
|
|
350
|
+
"reasoningEffort": "high"
|
|
351
|
+
}
|
|
352
|
+
},
|
|
353
|
+
"browser": {
|
|
354
|
+
"headed": false,
|
|
355
|
+
"sessionName": "pqa",
|
|
356
|
+
"defaultTimeout": 25000
|
|
357
|
+
},
|
|
358
|
+
"skills": {
|
|
359
|
+
"dirs": ["skills", ".agents/skills"],
|
|
360
|
+
"preloads": ["core"]
|
|
361
|
+
},
|
|
362
|
+
"agent": {
|
|
363
|
+
"maxTurns": 200,
|
|
364
|
+
"bashTimeoutMs": 120000
|
|
365
|
+
},
|
|
366
|
+
"auth": {
|
|
367
|
+
"admin": {
|
|
368
|
+
"scenario": "login-admin",
|
|
369
|
+
"statePath": ".pqa/auth/admin.json"
|
|
370
|
+
}
|
|
371
|
+
},
|
|
372
|
+
"healing": {
|
|
373
|
+
"enabled": true,
|
|
374
|
+
"maxRecoveryTurns": 2,
|
|
375
|
+
"recoverOnUnknown": false,
|
|
376
|
+
"transientPatterns": [
|
|
377
|
+
"timeout",
|
|
378
|
+
"timed out",
|
|
379
|
+
"not found",
|
|
380
|
+
"waiting for",
|
|
381
|
+
"navigation",
|
|
382
|
+
"net::",
|
|
383
|
+
"target closed",
|
|
384
|
+
"detached",
|
|
385
|
+
"stale",
|
|
386
|
+
"interrupted"
|
|
387
|
+
]
|
|
388
|
+
},
|
|
389
|
+
"recorder": {
|
|
390
|
+
"bridgePort": 17321,
|
|
391
|
+
"outputDir": ".pqa/recordings",
|
|
392
|
+
"defaultTags": ["recorded"]
|
|
393
|
+
},
|
|
394
|
+
"cache": {
|
|
395
|
+
"dir": ".pqa/cache",
|
|
396
|
+
"enabled": true
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
```
|
|
400
|
+
|
|
401
|
+
## CLI
|
|
402
|
+
|
|
403
|
+
| Command | Description |
|
|
404
|
+
| --- | --- |
|
|
405
|
+
| `pqa config <key> <value>` | Set a value in `pqa.config.json` |
|
|
406
|
+
| `pqa run [globs]` | Run scenarios (headless by default) |
|
|
407
|
+
| `pqa clear-cache [scenario]` | Clear scenario replay cache |
|
|
408
|
+
| `pqa debug [globs]` | Verbose debug run (headed by default, supports `--tag` / `--tags`) |
|
|
409
|
+
| `pqa skills list` | List discovered skills |
|
|
410
|
+
| `pqa skills show <name>` | Print skill body |
|
|
411
|
+
| `pqa skills sync` | Re-vendor agent-browser skill (dev repo only) |
|
|
412
|
+
| `pqa auth list` | List cached auth profiles in the auth store |
|
|
413
|
+
| `pqa auth clear [profile]` | Clear cached auth state |
|
|
414
|
+
| `pqa auth save <name>` | Run the configured auth scenario and save state |
|
|
415
|
+
| `pqa analyze [run...]` | Heuristic + LLM analysis, interactive patch review (REPL); multi-run flaky detection with `--last N` |
|
|
416
|
+
| `pqa record start` | Start headed recording session (browser + event bridge) |
|
|
417
|
+
| `pqa record note <text>` | Add a comment to the active recording |
|
|
418
|
+
| `pqa record checkpoint <text>` | Add a Then-section hint |
|
|
419
|
+
| `pqa record stop` | Stop recording and generate `scenarios/recorded/*.md` via LLM |
|
|
420
|
+
| `pqa record generate <dir>` | Regenerate scenario markdown from a saved recording |
|
|
421
|
+
|
|
422
|
+
Tag filters on `run` and `debug` can express AND/OR/NOT combinations:
|
|
423
|
+
|
|
424
|
+
```bash
|
|
425
|
+
# AND: scenario must have both tags
|
|
426
|
+
pqa run scenarios/**/*.md --tags smoke,checkout
|
|
427
|
+
|
|
428
|
+
# AND with NOT: scenario must have p0 and must not have smoke
|
|
429
|
+
pqa run scenarios/**/*.md --tags p0,!smoke
|
|
430
|
+
|
|
431
|
+
# OR: scenario may have either tag
|
|
432
|
+
pqa run scenarios/**/*.md --tag smoke --tag checkout
|
|
433
|
+
|
|
434
|
+
# OR with NOT: scenario either lacks p0 or has smoke
|
|
435
|
+
pqa run scenarios/**/*.md --tag !p0 --tag smoke
|
|
436
|
+
|
|
437
|
+
# Combined: (smoke AND checkout) OR auth
|
|
438
|
+
pqa run scenarios/**/*.md --tags smoke,checkout --tag auth
|
|
439
|
+
```
|
|
440
|
+
|
|
441
|
+
Use `--auth-refresh` on `run` / `debug` to re-run auth scenarios and refresh the store.
|
|
442
|
+
|
|
443
|
+
## Scenario replay cache
|
|
444
|
+
|
|
445
|
+
After a scenario passes, PQA runs a secondary LLM pass on the run transcript to produce **replay hints** under `.pqa/cache/<scenario-name>/` (`hints.md` + `meta.json`). On the next run, those hints are injected into the agent system prompt (like a skill) so the agent can follow proven `agent-browser` paths and avoid repeating costly recovery loops.
|
|
446
|
+
|
|
447
|
+
```bash
|
|
448
|
+
# First run: agent executes; hints are generated on pass
|
|
449
|
+
pqa run scenarios/lapresse/homepage-smoke.md
|
|
450
|
+
|
|
451
|
+
# Second run: agent runs with cached hints (if scenario content unchanged)
|
|
452
|
+
pqa run scenarios/lapresse/homepage-smoke.md
|
|
453
|
+
|
|
454
|
+
# Skip cache read/write
|
|
455
|
+
pqa run scenarios/**/*.md --no-cache
|
|
456
|
+
|
|
457
|
+
# Clear one or all caches
|
|
458
|
+
pqa clear-cache lapresse-homepage-smoke
|
|
459
|
+
pqa clear-cache
|
|
460
|
+
```
|
|
461
|
+
|
|
462
|
+
Cache is **invalidated** when the effective scenario content changes (Goal, Steps, Then, frontmatter, and expanded includes — detected via content hash). Hints are **merged and refined** on each subsequent pass. Failed runs do not update the cache.
|
|
463
|
+
|
|
464
|
+
Config (optional):
|
|
465
|
+
|
|
466
|
+
```json
|
|
467
|
+
{
|
|
468
|
+
"cache": {
|
|
469
|
+
"dir": ".pqa/cache",
|
|
470
|
+
"enabled": true
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
```
|
|
474
|
+
|
|
475
|
+
## Recording scenarios
|
|
476
|
+
|
|
477
|
+
Record user actions and generate a draft scenario markdown file:
|
|
478
|
+
|
|
479
|
+
```bash
|
|
480
|
+
pqa record start --url http://localhost:3000/projects
|
|
481
|
+
pqa record note "intentionally invalid date"
|
|
482
|
+
# interact in the browser
|
|
483
|
+
pqa record checkpoint 'page shows "Projects"'
|
|
484
|
+
pqa record stop --name my-flow
|
|
485
|
+
pqa debug scenarios/recorded/my-flow.md --verbose --headed
|
|
486
|
+
```
|
|
487
|
+
|
|
488
|
+
Events are stored under `.pqa/recordings/<timestamp>/events.jsonl`. On each interaction, the bridge runs `agent-browser snapshot -i`, matches the target to a snapshot ref (`snapshot.ref`, `snapshot.description`), and saves the tree under `snapshots/<ts>.json`. A background bridge process keeps receiving browser events until `pqa record stop` (so you can run `record note` / `record checkpoint` in another terminal while clicking in the browser). Generation uses the same LLM config as test runs (`prompt/RECORD.md`). Recorder options: see [`recorder`](#recorder-object-optional) in Configuration.
|
|
489
|
+
|
|
490
|
+
**Chrome extension (WIP):** load unpacked from [recorder-extension/](recorder-extension/README.md), run `pqa record start --connect 9222`, and use the popup for notes/checkpoints.
|
|
491
|
+
|
|
492
|
+
**Exit codes:** `0` pass · `1` failure · `2` config/harness error
|
|
493
|
+
|
|
494
|
+
## System prompt & skills
|
|
495
|
+
|
|
496
|
+
| File / skill | Role |
|
|
497
|
+
| --- | --- |
|
|
498
|
+
| [prompt/SYSTEM.md](prompt/SYSTEM.md) | Agent system prompt (workflow, verdict schema, rules) |
|
|
499
|
+
| `core` | Vendored agent-browser skill at `skills/agent-browser/` (bundled with the package) |
|
|
500
|
+
|
|
501
|
+
`prompt/SYSTEM.md` is loaded as the system prompt; `core` is appended as a supplemental skill. Browser control stays in bash — the agent runs `agent-browser` commands directly.
|
|
502
|
+
|
|
503
|
+
The system prompt enforces an **Observe-Act-Verify loop**: snapshot before each UI interaction, one interaction command per bash call, re-snapshot after page changes, and targeted reasoning only at ambiguous refs, failures, or before the final verdict. See [prompt/SYSTEM.md](prompt/SYSTEM.md) for details.
|
|
504
|
+
|
|
505
|
+
## Auth (hybrid authStore)
|
|
506
|
+
|
|
507
|
+
Map auth profiles to on-demand login scenarios via the [`auth`](#auth-object) config block. See [scenario format — Auth](prompt/references/scenario-format.md#auth-hybrid-authstore).
|
|
508
|
+
|
|
509
|
+
When a consumer scenario uses `auth: admin`, the harness loads cached state from `.pqa/auth/` or runs `login-admin` once, saves browser state, then continues.
|
|
510
|
+
|
|
511
|
+
```bash
|
|
512
|
+
# Inspect / invalidate cache
|
|
513
|
+
pqa auth list
|
|
514
|
+
pqa auth clear admin
|
|
515
|
+
|
|
516
|
+
# Force fresh login
|
|
517
|
+
pqa run scenarios/**/*.md --auth-refresh
|
|
518
|
+
```
|
|
519
|
+
|
|
520
|
+
**CI:** pass test credentials as GitHub Secrets → env vars (`PQA_TEST_EMAIL`, etc.) referenced in auth scenario Steps. Optionally pre-seed state from a base64 secret before the run.
|
|
521
|
+
|
|
522
|
+
Legacy manual capture (runs the configured auth scenario):
|
|
523
|
+
|
|
524
|
+
```bash
|
|
525
|
+
pqa auth save admin
|
|
526
|
+
```
|
|
527
|
+
|
|
528
|
+
## Self-healing (conservative)
|
|
529
|
+
|
|
530
|
+
When [`healing.enabled`](#healing-object-optional) is `true` (default), Prose-QA can:
|
|
531
|
+
|
|
532
|
+
1. **In-run recovery** — after a failed verdict, retry verification of failed checkpoints only (same browser session), for **transient** failures (timeouts, stale refs).
|
|
533
|
+
2. **Scenario retries** — `--retries N` with `--retries-policy transient` (default) re-runs the whole scenario only when the failure is classified transient. Use `--no-healing` for legacy behavior (any failure retries).
|
|
534
|
+
|
|
535
|
+
Checkpoints are never relaxed automatically. Passes after recovery are marked `healing.used: true` in reports.
|
|
536
|
+
|
|
537
|
+
```bash
|
|
538
|
+
# CI: one retry for flakes only
|
|
539
|
+
pqa run scenarios/**/*.md --retries 1 --retries-policy transient
|
|
540
|
+
|
|
541
|
+
# Analyze the latest run (interactive REPL)
|
|
542
|
+
pqa analyze
|
|
543
|
+
|
|
544
|
+
# Compare the 10 most recent runs for flaky scenarios
|
|
545
|
+
pqa analyze --last 10
|
|
546
|
+
```
|
|
547
|
+
|
|
548
|
+
All healing options: see [`healing`](#healing-object-optional) in Configuration.
|
|
549
|
+
|
|
550
|
+
## Reports
|
|
551
|
+
|
|
552
|
+
Runs write artifacts to `.pqa/runs/<runId>/`:
|
|
553
|
+
|
|
554
|
+
- `report.json` / `report.html` — summary
|
|
555
|
+
- `analyze.json` / `analyze-llm.json` — written by `pqa analyze` (single run)
|
|
556
|
+
- `.pqa/analyze/<timestamp>/analyze-flaky.json` / `analyze-llm.json` — multi-run flaky analysis
|
|
557
|
+
- `<scenario>/transcript.json` — bash commands + agent messages
|
|
558
|
+
- `<scenario>/verdict.json` — structured pass/fail
|
|
559
|
+
|
|
560
|
+
## CI
|
|
561
|
+
|
|
562
|
+
See [.github/workflows/smoke_tests.yml](.github/workflows/smoke_tests.yml). Unit tests run on every push. Optional smoke PQA runs require `ANTHROPIC_API_KEY` (or configure another provider via env).
|
|
563
|
+
|
|
564
|
+
## Security
|
|
565
|
+
|
|
566
|
+
See [SECURITY.md](SECURITY.md) for vulnerability reporting and guidance on run artifacts and credentials.
|
|
567
|
+
|
|
568
|
+
## License
|
|
569
|
+
|
|
570
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import type { BrowserEngine, LightpandaBrowserConfig } from "../types/config.js";
|
|
2
|
+
import type { BashEntry } from "../types/verdict.js";
|
|
3
|
+
/** Prepend local agent-browser CLI dirs so bash works without a global install. */
|
|
4
|
+
export declare function withAgentBrowserPath(cwd: string, env: NodeJS.ProcessEnv): NodeJS.ProcessEnv;
|
|
5
|
+
export declare function runBash(command: string, options: {
|
|
6
|
+
cwd: string;
|
|
7
|
+
timeoutMs: number;
|
|
8
|
+
env: NodeJS.ProcessEnv;
|
|
9
|
+
}): Promise<BashEntry>;
|
|
10
|
+
export declare function closeBrowserSession(options: {
|
|
11
|
+
cwd: string;
|
|
12
|
+
timeoutMs: number;
|
|
13
|
+
sessionName: string;
|
|
14
|
+
headed: boolean;
|
|
15
|
+
engine: BrowserEngine;
|
|
16
|
+
lightpanda?: LightpandaBrowserConfig;
|
|
17
|
+
verbose?: boolean;
|
|
18
|
+
}): Promise<void>;
|
|
19
|
+
export declare function closeAllBrowserSessions(options: {
|
|
20
|
+
cwd: string;
|
|
21
|
+
timeoutMs: number;
|
|
22
|
+
headed: boolean;
|
|
23
|
+
engine: BrowserEngine;
|
|
24
|
+
lightpanda?: LightpandaBrowserConfig;
|
|
25
|
+
verbose?: boolean;
|
|
26
|
+
}): Promise<void>;
|
|
27
|
+
export declare function prepareBrowserSession(options: {
|
|
28
|
+
cwd: string;
|
|
29
|
+
timeoutMs: number;
|
|
30
|
+
sessionName: string;
|
|
31
|
+
headed: boolean;
|
|
32
|
+
engine: BrowserEngine;
|
|
33
|
+
lightpanda?: LightpandaBrowserConfig;
|
|
34
|
+
profilePath?: string;
|
|
35
|
+
authStatePath?: string;
|
|
36
|
+
startUrl?: string;
|
|
37
|
+
verbose?: boolean;
|
|
38
|
+
}): Promise<{
|
|
39
|
+
startUrl: string;
|
|
40
|
+
}>;
|
|
41
|
+
export declare function buildBrowserEnv(config: {
|
|
42
|
+
cwd?: string;
|
|
43
|
+
headed: boolean;
|
|
44
|
+
sessionName: string;
|
|
45
|
+
engine?: BrowserEngine;
|
|
46
|
+
lightpanda?: LightpandaBrowserConfig;
|
|
47
|
+
authStatePath?: string;
|
|
48
|
+
authSavePath?: string;
|
|
49
|
+
profilePath?: string;
|
|
50
|
+
artifactDir: string;
|
|
51
|
+
}): NodeJS.ProcessEnv;
|
|
52
|
+
//# sourceMappingURL=bash.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bash.d.ts","sourceRoot":"","sources":["../../src/agent/bash.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,aAAa,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AACjF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAIrD,mFAAmF;AACnF,wBAAgB,oBAAoB,CAClC,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,CAAC,UAAU,GACrB,MAAM,CAAC,UAAU,CAUnB;AAED,wBAAsB,OAAO,CAC3B,OAAO,EAAE,MAAM,EACf,OAAO,EAAE;IACP,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC,UAAU,CAAC;CACxB,GACA,OAAO,CAAC,SAAS,CAAC,CAmCpB;AAED,wBAAsB,mBAAmB,CAAC,OAAO,EAAE;IACjD,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,aAAa,CAAC;IACtB,UAAU,CAAC,EAAE,uBAAuB,CAAC;IACrC,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,GAAG,OAAO,CAAC,IAAI,CAAC,CAiBhB;AAED,wBAAsB,uBAAuB,CAAC,OAAO,EAAE;IACrD,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,aAAa,CAAC;IACtB,UAAU,CAAC,EAAE,uBAAuB,CAAC;IACrC,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,GAAG,OAAO,CAAC,IAAI,CAAC,CAiBhB;AAeD,wBAAsB,qBAAqB,CAAC,OAAO,EAAE;IACnD,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,aAAa,CAAC;IACtB,UAAU,CAAC,EAAE,uBAAuB,CAAC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,GAAG,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC,CAmFhC;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE;IACtC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,UAAU,CAAC,EAAE,uBAAuB,CAAC;IACrC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;CACrB,GAAG,MAAM,CAAC,UAAU,CAuBpB"}
|