qualiow-exploratory-testing 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +92 -0
- package/data/domains/_default.md +181 -0
- package/data/domains/ecommerce.md +192 -0
- package/data/domains/fintech.md +200 -0
- package/data/domains/identity.md +88 -0
- package/data/domains/identity.yml +138 -0
- package/data/domains/marketing.md +194 -0
- package/data/domains/saas.md +197 -0
- package/data/knowledge/changelog.yml +86 -0
- package/data/knowledge/learned-patterns.md +172 -86
- package/data/knowledge/manifest.yml +211 -6
- package/data/knowledge/releases/v0.2.0/entries/technique-contract-narrowing.yml +150 -0
- package/data/knowledge/releases/v0.2.0/entries/technique-test-suite-audit.yml +116 -0
- package/data/knowledge/releases/v0.2.0/release.yml +20 -0
- package/data/knowledge/releases/v0.3.0/entries/technique-functional-diff-analysis.yml +119 -0
- package/data/knowledge/releases/v0.3.0/entries/technique-llm-output-verification.yml +102 -0
- package/data/knowledge/releases/v0.3.0/entries/technique-verification-mode-selection.yml +153 -0
- package/data/knowledge/releases/v0.3.0/release.yml +25 -0
- package/data/knowledge/releases/v0.4.0/entries/technique-authenticated-api-probing.yml +137 -0
- package/data/knowledge/releases/v0.4.0/entries/technique-environment-fingerprinting.yml +113 -0
- package/data/knowledge/releases/v0.4.0/entries/technique-silent-failure-audit.yml +86 -0
- package/data/knowledge/releases/v0.4.0/entries/technique-ui-api-differential.yml +97 -0
- package/data/knowledge/releases/v0.4.0/release.yml +28 -0
- package/data/knowledge/releases/v0.5.0/entries/technique-config-surface-verification.yml +77 -0
- package/data/knowledge/releases/v0.5.0/entries/technique-derived-value-verification.yml +81 -0
- package/data/knowledge/releases/v0.5.0/entries/technique-expected-behaviour-specification.yml +97 -0
- package/data/knowledge/releases/v0.5.0/entries/technique-presentation-integrity.yml +78 -0
- package/data/knowledge/releases/v0.5.0/entries/technique-release-readiness-verification.yml +110 -0
- package/data/knowledge/releases/v0.5.0/release.yml +33 -0
- package/data/targets/_example-backend.yml +123 -0
- package/data/templates/api-probe-matrix.md +99 -0
- package/data/templates/coverage-map.md +14 -0
- package/data/templates/expected-behaviour.md +93 -0
- package/dist/cli/index.js +79 -70
- package/dist/cli/index.js.map +1 -1
- package/dist/index.d.ts +90 -0
- package/dist/index.js +58 -49
- package/dist/index.js.map +1 -1
- package/docs/BACKEND-VERIFICATION.md +277 -0
- package/package.json +5 -3
- package/skills/qa-verify-backend/SKILL.md +163 -0
- package/skills/qa-verify-backend/phases/00-setup.md +97 -0
- package/skills/qa-verify-backend/phases/01-ac-decomposition.md +58 -0
- package/skills/qa-verify-backend/phases/02-static-review.md +71 -0
- package/skills/qa-verify-backend/phases/03-live-verification.md +62 -0
- package/skills/qa-verify-backend/phases/03b-api-verification.md +157 -0
- package/skills/qa-verify-backend/phases/04-e2e-trigger.md +66 -0
- package/skills/qa-verify-backend/phases/05-reporting.md +135 -0
- package/skills/qa-verify-backend/references/api-probes.md +141 -0
- package/skills/qa-verify-backend/references/aws-readonly-probes.md +156 -0
- package/skills/qa-verify-backend/references/environment-fingerprinting.md +104 -0
- package/skills/qa-verify-backend/references/payload-verification.md +112 -0
- package/skills/qa-verify-backend/references/release-readiness.md +97 -0
- package/skills/qa-verify-backend/references/safety-rules.md +90 -0
package/dist/index.d.ts
CHANGED
|
@@ -39,6 +39,49 @@ interface SafetyConfig {
|
|
|
39
39
|
no_file_upload?: boolean;
|
|
40
40
|
no_delete_actions?: boolean;
|
|
41
41
|
}
|
|
42
|
+
interface EnvironmentConfig {
|
|
43
|
+
kind: 'dev' | 'ephemeral' | 'staging' | 'production';
|
|
44
|
+
destroyed_automatically?: boolean;
|
|
45
|
+
}
|
|
46
|
+
interface BackendConfig {
|
|
47
|
+
provider?: 'aws';
|
|
48
|
+
/** Env var NAME holding the AWS profile — never the credential itself. */
|
|
49
|
+
aws_profile_env?: string;
|
|
50
|
+
region_env?: string;
|
|
51
|
+
region?: string;
|
|
52
|
+
/** Expected account id; the live lane stops if the caller identity differs. */
|
|
53
|
+
account_id?: string;
|
|
54
|
+
env_suffix?: string;
|
|
55
|
+
/** Logical name -> deployed resource name. Asserted, confirmed before use. */
|
|
56
|
+
resources?: Record<string, string>;
|
|
57
|
+
notes?: string;
|
|
58
|
+
}
|
|
59
|
+
interface ApiSurfaceConfig {
|
|
60
|
+
/** Defaults to the target's base_url when omitted. */
|
|
61
|
+
base_url?: string;
|
|
62
|
+
auth?: 'session-cookie' | 'bearer-env' | 'none';
|
|
63
|
+
/** Env var NAME holding a bearer token — never the token itself. */
|
|
64
|
+
token_env?: string;
|
|
65
|
+
/** Persistent browser profile directory holding the authenticated session. */
|
|
66
|
+
browser_profile?: string;
|
|
67
|
+
/** Endpoint returning the deployed build, used to fingerprint the environment. */
|
|
68
|
+
version_endpoint?: string;
|
|
69
|
+
/** Logical name -> "VERB /path". */
|
|
70
|
+
endpoints?: Record<string, string>;
|
|
71
|
+
/** The only endpoints the read-only API lane may call without asking. */
|
|
72
|
+
probe_allowlist?: string[];
|
|
73
|
+
/** Other target ids to run the same matrix against, for parity comparison. */
|
|
74
|
+
parity_targets?: string[];
|
|
75
|
+
/** Flag name -> where its deployed value is declared for this environment. */
|
|
76
|
+
feature_flags?: Record<string, string>;
|
|
77
|
+
notes?: string;
|
|
78
|
+
}
|
|
79
|
+
interface SourceBranchConfig {
|
|
80
|
+
repo_path: string;
|
|
81
|
+
branch: string;
|
|
82
|
+
base_branch?: string;
|
|
83
|
+
components?: Record<string, string>;
|
|
84
|
+
}
|
|
42
85
|
interface TargetConfig {
|
|
43
86
|
id: string;
|
|
44
87
|
name: string;
|
|
@@ -48,6 +91,10 @@ interface TargetConfig {
|
|
|
48
91
|
browser: BrowserConfig;
|
|
49
92
|
scope: ScopeConfig;
|
|
50
93
|
safety?: SafetyConfig;
|
|
94
|
+
environment?: EnvironmentConfig;
|
|
95
|
+
backend?: BackendConfig;
|
|
96
|
+
api?: ApiSurfaceConfig;
|
|
97
|
+
source?: SourceBranchConfig;
|
|
51
98
|
notes?: string;
|
|
52
99
|
}
|
|
53
100
|
interface MobileDeviceConfig {
|
|
@@ -290,6 +337,49 @@ declare const WebTargetConfigSchema: z.ZodObject<{
|
|
|
290
337
|
no_file_upload: z.ZodOptional<z.ZodBoolean>;
|
|
291
338
|
no_delete_actions: z.ZodOptional<z.ZodBoolean>;
|
|
292
339
|
}, z.core.$strip>>;
|
|
340
|
+
environment: z.ZodOptional<z.ZodObject<{
|
|
341
|
+
kind: z.ZodEnum<{
|
|
342
|
+
dev: "dev";
|
|
343
|
+
ephemeral: "ephemeral";
|
|
344
|
+
staging: "staging";
|
|
345
|
+
production: "production";
|
|
346
|
+
}>;
|
|
347
|
+
destroyed_automatically: z.ZodOptional<z.ZodBoolean>;
|
|
348
|
+
}, z.core.$strip>>;
|
|
349
|
+
backend: z.ZodOptional<z.ZodObject<{
|
|
350
|
+
provider: z.ZodOptional<z.ZodEnum<{
|
|
351
|
+
aws: "aws";
|
|
352
|
+
}>>;
|
|
353
|
+
aws_profile_env: z.ZodOptional<z.ZodString>;
|
|
354
|
+
region_env: z.ZodOptional<z.ZodString>;
|
|
355
|
+
region: z.ZodOptional<z.ZodString>;
|
|
356
|
+
account_id: z.ZodOptional<z.ZodString>;
|
|
357
|
+
env_suffix: z.ZodOptional<z.ZodString>;
|
|
358
|
+
resources: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
359
|
+
notes: z.ZodOptional<z.ZodString>;
|
|
360
|
+
}, z.core.$strip>>;
|
|
361
|
+
api: z.ZodOptional<z.ZodObject<{
|
|
362
|
+
base_url: z.ZodOptional<z.ZodString>;
|
|
363
|
+
auth: z.ZodOptional<z.ZodEnum<{
|
|
364
|
+
none: "none";
|
|
365
|
+
"session-cookie": "session-cookie";
|
|
366
|
+
"bearer-env": "bearer-env";
|
|
367
|
+
}>>;
|
|
368
|
+
token_env: z.ZodOptional<z.ZodString>;
|
|
369
|
+
browser_profile: z.ZodOptional<z.ZodString>;
|
|
370
|
+
version_endpoint: z.ZodOptional<z.ZodString>;
|
|
371
|
+
endpoints: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
372
|
+
probe_allowlist: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
373
|
+
parity_targets: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
374
|
+
feature_flags: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
375
|
+
notes: z.ZodOptional<z.ZodString>;
|
|
376
|
+
}, z.core.$strip>>;
|
|
377
|
+
source: z.ZodOptional<z.ZodObject<{
|
|
378
|
+
repo_path: z.ZodString;
|
|
379
|
+
branch: z.ZodString;
|
|
380
|
+
base_branch: z.ZodOptional<z.ZodString>;
|
|
381
|
+
components: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
382
|
+
}, z.core.$strip>>;
|
|
293
383
|
notes: z.ZodOptional<z.ZodString>;
|
|
294
384
|
}, z.core.$strip>;
|
|
295
385
|
declare const MobileDeviceConfigSchema: z.ZodObject<{
|
package/dist/index.js
CHANGED
|
@@ -49,6 +49,51 @@ var SafetyConfigSchema = z.object({
|
|
|
49
49
|
no_file_upload: z.boolean().optional(),
|
|
50
50
|
no_delete_actions: z.boolean().optional()
|
|
51
51
|
});
|
|
52
|
+
var EnvironmentConfigSchema = z.object({
|
|
53
|
+
kind: z.enum(["dev", "ephemeral", "staging", "production"]),
|
|
54
|
+
destroyed_automatically: z.boolean().optional()
|
|
55
|
+
});
|
|
56
|
+
var BackendConfigSchema = z.object({
|
|
57
|
+
provider: z.enum(["aws"]).optional(),
|
|
58
|
+
aws_profile_env: z.string().optional(),
|
|
59
|
+
region_env: z.string().optional(),
|
|
60
|
+
region: z.string().optional(),
|
|
61
|
+
// Expected account. The skill compares it to `sts get-caller-identity` and
|
|
62
|
+
// stops the live lane on a mismatch rather than probing someone else's account.
|
|
63
|
+
account_id: z.string().optional(),
|
|
64
|
+
env_suffix: z.string().optional(),
|
|
65
|
+
// Free-form logical name -> deployed resource name (config_table, dlq, ...).
|
|
66
|
+
resources: z.record(z.string(), z.string()).optional(),
|
|
67
|
+
notes: z.string().optional()
|
|
68
|
+
});
|
|
69
|
+
var ApiSurfaceConfigSchema = z.object({
|
|
70
|
+
// Defaults to the target's base_url when omitted.
|
|
71
|
+
base_url: z.string().optional(),
|
|
72
|
+
auth: z.enum(["session-cookie", "bearer-env", "none"]).optional(),
|
|
73
|
+
// NAME of the env var holding a bearer token, for `auth: bearer-env`.
|
|
74
|
+
token_env: z.string().optional(),
|
|
75
|
+
// Persistent browser profile directory holding the authenticated session,
|
|
76
|
+
// e.g. `.auth/my-service-dev-profile`. Gitignored.
|
|
77
|
+
browser_profile: z.string().optional(),
|
|
78
|
+
// Endpoint returning the deployed build, used to fingerprint the environment.
|
|
79
|
+
version_endpoint: z.string().optional(),
|
|
80
|
+
// Logical name -> "VERB /path", e.g. search: "POST /api/v1/search".
|
|
81
|
+
endpoints: z.record(z.string(), z.string()).optional(),
|
|
82
|
+
// The ONLY endpoints the read-only API lane may call without asking.
|
|
83
|
+
probe_allowlist: z.array(z.string()).optional(),
|
|
84
|
+
// Other target ids to run the same matrix against, for parity comparison.
|
|
85
|
+
parity_targets: z.array(z.string()).optional(),
|
|
86
|
+
// Flag name -> where its deployed value is declared for this environment.
|
|
87
|
+
// A flag can select between two implementations inside one identical build.
|
|
88
|
+
feature_flags: z.record(z.string(), z.string()).optional(),
|
|
89
|
+
notes: z.string().optional()
|
|
90
|
+
});
|
|
91
|
+
var SourceBranchConfigSchema = z.object({
|
|
92
|
+
repo_path: z.string(),
|
|
93
|
+
branch: z.string(),
|
|
94
|
+
base_branch: z.string().optional(),
|
|
95
|
+
components: z.record(z.string(), z.string()).optional()
|
|
96
|
+
});
|
|
52
97
|
var WebTargetConfigSchema = z.object({
|
|
53
98
|
id: z.string().min(1),
|
|
54
99
|
name: z.string().min(1),
|
|
@@ -60,6 +105,12 @@ var WebTargetConfigSchema = z.object({
|
|
|
60
105
|
browser: BrowserConfigSchema,
|
|
61
106
|
scope: ScopeConfigSchema,
|
|
62
107
|
safety: SafetyConfigSchema.optional(),
|
|
108
|
+
// Optional blocks consumed by /qa-verify-backend. A target without them is
|
|
109
|
+
// still a valid /qa-explore target.
|
|
110
|
+
environment: EnvironmentConfigSchema.optional(),
|
|
111
|
+
backend: BackendConfigSchema.optional(),
|
|
112
|
+
api: ApiSurfaceConfigSchema.optional(),
|
|
113
|
+
source: SourceBranchConfigSchema.optional(),
|
|
63
114
|
notes: z.string().optional()
|
|
64
115
|
});
|
|
65
116
|
var MobileDeviceConfigSchema = z.object({
|
|
@@ -540,8 +591,6 @@ async function parseSession(sessionDir) {
|
|
|
540
591
|
}
|
|
541
592
|
|
|
542
593
|
// src/formatters/html-report.ts
|
|
543
|
-
import { readFileSync as readFileSync4, existsSync as existsSync3 } from "fs";
|
|
544
|
-
import { resolve as resolve2, extname } from "path";
|
|
545
594
|
import Handlebars from "handlebars";
|
|
546
595
|
function extractReportMeta(report, sessionId) {
|
|
547
596
|
const meta = {
|
|
@@ -619,17 +668,6 @@ var HTML_TEMPLATE = `<!DOCTYPE html>
|
|
|
619
668
|
a { color: #3498db; text-decoration: none; }
|
|
620
669
|
a:hover { text-decoration: underline; }
|
|
621
670
|
|
|
622
|
-
.screenshot-container { margin: 0.75rem 0; }
|
|
623
|
-
.screenshot-img {
|
|
624
|
-
max-width: 100%;
|
|
625
|
-
border-radius: 6px;
|
|
626
|
-
border: 1px solid #1e2a3a;
|
|
627
|
-
cursor: pointer;
|
|
628
|
-
transition: transform 0.2s;
|
|
629
|
-
}
|
|
630
|
-
.screenshot-img:hover { transform: scale(1.02); }
|
|
631
|
-
.screenshot-label { font-size: 0.8rem; color: #8899aa; margin-top: 0.25rem; }
|
|
632
|
-
|
|
633
671
|
.container { max-width: 960px; margin: 0 auto; }
|
|
634
672
|
|
|
635
673
|
/* Header */
|
|
@@ -917,11 +955,8 @@ var HTML_TEMPLATE = `<!DOCTYPE html>
|
|
|
917
955
|
|
|
918
956
|
{{#if this.hasEvidence}}
|
|
919
957
|
<div class="bug-section-label">Evidence</div>
|
|
920
|
-
{{#each this.
|
|
921
|
-
<
|
|
922
|
-
<img src="{{this.dataUri}}" alt="{{this.filename}}" class="screenshot-img" />
|
|
923
|
-
<p class="screenshot-label">{{this.filename}}</p>
|
|
924
|
-
</div>
|
|
958
|
+
{{#each this.evidence.screenshots}}
|
|
959
|
+
<p class="evidence-item">Screenshot: {{this}}</p>
|
|
925
960
|
{{/each}}
|
|
926
961
|
{{#each this.evidence.console_errors}}
|
|
927
962
|
<p class="evidence-item">Console: {{this}}</p>
|
|
@@ -995,37 +1030,11 @@ async function generateHtmlReport(sessionDir) {
|
|
|
995
1030
|
for (const bug of session.bugs) {
|
|
996
1031
|
counts[bug.severity]++;
|
|
997
1032
|
}
|
|
998
|
-
const bugsForTemplate = session.bugs.map((bug) => {
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
resolve2(sessionDir, "screenshots", cleaned),
|
|
1004
|
-
resolve2(sessionDir, cleaned.replace(/^screenshots\//, "")),
|
|
1005
|
-
resolve2(sessionDir, "screenshots", cleaned.replace(/^screenshots\//, ""))
|
|
1006
|
-
];
|
|
1007
|
-
for (const imgPath of possiblePaths) {
|
|
1008
|
-
if (existsSync3(imgPath)) {
|
|
1009
|
-
try {
|
|
1010
|
-
const ext = extname(imgPath).toLowerCase().replace(".", "");
|
|
1011
|
-
const mime = ext === "jpg" ? "image/jpeg" : `image/${ext || "png"}`;
|
|
1012
|
-
const data = readFileSync4(imgPath).toString("base64");
|
|
1013
|
-
const filename = imgPath.split("/").pop() || screenshotRef;
|
|
1014
|
-
return { dataUri: `data:${mime};base64,${data}`, filename };
|
|
1015
|
-
} catch {
|
|
1016
|
-
return null;
|
|
1017
|
-
}
|
|
1018
|
-
}
|
|
1019
|
-
}
|
|
1020
|
-
return null;
|
|
1021
|
-
}).filter(Boolean);
|
|
1022
|
-
return {
|
|
1023
|
-
...bug,
|
|
1024
|
-
business_impact_summary: summarizeBusinessImpact(bug.business_impact),
|
|
1025
|
-
hasEvidence: embeddedScreenshots.length > 0 || bug.evidence.console_errors.length > 0,
|
|
1026
|
-
embeddedScreenshots
|
|
1027
|
-
};
|
|
1028
|
-
});
|
|
1033
|
+
const bugsForTemplate = session.bugs.map((bug) => ({
|
|
1034
|
+
...bug,
|
|
1035
|
+
business_impact_summary: summarizeBusinessImpact(bug.business_impact),
|
|
1036
|
+
hasEvidence: bug.evidence.screenshots.length > 0 || bug.evidence.console_errors.length > 0
|
|
1037
|
+
}));
|
|
1029
1038
|
const html = compiledTemplate({
|
|
1030
1039
|
meta,
|
|
1031
1040
|
totalBugs: session.bugs.length,
|