universal-dev-standards 5.4.0 → 5.6.0

package/bundled/core/release-quality-manifest.md

@@ -0,0 +1,193 @@
+# Release Quality Manifest
+
+## Overview
+
+A Release Quality Manifest (RQM) is a machine-readable document generated automatically by CI for every release. It aggregates the results of all quality gates into a single artifact that serves as the authoritative evidence of release readiness — both for internal go/no-go automation and for customer audits.
+
+## Why a Manifest?
+
+Without a manifest, quality evidence is scattered across CI logs, coverage HTML reports, SARIF files, and container scan summaries. When a customer asks "how was this release tested?", the answer is either "trust us" or a 45-minute manual aggregation exercise.
+
+A Release Quality Manifest makes quality evidence:
+- **Aggregated**: one file, all gates
+- **Machine-readable**: downstream tooling can parse and enforce
+- **Timestamped and commit-pinned**: tied to a specific release artifact
+- **Customer-shareable**: ready to attach to a release package
+
+## Schema
+
+The RQM now covers **16 quality dimensions** matching `release-readiness-gate.md`. Automated gates appear here; human-verified gates appear in the Release Readiness Sign-off document.
+
+```yaml
+release: vibeops-commercial-1.2.0
+generated_at: "2026-05-05T04:00:00Z"
+commit: "abc1234"
+gates:
+  # ── Automated quality gates ──────────────────────────────
+  unit_coverage:
+    actual: "73%"
+    target: "80%"
+    status: warn        # within 10pp of target → warn, not fail
+  mutation_score:
+    actual: "62%"
+    target: "60%"
+    status: pass
+  sca_critical_cve:
+    actual: 0
+    target: 0
+    status: pass
+  sca_high_cve:
+    actual: 0
+    target: 0
+    status: pass
+  sast_high:
+    actual: 0
+    target: 0
+    status: pass
+  e2e_pass_rate:
+    actual: "96%"
+    target: "95%"
+    status: pass
+  container_cve_critical:
+    actual: 0
+    target: 0
+    status: pass
+  image_signed:
+    actual: true
+    target: true
+    status: pass
+  sbom_present:
+    actual: true
+    target: true
+    status: pass
+  # ── Extended dimensions (aligned with release-readiness-gate.md) ──
+  a11y_critical:             # Dimension 3: axe-core critical violations
+    actual: 0
+    target: 0
+    status: pass
+  a11y_serious:              # Dimension 3: axe-core serious violations
+    actual: 0
+    target: 0
+    status: pass
+  contract_drift:            # Dimension 4: consumer contracts failing (n/a if no consumers)
+    actual: 0
+    target: 0
+    status: pass             # or "n/a" if no API consumers
+  cross_flow_cuj_pass_rate:  # Dimension 6: critical user journey pass rate
+    actual: "100%"
+    target: "95%"
+    status: pass
+  browser_tier1_pass_rate:   # Dimension 9: Tier-1 browser matrix (n/a for non-frontend)
+    actual: "100%"
+    target: "100%"
+    status: pass             # or "n/a" for CLI/backend
+  capacity_headroom_cpu_pct: # Dimension 10: CPU headroom at projected peak (n/a for small projects)
+    actual: "42%"
+    target: "30%"
+    status: pass             # or "n/a" for small-scale projects
+  smoke_pass_rate:           # Dimension 14: post-deploy smoke (populated after staging deploy)
+    actual: "100%"
+    target: "100%"
+    status: pass
+  flow_gate_report:          # Dimension 16: Multi-Gate Flow verification
+    gate_0_complete: true    # all flows with ≥3 steps have §2.4 + §9.4 filled
+    gate_1_pr_coverage: true # all PRs touching flows include terminal-state tests
+    gate_3_ci_pass: true     # Decision Table CI all green; branch coverage ≥ 90%
+    gate_4_uat_signoff: true # UAT sign-off table signed
+    status: pass
+overall: WARN   # worst gate status across all dimensions (2 warns, no fails)
+```
+
+## Status Semantics
+
+| Status | Meaning | Action |
+|--------|---------|--------|
+| `pass` | Meets or exceeds target | None required |
+| `warn` | Within acceptable deviation (see per-gate policy) | Document reason; no release block |
+| `fail` | Below hard minimum | **Blocks release** |
+
+### Per-Gate Hard Minimums
+
+| Gate | Warn Band | Fail Threshold | Release Readiness Dimension |
+|------|-----------|----------------|----------------------------|
+| unit_coverage | target - 10pp to target | below target - 10pp | (core RQM) |
+| mutation_score | target - 5pp to target | below target - 5pp | (core RQM) |
+| sca_critical_cve | — | any critical CVE = fail | Dim 2 (Security) |
+| container_cve_critical | — | any critical CVE = fail | Dim 2 (Security) |
+| e2e_pass_rate | target - 3pp to target | below target - 3pp | (core RQM) |
+| a11y_critical | — | > 0 = fail | Dim 3 (a11y) |
+| a11y_serious | project threshold | project threshold + 1-2 | Dim 3 (a11y) |
+| contract_drift | — | any red consumer contract = fail (if n/a: skip) | Dim 4 (Contract) |
+| cross_flow_cuj_pass_rate | 90–95% | < 90% | Dim 6 (Cross-flow Regression) |
+| browser_tier1_pass_rate | — | < 100% (if n/a: skip) | Dim 9 (Browser Compat) |
+| capacity_headroom_cpu_pct | 20–30% | < 20% (if n/a: skip) | Dim 10 (Capacity) |
+| smoke_pass_rate | — | any smoke failure = fail | Dim 14 (Smoke) |
+| flow_gate_report | gate_3_ci_pass=false | gate_0_complete=false OR gate_4_uat_signoff=false | Dim 16 (Multi-Gate Flow) |
+
+## Automated Generation
+
+Generate the manifest in CI after all gate jobs complete:
+
+```bash
+#!/usr/bin/env bash
+# scripts/generate-quality-manifest.sh
+set -euo pipefail
+
+COVERAGE=$(node -e "
+  const r = JSON.parse(require('fs').readFileSync('coverage/coverage-summary.json'));
+  console.log(r.total.lines.pct.toFixed(1) + '%')
+")
+
+MUTATION=$(node -e "
+  const r = JSON.parse(require('fs').readFileSync('reports/mutation/mutation-testing-report.json'));
+  console.log(r.metrics.mutationScore.toFixed(1) + '%')
+")
+
+CRITICAL_CVE=$(jq '[.Results[]?.Vulnerabilities[]? | select(.Severity == "CRITICAL")] | length' trivy-report.json)
+
+cat > quality-manifest.yaml <<YAML
+release: ${RELEASE_TAG}
+generated_at: "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+commit: "${GITHUB_SHA:-$(git rev-parse HEAD)}"
+gates:
+  unit_coverage:
+    actual: "${COVERAGE}"
+    target: "80%"
+    status: $([ $(echo "$COVERAGE" | tr -d '%') -ge 80 ] && echo pass || echo warn)
+  sca_critical_cve:
+    actual: ${CRITICAL_CVE}
+    target: 0
+    status: $([ "$CRITICAL_CVE" -eq 0 ] && echo pass || echo fail)
+overall: $(grep -q "fail" quality-manifest.yaml && echo FAIL || grep -q "warn" quality-manifest.yaml && echo WARN || echo PASS)
+YAML
+```
+
+## Customer-Facing Summary
+
+Generate a Markdown table alongside the YAML for inclusion in release notes:
+
+```markdown
+## Release Quality Gates — vibeops-commercial-1.2.0
+
+| Gate | Actual | Target | Status |
+|------|--------|--------|--------|
+| Unit Test Coverage | 73% | 80% | ⚠️ WARN |
+| Mutation Score | 62% | 60% | ✅ PASS |
+| Critical CVEs | 0 | 0 | ✅ PASS |
+...
+| **Overall** | | | ⚠️ WARN |
+```
+
+## Anti-Patterns
+
+- **Manually authoring the manifest** — defeats the purpose; must be generated from tool outputs
+- **Using warn for critical security gates** — `sca_critical_cve` and `container_cve_critical` are binary
+- **Generating the manifest before all gates have run** — values must reflect actual results, not estimates
+- **Not attaching the manifest to the release artifact** — a manifest in git history is not accessible to customers
+
+## See Also
+
+- `verification-evidence.ai.yaml` — audit evidence principles
+- `supply-chain-attestation.ai.yaml` — SBOM and provenance
+- `testing.ai.yaml` — overall test strategy
+- `deployment-standards.ai.yaml` — release gate integration

package/bundled/core/release-readiness-gate.md

@@ -0,0 +1,184 @@
+# Release Readiness Gate
+
+> **Language**: English | [繁體中文](../locales/zh-TW/core/release-readiness-gate.md)
+
+**Version**: 1.0.0
+**Last Updated**: 2026-05-05
+**Applicability**: All software projects preparing a production release
+**Scope**: universal
+**Industry Standards**: ISO/IEC 25010 (Product Quality), ISTQB Advanced Test Manager
+**References**: `core/release-quality-manifest.md`, `core/flow-based-testing.md`
+
+---
+
+## Purpose
+
+This standard defines a **single, aggregated Release Readiness Gate** that unifies all quality dimensions into one explicit go/no-go decision before production deployment.
+
+Without this gate, quality evidence is spread across 16+ separate standards. Teams pass individual checks but ship with unverified dimensions, because no one document says "you must pass *all of these* before release."
+
+The Release Readiness Gate:
+- **Aggregates** 16 quality dimensions into a tiered checklist
+- **Connects** human sign-off (this document) to machine-readable evidence (`release-quality-manifest.md`)
+- **Distinguishes** blocking criteria from advisory warnings
+- **Scales** via Tier-1 / Tier-2 / Tier-3 classification to fit projects of different types and risk levels
+
+---
+
+## Relationship to Release Quality Manifest (RQM)
+
+| Artifact | Format | Audience | Purpose |
+|----------|--------|----------|---------|
+| **Release Readiness Sign-off** (this document's template) | Markdown checklist | Humans (PM, QA, Eng Lead, Business) | Go/no-go decision, accountability, audit trail |
+| **Release Quality Manifest** (`release-quality-manifest.md`) | YAML/JSON | CI, tooling, customers | Machine-readable aggregation, automated gate enforcement |
+
+These two artifacts are generated **in parallel** for every release. The Sign-off covers human-verified dimensions; the RQM covers automated dimensions. Both must be `PASS` / `WARN` (never `FAIL`) before production deployment.
+
+---
+
+## Tier Classification
+
+| Tier | Requirement | Miss = ? | Who Applies |
+|------|-------------|---------|-------------|
+| **Tier-1** | Must pass; release blocked if `FAIL` | Hard block | All projects |
+| **Tier-2** | Should pass; `WARN` documented with rationale; no block | Documented WARN | All projects |
+| **Tier-3** | Applicable when feature set or domain requires it; `N/A` is valid | N/A accepted | Depends on project type |
+
+---
+
+## 16-Dimension Release Readiness Matrix
+
+| # | Dimension | Tier | Gate Type | Blocking Criterion | Evidence | Standard | Responsible |
+|---|-----------|------|-----------|-------------------|----------|---------|-------------|
+| 1 | **Performance / Load** | 2 | Automated | p95 latency regression > 10%; headroom < 20% | Load test report | `performance-standards.md` | Eng Lead + SRE |
+| 2 | **Security** (SAST/DAST/SCA/secrets) | 1 | Automated | Any Critical/High CVE, SAST High unfixed, secret in diff | SARIF, Trivy, SBOM | `pipeline-security-gates.md` | SecEng / Eng Lead |
+| 3 | **Accessibility (a11y)** | 2 | Automated + Manual | axe-core critical > 0; keyboard nav path broken | axe report, screen reader log | `accessibility-standards.md` §Release-Blocking Threshold | QA + UX |
+| 4 | **API / Contract Testing** | 3 | Automated | Upstream consumer contract red; N-1 compat broken | Pact broker report | `contract-testing-standards.md` | API owner |
+| 5 | **Database Migration** | 1 | Automated | up/rollback/idempotency test fails; data-preservation test fails | `data-migration-testing.md` gate results | `data-migration-testing.md` | DB Lead |
+| 6 | **Cross-flow Regression** | 2 | Automated | Critical user journey pass rate < 95%; business-critical flow combo fails | Cross-flow regression report | `cross-flow-regression.md` | QA Lead |
+| 7 | **Operational Readiness** | 1 | Manual | Runbook missing; alerting unconfigured; no rollback procedure | Runbook link, alert rule review | `runbook-standards.md`, `alerting-standards.md` | SRE / Ops |
+| 8 | **Localization / i18n** | 2 | Automated | MISSING or MAJOR i18n gap in release (semver gap) | `check-translation-sync.sh` output | `translation-lifecycle-standards.md` | i18n Lead |
+| 9 | **Browser / Device Compatibility** | 3 | Automated | Tier-1 browser/device pass rate < 100% | Playwright matrix report | `browser-compatibility-standards.md` | Frontend QA |
+| 10 | **Capacity Sign-off** | 3 | Manual | Headroom < 30% at projected peak; no Eng+SRE sign-off | Capacity forecast + sign-off | `performance-standards.md` §Per-Release Capacity Sign-off | SRE + Eng Lead |
+| 11 | **Compliance / Privacy** | 3 | Manual | GDPR/CCPA violation; audit log missing; retention policy broken | Privacy review checklist | `privacy-standards.md` | DPO / Legal |
+| 12 | **Documentation Completeness** | 2 | Manual | CHANGELOG missing for release; customer-facing docs not updated | CHANGELOG diff, docs review | `changelog-standards.md`, `documentation-lifecycle.md` | Tech Writer / PM |
+| 13 | **Rollback / Disaster Recovery** | 1 | Manual | No tested rollback procedure for this release; RTO > threshold | DR drill record; rollback script | `rollback-standards.md`, `disaster-recovery-drill.md` | SRE |
+| 14 | **Production Smoke / Canary** | 1 | Automated | Post-deploy smoke fails; canary error rate > SLO | Smoke test results; canary dashboard | `smoke-test.md`, `cd-deployment-strategies.md` | SRE / DevOps |
+| 15 | **Feature Flag Governance** | 2 | Manual | Default state not reviewed; kill-switch not tested | Flag audit checklist | `feature-flag-standards.md` | PM + Eng Lead |
+| 16 | **Multi-Gate Flow Verification** | 2 | Automated + Manual | Gate 0 missing for any flow with ≥ 3 steps; Gate 3 CI fail; Gate 4 UAT sign-off missing | `flow_gate_report.json`; UAT sign-off table | `flow-based-testing.md` §Multi-Gate | QA Lead + Business |
+
+> **Note on Tier-3**: Mark as `N/A` when not applicable (e.g., browser matrix for a CLI tool; contract testing for a standalone service with no API consumers). `N/A` requires a rationale comment in the sign-off.
+
+---
+
+## Release Readiness Sign-off Template
+
+> Copy this template for each release. File as `.release-readiness/<version>.md` in the repo root, or attach to the release artifact.
+
+```markdown
+# Release Readiness Sign-off
+
+**Release**: [tag/version]
+**Date**: [YYYY-MM-DD]
+**Environment**: Pre-Production → Production
+**RQM Artifact**: [link or commit SHA]
+
+## Tier-1 Gates (ALL must be PASS)
+
+| # | Dimension | Status | Evidence | Sign-off |
+|---|-----------|--------|----------|---------|
+| 2 | Security (SAST/DAST/SCA) | PASS / FAIL | [link] | [name] |
+| 5 | Database Migration | PASS / FAIL | [link] | [name] |
+| 7 | Operational Readiness | PASS / FAIL | [link] | [name] |
+| 13 | Rollback / DR | PASS / FAIL | [link] | [name] |
+| 14 | Production Smoke/Canary | PASS / FAIL | [link] | [name] |
+
+## Tier-2 Gates (WARN must have rationale)
+
+| # | Dimension | Status | Evidence | Rationale (if WARN) | Sign-off |
+|---|-----------|--------|----------|---------------------|---------|
+| 1 | Performance / Load | PASS / WARN / FAIL | [link] | | [name] |
+| 3 | Accessibility | PASS / WARN / FAIL | [link] | | [name] |
+| 6 | Cross-flow Regression | PASS / WARN / FAIL | [link] | | [name] |
+| 8 | Localization / i18n | PASS / WARN / FAIL | [link] | | [name] |
+| 12 | Documentation | PASS / WARN / FAIL | [link] | | [name] |
+| 15 | Feature Flag Governance | PASS / WARN / FAIL | [link] | | [name] |
+| 16 | Multi-Gate Flow Verification | PASS / WARN / FAIL | [link] | | [name] |
+
+## Tier-3 Gates (N/A with rationale allowed)
+
+| # | Dimension | Status | Evidence | Rationale (if N/A) | Sign-off |
+|---|-----------|--------|----------|---------------------|---------|
+| 4 | API / Contract Testing | PASS / WARN / N/A | [link] | | [name] |
+| 9 | Browser / Device Compat | PASS / WARN / N/A | [link] | | [name] |
+| 10 | Capacity Sign-off | PASS / WARN / N/A | [link] | | [name] |
+| 11 | Compliance / Privacy | PASS / WARN / N/A | [link] | | [name] |
+
+## Overall Decision
+
+- [ ] **GO** — All Tier-1 PASS; all WARN documented; all N/A have rationale
+- [ ] **NO-GO** — One or more Tier-1 FAIL, or undocumented WARN
+
+**Decision made by**: [name, role]
+**Date**: [YYYY-MM-DD]
+```
+
+---
+
+## Status Semantics
+
+| Status | Meaning | Release Impact |
+|--------|---------|----------------|
+| `PASS` | Meets or exceeds all criteria | None |
+| `WARN` | Below target but above hard minimum; rationale documented | Allowed; logged |
+| `FAIL` | Below hard minimum; unresolved | **Blocks release** |
+| `N/A` | Dimension not applicable to this project/release; rationale documented | Allowed |
+
+---
+
+## When to Create the Sign-off
+
+| Milestone | Action |
+|-----------|--------|
+| Release candidate tagged | Create `.release-readiness/<version>.md` from template; fill evidence links |
+| Pre-UAT deployment | Gate 3 CI results populated; Tier-1 automated gates verified |
+| UAT sign-off (Gate 4) | Tier-3 manual gates completed; Multi-Gate Flow row finalized |
+| Production deployment decision | Overall GO/NO-GO decision signed by release owner |
+
+The sign-off is **not** an afterthought — Gate 0 (PRD completeness) and Gate 1 (PR-level tests) must be satisfied long before the sign-off document is created. The sign-off aggregates evidence that was being collected throughout the release cycle.
+
+---
+
+## Anti-Patterns
+
+- **Creating the sign-off the day of deployment** — evidence should be collected incrementally throughout the release cycle
+- **Marking WARN without rationale** — WARN without documented reason is functionally equivalent to ignoring the gate
+- **Skipping Tier-3 entirely without N/A rationale** — if browser testing is omitted for a web app, that must be explicitly justified
+- **Treating the Sign-off as a rubber stamp** — every row requires a named sign-off owner; anonymous collective ownership means no real accountability
+- **Using a shared sign-off for multiple releases** — one sign-off per release tag; do not reuse across versions
+
+---
+
+## See Also
+
+- `release-quality-manifest.md` — machine-readable RQM (the automated counterpart to this sign-off)
+- `flow-based-testing.md` — Multi-Gate Flow Model (Dimension 16)
+- `branch-completion.md` — branch-level gate (prerequisite; not equivalent to release readiness)
+- `verification-evidence.md` — evidence standards (all evidence links must meet this standard)
+- `deployment-standards.md` — post-deploy gate integration
+
+---
+
+## Version History
+
+| Version | Date | Changes |
+|---------|------|---------|
+| 1.0.0 | 2026-05-05 | Initial release: 16-dimension matrix, tiered sign-off template, RQM integration |
+
+---
+
+## License
+
+This standard is released under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).
+
+**Source**: [universal-dev-standards](https://github.com/AsiaOstrich/universal-dev-standards)

package/bundled/core/replay-test.md

@@ -0,0 +1,86 @@
+# Replay Test Standards
+
+## Overview
+
+AI agent systems interact with users through complex multi-step pipelines. When a customer reports unexpected behaviour, reproducing the exact failure is often difficult — the model output may be non-deterministic, the environment may have changed, or the exact inputs may be unclear. Golden fixture replay solves this by serialising the exact inputs and expected outputs at time of discovery, enabling deterministic regression tests.
+
+## Fixture Format
+
+```json
+{
+  "meta": {
+    "recorded": "2026-05-05",
+    "source": "customer-report | ci-regression | red-team | incident",
+    "description": "Human-readable description of what this tests"
+  },
+  "input": { /* exact component input */ },
+  "expected": { /* expected output fields to assert */ }
+}
+```
+
+## Fixture Naming
+
+`<component>-<outcome>-<description>.json`
+
+| Good | Bad |
+|------|-----|
+| `guardian-deny-prod-drop-table.json` | `test1.json` |
+| `guardian-allow-dev-npm-test.json` | `fixture.json` |
+| `guardian-hitl-prod-irreversible.json` | `scenario_3.json` |
+
+## Replay Test Implementation (Vitest)
+
+```typescript
+// SPDX-License-Identifier: AGPL-3.0-only
+import { readdirSync, readFileSync } from "fs"
+import { join } from "path"
+import { describe, it, expect } from "vitest"
+import { scoreReviewable } from "../scoring/risk-engine.js"
+
+const FIXTURES_DIR = join(__dirname, "..", "__fixtures__")
+
+interface ReplayFixture {
+  meta: { recorded: string; source: string; description: string }
+  input: Parameters<typeof scoreReviewable>[0]
+  expected: { decision: string }
+}
+
+function deriveDecision(score: number): string {
+  if (score >= 76) return "DENY"
+  if (score >= 51) return "REQUIRE_HITL"
+  return "ALLOW"
+}
+
+describe("Guardian replay fixtures", () => {
+  const fixtures = readdirSync(FIXTURES_DIR)
+    .filter(f => f.endsWith(".json"))
+    .map(f => ({
+      name: f,
+      fixture: JSON.parse(readFileSync(join(FIXTURES_DIR, f), "utf-8")) as ReplayFixture,
+    }))
+
+  for (const { name, fixture } of fixtures) {
+    it(`[${fixture.meta.source}] ${fixture.meta.description}`, () => {
+      const result = scoreReviewable(fixture.input)
+      const decision = deriveDecision(result.score)
+      expect(decision).toBe(fixture.expected.decision)
+    })
+  }
+})
+```
+
+## Bug Regression Workflow
+
+1. Customer reports unexpected Guardian verdict
+2. Capture the exact `Reviewable` input (from audit logs)
+3. Create fixture file: `guardian-<outcome>-<description>.json`
+4. Reproduce failure locally (test should fail)
+5. Fix the bug
+6. Confirm test passes
+7. The fixture now permanently prevents regression
+
+## Related Standards
+
+- [Adversarial Test Standards](adversarial-test.md) — red-team corpus
+- [Verification Evidence Standards](verification-evidence.md) — AC traceability
+- [Testing Standards](testing.md) — overall test pyramid