vouchington-tooling 0.27.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/cli/index.mjs +4 -0
- package/dist/cli/parse.d.mts +1 -1
- package/dist/cli/parse.mjs +1 -0
- package/dist/cli/usage.d.mts +1 -1
- package/dist/cli/usage.mjs +2 -0
- package/package.json +3 -3
- package/scripts/gha/gha-collaborator-trust.sh +75 -0
- package/skills/bounded-iteration/SKILL.md +51 -0
- package/skills/bounded-iteration/references/sweep-patterns.md +342 -0
- package/skills/manifest.json +9 -3
- package/skills/postgres-node-performance-tuning/SKILL.md +2 -1
- package/skills/postgres-schema-design/SKILL.md +2 -0
package/README.md
CHANGED
|
@@ -80,6 +80,7 @@ vouchington install-playwright-chromium-arm64
|
|
|
80
80
|
vouchington ghcr-package-retention example%2Fapi
|
|
81
81
|
vouchington harness-admission-lane 4
|
|
82
82
|
vouchington harness-assert-gates HARNESS_DISPATCH_ENABLED HARNESS_SHEPHERD_ENABLED
|
|
83
|
+
echo '[{"login":"octocat","type":"User"}]' | vouchington gha-collaborator-trust owner/repo --allow-bot github-actions[bot]
|
|
83
84
|
vouchington nuget-central-version trusted.props candidate.props metadata.json out.props
|
|
84
85
|
vouchington swift-semantic-equal BASE HEAD App.swift
|
|
85
86
|
vouchington post-review
|
package/dist/cli/index.mjs
CHANGED
|
@@ -73,6 +73,10 @@ const SCRIPT_PATHS = {
|
|
|
73
73
|
path: 'scripts/gha/harness-admission-lane.sh',
|
|
74
74
|
},
|
|
75
75
|
'harness-assert-gates': { command: 'bash', path: 'scripts/gha/harness-assert-gates.sh' },
|
|
76
|
+
'gha-collaborator-trust': {
|
|
77
|
+
command: 'bash',
|
|
78
|
+
path: 'scripts/gha/gha-collaborator-trust.sh',
|
|
79
|
+
},
|
|
76
80
|
};
|
|
77
81
|
export function runCli(argv = process.argv) {
|
|
78
82
|
const parsed = parseCli(argv);
|
package/dist/cli/parse.d.mts
CHANGED
|
@@ -89,7 +89,7 @@ export type ParsedCli = {
|
|
|
89
89
|
workflowDirectories?: string[];
|
|
90
90
|
actionDirectories?: string[];
|
|
91
91
|
} | ParsedGhaRuntimeAudit | ParsedGhaArtifactsCleanup;
|
|
92
|
-
export type ScriptCommand = 'gha-output' | 'gha-needs-results' | 'download-with-diagnostics' | 'download-optional-run-artifacts' | 'host-pressure-diagnostics' | 'allocate-browser-safe-ports' | 'diagnose-port-collision' | 'prepare-trivy-db' | 'check-cache-size' | 'make-shard-matrix' | 'load-runner-env' | 'clean-workspace' | 'install-github-release' | 'run-with-timeout' | 'lint-links' | 'materialize-pr-context' | 'wait-for-apt-locks' | 'install-playwright-chromium-arm64' | 'ghcr-package-retention' | 'harness-admission-lane' | 'harness-assert-gates';
|
|
92
|
+
export type ScriptCommand = 'gha-output' | 'gha-needs-results' | 'download-with-diagnostics' | 'download-optional-run-artifacts' | 'host-pressure-diagnostics' | 'allocate-browser-safe-ports' | 'diagnose-port-collision' | 'prepare-trivy-db' | 'check-cache-size' | 'make-shard-matrix' | 'load-runner-env' | 'clean-workspace' | 'install-github-release' | 'run-with-timeout' | 'lint-links' | 'materialize-pr-context' | 'wait-for-apt-locks' | 'install-playwright-chromium-arm64' | 'ghcr-package-retention' | 'harness-admission-lane' | 'harness-assert-gates' | 'gha-collaborator-trust';
|
|
93
93
|
export declare function parseCli(argv: readonly string[]): ParsedCli;
|
|
94
94
|
export declare function parseCommandHelp(command: string | undefined, rest: readonly string[]): {
|
|
95
95
|
kind: 'command-help';
|
package/dist/cli/parse.mjs
CHANGED
package/dist/cli/usage.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export declare const USAGE = "Usage: vouchington <command> [options]\n\nCommands:\n runner-port-policy Print or validate a runner port policy\n with-host-lock Run a command under a host-wide lock\n agent-harness-config Apply classifier-auto + sandbox keys to agent harnesses\n gha-runtime-audit Audit successful GitHub Actions job runtimes\n require-up-to-date Require HEAD to include a fetched remote branch\n gitleaks-directory-scan Scan a directory with Gitleaks\n ast-grep-examples Run AST-grep rule examples\n ast-grep-pack Print shipped unconditional AST-grep pack paths as JSON\n gha-workspace-policy Check GitHub Actions workspace safety policy\n gha-output Write a collision-safe multiline GITHUB_OUTPUT record\n gha-needs-results Fail if required GitHub Actions job results failed\n download-with-diagnostics Download a URL and report HTTP status on failure\n download-optional-run-artifacts Download optional artifacts from the current run\n host-pressure-diagnostics Print a bounded host memory/OOM/PSI snapshot\n allocate-browser-safe-ports Allocate Fetch-safe localhost ports\n diagnose-port-collision Capture bounded localhost port diagnostics\n prepare-trivy-db Download the Trivy vulnerability database\n gha-artifacts-cleanup Delete classified GitHub Actions artifacts\n http-origin Validate an optional HTTP(S) origin\n vitest-blob-manifest Stamp a vitest-blob-manifest:v1 identity file\n vitest-report-attempt Write or read a Vitest report-attempt marker\n prepare-vitest-reports Validate and select Vitest report JSON files\n pnpm-install Install a pnpm workspace with retry and release-age fail-fast\n check-cache-size Measure a path and decide whether to save a GHA cache\n make-shard-matrix Emit a [1..N] GitHub Actions shard matrix\n load-runner-env Overlay a runner env file onto GITHUB_ENV with injection guards\n clean-workspace Reset a persistent-runner workspace with a fork-PR trust gate\n install-github-release Download a checksum-verified GitHub Release binary\n run-with-timeout Run a command with GNU timeout or a Perl fallback\n lint-links Two-pass lychee: internal links fail, external warn\n materialize-pr-context Dump PR title/body/files/diff/comments and #N crawl\n wait-for-apt-locks Wait until apt/dpkg lock files are free\n install-playwright-chromium-arm64 Install Playwright Chromium from browsers.json\n ghcr-package-retention Delete old GHCR package versions past KEEP_MIN\n harness-admission-lane Compute a GITHUB_RUN_ID admission lane for fleet fan-out\n harness-assert-gates Fail if any named HARNESS_*_ENABLED gate is enabled\n nuget-central-version Validate a Directory.Packages.props PackageVersion delta\n swift-semantic-equal Compare Swift sources ignoring comments and whitespace\n post-review Post one COMMENT review from a staged payload file\n stage-review-payload Validate a review payload file into a staging directory\n retrospective-transcript Format facts from Claude-compatible, Codex, or Grok transcripts\n link-skill Link one packaged skill into an explicit consumer directory\n retrospective-facts Gather immutable facts for a retrospective\n agent-blackboard Probe and journal an Agent Blackboard deployment\n mcp Serve the agent-blackboard journal tools over stdio (MCP)\n\nOptions:\n -h, --help Show this help\n -v, --version Print the package version\n\nrunner-port-policy\n (no args) Print the shipped policy as JSON\n --file <path> Validate and print a policy file\n --reserved <port> Print true if the port is reserved\n\nwith-host-lock\n --name <family>\n [--slots <n>]\n --timeout-seconds <n>\n [--command-timeout-seconds <n>]\n [--failure-diagnostics <absolute-script>]\n [--on-acquire-timeout fail|run-unlocked]\n -- <command> [args...]\n\nagent-harness-config dump\nagent-harness-config check|apply [--global] [--repo PATH]... [--harness claude|codex|grok|cursor]...\n [--home PATH]\n\ngha-runtime-audit\n [--repository owner/name] Default GITHUB_REPOSITORY\n [--branch main]\n [--median-threshold-floor <seconds>] Off unless set. Must be below the ceiling\n [--median-threshold-ceiling <seconds>] Default 360. Flags a five-sample median above this\n --pr-workflow <name|/regex/> Repeatable\n --push-workflow <name|/regex/> Repeatable\n\nrequire-up-to-date --remote <name> --branch <name>\ngitleaks-directory-scan --config <path> [--directory <path>]\nast-grep-examples --rules <directory> --config <path>\nast-grep-pack\ngha-workspace-policy [--root <directory>] [--workflow-directory <directory>] [--action-directory <directory>]\n\ngha-output <name>\ngha-needs-results [label]\ndownload-with-diagnostics <url> <destination> [-- curl-args...]\ndownload-optional-run-artifacts (--name <name>... | --pattern <pattern>) --dir <directory>\nhost-pressure-diagnostics\nallocate-browser-safe-ports <count> [--policy path] [--forbidden-ports path]\ndiagnose-port-collision [--ports \"2200 2216\"] [--output-dir PATH]\nprepare-trivy-db\ngha-artifacts-cleanup run --run-id <id> [--keep-pattern glob] [--delete-pattern glob] [--patterns-file json]\ngha-artifacts-cleanup sweep --older-than-hours <n> [--keep-pattern glob] [--delete-pattern glob] [--patterns-file json]\nhttp-origin [--field NAME] [value]\nvitest-blob-manifest <suite> [reports-directory]\nvitest-report-attempt <write DIRECTORY SUITE|read ROOT>\nprepare-vitest-reports [primary-directory] [fallback-directory] [output-directory]\npnpm-install --runner-lifecycle persistent|ephemeral|ephemeral-full --install-scripts true|false\ncheck-cache-size <path> <max-bytes> <label>\nmake-shard-matrix <total>\nload-runner-env\nclean-workspace\ninstall-github-release --repo owner/name --version X --asset 'name-{platform}.tar.gz' --bin name [--tag-prefix PREFIX] [--expected-sha256 SHA256] [--no-checksum] [--checksums-asset NAME] [--version-flag FLAG] [--bin-dir DIR]\nrun-with-timeout <timeout-seconds> <kill-after-seconds> <command...>\nlint-links [--offline] [--config PATH] [--glob PATTERN] [files...]\nmaterialize-pr-context\nwait-for-apt-locks\ninstall-playwright-chromium-arm64 [name:archive...]\nghcr-package-retention <url-encoded-package>...\nharness-admission-lane <lanes>\nharness-assert-gates <gate>...\nnuget-central-version <trusted-props> <candidate-props> <metadata-json> <output-props>\nswift-semantic-equal <base> <head> <file.swift>\npost-review\nstage-review-payload optional|required <source> <destination>\nretrospective-transcript [--session-id ID] [--jsonl PATH] [--projects-dir PATH] [--codex-sessions-dir PATH] [--grok-sessions-dir PATH]\nlink-skill <name> --source-root <skills-dir> --target-root <consumer-skills-dir> Link a packaged or repository-local skill\nretrospective-facts (--pr NUMBER | --branch NAME | --no-pr) [--repo OWNER/NAME] [--raw]\nagent-blackboard probe\nagent-blackboard journal append --mode interactive|autonomous --source-event-id ID --work-outcome OUTCOME --coverage-status STATUS --session-id UUID --agent NAME --version VERSION --file PATH --repository OWNER/NAME [--repository OWNER/NAME ...] [--parent-session-id UUID] [--timestamp ISO8601] [--coverage-source SOURCE,...] [--dropped-count N] [--outbox-directory PATH]\nagent-blackboard journal entries --session-id UUID\nagent-blackboard journal flush --outbox-directory PATH\nInteractive journal append requires --outbox-directory PATH.\nagent-blackboard snapshot partition --snapshot PATH --checksum SHA256 --counts '{\"sessions\":N,\"entries\":N,\"records\":N,\"bytes\":N}'\nagent-blackboard snapshot cleanup [--snapshot PATH] [--partition-directory PATH --receipt JSON]\nmcp\n (no args) Serve journal_append, journal_entries, outbox_status, outbox_flush,\n session_ensure, snapshot_export, and session_archive over stdio.\n Needs the optional @modelcontextprotocol/sdk peer (and zod).\n Launch it from inside the repository's git worktree.\n";
|
|
1
|
+
export declare const USAGE = "Usage: vouchington <command> [options]\n\nCommands:\n runner-port-policy Print or validate a runner port policy\n with-host-lock Run a command under a host-wide lock\n agent-harness-config Apply classifier-auto + sandbox keys to agent harnesses\n gha-runtime-audit Audit successful GitHub Actions job runtimes\n require-up-to-date Require HEAD to include a fetched remote branch\n gitleaks-directory-scan Scan a directory with Gitleaks\n ast-grep-examples Run AST-grep rule examples\n ast-grep-pack Print shipped unconditional AST-grep pack paths as JSON\n gha-workspace-policy Check GitHub Actions workspace safety policy\n gha-output Write a collision-safe multiline GITHUB_OUTPUT record\n gha-needs-results Fail if required GitHub Actions job results failed\n download-with-diagnostics Download a URL and report HTTP status on failure\n download-optional-run-artifacts Download optional artifacts from the current run\n host-pressure-diagnostics Print a bounded host memory/OOM/PSI snapshot\n allocate-browser-safe-ports Allocate Fetch-safe localhost ports\n diagnose-port-collision Capture bounded localhost port diagnostics\n prepare-trivy-db Download the Trivy vulnerability database\n gha-artifacts-cleanup Delete classified GitHub Actions artifacts\n http-origin Validate an optional HTTP(S) origin\n vitest-blob-manifest Stamp a vitest-blob-manifest:v1 identity file\n vitest-report-attempt Write or read a Vitest report-attempt marker\n prepare-vitest-reports Validate and select Vitest report JSON files\n pnpm-install Install a pnpm workspace with retry and release-age fail-fast\n check-cache-size Measure a path and decide whether to save a GHA cache\n make-shard-matrix Emit a [1..N] GitHub Actions shard matrix\n load-runner-env Overlay a runner env file onto GITHUB_ENV with injection guards\n clean-workspace Reset a persistent-runner workspace with a fork-PR trust gate\n install-github-release Download a checksum-verified GitHub Release binary\n run-with-timeout Run a command with GNU timeout or a Perl fallback\n lint-links Two-pass lychee: internal links fail, external warn\n materialize-pr-context Dump PR title/body/files/diff/comments and #N crawl\n wait-for-apt-locks Wait until apt/dpkg lock files are free\n install-playwright-chromium-arm64 Install Playwright Chromium from browsers.json\n ghcr-package-retention Delete old GHCR package versions past KEEP_MIN\n harness-admission-lane Compute a GITHUB_RUN_ID admission lane for fleet fan-out\n harness-assert-gates Fail if any named HARNESS_*_ENABLED gate is enabled\n gha-collaborator-trust Classify authors as write-access collaborators or not\n nuget-central-version Validate a Directory.Packages.props PackageVersion delta\n swift-semantic-equal Compare Swift sources ignoring comments and whitespace\n post-review Post one COMMENT review from a staged payload file\n stage-review-payload Validate a review payload file into a staging directory\n retrospective-transcript Format facts from Claude-compatible, Codex, or Grok transcripts\n link-skill Link one packaged skill into an explicit consumer directory\n retrospective-facts Gather immutable facts for a retrospective\n agent-blackboard Probe and journal an Agent Blackboard deployment\n mcp Serve the agent-blackboard journal tools over stdio (MCP)\n\nOptions:\n -h, --help Show this help\n -v, --version Print the package version\n\nrunner-port-policy\n (no args) Print the shipped policy as JSON\n --file <path> Validate and print a policy file\n --reserved <port> Print true if the port is reserved\n\nwith-host-lock\n --name <family>\n [--slots <n>]\n --timeout-seconds <n>\n [--command-timeout-seconds <n>]\n [--failure-diagnostics <absolute-script>]\n [--on-acquire-timeout fail|run-unlocked]\n -- <command> [args...]\n\nagent-harness-config dump\nagent-harness-config check|apply [--global] [--repo PATH]... [--harness claude|codex|grok|cursor]...\n [--home PATH]\n\ngha-runtime-audit\n [--repository owner/name] Default GITHUB_REPOSITORY\n [--branch main]\n [--median-threshold-floor <seconds>] Off unless set. Must be below the ceiling\n [--median-threshold-ceiling <seconds>] Default 360. Flags a five-sample median above this\n --pr-workflow <name|/regex/> Repeatable\n --push-workflow <name|/regex/> Repeatable\n\nrequire-up-to-date --remote <name> --branch <name>\ngitleaks-directory-scan --config <path> [--directory <path>]\nast-grep-examples --rules <directory> --config <path>\nast-grep-pack\ngha-workspace-policy [--root <directory>] [--workflow-directory <directory>] [--action-directory <directory>]\n\ngha-output <name>\ngha-needs-results [label]\ndownload-with-diagnostics <url> <destination> [-- curl-args...]\ndownload-optional-run-artifacts (--name <name>... | --pattern <pattern>) --dir <directory>\nhost-pressure-diagnostics\nallocate-browser-safe-ports <count> [--policy path] [--forbidden-ports path]\ndiagnose-port-collision [--ports \"2200 2216\"] [--output-dir PATH]\nprepare-trivy-db\ngha-artifacts-cleanup run --run-id <id> [--keep-pattern glob] [--delete-pattern glob] [--patterns-file json]\ngha-artifacts-cleanup sweep --older-than-hours <n> [--keep-pattern glob] [--delete-pattern glob] [--patterns-file json]\nhttp-origin [--field NAME] [value]\nvitest-blob-manifest <suite> [reports-directory]\nvitest-report-attempt <write DIRECTORY SUITE|read ROOT>\nprepare-vitest-reports [primary-directory] [fallback-directory] [output-directory]\npnpm-install --runner-lifecycle persistent|ephemeral|ephemeral-full --install-scripts true|false\ncheck-cache-size <path> <max-bytes> <label>\nmake-shard-matrix <total>\nload-runner-env\nclean-workspace\ninstall-github-release --repo owner/name --version X --asset 'name-{platform}.tar.gz' --bin name [--tag-prefix PREFIX] [--expected-sha256 SHA256] [--no-checksum] [--checksums-asset NAME] [--version-flag FLAG] [--bin-dir DIR]\nrun-with-timeout <timeout-seconds> <kill-after-seconds> <command...>\nlint-links [--offline] [--config PATH] [--glob PATTERN] [files...]\nmaterialize-pr-context\nwait-for-apt-locks\ninstall-playwright-chromium-arm64 [name:archive...]\nghcr-package-retention <url-encoded-package>...\nharness-admission-lane <lanes>\nharness-assert-gates <gate>...\ngha-collaborator-trust <owner/repo> [--allow-bot <login>]... < authors.json\nnuget-central-version <trusted-props> <candidate-props> <metadata-json> <output-props>\nswift-semantic-equal <base> <head> <file.swift>\npost-review\nstage-review-payload optional|required <source> <destination>\nretrospective-transcript [--session-id ID] [--jsonl PATH] [--projects-dir PATH] [--codex-sessions-dir PATH] [--grok-sessions-dir PATH]\nlink-skill <name> --source-root <skills-dir> --target-root <consumer-skills-dir> Link a packaged or repository-local skill\nretrospective-facts (--pr NUMBER | --branch NAME | --no-pr) [--repo OWNER/NAME] [--raw]\nagent-blackboard probe\nagent-blackboard journal append --mode interactive|autonomous --source-event-id ID --work-outcome OUTCOME --coverage-status STATUS --session-id UUID --agent NAME --version VERSION --file PATH --repository OWNER/NAME [--repository OWNER/NAME ...] [--parent-session-id UUID] [--timestamp ISO8601] [--coverage-source SOURCE,...] [--dropped-count N] [--outbox-directory PATH]\nagent-blackboard journal entries --session-id UUID\nagent-blackboard journal flush --outbox-directory PATH\nInteractive journal append requires --outbox-directory PATH.\nagent-blackboard snapshot partition --snapshot PATH --checksum SHA256 --counts '{\"sessions\":N,\"entries\":N,\"records\":N,\"bytes\":N}'\nagent-blackboard snapshot cleanup [--snapshot PATH] [--partition-directory PATH --receipt JSON]\nmcp\n (no args) Serve journal_append, journal_entries, outbox_status, outbox_flush,\n session_ensure, snapshot_export, and session_archive over stdio.\n Needs the optional @modelcontextprotocol/sdk peer (and zod).\n Launch it from inside the repository's git worktree.\n";
|
|
2
2
|
export declare function printUsage(stream?: NodeJS.WritableStream): void;
|
|
3
3
|
export declare function commandNames(usage?: string): string[];
|
|
4
4
|
export declare function commandUsage(command: string, usage?: string): string;
|
package/dist/cli/usage.mjs
CHANGED
|
@@ -37,6 +37,7 @@ Commands:
|
|
|
37
37
|
ghcr-package-retention Delete old GHCR package versions past KEEP_MIN
|
|
38
38
|
harness-admission-lane Compute a GITHUB_RUN_ID admission lane for fleet fan-out
|
|
39
39
|
harness-assert-gates Fail if any named HARNESS_*_ENABLED gate is enabled
|
|
40
|
+
gha-collaborator-trust Classify authors as write-access collaborators or not
|
|
40
41
|
nuget-central-version Validate a Directory.Packages.props PackageVersion delta
|
|
41
42
|
swift-semantic-equal Compare Swift sources ignoring comments and whitespace
|
|
42
43
|
post-review Post one COMMENT review from a staged payload file
|
|
@@ -111,6 +112,7 @@ install-playwright-chromium-arm64 [name:archive...]
|
|
|
111
112
|
ghcr-package-retention <url-encoded-package>...
|
|
112
113
|
harness-admission-lane <lanes>
|
|
113
114
|
harness-assert-gates <gate>...
|
|
115
|
+
gha-collaborator-trust <owner/repo> [--allow-bot <login>]... < authors.json
|
|
114
116
|
nuget-central-version <trusted-props> <candidate-props> <metadata-json> <output-props>
|
|
115
117
|
swift-semantic-equal <base> <head> <file.swift>
|
|
116
118
|
post-review
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "vouchington-tooling",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.29.0",
|
|
4
4
|
"description": "Vouchington CLI and extractable tooling libraries.",
|
|
5
5
|
"homepage": "https://github.com/vouchington/vouchington-tooling/tree/main/packages/vouchington-tooling#readme",
|
|
6
6
|
"bugs": {
|
|
@@ -346,8 +346,8 @@
|
|
|
346
346
|
"dependencies": {
|
|
347
347
|
"@types/mdast": "4.0.4",
|
|
348
348
|
"@types/unist": "3.0.3",
|
|
349
|
-
"csv-parse": "7.0.
|
|
350
|
-
"csv-stringify": "6.
|
|
349
|
+
"csv-parse": "7.0.3",
|
|
350
|
+
"csv-stringify": "6.9.0",
|
|
351
351
|
"dockerfile-ast": "0.7.1",
|
|
352
352
|
"picomatch": "4.0.7",
|
|
353
353
|
"remark": "15.0.1",
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Classify GitHub authors by their live permission on one repository.
|
|
3
|
+
#
|
|
4
|
+
# usage: gha-collaborator-trust.sh <owner/repo> [--allow-bot <login>]... < authors.json
|
|
5
|
+
# stdin: JSON array of {"login": string, "type": string} (GitHub user objects work as-is).
|
|
6
|
+
# stdout: {"trusted": [login...], "untrusted": [login...]}, each sorted and deduplicated.
|
|
7
|
+
#
|
|
8
|
+
# A user is trusted when the legacy `permission` field is admin or write (maintain maps to
|
|
9
|
+
# write). A GitHub App bot (type "Bot") is trusted only when its login is passed with
|
|
10
|
+
# --allow-bot, because an app with only issue or pull-request write access can still comment.
|
|
11
|
+
# A definitive 404 means no permission. Any other lookup failure exits non-zero so callers
|
|
12
|
+
# fail closed.
|
|
13
|
+
set -euo pipefail
|
|
14
|
+
|
|
15
|
+
usage() {
|
|
16
|
+
echo "usage: gha-collaborator-trust.sh <owner/repo> [--allow-bot <login>]... < authors.json" >&2
|
|
17
|
+
exit 2
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
[ "$#" -ge 1 ] || usage
|
|
21
|
+
repository="$1"
|
|
22
|
+
shift
|
|
23
|
+
[[ "$repository" =~ ^[A-Za-z0-9_-][A-Za-z0-9_.-]*/[A-Za-z0-9_-][A-Za-z0-9_.-]*$ ]] || usage
|
|
24
|
+
|
|
25
|
+
allowed_bots=$'\n'
|
|
26
|
+
while [ "$#" -gt 0 ]; do
|
|
27
|
+
if [ "$1" != "--allow-bot" ] || [ "$#" -lt 2 ] || ! [[ "$2" =~ ^[A-Za-z0-9-]+\[bot\]$ ]]; then
|
|
28
|
+
usage
|
|
29
|
+
fi
|
|
30
|
+
allowed_bots+="$2"$'\n'
|
|
31
|
+
shift 2
|
|
32
|
+
done
|
|
33
|
+
|
|
34
|
+
input="$(cat)"
|
|
35
|
+
if ! jq -e '
|
|
36
|
+
type == "array" and all(.[];
|
|
37
|
+
type == "object"
|
|
38
|
+
and (.login | type) == "string" and (.login | length) > 0
|
|
39
|
+
and ((.type | type) == "string" or .type == null))' >/dev/null 2>&1 <<<"$input"; then
|
|
40
|
+
echo "::error::gha-collaborator-trust: stdin must be a JSON array of {login, type}" >&2
|
|
41
|
+
exit 2
|
|
42
|
+
fi
|
|
43
|
+
rows="$(jq -r 'unique_by(.login) | .[] | [.login, (.type // "")] | @tsv' <<<"$input")"
|
|
44
|
+
|
|
45
|
+
results="$(mktemp)"
|
|
46
|
+
errors="$(mktemp)"
|
|
47
|
+
trap 'rm -f "$results" "$errors"' EXIT
|
|
48
|
+
|
|
49
|
+
while IFS=$'\t' read -r login type; do
|
|
50
|
+
[ -n "$login" ] || continue
|
|
51
|
+
verdict=untrusted
|
|
52
|
+
if [ "$type" = "Bot" ]; then
|
|
53
|
+
if [[ "$allowed_bots" == *$'\n'"$login"$'\n'* ]]; then
|
|
54
|
+
verdict=trusted
|
|
55
|
+
fi
|
|
56
|
+
elif [[ "$login" =~ ^[A-Za-z0-9-]+$ ]]; then
|
|
57
|
+
if permission="$(gh api "repos/$repository/collaborators/$login/permission" --jq '.permission' 2>"$errors")"; then
|
|
58
|
+
case "$permission" in
|
|
59
|
+
admin | write) verdict=trusted ;;
|
|
60
|
+
esac
|
|
61
|
+
elif ! grep -q 'HTTP 404' "$errors"; then
|
|
62
|
+
cat "$errors" >&2
|
|
63
|
+
echo "::error::gha-collaborator-trust: permission lookup failed for $login" >&2
|
|
64
|
+
exit 1
|
|
65
|
+
fi
|
|
66
|
+
fi
|
|
67
|
+
printf '%s\t%s\n' "$verdict" "$login" >>"$results"
|
|
68
|
+
done <<<"$rows"
|
|
69
|
+
|
|
70
|
+
jq -Rn '
|
|
71
|
+
[inputs | split("\t")] as $rows
|
|
72
|
+
| {
|
|
73
|
+
trusted: [$rows[] | select(.[0] == "trusted") | .[1]] | sort,
|
|
74
|
+
untrusted: [$rows[] | select(.[0] == "untrusted") | .[1]] | sort
|
|
75
|
+
}' <"$results"
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: bounded-iteration
|
|
3
|
+
description: Bound the rows each run reads when writing or reviewing scheduled jobs, reconcilers, cleanups, dispatchers, backfills, exports, list endpoints, or any loop that reads rows.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Bounded iteration
|
|
7
|
+
|
|
8
|
+
Read applicable `AGENTS.md` and the repository's local database adapter when present.
|
|
9
|
+
Local policy overrides these portable conventions.
|
|
10
|
+
|
|
11
|
+
A run's cost is the rows it reads and the work it starts, not the memory it holds. Read
|
|
12
|
+
[sweep patterns](references/sweep-patterns.md) for worked SQL and loop shapes.
|
|
13
|
+
|
|
14
|
+
1. **Choose work, not entities.** Read only rows that need work, through a predicate an index
|
|
15
|
+
serves: a work-item or dirty-marker row, a parent id, a due timestamp with a partial index, or
|
|
16
|
+
a time or UUIDv7 window past a high-water mark. Column-to-column comparisons, `OR` of unrelated
|
|
17
|
+
conditions, and `EXISTS` as the only condition are not filters. (`postgres-required-predicates`)
|
|
18
|
+
2. **Cap every run.** Every loop has a per-run cap (batches, rows, or a deadline), reports whether
|
|
19
|
+
work remains, and resumes from a persisted position. Never loop until empty, and never default
|
|
20
|
+
a cap to infinity.
|
|
21
|
+
3. **Bound every statement.** A SELECT, UPDATE, or DELETE that can touch many rows has a `LIMIT`.
|
|
22
|
+
For writes the `LIMIT` goes in a CTE, with `FOR UPDATE SKIP LOCKED` when workers compete.
|
|
23
|
+
(`postgres-lock-ordering`)
|
|
24
|
+
4. **A stream is not a bound.** A cursor or stream bounds memory, not work. Cap the rows streamed
|
|
25
|
+
per run.
|
|
26
|
+
5. **Resume without restarting.** Keep a keyset position on the UUIDv7 key or a
|
|
27
|
+
`<verb>_through_at` time position, fix the sweep's end when it starts, and rescan an overlap
|
|
28
|
+
when insert order differs from commit order. Cursor tables follow
|
|
29
|
+
[table shapes](../postgres-schema-design/references/table-shapes.md).
|
|
30
|
+
(`postgres-generated-column-predicates`)
|
|
31
|
+
6. **No unbounded fan-out.** Use `Promise.all` only over a bounded result, with bounded
|
|
32
|
+
concurrency.
|
|
33
|
+
7. **Keyset pagination.** No `OFFSET`. Clamp the client's page size on the server. The API schema
|
|
34
|
+
keeps a static ceiling; the default and the effective maximum are runtime-tunable but never
|
|
35
|
+
above it. (`postgres-no-offset`)
|
|
36
|
+
8. **Tunables are runtime configuration.** Batch sizes, page sizes, caps, and windows come from
|
|
37
|
+
the consumer's runtime configuration with a validated default and a hard maximum, and are bound
|
|
38
|
+
as SQL parameters. `LIMIT 1` and an external API's page maximum are contracts, not tunables.
|
|
39
|
+
9. **Work that needs every key** (rebuilding a membership filter, a reindex) keeps derived state
|
|
40
|
+
current on write. A full pass is an explicit, capped, resumable backfill, not a scheduled pass
|
|
41
|
+
over everything.
|
|
42
|
+
10. **Verify with a plan.** EXPLAIN at representative cardinality shows no Seq Scan on an
|
|
43
|
+
unbounded table, and an index that supplies the `ORDER BY`, so the scan stops at the
|
|
44
|
+
`LIMIT` instead of reading and sorting every match.
|
|
45
|
+
|
|
46
|
+
For work-item, lease, and cursor tables, read [postgres-schema-design](../postgres-schema-design/SKILL.md).
|
|
47
|
+
For query shape and streaming, read [performance patterns](../postgres-node-performance-tuning/references/performance-patterns.md).
|
|
48
|
+
For UUIDv7 windows and pruning, use [UUIDv7 partitioning](../postgres-partitioning-uuid-v7/SKILL.md).
|
|
49
|
+
Names in parentheses are generic `jonathanong/no-mistakes` rules; check that the installed release
|
|
50
|
+
has one before relying on it. No generic rule checks run caps, streams, or fan-out.
|
|
51
|
+
Consumer policy owns run caps, tunable names and ceilings, schedules, and enforcement configuration.
|
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
# Sweep patterns
|
|
2
|
+
|
|
3
|
+
Synthetic examples for [bounded iteration](../SKILL.md). `db.query` stands for the consumer's
|
|
4
|
+
executor and `config` for its runtime configuration; every cap in SQL is a bound parameter.
|
|
5
|
+
`min_uuidv7(timestamptz)` stands for the consumer's helper that returns the smallest UUIDv7 for a
|
|
6
|
+
timestamp, like the lower-bound helper in
|
|
7
|
+
[performance patterns](../../postgres-node-performance-tuning/references/performance-patterns.md).
|
|
8
|
+
Comments on tables and columns are omitted for space.
|
|
9
|
+
|
|
10
|
+
## Due timestamp, partial index, capped claim
|
|
11
|
+
|
|
12
|
+
A work-item row exists only while work is pending, so the index covers only rows a claim can pick:
|
|
13
|
+
|
|
14
|
+
```sql
|
|
15
|
+
CREATE INDEX idx_invoice_delivery_work_items__available_at
|
|
16
|
+
ON invoice_delivery_work_items (available_at, id)
|
|
17
|
+
WHERE completed_at IS NULL;
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
One claim takes at most `$1` due rows. Competing workers skip rows another worker holds, and the
|
|
21
|
+
lease moves `available_at` to its expiry, so an abandoned claim is due again through the same index
|
|
22
|
+
without an `OR` in the predicate:
|
|
23
|
+
|
|
24
|
+
```sql
|
|
25
|
+
WITH due AS (
|
|
26
|
+
SELECT id
|
|
27
|
+
FROM invoice_delivery_work_items
|
|
28
|
+
WHERE completed_at IS NULL AND available_at <= now()
|
|
29
|
+
ORDER BY available_at, id
|
|
30
|
+
LIMIT $1
|
|
31
|
+
FOR UPDATE SKIP LOCKED
|
|
32
|
+
)
|
|
33
|
+
UPDATE invoice_delivery_work_items AS item
|
|
34
|
+
SET lease_token = gen_random_uuid(),
|
|
35
|
+
leased_at = now(),
|
|
36
|
+
lease_expires_at = now() + $2::interval,
|
|
37
|
+
available_at = now() + $2::interval,
|
|
38
|
+
attempt_count = item.attempt_count + 1
|
|
39
|
+
FROM due
|
|
40
|
+
WHERE item.id = due.id
|
|
41
|
+
RETURNING item.id, item.invoice_id, item.lease_token;
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Renewal moves `lease_expires_at` and `available_at` to the same new deadline, so a renewed lease
|
|
45
|
+
is not due. Renewal, completion, and failure all add `AND lease_token = $mine`; see
|
|
46
|
+
[table shapes](../../postgres-schema-design/references/table-shapes.md).
|
|
47
|
+
|
|
48
|
+
## Dirty marker or work item
|
|
49
|
+
|
|
50
|
+
Do not walk every order to find the few that need reconciling. The writer that changes an order
|
|
51
|
+
marks it in the same transaction, and the reconciler claims only marks, as above:
|
|
52
|
+
|
|
53
|
+
```sql
|
|
54
|
+
CREATE TABLE order_reconcile_work_items (
|
|
55
|
+
id uuid PRIMARY KEY DEFAULT uuidv7(),
|
|
56
|
+
order_id uuid NOT NULL UNIQUE REFERENCES orders (id) ON DELETE CASCADE,
|
|
57
|
+
available_at timestamptz NOT NULL DEFAULT now(),
|
|
58
|
+
lease_token uuid,
|
|
59
|
+
leased_at timestamptz,
|
|
60
|
+
lease_expires_at timestamptz,
|
|
61
|
+
attempt_count integer NOT NULL DEFAULT 0,
|
|
62
|
+
generation integer NOT NULL DEFAULT 0,
|
|
63
|
+
completed_at timestamptz
|
|
64
|
+
);
|
|
65
|
+
|
|
66
|
+
INSERT INTO order_reconcile_work_items (order_id)
|
|
67
|
+
VALUES ($1)
|
|
68
|
+
ON CONFLICT (order_id) DO UPDATE
|
|
69
|
+
SET available_at = GREATEST(order_reconcile_work_items.available_at, now()),
|
|
70
|
+
completed_at = NULL,
|
|
71
|
+
generation = order_reconcile_work_items.generation + 1;
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
The claim above applies with `order_id` for `invoice_id`, and its `RETURNING` also includes
|
|
75
|
+
`generation`, so `$claimed` is captured in the same statement as the lease. Completion adds
|
|
76
|
+
`AND generation = $claimed`. A re-mark never makes a leased row due early, since
|
|
77
|
+
the claim moved `available_at` to the lease expiry. A completion that finds a newer generation
|
|
78
|
+
releases the lease with `available_at = now()`, and the next claim takes the row again. Run cost
|
|
79
|
+
follows the number of changes, not the number of orders. A job fanned out per account reads by
|
|
80
|
+
parent id, `WHERE account_id = $1`, for the one account it was started for.
|
|
81
|
+
|
|
82
|
+
## High-water-mark window with an overlap
|
|
83
|
+
|
|
84
|
+
An export reads `invoices` added since the last sweep. The cursor row holds the position and the
|
|
85
|
+
sweep's fixed end, and the job seeds its own row:
|
|
86
|
+
|
|
87
|
+
```sql
|
|
88
|
+
CREATE TABLE invoice_export_cursors (
|
|
89
|
+
is_singleton boolean PRIMARY KEY DEFAULT true CHECK (is_singleton),
|
|
90
|
+
cursor_invoice_id uuid,
|
|
91
|
+
sweep_upper_bound_invoice_id uuid
|
|
92
|
+
);
|
|
93
|
+
|
|
94
|
+
INSERT INTO invoice_export_cursors DEFAULT VALUES ON CONFLICT DO NOTHING;
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
A UUIDv7 is assigned at insert, not commit, so a row below the last sweep's end can commit after
|
|
98
|
+
that sweep passed it. A new sweep starts an overlap before the previous end and fixes its own end
|
|
99
|
+
now, only when the previous sweep reached its end. `$1` is the overlap, and NULL means the start,
|
|
100
|
+
so the first sweep begins at the start. The overlap must exceed the longest insert-to-commit
|
|
101
|
+
delay, which the consumer enforces, for example with a transaction timeout; a row that commits
|
|
102
|
+
after the overlap has passed is missed for good. Work that cannot tolerate a miss writes a
|
|
103
|
+
work-item row in the same transaction instead:
|
|
104
|
+
|
|
105
|
+
```sql
|
|
106
|
+
UPDATE invoice_export_cursors
|
|
107
|
+
SET cursor_invoice_id =
|
|
108
|
+
min_uuidv7(uuid_extract_timestamp(sweep_upper_bound_invoice_id) - $1::interval),
|
|
109
|
+
sweep_upper_bound_invoice_id = min_uuidv7(now())
|
|
110
|
+
WHERE is_singleton
|
|
111
|
+
AND cursor_invoice_id IS NOT DISTINCT FROM sweep_upper_bound_invoice_id;
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
Each batch moves strictly forward inside the sweep, where `$1` is the cursor (the nil UUID for the
|
|
115
|
+
start), `$2` the sweep's end, and `$3` the batch size:
|
|
116
|
+
|
|
117
|
+
```sql
|
|
118
|
+
SELECT id, account_id, total_cents
|
|
119
|
+
FROM invoices
|
|
120
|
+
WHERE id > $1 AND id <= $2
|
|
121
|
+
ORDER BY id
|
|
122
|
+
LIMIT $3;
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Persist the last `id` as the cursor after each batch with a compare-and-set, so a stale or
|
|
126
|
+
overlapping worker cannot move the position backward or into another sweep. `$1` is the new
|
|
127
|
+
position, `$2` the cursor this batch started from (NULL at the start), and `$3` the sweep's end.
|
|
128
|
+
Zero updated rows means another worker moved on, so the run stops:
|
|
129
|
+
|
|
130
|
+
```sql
|
|
131
|
+
UPDATE invoice_export_cursors
|
|
132
|
+
SET cursor_invoice_id = $1
|
|
133
|
+
WHERE is_singleton
|
|
134
|
+
AND cursor_invoice_id IS NOT DISTINCT FROM $2::uuid
|
|
135
|
+
AND sweep_upper_bound_invoice_id = $3;
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
A short batch ends the sweep: set the cursor to the sweep's end the same way, and the next run
|
|
139
|
+
starts a new sweep. Rows in the overlap are read twice, so the export must be idempotent.
|
|
140
|
+
|
|
141
|
+
A time position works the same way on a column the database sets at insert, such as
|
|
142
|
+
`received_at timestamptz NOT NULL DEFAULT clock_timestamp()`, with the same overlap rule. Never use a business
|
|
143
|
+
time such as `occurred_at`: a late-arriving row carries an old timestamp and falls behind every
|
|
144
|
+
later window. Avoid `now()` too, because it records the transaction's start, not the insert. Fix the window's end when the window opens, keep a composite `(time, id)` keyset
|
|
145
|
+
position inside it, and read
|
|
146
|
+
`WHERE (received_at, id) > ($1, $2) AND received_at <= $3 ORDER BY received_at, id`, where `$3` is
|
|
147
|
+
the window's end. Rows that share a timestamp are neither skipped nor repeated, and new arrivals
|
|
148
|
+
wait for the next window. Advance `<verb>_through_at` to the window's end only after the window is
|
|
149
|
+
drained, and start the next window an overlap earlier.
|
|
150
|
+
|
|
151
|
+
## Capped batch loop with `hasMore`
|
|
152
|
+
|
|
153
|
+
The run stops at a batch cap or a deadline and says whether work remains. A short batch is the
|
|
154
|
+
only proof that nothing is left. Rows a `SKIP LOCKED` claim passed over are held by another
|
|
155
|
+
claimer, who owns them; if that claim rolls back, they are due again on the next scheduled run,
|
|
156
|
+
so a short batch still ends this run's backlog. A cap that stops the run reports
|
|
157
|
+
`hasMore: true`, even when the last batch happened to drain the work; the continuation finds
|
|
158
|
+
nothing and stops:
|
|
159
|
+
|
|
160
|
+
```ts
|
|
161
|
+
interface RunResult {
|
|
162
|
+
processed: number
|
|
163
|
+
hasMore: boolean
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
export async function runOrderReconcile(config: ReconcileConfig): Promise<RunResult> {
|
|
167
|
+
const deadline = Date.now() + config.runDeadlineMs
|
|
168
|
+
let processed = 0
|
|
169
|
+
for (let batch = 0; batch < config.maxBatchesPerRun; batch += 1) {
|
|
170
|
+
if (Date.now() >= deadline) return { processed, hasMore: true }
|
|
171
|
+
const items = await claimDueItems(config.batchSize, config.leaseInterval)
|
|
172
|
+
await reconcileOrders(items)
|
|
173
|
+
processed += items.length
|
|
174
|
+
if (items.length < config.batchSize) return { processed, hasMore: false }
|
|
175
|
+
}
|
|
176
|
+
return { processed, hasMore: true }
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
export async function orderReconcileJob(config: ReconcileConfig): Promise<void> {
|
|
180
|
+
const { hasMore } = await runOrderReconcile(config)
|
|
181
|
+
if (hasMore) await enqueueContinuation('order-reconcile')
|
|
182
|
+
}
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
The persisted position is the work-item table itself: claimed and completed rows leave the due
|
|
186
|
+
set. A cursor sweep persists its position after each batch. The job asks for a continuation
|
|
187
|
+
instead of looping on.
|
|
188
|
+
|
|
189
|
+
## Tunables
|
|
190
|
+
|
|
191
|
+
Read each cap from runtime configuration with a validated default and a hard maximum. A bad value
|
|
192
|
+
fails at startup, and no cap is unset or infinite:
|
|
193
|
+
|
|
194
|
+
```ts
|
|
195
|
+
interface Bound {
|
|
196
|
+
default: number
|
|
197
|
+
max: number
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
export function readBound(name: string, raw: string | undefined, bound: Bound): number {
|
|
201
|
+
const value = raw === undefined ? bound.default : Number(raw)
|
|
202
|
+
if (!Number.isInteger(value) || value < 1 || value > bound.max) {
|
|
203
|
+
throw new RangeError(`${name} must be an integer from 1 to ${bound.max}`)
|
|
204
|
+
}
|
|
205
|
+
return value
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
const batchSize = readBound('ORDER_RECONCILE_BATCH_SIZE', env.ORDER_RECONCILE_BATCH_SIZE, {
|
|
209
|
+
default: 200,
|
|
210
|
+
max: 1000,
|
|
211
|
+
})
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
Bind the value as a parameter, `LIMIT $1`, never a literal in SQL text. `LIMIT 1` for a single-row
|
|
215
|
+
lookup and an external API's documented page maximum are contracts, not tunables.
|
|
216
|
+
|
|
217
|
+
## List endpoint
|
|
218
|
+
|
|
219
|
+
The API schema keeps a static ceiling. The runtime default and effective maximum are tunable but
|
|
220
|
+
never above it, and the server rejects an invalid size and clamps a large one:
|
|
221
|
+
|
|
222
|
+
```ts
|
|
223
|
+
const API_MAX_PAGE_SIZE = 100
|
|
224
|
+
|
|
225
|
+
export function pageSize(requested: number | undefined, config: PageConfig): number {
|
|
226
|
+
const ceiling = Math.min(config.maxPageSize, API_MAX_PAGE_SIZE)
|
|
227
|
+
const value = requested ?? config.defaultPageSize
|
|
228
|
+
if (!Number.isInteger(value) || value < 1) {
|
|
229
|
+
throw new RangeError('Page size must be a positive integer')
|
|
230
|
+
}
|
|
231
|
+
return Math.min(value, ceiling)
|
|
232
|
+
}
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
Page by keyset on the key the index serves. Fetch `pageSize + 1` rows to learn whether another page
|
|
236
|
+
exists, return `pageSize`, and encode the last returned `id` as the next cursor. The first page
|
|
237
|
+
omits `id < $2`:
|
|
238
|
+
|
|
239
|
+
```sql
|
|
240
|
+
SELECT id, total_cents
|
|
241
|
+
FROM orders
|
|
242
|
+
WHERE account_id = $1 AND id < $2
|
|
243
|
+
ORDER BY id DESC
|
|
244
|
+
LIMIT $3;
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
## Bounded DELETE through a CTE
|
|
248
|
+
|
|
249
|
+
DELETE has no `LIMIT`, so select the ids in a CTE and delete by join. The partial index serves the
|
|
250
|
+
retention predicate, and `$1` is the cutoff and `$2` the batch size:
|
|
251
|
+
|
|
252
|
+
```sql
|
|
253
|
+
CREATE INDEX idx_invoice_delivery_work_items__completed_at
|
|
254
|
+
ON invoice_delivery_work_items (completed_at, id)
|
|
255
|
+
WHERE completed_at IS NOT NULL;
|
|
256
|
+
|
|
257
|
+
WITH expired AS (
|
|
258
|
+
SELECT id
|
|
259
|
+
FROM invoice_delivery_work_items
|
|
260
|
+
WHERE completed_at IS NOT NULL AND completed_at < $1
|
|
261
|
+
ORDER BY completed_at, id
|
|
262
|
+
LIMIT $2
|
|
263
|
+
FOR UPDATE SKIP LOCKED
|
|
264
|
+
)
|
|
265
|
+
DELETE FROM invoice_delivery_work_items AS item
|
|
266
|
+
USING expired
|
|
267
|
+
WHERE item.id = expired.id;
|
|
268
|
+
```
|
|
269
|
+
|
|
270
|
+
Report `hasMore` when the deleted row count equals the batch size, and let the next run continue.
|
|
271
|
+
|
|
272
|
+
## Deduplicate against a large table
|
|
273
|
+
|
|
274
|
+
Do not load every existing key to diff against a candidate list. Ask the table about the
|
|
275
|
+
candidates only, in chunks whose size is a bounded tunable. The candidates themselves come from a
|
|
276
|
+
capped page:
|
|
277
|
+
|
|
278
|
+
```sql
|
|
279
|
+
SELECT external_ref
|
|
280
|
+
FROM order_imports
|
|
281
|
+
WHERE account_id = $1 AND external_ref = ANY($2::text[]);
|
|
282
|
+
```
|
|
283
|
+
|
|
284
|
+
```ts
|
|
285
|
+
const known = new Set<string>()
|
|
286
|
+
for (const chunk of chunks(candidates, config.lookupChunkSize)) {
|
|
287
|
+
const rows = await db.query(findKnownRefsSql, [accountId, chunk])
|
|
288
|
+
for (const row of rows) known.add(row.external_ref)
|
|
289
|
+
}
|
|
290
|
+
const fresh = candidates.filter((ref) => !known.has(ref))
|
|
291
|
+
```
|
|
292
|
+
|
|
293
|
+
When the goal is only to insert the new rows, `INSERT ... ON CONFLICT DO NOTHING RETURNING` skips
|
|
294
|
+
the lookup.
|
|
295
|
+
|
|
296
|
+
## Work that needs every key
|
|
297
|
+
|
|
298
|
+
A membership filter or a reindex needs every key, but not on every run. Keep the derived state
|
|
299
|
+
current on write: the insert or a dirty marker updates it. A full rebuild is a backfill started on
|
|
300
|
+
purpose, with a row cap per run and a keyset cursor to resume from, as in the high-water-mark
|
|
301
|
+
window. A scheduler never starts it as a pass over everything.
|
|
302
|
+
|
|
303
|
+
## Verify with a plan
|
|
304
|
+
|
|
305
|
+
Run each statement with representative parameters against representative table sizes, because a
|
|
306
|
+
small table scans sequentially for good reason. The `due` query of the claim above should plan as
|
|
307
|
+
a `Limit` over a `LockRows` over an index scan on the partial index, with no `Sort` between the
|
|
308
|
+
scan and the `Limit`. Read the rows each node returns, not only the scan type: an index scan that
|
|
309
|
+
feeds a `Sort` below the `Limit` still reads every match. A `Seq Scan` on a table that grows is
|
|
310
|
+
also a finding:
|
|
311
|
+
|
|
312
|
+
```sql
|
|
313
|
+
PREPARE claim_due (integer) AS
|
|
314
|
+
SELECT id
|
|
315
|
+
FROM invoice_delivery_work_items
|
|
316
|
+
WHERE completed_at IS NULL AND available_at <= now()
|
|
317
|
+
ORDER BY available_at, id
|
|
318
|
+
LIMIT $1
|
|
319
|
+
FOR UPDATE SKIP LOCKED;
|
|
320
|
+
|
|
321
|
+
BEGIN;
|
|
322
|
+
EXPLAIN (ANALYZE, BUFFERS) EXECUTE claim_due (200);
|
|
323
|
+
ROLLBACK;
|
|
324
|
+
```
|
|
325
|
+
|
|
326
|
+
`EXPLAIN ANALYZE` runs the statement, so check the claim `UPDATE` or the `DELETE` inside a
|
|
327
|
+
transaction that you roll back, or on a disposable copy. Under contention, `SKIP LOCKED` passes over rows other claimers hold, and those rows sit under the
|
|
328
|
+
`Limit`, so the scan reads the cap plus the rows concurrent claims hold. That stays bounded by the
|
|
329
|
+
number of workers times the batch size when claims are short, so keep the claim transaction to the
|
|
330
|
+
claim statement, and check the index scan's actual rows while several workers claim at once.
|
|
331
|
+
|
|
332
|
+
## Anti-patterns
|
|
333
|
+
|
|
334
|
+
| Anti-pattern | Why it is unbounded | Bounded shape |
|
|
335
|
+
| ----------------------------------------------------------- | ------------------------------------------------------------------------ | ------------------------------------------------------------ |
|
|
336
|
+
| Loop until empty: `while (true)` claim, process, break on 0 | New and failing rows keep the run alive; the run has no end | A batch or deadline cap, `hasMore`, and a continuation |
|
|
337
|
+
| A stream with no cap: `for await` over every account | Memory stays flat while run time and database work grow with the table | Stream a keyset window and cap the rows per run |
|
|
338
|
+
| A key-only walk: select every account id, then check each | Touches every entity to find the few with work | Read work items, dirty markers, or a due timestamp |
|
|
339
|
+
| A literal `LIMIT 500` in SQL text | Cannot be tuned per environment or during an incident | A validated tunable with a hard maximum, bound as `LIMIT $1` |
|
|
340
|
+
| `Promise.all(accounts.map(sync))` over a result of any size | One task per row exhausts the pool or the upstream API | A bounded batch with bounded concurrency |
|
|
341
|
+
| `OFFSET` pagination | Every page rescans and discards the rows before it | Keyset on the indexed key |
|
|
342
|
+
| `UPDATE` or `DELETE` with a predicate but no `LIMIT` | One statement locks and rewrites an unbounded row set in one transaction | A CTE that selects a capped, `SKIP LOCKED` id set |
|
package/skills/manifest.json
CHANGED
|
@@ -21,6 +21,12 @@
|
|
|
21
21
|
"pluginVersion": "0.11.0",
|
|
22
22
|
"path": "blackboard/SKILL.md"
|
|
23
23
|
},
|
|
24
|
+
{
|
|
25
|
+
"name": "bounded-iteration",
|
|
26
|
+
"plugin": "vouchington-database",
|
|
27
|
+
"pluginVersion": "0.4.0",
|
|
28
|
+
"path": "bounded-iteration/SKILL.md"
|
|
29
|
+
},
|
|
24
30
|
{
|
|
25
31
|
"name": "dependabot",
|
|
26
32
|
"plugin": "vouchington-workflow",
|
|
@@ -103,19 +109,19 @@
|
|
|
103
109
|
{
|
|
104
110
|
"name": "postgres-node-performance-tuning",
|
|
105
111
|
"plugin": "vouchington-database",
|
|
106
|
-
"pluginVersion": "0.
|
|
112
|
+
"pluginVersion": "0.4.0",
|
|
107
113
|
"path": "postgres-node-performance-tuning/SKILL.md"
|
|
108
114
|
},
|
|
109
115
|
{
|
|
110
116
|
"name": "postgres-partitioning-uuid-v7",
|
|
111
117
|
"plugin": "vouchington-database",
|
|
112
|
-
"pluginVersion": "0.
|
|
118
|
+
"pluginVersion": "0.4.0",
|
|
113
119
|
"path": "postgres-partitioning-uuid-v7/SKILL.md"
|
|
114
120
|
},
|
|
115
121
|
{
|
|
116
122
|
"name": "postgres-schema-design",
|
|
117
123
|
"plugin": "vouchington-database",
|
|
118
|
-
"pluginVersion": "0.
|
|
124
|
+
"pluginVersion": "0.4.0",
|
|
119
125
|
"path": "postgres-schema-design/SKILL.md"
|
|
120
126
|
},
|
|
121
127
|
{
|
|
@@ -12,7 +12,8 @@ Local policy overrides these defaults; do not reload an already-read adapter or
|
|
|
12
12
|
Measure the query plan and the workload shape before changing code.
|
|
13
13
|
|
|
14
14
|
- Select only the required columns.
|
|
15
|
-
- Bound each result set. Paginate or stream a large read.
|
|
15
|
+
- Bound each result set. Paginate or stream a large read. A stream bounds memory, not work: cap
|
|
16
|
+
the rows per run as in [bounded iteration](../bounded-iteration/SKILL.md).
|
|
16
17
|
- Batch writes inside an explicit transaction limit.
|
|
17
18
|
- Keep connection-pool usage bounded.
|
|
18
19
|
- Validate a query change at representative cardinality. Watch latency, memory, lock time, and
|
|
@@ -18,4 +18,6 @@ Local policy overrides these portable conventions.
|
|
|
18
18
|
|
|
19
19
|
For query shape, read [performance patterns](../postgres-node-performance-tuning/references/performance-patterns.md).
|
|
20
20
|
For partition lifecycle, use [UUIDv7 partitioning](../postgres-partitioning-uuid-v7/SKILL.md).
|
|
21
|
+
For how jobs choose, cap, and resume over work-item and cursor tables, read
|
|
22
|
+
[bounded iteration](../bounded-iteration/SKILL.md).
|
|
21
23
|
Consumer policy owns lifecycle, retention, migration strategy, and enforcement configuration.
|