sentinelayer-cli 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -2
- package/package.json +4 -4
- package/src/agents/ai-governance/index.js +12 -0
- package/src/agents/ai-governance/tools/base.js +171 -0
- package/src/agents/ai-governance/tools/eval-regression.js +47 -0
- package/src/agents/ai-governance/tools/hitl-audit.js +81 -0
- package/src/agents/ai-governance/tools/index.js +52 -0
- package/src/agents/ai-governance/tools/prompt-drift.js +42 -0
- package/src/agents/ai-governance/tools/provenance-check.js +69 -0
- package/src/agents/backend/index.js +12 -0
- package/src/agents/backend/tools/base.js +189 -0
- package/src/agents/backend/tools/circuit-breaker-check.js +123 -0
- package/src/agents/backend/tools/idempotency-audit.js +105 -0
- package/src/agents/backend/tools/index.js +87 -0
- package/src/agents/backend/tools/retry-audit.js +132 -0
- package/src/agents/backend/tools/timeout-audit.js +144 -0
- package/src/agents/code-quality/index.js +12 -0
- package/src/agents/code-quality/tools/base.js +159 -0
- package/src/agents/code-quality/tools/complexity-measure.js +197 -0
- package/src/agents/code-quality/tools/coupling-analysis.js +81 -0
- package/src/agents/code-quality/tools/cycle-detect.js +49 -0
- package/src/agents/code-quality/tools/dep-graph.js +196 -0
- package/src/agents/code-quality/tools/index.js +89 -0
- package/src/agents/data-layer/index.js +12 -0
- package/src/agents/data-layer/tools/base.js +181 -0
- package/src/agents/data-layer/tools/index-audit.js +165 -0
- package/src/agents/data-layer/tools/index.js +83 -0
- package/src/agents/data-layer/tools/migration-scan.js +135 -0
- package/src/agents/data-layer/tools/query-explain.js +120 -0
- package/src/agents/data-layer/tools/tenancy-scan.js +166 -0
- package/src/agents/documentation/index.js +12 -0
- package/src/agents/documentation/tools/api-diff.js +91 -0
- package/src/agents/documentation/tools/base.js +151 -0
- package/src/agents/documentation/tools/dead-link-check.js +58 -0
- package/src/agents/documentation/tools/docstring-coverage.js +78 -0
- package/src/agents/documentation/tools/index.js +52 -0
- package/src/agents/documentation/tools/readme-freshness.js +61 -0
- package/src/agents/envelope/fix-cycle.js +45 -0
- package/src/agents/envelope/index.js +31 -0
- package/src/agents/envelope/loop.js +150 -0
- package/src/agents/envelope/pulse.js +18 -0
- package/src/agents/envelope/stream.js +40 -0
- package/src/agents/infrastructure/index.js +12 -0
- package/src/agents/infrastructure/tools/base.js +171 -0
- package/src/agents/infrastructure/tools/checkov-run.js +32 -0
- package/src/agents/infrastructure/tools/drift-detect.js +59 -0
- package/src/agents/infrastructure/tools/iam-least-priv-check.js +78 -0
- package/src/agents/infrastructure/tools/index.js +52 -0
- package/src/agents/infrastructure/tools/tflint-run.js +31 -0
- package/src/agents/jules/loop.js +7 -4
- package/src/agents/jules/swarm/sub-agent.js +5 -1
- package/src/agents/jules/tools/auth-audit.js +10 -1
- package/src/agents/mode.js +113 -0
- package/src/agents/observability/index.js +12 -0
- package/src/agents/observability/tools/alert-audit.js +39 -0
- package/src/agents/observability/tools/base.js +181 -0
- package/src/agents/observability/tools/dashboard-gap.js +42 -0
- package/src/agents/observability/tools/index.js +54 -0
- package/src/agents/observability/tools/log-schema-check.js +74 -0
- package/src/agents/observability/tools/span-coverage.js +74 -0
- package/src/agents/persona-visuals.js +38 -0
- package/src/agents/release/index.js +12 -0
- package/src/agents/release/tools/base.js +181 -0
- package/src/agents/release/tools/changelog-diff.js +86 -0
- package/src/agents/release/tools/feature-flag-audit.js +126 -0
- package/src/agents/release/tools/index.js +61 -0
- package/src/agents/release/tools/rollback-verify.js +129 -0
- package/src/agents/release/tools/semver-check.js +109 -0
- package/src/agents/reliability/index.js +12 -0
- package/src/agents/reliability/tools/backpressure-check.js +129 -0
- package/src/agents/reliability/tools/base.js +181 -0
- package/src/agents/reliability/tools/chaos-probe.js +109 -0
- package/src/agents/reliability/tools/graceful-degradation-check.js +114 -0
- package/src/agents/reliability/tools/health-check-audit.js +111 -0
- package/src/agents/reliability/tools/index.js +87 -0
- package/src/agents/run-persona.js +109 -0
- package/src/agents/security/index.js +12 -0
- package/src/agents/security/tools/authz-audit.js +134 -0
- package/src/agents/security/tools/base.js +190 -0
- package/src/agents/security/tools/crypto-review.js +175 -0
- package/src/agents/security/tools/index.js +97 -0
- package/src/agents/security/tools/sast-scan.js +175 -0
- package/src/agents/security/tools/secrets-scan.js +216 -0
- package/src/agents/supply-chain/index.js +12 -0
- package/src/agents/supply-chain/tools/attestation-check.js +42 -0
- package/src/agents/supply-chain/tools/base.js +151 -0
- package/src/agents/supply-chain/tools/index.js +52 -0
- package/src/agents/supply-chain/tools/lockfile-integrity.js +73 -0
- package/src/agents/supply-chain/tools/package-verify.js +56 -0
- package/src/agents/supply-chain/tools/sbom-diff.js +34 -0
- package/src/agents/testing/index.js +12 -0
- package/src/agents/testing/tools/base.js +202 -0
- package/src/agents/testing/tools/coverage-gap.js +144 -0
- package/src/agents/testing/tools/flake-detect.js +125 -0
- package/src/agents/testing/tools/index.js +85 -0
- package/src/agents/testing/tools/mutation-test.js +143 -0
- package/src/agents/testing/tools/snapshot-diff.js +103 -0
- package/src/auth/gate.js +65 -37
- package/src/cli.js +1 -1
- package/src/commands/chat.js +3 -10
- package/src/commands/legacy-args.js +10 -0
- package/src/commands/omargate.js +36 -2
- package/src/commands/persona.js +46 -1
- package/src/commands/scan.js +3 -10
- package/src/commands/session.js +654 -6
- package/src/commands/spec.js +3 -10
- package/src/coord/events-log.js +141 -0
- package/src/coord/handshake.js +719 -0
- package/src/coord/index.js +35 -0
- package/src/coord/paths.js +84 -0
- package/src/coord/priority.js +62 -0
- package/src/coord/tarjan.js +157 -0
- package/src/cost/tokenizer.js +160 -0
- package/src/cost/tracker.js +61 -0
- package/src/daemon/artifact-lineage.js +362 -0
- package/src/daemon/assignment-ledger.js +117 -0
- package/src/daemon/ast-drift.js +496 -0
- package/src/daemon/ingest-refresh.js +69 -2
- package/src/ingest/engine.js +15 -0
- package/src/ingest/ownership.js +380 -0
- package/src/legacy-cli.js +68 -1
- package/src/orchestrator/kai-chen.js +126 -0
- package/src/review/ai-review.js +3 -10
- package/src/review/compliance-pack.js +389 -0
- package/src/review/investor-dd-config.js +54 -0
- package/src/review/investor-dd-file-loop.js +303 -0
- package/src/review/investor-dd-file-router.js +406 -0
- package/src/review/investor-dd-html-report.js +233 -0
- package/src/review/investor-dd-notification.js +120 -0
- package/src/review/investor-dd-orchestrator.js +405 -0
- package/src/review/investor-dd-persona-runner.js +275 -0
- package/src/review/live-validator.js +253 -0
- package/src/review/omargate-orchestrator.js +90 -2
- package/src/review/persona-prompts.js +244 -56
- package/src/review/reconciliation-rules.js +329 -0
- package/src/review/reproducibility-chain.js +136 -0
- package/src/review/scan-modes.js +102 -3
- package/src/session/agent-registry.js +7 -0
- package/src/session/analytics.js +479 -0
- package/src/session/daemon.js +609 -14
- package/src/session/file-locks.js +666 -0
- package/src/session/paths.js +4 -0
- package/src/session/recap.js +567 -0
- package/src/session/redact.js +82 -0
- package/src/session/runtime-bridge.js +24 -1
- package/src/session/scoring.js +406 -0
- package/src/session/setup-guides.js +304 -0
- package/src/session/store.js +318 -2
- package/src/session/stream.js +9 -1
- package/src/session/sync.js +753 -0
- package/src/session/tasks.js +1054 -0
- package/src/session/templates.js +188 -0
- package/src/swarm/runtime.js +1 -8
package/README.md
CHANGED
|
@@ -99,6 +99,19 @@ Inputs for non-interactive mode:
|
|
|
99
99
|
- `package.json` (adds `sentinel:start`, `sentinel:omargate`, `sentinel:omargate:json`, `sentinel:audit`, `sentinel:audit:json`, `sentinel:persona:*`, `sentinel:apply` when missing)
|
|
100
100
|
- `.env` with `SENTINELAYER_TOKEN` (or API-provided secret name) in managed auth mode
|
|
101
101
|
|
|
102
|
+
## Multi-Agent Session Workflow
|
|
103
|
+
|
|
104
|
+
Sentinelayer includes a deterministic session coordination surface for multi-agent coding loops:
|
|
105
|
+
|
|
106
|
+
- session event stream and replay (`start`, `join`, `say`, `read`, `status`, `leave`, `list`, `kill`)
|
|
107
|
+
- agent lifecycle controls (join/heartbeat/leave/kill)
|
|
108
|
+
- recap and context briefing for late-joining agents
|
|
109
|
+
- analytics + lineage artifacts at session closeout
|
|
110
|
+
|
|
111
|
+
Read the full guide: [docs/sessions.md](docs/sessions.md)
|
|
112
|
+
|
|
113
|
+
For strategy context, see the long-form blog draft: [docs/blog/slack-for-ai-coding-agents.md](docs/blog/slack-for-ai-coding-agents.md)
|
|
114
|
+
|
|
102
115
|
## Advanced options
|
|
103
116
|
|
|
104
117
|
When `Advanced options?` is enabled:
|
|
@@ -647,9 +660,17 @@ The CLI now includes policy-pack selection commands:
|
|
|
647
660
|
Built-in packs: `community` (default), `strict`, `compliance-soc2`, `compliance-hipaa`.
|
|
648
661
|
Policy selection is stored in config (`defaultPolicyPack`) and applied during `scan init` / `scan validate` / `scan precheck` profile resolution.
|
|
649
662
|
|
|
650
|
-
## AIdenID CLI
|
|
663
|
+
## AIdenID CLI surface — Clearance Layer for Agentic Access
|
|
664
|
+
|
|
665
|
+
AIdenID is the **clearance layer for agentic access** — the site-owned,
|
|
666
|
+
per-request decision layer for AI-agent traffic (`allow | throttle |
|
|
667
|
+
queue | sandbox | deny | price_required`). The `sl ai` CLI surface in
|
|
668
|
+
Sentinelayer drives the **identity-issuance side** of that layer:
|
|
669
|
+
scoped agent identities, intent-bound delegation, verification
|
|
670
|
+
interception, and lifecycle control. Sites that want to decide per
|
|
671
|
+
request consume the identities this CLI provisions.
|
|
651
672
|
|
|
652
|
-
The CLI
|
|
673
|
+
The CLI includes an `sl ai` surface for AIdenID identity provisioning:
|
|
653
674
|
|
|
654
675
|
- `sl ai provision-email --json` (dry-run artifact generation)
|
|
655
676
|
- `sl ai provision-email --execute --api-key <key> --org-id <id> --project-id <id>` (live API call)
|
package/package.json
CHANGED
|
@@ -1,15 +1,16 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "sentinelayer-cli",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.2",
|
|
4
4
|
"description": "Scaffold Sentinelayer spec/prompt/guide artifacts with secure browser auth and token bootstrap.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"scripts": {
|
|
7
|
-
"check": "node
|
|
7
|
+
"check": "node scripts/check.mjs",
|
|
8
|
+
"docs:build": "node scripts/docs-build.mjs",
|
|
8
9
|
"test": "npm run test:unit && npm run test:e2e",
|
|
9
10
|
"test:unit": "node --test tests/unit*.test.mjs",
|
|
10
11
|
"test:e2e": "node --test tests/e2e.test.mjs",
|
|
11
12
|
"test:coverage": "c8 node --test tests/unit*.test.mjs",
|
|
12
|
-
"verify": "npm run check && npm run test:e2e && npm run test:coverage && npm pack --dry-run"
|
|
13
|
+
"verify": "npm run check && npm run docs:build && npm run test:e2e && npm run test:coverage && npm pack --dry-run"
|
|
13
14
|
},
|
|
14
15
|
"bin": {
|
|
15
16
|
"sentinelayer-cli": "bin/sentinelayer-cli.js",
|
|
@@ -61,4 +62,3 @@
|
|
|
61
62
|
"license-checker-rseidelsohn": "4.4.2"
|
|
62
63
|
}
|
|
63
64
|
}
|
|
64
|
-
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
// Amina (ai-governance persona) — barrel export (#A24).
|
|
2
|
+
|
|
3
|
+
export {
|
|
4
|
+
AI_GOVERNANCE_TOOLS,
|
|
5
|
+
AI_GOVERNANCE_TOOL_IDS,
|
|
6
|
+
dispatchAiGovernanceTool,
|
|
7
|
+
runAllAiGovernanceTools,
|
|
8
|
+
runEvalRegression,
|
|
9
|
+
runHitlAudit,
|
|
10
|
+
runProvenanceCheck,
|
|
11
|
+
runPromptDrift,
|
|
12
|
+
} from "./tools/index.js";
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
// Shared helpers for Amina's (ai-governance) domain tools (#A24).
|
|
2
|
+
|
|
3
|
+
import fsp from "node:fs/promises";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import process from "node:process";
|
|
6
|
+
|
|
7
|
+
import ignore from "ignore";
|
|
8
|
+
|
|
9
|
+
const DEFAULT_IGNORED_DIRS = new Set([
|
|
10
|
+
".git",
|
|
11
|
+
"node_modules",
|
|
12
|
+
".venv",
|
|
13
|
+
".next",
|
|
14
|
+
"dist",
|
|
15
|
+
"build",
|
|
16
|
+
"coverage",
|
|
17
|
+
".sentinelayer",
|
|
18
|
+
".sentinel",
|
|
19
|
+
".turbo",
|
|
20
|
+
".idea",
|
|
21
|
+
".vscode",
|
|
22
|
+
"__pycache__",
|
|
23
|
+
".cache",
|
|
24
|
+
]);
|
|
25
|
+
const MAX_FILE_SIZE_BYTES = 1024 * 1024;
|
|
26
|
+
const SEVERITIES = Object.freeze(["P0", "P1", "P2", "P3"]);
|
|
27
|
+
|
|
28
|
+
export function toPosix(value) {
|
|
29
|
+
return String(value || "").replace(/\\/g, "/");
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function createFinding({
|
|
33
|
+
severity,
|
|
34
|
+
kind,
|
|
35
|
+
file,
|
|
36
|
+
line = 0,
|
|
37
|
+
evidence = "",
|
|
38
|
+
rootCause = "",
|
|
39
|
+
recommendedFix = "",
|
|
40
|
+
confidence = null,
|
|
41
|
+
tool = "",
|
|
42
|
+
persona = "ai-governance",
|
|
43
|
+
} = {}) {
|
|
44
|
+
const normalizedSeverity = SEVERITIES.includes(String(severity || "").toUpperCase())
|
|
45
|
+
? String(severity).toUpperCase()
|
|
46
|
+
: "P2";
|
|
47
|
+
return {
|
|
48
|
+
persona,
|
|
49
|
+
tool: String(tool || "").trim(),
|
|
50
|
+
kind: String(kind || "").trim() || "ai-governance",
|
|
51
|
+
severity: normalizedSeverity,
|
|
52
|
+
file: toPosix(file || ""),
|
|
53
|
+
line: Number.isFinite(Number(line)) ? Math.max(0, Math.floor(Number(line))) : 0,
|
|
54
|
+
evidence: String(evidence || "").trim().slice(0, 400),
|
|
55
|
+
rootCause: String(rootCause || "").trim(),
|
|
56
|
+
recommendedFix: String(recommendedFix || "").trim(),
|
|
57
|
+
confidence:
|
|
58
|
+
confidence === null || confidence === undefined
|
|
59
|
+
? null
|
|
60
|
+
: Math.max(0, Math.min(1, Number(confidence) || 0)),
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
async function readIgnorePatterns(filePath) {
|
|
65
|
+
try {
|
|
66
|
+
const raw = await fsp.readFile(filePath, "utf-8");
|
|
67
|
+
return String(raw || "")
|
|
68
|
+
.split(/\r?\n/)
|
|
69
|
+
.map((line) => line.trim())
|
|
70
|
+
.filter((line) => line && !line.startsWith("#"));
|
|
71
|
+
} catch (err) {
|
|
72
|
+
if (err && typeof err === "object" && err.code === "ENOENT") {
|
|
73
|
+
return [];
|
|
74
|
+
}
|
|
75
|
+
throw err;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
async function createIgnoreMatcher(rootPath) {
|
|
80
|
+
const matcher = ignore();
|
|
81
|
+
const gitignore = await readIgnorePatterns(path.join(rootPath, ".gitignore"));
|
|
82
|
+
const sentinel = await readIgnorePatterns(
|
|
83
|
+
path.join(rootPath, ".sentinelayerignore")
|
|
84
|
+
);
|
|
85
|
+
matcher.add([...gitignore, ...sentinel]);
|
|
86
|
+
return (relativePath, isDirectory) => {
|
|
87
|
+
const normalized = toPosix(relativePath);
|
|
88
|
+
if (!normalized) {
|
|
89
|
+
return false;
|
|
90
|
+
}
|
|
91
|
+
const candidate = isDirectory ? `${normalized}/` : normalized;
|
|
92
|
+
return matcher.ignores(candidate);
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export function findLineMatches(content, pattern) {
|
|
97
|
+
const text = String(content || "");
|
|
98
|
+
const global = new RegExp(
|
|
99
|
+
pattern.source,
|
|
100
|
+
pattern.flags.includes("g") ? pattern.flags : `${pattern.flags}g`
|
|
101
|
+
);
|
|
102
|
+
const matches = [];
|
|
103
|
+
let match;
|
|
104
|
+
while ((match = global.exec(text)) !== null) {
|
|
105
|
+
const lineIndex = text.slice(0, match.index).split(/\r?\n/).length;
|
|
106
|
+
matches.push({ index: match.index, line: lineIndex, match: match[0] });
|
|
107
|
+
}
|
|
108
|
+
return matches;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export function getLineContent(content, line) {
|
|
112
|
+
const lines = String(content || "").split(/\r?\n/);
|
|
113
|
+
return (lines[Math.max(0, (Number(line) || 1) - 1)] || "").trim();
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export async function* walkRepoFiles({
|
|
117
|
+
rootPath = process.cwd(),
|
|
118
|
+
extensions = new Set(),
|
|
119
|
+
maxFileSize = MAX_FILE_SIZE_BYTES,
|
|
120
|
+
} = {}) {
|
|
121
|
+
const resolvedRoot = path.resolve(rootPath);
|
|
122
|
+
const ignoreMatcher = await createIgnoreMatcher(resolvedRoot);
|
|
123
|
+
const wantedExtensions =
|
|
124
|
+
extensions instanceof Set
|
|
125
|
+
? extensions
|
|
126
|
+
: new Set(Array.isArray(extensions) ? extensions : []);
|
|
127
|
+
const stack = [resolvedRoot];
|
|
128
|
+
while (stack.length > 0) {
|
|
129
|
+
const current = stack.pop();
|
|
130
|
+
let entries = [];
|
|
131
|
+
try {
|
|
132
|
+
entries = await fsp.readdir(current, { withFileTypes: true });
|
|
133
|
+
} catch {
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
for (const entry of entries) {
|
|
137
|
+
const fullPath = path.join(current, entry.name);
|
|
138
|
+
const relativePath = toPosix(path.relative(resolvedRoot, fullPath));
|
|
139
|
+
if (entry.isDirectory()) {
|
|
140
|
+
if (!relativePath || DEFAULT_IGNORED_DIRS.has(entry.name)) {
|
|
141
|
+
continue;
|
|
142
|
+
}
|
|
143
|
+
if (ignoreMatcher(relativePath, true)) {
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
stack.push(fullPath);
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
if (!entry.isFile()) {
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
if (ignoreMatcher(relativePath, false)) {
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
155
|
+
const ext = path.extname(entry.name).toLowerCase();
|
|
156
|
+
if (wantedExtensions.size > 0 && !wantedExtensions.has(ext) && !wantedExtensions.has("")) {
|
|
157
|
+
continue;
|
|
158
|
+
}
|
|
159
|
+
let stat = null;
|
|
160
|
+
try {
|
|
161
|
+
stat = await fsp.stat(fullPath);
|
|
162
|
+
} catch {
|
|
163
|
+
stat = null;
|
|
164
|
+
}
|
|
165
|
+
if (!stat || stat.size > maxFileSize) {
|
|
166
|
+
continue;
|
|
167
|
+
}
|
|
168
|
+
yield { fullPath, relativePath, stat };
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
// eval-regression — advise when LLM use is present but no eval suite (#A24).
|
|
2
|
+
|
|
3
|
+
import fsp from "node:fs/promises";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
|
|
6
|
+
import { createFinding, toPosix, walkRepoFiles } from "./base.js";
|
|
7
|
+
|
|
8
|
+
const LLM_SIGNALS = [
|
|
9
|
+
/openai|anthropic|gemini|bedrock/i,
|
|
10
|
+
/Messages\.create|ChatCompletion|chat\.completions|generateContent/,
|
|
11
|
+
/createMultiProviderApiClient/,
|
|
12
|
+
];
|
|
13
|
+
|
|
14
|
+
export async function runEvalRegression({ rootPath } = {}) {
|
|
15
|
+
const resolvedRoot = path.resolve(String(rootPath || "."));
|
|
16
|
+
let hasLlm = false;
|
|
17
|
+
let hasEval = false;
|
|
18
|
+
for await (const { fullPath, relativePath } of walkRepoFiles({
|
|
19
|
+
rootPath: resolvedRoot,
|
|
20
|
+
extensions: new Set([".js", ".ts", ".tsx", ".jsx", ".mjs", ".cjs", ".py", ".yaml", ".yml", ".json"]),
|
|
21
|
+
})) {
|
|
22
|
+
const rel = toPosix(relativePath);
|
|
23
|
+
if (/(^|\/)(evals?|evaluations?)\//i.test(rel)) hasEval = true;
|
|
24
|
+
if (/(^|\/)promptfoo\./i.test(rel) || /(^|\/)\.promptfoo\./.test(rel)) hasEval = true;
|
|
25
|
+
try {
|
|
26
|
+
const content = await fsp.readFile(fullPath, "utf-8");
|
|
27
|
+
if (LLM_SIGNALS.some((p) => p.test(content))) hasLlm = true;
|
|
28
|
+
} catch {
|
|
29
|
+
/* skip */
|
|
30
|
+
}
|
|
31
|
+
if (hasLlm && hasEval) break;
|
|
32
|
+
}
|
|
33
|
+
if (!hasLlm || hasEval) return [];
|
|
34
|
+
return [
|
|
35
|
+
createFinding({
|
|
36
|
+
tool: "eval-regression",
|
|
37
|
+
kind: "ai-governance.no-eval-suite",
|
|
38
|
+
severity: "P1",
|
|
39
|
+
file: "",
|
|
40
|
+
line: 0,
|
|
41
|
+
evidence: "LLM usage detected (openai/anthropic/gemini) but no evals/ or promptfoo config",
|
|
42
|
+
rootCause: "Without a regression eval suite, prompt edits and model upgrades silently change behavior in production.",
|
|
43
|
+
recommendedFix: "Ship an evals/ directory with promptfoo / lm-evaluation-harness test cases. Run on every prompt change and model version bump.",
|
|
44
|
+
confidence: 0.7,
|
|
45
|
+
}),
|
|
46
|
+
];
|
|
47
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
// hitl-audit — advise when LLM output is acted on without human-in-the-loop (#A24).
|
|
2
|
+
|
|
3
|
+
import fsp from "node:fs/promises";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
|
|
6
|
+
import { createFinding, findLineMatches, getLineContent, toPosix, walkRepoFiles } from "./base.js";
|
|
7
|
+
|
|
8
|
+
const CODE_EXTENSIONS = new Set([".js", ".jsx", ".ts", ".tsx", ".mjs", ".cjs", ".py"]);
|
|
9
|
+
|
|
10
|
+
const LLM_CALL_PATTERNS = [
|
|
11
|
+
/Messages\.create\s*\(/,
|
|
12
|
+
/chat\.completions\.create\s*\(/,
|
|
13
|
+
/generateContent\s*\(/,
|
|
14
|
+
/createMultiProviderApiClient\s*\(/,
|
|
15
|
+
];
|
|
16
|
+
|
|
17
|
+
const ACTION_PATTERNS = [
|
|
18
|
+
/exec(?:Sync)?\s*\(/,
|
|
19
|
+
/spawn(?:Sync)?\s*\(/,
|
|
20
|
+
/fs\.(?:unlink|unlinkSync|rm|rmSync|writeFile|writeFileSync|rename|renameSync)\s*\(/,
|
|
21
|
+
/db\.(?:update|delete|drop|truncate|raw)\s*\(/,
|
|
22
|
+
/fetch\s*\([^)]*method\s*:\s*['"](?:POST|PUT|DELETE|PATCH)['"]/,
|
|
23
|
+
];
|
|
24
|
+
|
|
25
|
+
const APPROVAL_SIGNALS = [
|
|
26
|
+
/human[_-]?in[_-]?(?:the[_-]?)?loop|HITL/i,
|
|
27
|
+
/await\s+(?:confirm|approval|operatorApprov|humanReview)/i,
|
|
28
|
+
/requires?[_-]?approval|needs[_-]?approval/i,
|
|
29
|
+
/await\s+prompts?\s*\(/,
|
|
30
|
+
];
|
|
31
|
+
|
|
32
|
+
export async function runHitlAudit({ rootPath, files = null } = {}) {
|
|
33
|
+
const resolvedRoot = path.resolve(String(rootPath || "."));
|
|
34
|
+
const iterator =
|
|
35
|
+
Array.isArray(files) && files.length > 0
|
|
36
|
+
? iterateExplicitFiles(resolvedRoot, files)
|
|
37
|
+
: walkRepoFiles({ rootPath: resolvedRoot, extensions: CODE_EXTENSIONS });
|
|
38
|
+
|
|
39
|
+
const findings = [];
|
|
40
|
+
for await (const { fullPath, relativePath } of iterator) {
|
|
41
|
+
let content;
|
|
42
|
+
try {
|
|
43
|
+
content = await fsp.readFile(fullPath, "utf-8");
|
|
44
|
+
} catch {
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
const hasLlm = LLM_CALL_PATTERNS.some((p) => p.test(content));
|
|
48
|
+
if (!hasLlm) continue;
|
|
49
|
+
const hasAction = ACTION_PATTERNS.some((p) => p.test(content));
|
|
50
|
+
if (!hasAction) continue;
|
|
51
|
+
const hasApproval = APPROVAL_SIGNALS.some((p) => p.test(content));
|
|
52
|
+
if (hasApproval) continue;
|
|
53
|
+
const match = findLineMatches(content, ACTION_PATTERNS[0])[0] ||
|
|
54
|
+
findLineMatches(content, ACTION_PATTERNS[1])[0] ||
|
|
55
|
+
findLineMatches(content, ACTION_PATTERNS[2])[0];
|
|
56
|
+
findings.push(
|
|
57
|
+
createFinding({
|
|
58
|
+
tool: "hitl-audit",
|
|
59
|
+
kind: "ai-governance.no-hitl",
|
|
60
|
+
severity: "P1",
|
|
61
|
+
file: toPosix(relativePath),
|
|
62
|
+
line: match?.line || 1,
|
|
63
|
+
evidence: getLineContent(content, match?.line || 1),
|
|
64
|
+
rootCause: "File calls an LLM then takes a destructive / mutating action with no human-in-the-loop confirmation. A jailbroken prompt becomes an arbitrary operation.",
|
|
65
|
+
recommendedFix: "Gate high-impact actions on explicit operator approval (`await confirmWithOperator(plan)`), or run them inside a sandbox with narrow permissions and a review queue.",
|
|
66
|
+
confidence: 0.5,
|
|
67
|
+
})
|
|
68
|
+
);
|
|
69
|
+
}
|
|
70
|
+
return findings;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
async function* iterateExplicitFiles(resolvedRoot, files) {
|
|
74
|
+
for (const file of files) {
|
|
75
|
+
const trimmed = String(file || "").trim();
|
|
76
|
+
if (!trimmed) continue;
|
|
77
|
+
const fullPath = path.isAbsolute(trimmed) ? trimmed : path.join(resolvedRoot, trimmed);
|
|
78
|
+
const relativePath = path.relative(resolvedRoot, fullPath).replace(/\\/g, "/");
|
|
79
|
+
yield { fullPath, relativePath };
|
|
80
|
+
}
|
|
81
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
// Amina (ai-governance persona) domain-tool registry (#A24).
|
|
2
|
+
|
|
3
|
+
import { runEvalRegression } from "./eval-regression.js";
|
|
4
|
+
import { runHitlAudit } from "./hitl-audit.js";
|
|
5
|
+
import { runProvenanceCheck } from "./provenance-check.js";
|
|
6
|
+
import { runPromptDrift } from "./prompt-drift.js";
|
|
7
|
+
|
|
8
|
+
export const AI_GOVERNANCE_TOOLS = Object.freeze({
|
|
9
|
+
"eval-regression": {
|
|
10
|
+
id: "eval-regression",
|
|
11
|
+
description: "Advise when LLM usage is detected but no evals/ or promptfoo config exists.",
|
|
12
|
+
schema: { type: "object", properties: { rootPath: { type: "string" } } },
|
|
13
|
+
handler: runEvalRegression,
|
|
14
|
+
},
|
|
15
|
+
"prompt-drift": {
|
|
16
|
+
id: "prompt-drift",
|
|
17
|
+
description: "Flag prompt files under prompts/ / ai/ / llm/ that lack a version header.",
|
|
18
|
+
schema: { type: "object", properties: { rootPath: { type: "string" } } },
|
|
19
|
+
handler: runPromptDrift,
|
|
20
|
+
},
|
|
21
|
+
"hitl-audit": {
|
|
22
|
+
id: "hitl-audit",
|
|
23
|
+
description: "Flag files that call an LLM and then run a destructive action without human-in-the-loop approval signals.",
|
|
24
|
+
schema: { type: "object", properties: { rootPath: { type: "string" }, files: { type: "array", items: { type: "string" } } } },
|
|
25
|
+
handler: runHitlAudit,
|
|
26
|
+
},
|
|
27
|
+
"provenance-check": {
|
|
28
|
+
id: "provenance-check",
|
|
29
|
+
description: "Flag generateContent / composeEmail / LLM-generated content without provenance (ai-generated header, C2PA, watermark).",
|
|
30
|
+
schema: { type: "object", properties: { rootPath: { type: "string" }, files: { type: "array", items: { type: "string" } } } },
|
|
31
|
+
handler: runProvenanceCheck,
|
|
32
|
+
},
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
export const AI_GOVERNANCE_TOOL_IDS = Object.freeze(Object.keys(AI_GOVERNANCE_TOOLS));
|
|
36
|
+
|
|
37
|
+
export async function dispatchAiGovernanceTool(toolId, args = {}) {
|
|
38
|
+
const tool = AI_GOVERNANCE_TOOLS[toolId];
|
|
39
|
+
if (!tool) throw new Error(`Unknown ai-governance tool: ${toolId}`);
|
|
40
|
+
return tool.handler(args);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export async function runAllAiGovernanceTools({ rootPath, files = null } = {}) {
|
|
44
|
+
const findings = [];
|
|
45
|
+
for (const toolId of AI_GOVERNANCE_TOOL_IDS) {
|
|
46
|
+
const out = await dispatchAiGovernanceTool(toolId, { rootPath, files });
|
|
47
|
+
findings.push(...out);
|
|
48
|
+
}
|
|
49
|
+
return findings;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export { runEvalRegression, runHitlAudit, runProvenanceCheck, runPromptDrift };
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
// prompt-drift — advise when prompt files aren't versioned (#A24).
|
|
2
|
+
|
|
3
|
+
import fsp from "node:fs/promises";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
|
|
6
|
+
import { createFinding, toPosix, walkRepoFiles } from "./base.js";
|
|
7
|
+
|
|
8
|
+
export async function runPromptDrift({ rootPath } = {}) {
|
|
9
|
+
const resolvedRoot = path.resolve(String(rootPath || "."));
|
|
10
|
+
const findings = [];
|
|
11
|
+
for await (const { fullPath, relativePath } of walkRepoFiles({
|
|
12
|
+
rootPath: resolvedRoot,
|
|
13
|
+
})) {
|
|
14
|
+
const rel = toPosix(relativePath);
|
|
15
|
+
if (!/(^|\/)(prompts?|ai|llm)\//i.test(rel)) continue;
|
|
16
|
+
if (!/\.(md|txt|ya?ml|json|prompt)$/i.test(rel)) continue;
|
|
17
|
+
let content;
|
|
18
|
+
try {
|
|
19
|
+
content = await fsp.readFile(fullPath, "utf-8");
|
|
20
|
+
} catch {
|
|
21
|
+
continue;
|
|
22
|
+
}
|
|
23
|
+
const hasVersion = /version\s*[:=]\s*['"]?v?\d+\.\d+/i.test(content) ||
|
|
24
|
+
/---[\s\S]*?version:[\s\S]*?---/.test(content);
|
|
25
|
+
if (!hasVersion) {
|
|
26
|
+
findings.push(
|
|
27
|
+
createFinding({
|
|
28
|
+
tool: "prompt-drift",
|
|
29
|
+
kind: "ai-governance.unversioned-prompt",
|
|
30
|
+
severity: "P2",
|
|
31
|
+
file: rel,
|
|
32
|
+
line: 0,
|
|
33
|
+
evidence: "No version header in prompt file",
|
|
34
|
+
rootCause: "Prompts without explicit versions make it impossible to roll back behavior changes or compare eval runs across versions.",
|
|
35
|
+
recommendedFix: "Add a `version: 1.2.0` frontmatter header. Bump on every edit. Pair with eval-regression runs keyed on version.",
|
|
36
|
+
confidence: 0.55,
|
|
37
|
+
})
|
|
38
|
+
);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
return findings;
|
|
42
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
// provenance-check — advise when AI-generated content lacks provenance signals (#A24).
|
|
2
|
+
|
|
3
|
+
import fsp from "node:fs/promises";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
|
|
6
|
+
import { createFinding, findLineMatches, getLineContent, toPosix, walkRepoFiles } from "./base.js";
|
|
7
|
+
|
|
8
|
+
const CODE_EXTENSIONS = new Set([".js", ".jsx", ".ts", ".tsx", ".mjs", ".cjs", ".py"]);
|
|
9
|
+
|
|
10
|
+
const GENERATION_PATTERNS = [
|
|
11
|
+
/\bgenerate(?:Content|Response|Completion|Text|Draft|Message)?\s*\(/,
|
|
12
|
+
/\bcompose(?:Email|Message|Reply)\s*\(/,
|
|
13
|
+
/\bllm[._](?:complete|generate)\s*\(/,
|
|
14
|
+
];
|
|
15
|
+
|
|
16
|
+
const PROVENANCE_SIGNALS = [
|
|
17
|
+
/ai[_-]?generated|generated[_-]?by[_-]?ai/i,
|
|
18
|
+
/provenance|attribution/i,
|
|
19
|
+
/X-AI-Generated|x_ai_generated/,
|
|
20
|
+
/watermark/i,
|
|
21
|
+
/c2pa/i,
|
|
22
|
+
];
|
|
23
|
+
|
|
24
|
+
export async function runProvenanceCheck({ rootPath, files = null } = {}) {
|
|
25
|
+
const resolvedRoot = path.resolve(String(rootPath || "."));
|
|
26
|
+
const iterator =
|
|
27
|
+
Array.isArray(files) && files.length > 0
|
|
28
|
+
? iterateExplicitFiles(resolvedRoot, files)
|
|
29
|
+
: walkRepoFiles({ rootPath: resolvedRoot, extensions: CODE_EXTENSIONS });
|
|
30
|
+
|
|
31
|
+
const findings = [];
|
|
32
|
+
for await (const { fullPath, relativePath } of iterator) {
|
|
33
|
+
let content;
|
|
34
|
+
try {
|
|
35
|
+
content = await fsp.readFile(fullPath, "utf-8");
|
|
36
|
+
} catch {
|
|
37
|
+
continue;
|
|
38
|
+
}
|
|
39
|
+
const matches = GENERATION_PATTERNS.flatMap((p) => findLineMatches(content, p));
|
|
40
|
+
if (matches.length === 0) continue;
|
|
41
|
+
const hasProvenance = PROVENANCE_SIGNALS.some((p) => p.test(content));
|
|
42
|
+
if (hasProvenance) continue;
|
|
43
|
+
const first = matches.sort((a, b) => a.line - b.line)[0];
|
|
44
|
+
findings.push(
|
|
45
|
+
createFinding({
|
|
46
|
+
tool: "provenance-check",
|
|
47
|
+
kind: "ai-governance.no-provenance",
|
|
48
|
+
severity: "P2",
|
|
49
|
+
file: toPosix(relativePath),
|
|
50
|
+
line: first.line,
|
|
51
|
+
evidence: getLineContent(content, first.line),
|
|
52
|
+
rootCause: "AI-generated content shipped without provenance metadata (ai-generated header, attribution line, C2PA manifest). Downstream can't tell it from human output.",
|
|
53
|
+
recommendedFix: "Tag generated content with an AI-generated marker (HTTP header, Markdown frontmatter, or C2PA manifest for images). Regulated domains (health, legal, elections) often require this by law.",
|
|
54
|
+
confidence: 0.5,
|
|
55
|
+
})
|
|
56
|
+
);
|
|
57
|
+
}
|
|
58
|
+
return findings;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
async function* iterateExplicitFiles(resolvedRoot, files) {
|
|
62
|
+
for (const file of files) {
|
|
63
|
+
const trimmed = String(file || "").trim();
|
|
64
|
+
if (!trimmed) continue;
|
|
65
|
+
const fullPath = path.isAbsolute(trimmed) ? trimmed : path.join(resolvedRoot, trimmed);
|
|
66
|
+
const relativePath = path.relative(resolvedRoot, fullPath).replace(/\\/g, "/");
|
|
67
|
+
yield { fullPath, relativePath };
|
|
68
|
+
}
|
|
69
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
// Maya (backend persona) — barrel export (#A14).
|
|
2
|
+
|
|
3
|
+
export {
|
|
4
|
+
BACKEND_TOOLS,
|
|
5
|
+
BACKEND_TOOL_IDS,
|
|
6
|
+
dispatchBackendTool,
|
|
7
|
+
runAllBackendTools,
|
|
8
|
+
runCircuitBreakerCheck,
|
|
9
|
+
runIdempotencyAudit,
|
|
10
|
+
runRetryAudit,
|
|
11
|
+
runTimeoutAudit,
|
|
12
|
+
} from "./tools/index.js";
|