pi-evalset-lab 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/README.md +59 -1
- package/examples/evalset-compare-sample-embedded.html +142 -0
- package/examples/evalset-compare-sample.png +0 -0
- package/package.json +38 -6
- package/scripts/export-evalset-report-html.mjs +356 -0
- package/.copier-answers.yml +0 -5
- package/.githooks/pre-commit +0 -12
- package/.github/CODEOWNERS +0 -12
- package/.github/ISSUE_TEMPLATE/bug-report.yml +0 -63
- package/.github/ISSUE_TEMPLATE/config.yml +0 -5
- package/.github/ISSUE_TEMPLATE/docs.yml +0 -39
- package/.github/ISSUE_TEMPLATE/feature-request.yml +0 -41
- package/.github/VOUCHED.td +0 -8
- package/.github/dependabot.yml +0 -13
- package/.github/pull_request_template.md +0 -34
- package/.github/workflows/ci.yml +0 -37
- package/.github/workflows/publish.yml +0 -60
- package/.github/workflows/release-please.yml +0 -25
- package/.github/workflows/vouch-check-pr.yml +0 -29
- package/.github/workflows/vouch-manage.yml +0 -34
- package/.pi/extensions/startup-intake-router.ts +0 -151
- package/.pi/prompts/init-project-docs.md +0 -32
- package/.release-please-config.json +0 -11
- package/.release-please-manifest.json +0 -3
- package/AGENTS.md +0 -39
- package/CODE_OF_CONDUCT.md +0 -50
- package/CONTRIBUTING.md +0 -28
- package/NEXT_SESSION_PROMPT.md +0 -14
- package/SECURITY.md +0 -34
- package/SUPPORT.md +0 -37
- package/docs/dev/CONTRIBUTING.md +0 -37
- package/docs/dev/EXTENSION_SOP.md +0 -43
- package/docs/dev/next_steps.md +0 -17
- package/docs/dev/plans/001-initial-plan.md +0 -24
- package/docs/dev/status.md +0 -21
- package/docs/org/operating_model.md +0 -39
- package/docs/org/project-docs-intake.questions.json +0 -60
- package/docs/project/foundation.md +0 -28
- package/docs/project/incentives.md +0 -17
- package/docs/project/resources.md +0 -26
- package/docs/project/skills.md +0 -17
- package/docs/project/strategic_goals.md +0 -18
- package/docs/project/tactical_goals.md +0 -39
- package/docs/project/vision.md +0 -21
- package/external/.gitkeep +0 -0
- package/ontology/.gitkeep +0 -0
- package/policy/security-policy.json +0 -10
- package/prek.toml +0 -15
- package/scripts/docs-list.sh +0 -50
- package/scripts/init-project-docs.sh +0 -56
- package/scripts/install-hooks.sh +0 -13
- package/scripts/sync-to-live.sh +0 -91
- package/scripts/validate-structure.sh +0 -325
- package/src/.gitkeep +0 -0
- package/tests/.gitkeep +0 -0
|
@@ -1,325 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
set -euo pipefail
|
|
3
|
-
|
|
4
|
-
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
5
|
-
cd "$ROOT_DIR"
|
|
6
|
-
|
|
7
|
-
required_files=(
|
|
8
|
-
"README.md"
|
|
9
|
-
"CHANGELOG.md"
|
|
10
|
-
"SECURITY.md"
|
|
11
|
-
"CODE_OF_CONDUCT.md"
|
|
12
|
-
"SUPPORT.md"
|
|
13
|
-
"CONTRIBUTING.md"
|
|
14
|
-
"AGENTS.md"
|
|
15
|
-
".copier-answers.yml"
|
|
16
|
-
"prek.toml"
|
|
17
|
-
".github/CODEOWNERS"
|
|
18
|
-
".github/dependabot.yml"
|
|
19
|
-
".github/pull_request_template.md"
|
|
20
|
-
".github/VOUCHED.td"
|
|
21
|
-
".github/ISSUE_TEMPLATE/bug-report.yml"
|
|
22
|
-
".github/ISSUE_TEMPLATE/feature-request.yml"
|
|
23
|
-
".github/ISSUE_TEMPLATE/docs.yml"
|
|
24
|
-
".github/ISSUE_TEMPLATE/config.yml"
|
|
25
|
-
".github/workflows/ci.yml"
|
|
26
|
-
".github/workflows/release-please.yml"
|
|
27
|
-
".github/workflows/publish.yml"
|
|
28
|
-
".github/workflows/vouch-check-pr.yml"
|
|
29
|
-
".github/workflows/vouch-manage.yml"
|
|
30
|
-
".release-please-config.json"
|
|
31
|
-
".release-please-manifest.json"
|
|
32
|
-
"docs/org/operating_model.md"
|
|
33
|
-
"docs/org/project-docs-intake.questions.json"
|
|
34
|
-
"docs/project/foundation.md"
|
|
35
|
-
"docs/project/vision.md"
|
|
36
|
-
"docs/project/incentives.md"
|
|
37
|
-
"docs/project/resources.md"
|
|
38
|
-
"docs/project/skills.md"
|
|
39
|
-
"docs/project/strategic_goals.md"
|
|
40
|
-
"docs/project/tactical_goals.md"
|
|
41
|
-
"docs/dev/next_steps.md"
|
|
42
|
-
"docs/dev/status.md"
|
|
43
|
-
"docs/dev/CONTRIBUTING.md"
|
|
44
|
-
"docs/dev/EXTENSION_SOP.md"
|
|
45
|
-
".pi/extensions/startup-intake-router.ts"
|
|
46
|
-
".pi/prompts/init-project-docs.md"
|
|
47
|
-
"scripts/sync-to-live.sh"
|
|
48
|
-
"scripts/install-hooks.sh"
|
|
49
|
-
"scripts/init-project-docs.sh"
|
|
50
|
-
"scripts/docs-list.sh"
|
|
51
|
-
"scripts/validate-structure.sh"
|
|
52
|
-
".githooks/pre-commit"
|
|
53
|
-
"prompts/implementation-planning.md"
|
|
54
|
-
"prompts/security-review.md"
|
|
55
|
-
"prompts/init-project-docs.md"
|
|
56
|
-
)
|
|
57
|
-
|
|
58
|
-
required_dirs=(
|
|
59
|
-
".github"
|
|
60
|
-
".github/workflows"
|
|
61
|
-
".github/ISSUE_TEMPLATE"
|
|
62
|
-
"docs/org"
|
|
63
|
-
"docs/dev/plans"
|
|
64
|
-
"examples"
|
|
65
|
-
"external"
|
|
66
|
-
"ontology"
|
|
67
|
-
"policy"
|
|
68
|
-
"scripts"
|
|
69
|
-
"src"
|
|
70
|
-
"tests"
|
|
71
|
-
".pi"
|
|
72
|
-
".pi/extensions"
|
|
73
|
-
".pi/prompts"
|
|
74
|
-
".githooks"
|
|
75
|
-
"prompts"
|
|
76
|
-
)
|
|
77
|
-
|
|
78
|
-
required_executables=(
|
|
79
|
-
"scripts/sync-to-live.sh"
|
|
80
|
-
"scripts/install-hooks.sh"
|
|
81
|
-
"scripts/init-project-docs.sh"
|
|
82
|
-
"scripts/docs-list.sh"
|
|
83
|
-
"scripts/validate-structure.sh"
|
|
84
|
-
".githooks/pre-commit"
|
|
85
|
-
)
|
|
86
|
-
|
|
87
|
-
errors=0
|
|
88
|
-
|
|
89
|
-
for required_file in "${required_files[@]}"; do
|
|
90
|
-
if [[ ! -f "$required_file" ]]; then
|
|
91
|
-
echo "Missing required file: $required_file" >&2
|
|
92
|
-
((errors+=1))
|
|
93
|
-
fi
|
|
94
|
-
done
|
|
95
|
-
|
|
96
|
-
for required_dir in "${required_dirs[@]}"; do
|
|
97
|
-
if [[ ! -d "$required_dir" ]]; then
|
|
98
|
-
echo "Missing required directory: $required_dir" >&2
|
|
99
|
-
((errors+=1))
|
|
100
|
-
fi
|
|
101
|
-
done
|
|
102
|
-
|
|
103
|
-
for executable in "${required_executables[@]}"; do
|
|
104
|
-
if [[ ! -x "$executable" ]]; then
|
|
105
|
-
echo "Expected executable bit on: $executable" >&2
|
|
106
|
-
((errors+=1))
|
|
107
|
-
fi
|
|
108
|
-
done
|
|
109
|
-
|
|
110
|
-
plan_count=$(find "docs/dev/plans" -maxdepth 1 -type f -name "*.md" | wc -l | tr -d ' ')
|
|
111
|
-
if [[ "$plan_count" -lt 1 ]]; then
|
|
112
|
-
echo "docs/dev/plans must contain at least one markdown plan file" >&2
|
|
113
|
-
((errors+=1))
|
|
114
|
-
fi
|
|
115
|
-
|
|
116
|
-
for copier_key in "_src_path:" "repo_name:" "command_name:"; do
|
|
117
|
-
if ! grep -q "^${copier_key}" ".copier-answers.yml"; then
|
|
118
|
-
echo "Missing copier answer key in .copier-answers.yml: ${copier_key}" >&2
|
|
119
|
-
((errors+=1))
|
|
120
|
-
fi
|
|
121
|
-
done
|
|
122
|
-
|
|
123
|
-
placeholder_pattern='\{username\}|\{repo\}|\{discordInvite\}|\{@twitter\}'
|
|
124
|
-
placeholder_hits="$(grep -R -nE "$placeholder_pattern" .github || true)"
|
|
125
|
-
if [[ -n "$placeholder_hits" ]]; then
|
|
126
|
-
echo "Unresolved placeholders found under .github:" >&2
|
|
127
|
-
echo "$placeholder_hits" >&2
|
|
128
|
-
((errors+=1))
|
|
129
|
-
fi
|
|
130
|
-
|
|
131
|
-
vouch_ref="0e11a71bba23218a284d3ecca162e75a110fd7e3"
|
|
132
|
-
if ! grep -q "mitchellh/vouch/action/check-pr@${vouch_ref}" ".github/workflows/vouch-check-pr.yml"; then
|
|
133
|
-
echo "vouch-check-pr workflow must pin mitchellh/vouch/action/check-pr to ${vouch_ref}" >&2
|
|
134
|
-
((errors+=1))
|
|
135
|
-
fi
|
|
136
|
-
|
|
137
|
-
if ! grep -q "mitchellh/vouch/action/manage-by-issue@${vouch_ref}" ".github/workflows/vouch-manage.yml"; then
|
|
138
|
-
echo "vouch-manage workflow must pin mitchellh/vouch/action/manage-by-issue to ${vouch_ref}" >&2
|
|
139
|
-
((errors+=1))
|
|
140
|
-
fi
|
|
141
|
-
|
|
142
|
-
if grep -n "@main" .github/workflows/vouch-*.yml >/dev/null 2>&1; then
|
|
143
|
-
echo "vouch workflows must not use @main refs" >&2
|
|
144
|
-
((errors+=1))
|
|
145
|
-
fi
|
|
146
|
-
|
|
147
|
-
if ! grep -q "pull_request_target" ".github/workflows/vouch-check-pr.yml"; then
|
|
148
|
-
echo "vouch-check-pr workflow must trigger on pull_request_target" >&2
|
|
149
|
-
((errors+=1))
|
|
150
|
-
fi
|
|
151
|
-
|
|
152
|
-
if ! grep -q "require-vouch" ".github/workflows/vouch-check-pr.yml"; then
|
|
153
|
-
echo "vouch-check-pr workflow must set require-vouch" >&2
|
|
154
|
-
((errors+=1))
|
|
155
|
-
fi
|
|
156
|
-
|
|
157
|
-
if ! grep -q "auto-close" ".github/workflows/vouch-check-pr.yml"; then
|
|
158
|
-
echo "vouch-check-pr workflow must set auto-close" >&2
|
|
159
|
-
((errors+=1))
|
|
160
|
-
fi
|
|
161
|
-
|
|
162
|
-
if ! grep -q "issue_comment" ".github/workflows/vouch-manage.yml"; then
|
|
163
|
-
echo "vouch-manage workflow must trigger on issue_comment" >&2
|
|
164
|
-
((errors+=1))
|
|
165
|
-
fi
|
|
166
|
-
|
|
167
|
-
if ! grep -q "concurrency:" ".github/workflows/vouch-manage.yml" || ! grep -q "group: vouch-manage" ".github/workflows/vouch-manage.yml"; then
|
|
168
|
-
echo "vouch-manage workflow must define serialized concurrency" >&2
|
|
169
|
-
((errors+=1))
|
|
170
|
-
fi
|
|
171
|
-
|
|
172
|
-
if ! grep -q "vouched-file: .github/VOUCHED.td" ".github/workflows/vouch-manage.yml"; then
|
|
173
|
-
echo "vouch-manage workflow must target .github/VOUCHED.td" >&2
|
|
174
|
-
((errors+=1))
|
|
175
|
-
fi
|
|
176
|
-
|
|
177
|
-
if grep -q "@your-github-handle" ".github/CODEOWNERS"; then
|
|
178
|
-
echo ".github/CODEOWNERS must not keep @your-github-handle placeholder" >&2
|
|
179
|
-
((errors+=1))
|
|
180
|
-
fi
|
|
181
|
-
|
|
182
|
-
if ! grep -Eq "^github:[A-Za-z0-9][A-Za-z0-9-]*" ".github/VOUCHED.td"; then
|
|
183
|
-
echo ".github/VOUCHED.td must include at least one github maintainer entry" >&2
|
|
184
|
-
((errors+=1))
|
|
185
|
-
fi
|
|
186
|
-
|
|
187
|
-
if command -v node >/dev/null 2>&1; then
|
|
188
|
-
if ! node - <<'NODE'
|
|
189
|
-
const fs = require("node:fs");
|
|
190
|
-
|
|
191
|
-
let failed = false;
|
|
192
|
-
const fail = (msg) => {
|
|
193
|
-
console.error(msg);
|
|
194
|
-
failed = true;
|
|
195
|
-
};
|
|
196
|
-
|
|
197
|
-
try {
|
|
198
|
-
const qPath = "docs/org/project-docs-intake.questions.json";
|
|
199
|
-
const q = JSON.parse(fs.readFileSync(qPath, "utf8"));
|
|
200
|
-
if (!q.title || !Array.isArray(q.questions) || q.questions.length === 0) {
|
|
201
|
-
fail(`Invalid interview questions file: ${qPath}`);
|
|
202
|
-
}
|
|
203
|
-
} catch (error) {
|
|
204
|
-
fail(`Failed to parse interview questions file: ${error.message}`);
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
try {
|
|
208
|
-
const p = JSON.parse(fs.readFileSync("package.json", "utf8"));
|
|
209
|
-
if (!Array.isArray(p.keywords) || !p.keywords.includes("pi-package")) {
|
|
210
|
-
fail("package.json missing keywords entry: pi-package");
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
const ext = p.pi?.extensions;
|
|
214
|
-
if (!Array.isArray(ext) || ext.length < 1) {
|
|
215
|
-
fail("package.json missing pi.extensions array");
|
|
216
|
-
} else {
|
|
217
|
-
for (const entry of ext) {
|
|
218
|
-
const normalized = entry.replace(/^\.\//, "");
|
|
219
|
-
if (!fs.existsSync(normalized)) {
|
|
220
|
-
fail(`pi.extensions entry does not exist: ${entry}`);
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
const prompts = p.pi?.prompts;
|
|
226
|
-
if (!Array.isArray(prompts) || prompts.length < 1) {
|
|
227
|
-
fail("package.json missing pi.prompts array");
|
|
228
|
-
} else {
|
|
229
|
-
for (const entry of prompts) {
|
|
230
|
-
const normalized = entry.replace(/\/$/, "").replace(/^\.\//, "");
|
|
231
|
-
if (!fs.existsSync(normalized)) {
|
|
232
|
-
fail(`pi.prompts entry does not exist: ${entry}`);
|
|
233
|
-
}
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
const checkScript = p.scripts?.check;
|
|
238
|
-
if (checkScript !== "bash ./scripts/validate-structure.sh") {
|
|
239
|
-
fail("package.json scripts.check must be 'bash ./scripts/validate-structure.sh'");
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
const testScript = p.scripts?.test;
|
|
243
|
-
if (testScript !== "bash ./scripts/validate-structure.sh") {
|
|
244
|
-
fail("package.json scripts.test must be 'bash ./scripts/validate-structure.sh'");
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
const docsListScript = p.scripts?.["docs:list"];
|
|
248
|
-
if (docsListScript !== "bash ./scripts/docs-list.sh") {
|
|
249
|
-
fail("package.json scripts.docs:list must be 'bash ./scripts/docs-list.sh'");
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
const docsListWorkspaceScript = p.scripts?.["docs:list:workspace"];
|
|
253
|
-
if (docsListWorkspaceScript !== "bash ./scripts/docs-list.sh --workspace --discover") {
|
|
254
|
-
fail("package.json scripts.docs:list:workspace must be 'bash ./scripts/docs-list.sh --workspace --discover'");
|
|
255
|
-
}
|
|
256
|
-
|
|
257
|
-
const docsListJsonScript = p.scripts?.["docs:list:json"];
|
|
258
|
-
if (docsListJsonScript !== "bash ./scripts/docs-list.sh --json") {
|
|
259
|
-
fail("package.json scripts.docs:list:json must be 'bash ./scripts/docs-list.sh --json'");
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
const rpConfig = JSON.parse(fs.readFileSync(".release-please-config.json", "utf8"));
|
|
263
|
-
if (rpConfig["include-v-in-tag"] !== true) {
|
|
264
|
-
fail(".release-please-config.json must set include-v-in-tag=true");
|
|
265
|
-
}
|
|
266
|
-
if (!rpConfig.packages || !rpConfig.packages["."]) {
|
|
267
|
-
fail(".release-please-config.json must include packages['.']");
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
const rpManifest = JSON.parse(fs.readFileSync(".release-please-manifest.json", "utf8"));
|
|
271
|
-
if (!rpManifest["."]) {
|
|
272
|
-
fail(".release-please-manifest.json must include '.' version entry");
|
|
273
|
-
}
|
|
274
|
-
const versionPattern = /^\d+\.\d+\.\d+([-.][0-9A-Za-z.]+)?$/;
|
|
275
|
-
if (!versionPattern.test(rpManifest["."])) {
|
|
276
|
-
fail(".release-please-manifest.json '.' entry must match X.Y.Z");
|
|
277
|
-
}
|
|
278
|
-
if (rpManifest["."] !== p.version) {
|
|
279
|
-
fail(".release-please-manifest.json '.' entry must match package.json version");
|
|
280
|
-
}
|
|
281
|
-
} catch (error) {
|
|
282
|
-
fail(`Failed to validate package/release metadata: ${error.message}`);
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
process.exit(failed ? 1 : 0);
|
|
286
|
-
NODE
|
|
287
|
-
then
|
|
288
|
-
((errors+=1))
|
|
289
|
-
fi
|
|
290
|
-
fi
|
|
291
|
-
|
|
292
|
-
while IFS= read -r -d '' markdown_file; do
|
|
293
|
-
if [[ "$(head -n 1 "$markdown_file")" != "---" ]]; then
|
|
294
|
-
echo "Missing YAML frontmatter start in: $markdown_file" >&2
|
|
295
|
-
((errors+=1))
|
|
296
|
-
continue
|
|
297
|
-
fi
|
|
298
|
-
|
|
299
|
-
if ! grep -q "^system4d:" "$markdown_file"; then
|
|
300
|
-
echo "Missing system4d section in: $markdown_file" >&2
|
|
301
|
-
((errors+=1))
|
|
302
|
-
continue
|
|
303
|
-
fi
|
|
304
|
-
|
|
305
|
-
for key in container compass engine fog; do
|
|
306
|
-
if ! grep -q "^ $key:" "$markdown_file"; then
|
|
307
|
-
echo "Missing system4d.$key in: $markdown_file" >&2
|
|
308
|
-
((errors+=1))
|
|
309
|
-
fi
|
|
310
|
-
done
|
|
311
|
-
|
|
312
|
-
if [[ "$markdown_file" == "./prompts/"* || "$markdown_file" == "./.pi/prompts/"* ]]; then
|
|
313
|
-
if ! grep -q "^description:" "$markdown_file"; then
|
|
314
|
-
echo "Prompt template missing frontmatter description: $markdown_file" >&2
|
|
315
|
-
((errors+=1))
|
|
316
|
-
fi
|
|
317
|
-
fi
|
|
318
|
-
done < <(find . -type f -name "*.md" ! -path "./.git/*" ! -path "./node_modules/*" -print0)
|
|
319
|
-
|
|
320
|
-
if [[ "$errors" -gt 0 ]]; then
|
|
321
|
-
echo "Structure validation failed with $errors issue(s)." >&2
|
|
322
|
-
exit 1
|
|
323
|
-
fi
|
|
324
|
-
|
|
325
|
-
echo "Structure validation passed."
|
package/src/.gitkeep
DELETED
|
File without changes
|
package/tests/.gitkeep
DELETED
|
File without changes
|