pi-evalset-lab 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/CHANGELOG.md +11 -0
  2. package/README.md +59 -1
  3. package/examples/evalset-compare-sample-embedded.html +142 -0
  4. package/examples/evalset-compare-sample.png +0 -0
  5. package/package.json +38 -6
  6. package/scripts/export-evalset-report-html.mjs +356 -0
  7. package/.copier-answers.yml +0 -5
  8. package/.githooks/pre-commit +0 -12
  9. package/.github/CODEOWNERS +0 -12
  10. package/.github/ISSUE_TEMPLATE/bug-report.yml +0 -63
  11. package/.github/ISSUE_TEMPLATE/config.yml +0 -5
  12. package/.github/ISSUE_TEMPLATE/docs.yml +0 -39
  13. package/.github/ISSUE_TEMPLATE/feature-request.yml +0 -41
  14. package/.github/VOUCHED.td +0 -8
  15. package/.github/dependabot.yml +0 -13
  16. package/.github/pull_request_template.md +0 -34
  17. package/.github/workflows/ci.yml +0 -37
  18. package/.github/workflows/publish.yml +0 -60
  19. package/.github/workflows/release-please.yml +0 -25
  20. package/.github/workflows/vouch-check-pr.yml +0 -29
  21. package/.github/workflows/vouch-manage.yml +0 -34
  22. package/.pi/extensions/startup-intake-router.ts +0 -151
  23. package/.pi/prompts/init-project-docs.md +0 -32
  24. package/.release-please-config.json +0 -11
  25. package/.release-please-manifest.json +0 -3
  26. package/AGENTS.md +0 -39
  27. package/CODE_OF_CONDUCT.md +0 -50
  28. package/CONTRIBUTING.md +0 -28
  29. package/NEXT_SESSION_PROMPT.md +0 -14
  30. package/SECURITY.md +0 -34
  31. package/SUPPORT.md +0 -37
  32. package/docs/dev/CONTRIBUTING.md +0 -37
  33. package/docs/dev/EXTENSION_SOP.md +0 -43
  34. package/docs/dev/next_steps.md +0 -17
  35. package/docs/dev/plans/001-initial-plan.md +0 -24
  36. package/docs/dev/status.md +0 -21
  37. package/docs/org/operating_model.md +0 -39
  38. package/docs/org/project-docs-intake.questions.json +0 -60
  39. package/docs/project/foundation.md +0 -28
  40. package/docs/project/incentives.md +0 -17
  41. package/docs/project/resources.md +0 -26
  42. package/docs/project/skills.md +0 -17
  43. package/docs/project/strategic_goals.md +0 -18
  44. package/docs/project/tactical_goals.md +0 -39
  45. package/docs/project/vision.md +0 -21
  46. package/external/.gitkeep +0 -0
  47. package/ontology/.gitkeep +0 -0
  48. package/policy/security-policy.json +0 -10
  49. package/prek.toml +0 -15
  50. package/scripts/docs-list.sh +0 -50
  51. package/scripts/init-project-docs.sh +0 -56
  52. package/scripts/install-hooks.sh +0 -13
  53. package/scripts/sync-to-live.sh +0 -91
  54. package/scripts/validate-structure.sh +0 -325
  55. package/src/.gitkeep +0 -0
  56. package/tests/.gitkeep +0 -0
@@ -1,325 +0,0 @@
1
- #!/usr/bin/env bash
2
- set -euo pipefail
3
-
4
- ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
5
- cd "$ROOT_DIR"
6
-
7
- required_files=(
8
- "README.md"
9
- "CHANGELOG.md"
10
- "SECURITY.md"
11
- "CODE_OF_CONDUCT.md"
12
- "SUPPORT.md"
13
- "CONTRIBUTING.md"
14
- "AGENTS.md"
15
- ".copier-answers.yml"
16
- "prek.toml"
17
- ".github/CODEOWNERS"
18
- ".github/dependabot.yml"
19
- ".github/pull_request_template.md"
20
- ".github/VOUCHED.td"
21
- ".github/ISSUE_TEMPLATE/bug-report.yml"
22
- ".github/ISSUE_TEMPLATE/feature-request.yml"
23
- ".github/ISSUE_TEMPLATE/docs.yml"
24
- ".github/ISSUE_TEMPLATE/config.yml"
25
- ".github/workflows/ci.yml"
26
- ".github/workflows/release-please.yml"
27
- ".github/workflows/publish.yml"
28
- ".github/workflows/vouch-check-pr.yml"
29
- ".github/workflows/vouch-manage.yml"
30
- ".release-please-config.json"
31
- ".release-please-manifest.json"
32
- "docs/org/operating_model.md"
33
- "docs/org/project-docs-intake.questions.json"
34
- "docs/project/foundation.md"
35
- "docs/project/vision.md"
36
- "docs/project/incentives.md"
37
- "docs/project/resources.md"
38
- "docs/project/skills.md"
39
- "docs/project/strategic_goals.md"
40
- "docs/project/tactical_goals.md"
41
- "docs/dev/next_steps.md"
42
- "docs/dev/status.md"
43
- "docs/dev/CONTRIBUTING.md"
44
- "docs/dev/EXTENSION_SOP.md"
45
- ".pi/extensions/startup-intake-router.ts"
46
- ".pi/prompts/init-project-docs.md"
47
- "scripts/sync-to-live.sh"
48
- "scripts/install-hooks.sh"
49
- "scripts/init-project-docs.sh"
50
- "scripts/docs-list.sh"
51
- "scripts/validate-structure.sh"
52
- ".githooks/pre-commit"
53
- "prompts/implementation-planning.md"
54
- "prompts/security-review.md"
55
- "prompts/init-project-docs.md"
56
- )
57
-
58
- required_dirs=(
59
- ".github"
60
- ".github/workflows"
61
- ".github/ISSUE_TEMPLATE"
62
- "docs/org"
63
- "docs/dev/plans"
64
- "examples"
65
- "external"
66
- "ontology"
67
- "policy"
68
- "scripts"
69
- "src"
70
- "tests"
71
- ".pi"
72
- ".pi/extensions"
73
- ".pi/prompts"
74
- ".githooks"
75
- "prompts"
76
- )
77
-
78
- required_executables=(
79
- "scripts/sync-to-live.sh"
80
- "scripts/install-hooks.sh"
81
- "scripts/init-project-docs.sh"
82
- "scripts/docs-list.sh"
83
- "scripts/validate-structure.sh"
84
- ".githooks/pre-commit"
85
- )
86
-
87
- errors=0
88
-
89
- for required_file in "${required_files[@]}"; do
90
- if [[ ! -f "$required_file" ]]; then
91
- echo "Missing required file: $required_file" >&2
92
- ((errors+=1))
93
- fi
94
- done
95
-
96
- for required_dir in "${required_dirs[@]}"; do
97
- if [[ ! -d "$required_dir" ]]; then
98
- echo "Missing required directory: $required_dir" >&2
99
- ((errors+=1))
100
- fi
101
- done
102
-
103
- for executable in "${required_executables[@]}"; do
104
- if [[ ! -x "$executable" ]]; then
105
- echo "Expected executable bit on: $executable" >&2
106
- ((errors+=1))
107
- fi
108
- done
109
-
110
- plan_count=$(find "docs/dev/plans" -maxdepth 1 -type f -name "*.md" | wc -l | tr -d ' ')
111
- if [[ "$plan_count" -lt 1 ]]; then
112
- echo "docs/dev/plans must contain at least one markdown plan file" >&2
113
- ((errors+=1))
114
- fi
115
-
116
- for copier_key in "_src_path:" "repo_name:" "command_name:"; do
117
- if ! grep -q "^${copier_key}" ".copier-answers.yml"; then
118
- echo "Missing copier answer key in .copier-answers.yml: ${copier_key}" >&2
119
- ((errors+=1))
120
- fi
121
- done
122
-
123
- placeholder_pattern='\{username\}|\{repo\}|\{discordInvite\}|\{@twitter\}'
124
- placeholder_hits="$(grep -R -nE "$placeholder_pattern" .github || true)"
125
- if [[ -n "$placeholder_hits" ]]; then
126
- echo "Unresolved placeholders found under .github:" >&2
127
- echo "$placeholder_hits" >&2
128
- ((errors+=1))
129
- fi
130
-
131
- vouch_ref="0e11a71bba23218a284d3ecca162e75a110fd7e3"
132
- if ! grep -q "mitchellh/vouch/action/check-pr@${vouch_ref}" ".github/workflows/vouch-check-pr.yml"; then
133
- echo "vouch-check-pr workflow must pin mitchellh/vouch/action/check-pr to ${vouch_ref}" >&2
134
- ((errors+=1))
135
- fi
136
-
137
- if ! grep -q "mitchellh/vouch/action/manage-by-issue@${vouch_ref}" ".github/workflows/vouch-manage.yml"; then
138
- echo "vouch-manage workflow must pin mitchellh/vouch/action/manage-by-issue to ${vouch_ref}" >&2
139
- ((errors+=1))
140
- fi
141
-
142
- if grep -n "@main" .github/workflows/vouch-*.yml >/dev/null 2>&1; then
143
- echo "vouch workflows must not use @main refs" >&2
144
- ((errors+=1))
145
- fi
146
-
147
- if ! grep -q "pull_request_target" ".github/workflows/vouch-check-pr.yml"; then
148
- echo "vouch-check-pr workflow must trigger on pull_request_target" >&2
149
- ((errors+=1))
150
- fi
151
-
152
- if ! grep -q "require-vouch" ".github/workflows/vouch-check-pr.yml"; then
153
- echo "vouch-check-pr workflow must set require-vouch" >&2
154
- ((errors+=1))
155
- fi
156
-
157
- if ! grep -q "auto-close" ".github/workflows/vouch-check-pr.yml"; then
158
- echo "vouch-check-pr workflow must set auto-close" >&2
159
- ((errors+=1))
160
- fi
161
-
162
- if ! grep -q "issue_comment" ".github/workflows/vouch-manage.yml"; then
163
- echo "vouch-manage workflow must trigger on issue_comment" >&2
164
- ((errors+=1))
165
- fi
166
-
167
- if ! grep -q "concurrency:" ".github/workflows/vouch-manage.yml" || ! grep -q "group: vouch-manage" ".github/workflows/vouch-manage.yml"; then
168
- echo "vouch-manage workflow must define serialized concurrency" >&2
169
- ((errors+=1))
170
- fi
171
-
172
- if ! grep -q "vouched-file: .github/VOUCHED.td" ".github/workflows/vouch-manage.yml"; then
173
- echo "vouch-manage workflow must target .github/VOUCHED.td" >&2
174
- ((errors+=1))
175
- fi
176
-
177
- if grep -q "@your-github-handle" ".github/CODEOWNERS"; then
178
- echo ".github/CODEOWNERS must not keep @your-github-handle placeholder" >&2
179
- ((errors+=1))
180
- fi
181
-
182
- if ! grep -Eq "^github:[A-Za-z0-9][A-Za-z0-9-]*" ".github/VOUCHED.td"; then
183
- echo ".github/VOUCHED.td must include at least one github maintainer entry" >&2
184
- ((errors+=1))
185
- fi
186
-
187
- if command -v node >/dev/null 2>&1; then
188
- if ! node - <<'NODE'
189
- const fs = require("node:fs");
190
-
191
- let failed = false;
192
- const fail = (msg) => {
193
- console.error(msg);
194
- failed = true;
195
- };
196
-
197
- try {
198
- const qPath = "docs/org/project-docs-intake.questions.json";
199
- const q = JSON.parse(fs.readFileSync(qPath, "utf8"));
200
- if (!q.title || !Array.isArray(q.questions) || q.questions.length === 0) {
201
- fail(`Invalid interview questions file: ${qPath}`);
202
- }
203
- } catch (error) {
204
- fail(`Failed to parse interview questions file: ${error.message}`);
205
- }
206
-
207
- try {
208
- const p = JSON.parse(fs.readFileSync("package.json", "utf8"));
209
- if (!Array.isArray(p.keywords) || !p.keywords.includes("pi-package")) {
210
- fail("package.json missing keywords entry: pi-package");
211
- }
212
-
213
- const ext = p.pi?.extensions;
214
- if (!Array.isArray(ext) || ext.length < 1) {
215
- fail("package.json missing pi.extensions array");
216
- } else {
217
- for (const entry of ext) {
218
- const normalized = entry.replace(/^\.\//, "");
219
- if (!fs.existsSync(normalized)) {
220
- fail(`pi.extensions entry does not exist: ${entry}`);
221
- }
222
- }
223
- }
224
-
225
- const prompts = p.pi?.prompts;
226
- if (!Array.isArray(prompts) || prompts.length < 1) {
227
- fail("package.json missing pi.prompts array");
228
- } else {
229
- for (const entry of prompts) {
230
- const normalized = entry.replace(/\/$/, "").replace(/^\.\//, "");
231
- if (!fs.existsSync(normalized)) {
232
- fail(`pi.prompts entry does not exist: ${entry}`);
233
- }
234
- }
235
- }
236
-
237
- const checkScript = p.scripts?.check;
238
- if (checkScript !== "bash ./scripts/validate-structure.sh") {
239
- fail("package.json scripts.check must be 'bash ./scripts/validate-structure.sh'");
240
- }
241
-
242
- const testScript = p.scripts?.test;
243
- if (testScript !== "bash ./scripts/validate-structure.sh") {
244
- fail("package.json scripts.test must be 'bash ./scripts/validate-structure.sh'");
245
- }
246
-
247
- const docsListScript = p.scripts?.["docs:list"];
248
- if (docsListScript !== "bash ./scripts/docs-list.sh") {
249
- fail("package.json scripts.docs:list must be 'bash ./scripts/docs-list.sh'");
250
- }
251
-
252
- const docsListWorkspaceScript = p.scripts?.["docs:list:workspace"];
253
- if (docsListWorkspaceScript !== "bash ./scripts/docs-list.sh --workspace --discover") {
254
- fail("package.json scripts.docs:list:workspace must be 'bash ./scripts/docs-list.sh --workspace --discover'");
255
- }
256
-
257
- const docsListJsonScript = p.scripts?.["docs:list:json"];
258
- if (docsListJsonScript !== "bash ./scripts/docs-list.sh --json") {
259
- fail("package.json scripts.docs:list:json must be 'bash ./scripts/docs-list.sh --json'");
260
- }
261
-
262
- const rpConfig = JSON.parse(fs.readFileSync(".release-please-config.json", "utf8"));
263
- if (rpConfig["include-v-in-tag"] !== true) {
264
- fail(".release-please-config.json must set include-v-in-tag=true");
265
- }
266
- if (!rpConfig.packages || !rpConfig.packages["."]) {
267
- fail(".release-please-config.json must include packages['.']");
268
- }
269
-
270
- const rpManifest = JSON.parse(fs.readFileSync(".release-please-manifest.json", "utf8"));
271
- if (!rpManifest["."]) {
272
- fail(".release-please-manifest.json must include '.' version entry");
273
- }
274
- const versionPattern = /^\d+\.\d+\.\d+([-.][0-9A-Za-z.]+)?$/;
275
- if (!versionPattern.test(rpManifest["."])) {
276
- fail(".release-please-manifest.json '.' entry must match X.Y.Z");
277
- }
278
- if (rpManifest["."] !== p.version) {
279
- fail(".release-please-manifest.json '.' entry must match package.json version");
280
- }
281
- } catch (error) {
282
- fail(`Failed to validate package/release metadata: ${error.message}`);
283
- }
284
-
285
- process.exit(failed ? 1 : 0);
286
- NODE
287
- then
288
- ((errors+=1))
289
- fi
290
- fi
291
-
292
- while IFS= read -r -d '' markdown_file; do
293
- if [[ "$(head -n 1 "$markdown_file")" != "---" ]]; then
294
- echo "Missing YAML frontmatter start in: $markdown_file" >&2
295
- ((errors+=1))
296
- continue
297
- fi
298
-
299
- if ! grep -q "^system4d:" "$markdown_file"; then
300
- echo "Missing system4d section in: $markdown_file" >&2
301
- ((errors+=1))
302
- continue
303
- fi
304
-
305
- for key in container compass engine fog; do
306
- if ! grep -q "^ $key:" "$markdown_file"; then
307
- echo "Missing system4d.$key in: $markdown_file" >&2
308
- ((errors+=1))
309
- fi
310
- done
311
-
312
- if [[ "$markdown_file" == "./prompts/"* || "$markdown_file" == "./.pi/prompts/"* ]]; then
313
- if ! grep -q "^description:" "$markdown_file"; then
314
- echo "Prompt template missing frontmatter description: $markdown_file" >&2
315
- ((errors+=1))
316
- fi
317
- fi
318
- done < <(find . -type f -name "*.md" ! -path "./.git/*" ! -path "./node_modules/*" -print0)
319
-
320
- if [[ "$errors" -gt 0 ]]; then
321
- echo "Structure validation failed with $errors issue(s)." >&2
322
- exit 1
323
- fi
324
-
325
- echo "Structure validation passed."
package/src/.gitkeep DELETED
File without changes
package/tests/.gitkeep DELETED
File without changes