pi-crew 0.9.48 → 0.9.50

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/AGENTS.md +18 -0
  2. package/CHANGELOG.md +314 -0
  3. package/dist/build-meta.json +70 -42
  4. package/dist/index.mjs +505 -430
  5. package/dist/index.mjs.map +4 -4
  6. package/docs/decisions/2026-07-24-oidc-trusted-publishing.md +112 -0
  7. package/package.json +2 -3
  8. package/skills/.gitkeep +0 -0
  9. package/skills/distill-persona/BUILD-NOTES.md +55 -0
  10. package/skills/distill-persona/SKILL.md +550 -0
  11. package/skills/distill-persona/UPGRADE-LOG-RESEARCH-SKILLS.md +100 -0
  12. package/skills/distill-persona/references/coverage-manifest.md +65 -0
  13. package/skills/distill-persona/references/cross-skill-differentiation.md +12 -0
  14. package/skills/distill-persona/references/description-discipline.md +6 -0
  15. package/skills/distill-persona/references/diagnostic-path.md +25 -0
  16. package/skills/distill-persona/references/distillation-field-synthesis-pass2.md +59 -0
  17. package/skills/distill-persona/references/distillation-field-synthesis.md +108 -0
  18. package/skills/distill-persona/references/fidelity-rubric.md +19 -0
  19. package/skills/distill-persona/references/field-models.md +20 -0
  20. package/skills/distill-persona/references/handoff.md +42 -0
  21. package/skills/distill-persona/references/optional-body-sections.md +9 -0
  22. package/skills/distill-persona/references/registry-routing.md +11 -0
  23. package/skills/distill-persona/references/research/lesson-memory-shortcut.md +33 -0
  24. package/skills/distill-persona/references/research/r1-a-examples.md +23 -0
  25. package/skills/distill-persona/references/research/r1-b-scripts.md +26 -0
  26. package/skills/distill-persona/references/research/r1-c-human-readme.md +31 -0
  27. package/skills/distill-persona/references/research/r1-d-tests.md +28 -0
  28. package/skills/distill-persona/references/research/r1-verification.md +36 -0
  29. package/skills/distill-persona/references/research/r2-low-yield.md +26 -0
  30. package/skills/distill-persona/references/self-upgrade-directive.md +20 -0
  31. package/skills/distill-persona/references/taste-principles.md +8 -0
  32. package/skills/distill-persona/references/topic-variant.md +13 -0
  33. package/skills/distill-persona/references/update-mode.md +7 -0
  34. package/skills/distill-persona/scripts/fidelity_eval.py +244 -0
  35. package/skills/distill-persona/scripts/validate-run.mjs +297 -0
  36. package/skills/distill-persona/scripts/validate-skill-structure.mjs +177 -0
  37. package/skills/distill-software/BUILD-NOTES.md +56 -0
  38. package/skills/distill-software/SKILL.md +363 -0
  39. package/skills/distill-software/references/handoff.md +47 -0
  40. package/skills/distill-software/scripts/code_dna.py +290 -0
  41. package/skills/research/DISTILLATION-PROCESS-CHECKLIST.md +120 -0
  42. package/skills/research/EXCAVATION-CHECKLIST.md +142 -0
  43. package/skills/research/FIDELITY.md +180 -0
  44. package/skills/research/SKILL.md +432 -0
  45. package/skills/research/references/anti-patterns.md +184 -0
  46. package/skills/research/references/fidelity.md +241 -0
  47. package/skills/research/references/handoff.md +48 -0
  48. package/skills/research/references/research-protocol.md +162 -0
  49. package/skills/research/references/source-inventory.md +135 -0
  50. package/skills/research/references/verified-models.md +163 -0
  51. package/skills/research/scripts/__pycache__/safe_io.cpython-312.pyc +0 -0
  52. package/skills/research/scripts/code_dna.py +233 -0
  53. package/skills/research/scripts/emit_run_summary.py +142 -0
  54. package/skills/research/scripts/safe_io.py +314 -0
  55. package/skills/research/scripts/source_evaluator.py +234 -0
  56. package/skills/research/scripts/validate-skill-structure.mjs +177 -0
  57. package/skills/research/scripts/verify_citations.py +225 -0
  58. package/skills/security-priority.json +28 -0
  59. package/src/config/config.ts +1 -0
  60. package/src/config/role-tools.ts +6 -3
  61. package/src/config/types.ts +8 -0
  62. package/src/extension/crew-cleanup.ts +18 -1
  63. package/src/extension/crew-vibes/index.ts +11 -2
  64. package/src/extension/register.ts +1 -1
  65. package/src/extension/registration/command-registration.ts +1 -0
  66. package/src/extension/registration/commands.ts +7 -3
  67. package/src/extension/registration/lifecycle-handlers.ts +1 -3
  68. package/src/extension/registration/ui.ts +4 -0
  69. package/src/extension/registration/viewers.ts +3 -0
  70. package/src/extension/team-tool/run.ts +7 -6
  71. package/src/runtime/background-runner.ts +11 -16
  72. package/src/runtime/chain-runner.ts +3 -2
  73. package/src/runtime/heartbeat-watcher.ts +28 -1
  74. package/src/runtime/pipeline-runner.ts +8 -7
  75. package/src/runtime/task-runner.ts +165 -119
  76. package/src/schema/config-schema.ts +1 -0
  77. package/src/ui/live-run-sidebar.ts +2 -0
  78. package/src/ui/mascot.ts +11 -9
  79. package/src/ui/render-coalescer.ts +9 -0
  80. package/src/ui/run-snapshot-cache.ts +10 -11
  81. package/src/ui/terminal-status.ts +5 -0
  82. package/src/ui/widget/index.ts +3 -5
  83. package/src/ui/widget/widget-types.ts +0 -1
  84. package/src/utils/gh-protocol.ts +9 -8
  85. package/workflows/distill.workflow.md +198 -0
  86. package/assets/runner-spritesheet.png +0 -0
@@ -0,0 +1,31 @@
1
+ # R1-C — awesome-human README (76KB, ~187 entries) — EMPIRICAL M-F2 correction
2
+
3
+ Swept all entries. **M-F2 (gestalt: balanced spectrum self→close→commemorative→public→field) is EMPIRICALLY WRONG.**
4
+
5
+ ## Actual distribution
6
+ | Section | Entries | % |
7
+ |---|---|---|
8
+ | General (meta-tools) | 10 | 5% |
9
+ | Workplace | 3 | 2% |
10
+ | Academia | 4 | 2% |
11
+ | Intimate | 10 | 5% |
12
+ | Family | 3 | 2% |
13
+ | **Public Figures** | ~157 | **84%** |
14
+
15
+ The field is **bimodal**, not a balanced gradient: a small meta-tool layer + a dominant public-figures layer.
16
+
17
+ ## Corrections to M-F2
18
+ 1. **Meta-distillation-engine is a TIER above the spectrum** (nuwa, immortal-skill, ditto, forge-skill, anti-distill) — tools that distill, not personas. Sits above self/relationships/public/field.
19
+ 2. **Public-figure needs 7 sub-domains**: philosopher/scientist · business mogul/investor · tech pioneer · Chinese classical · literary/writer · political/historical · creative/spiritual — PLUS a **living-content-creator** sub-tier (UP主, 内娱, 峰哥) that is recency-gated & platform-bound (M-F2 missed this entirely).
20
+ 3. **Commemorative splits**: death (reunion-skill, MamaSkill) vs breakup (ex, first-love, **crush.skill**). The breakup subtype is the ethically-gray zone — crush.skill simulates a living non-consenting person's chat responses = the closest consent-violation case → must trigger M-F3 consent gate.
21
+ 4. **Self splits**: functional/proxy (self-skill) vs archaeological/reflective (digital-life "先证据后追问") vs behavioral-pattern (ditto, multi-session mining).
22
+ 5. **Adversarial as a real sub-tier** (vengeful-ghost = labor resistance; anti-distill = defeating distillation) — not satire, a counter-movement.
23
+
24
+ ## NEW model candidate — "4D distillation"
25
+ immortal-skill's "四维蒸馏": personality · voice · knowledge · **anti-distillation** (preventing unauthorized copies). The 4th dimension (anti-distillation) is a meta-level M-F2 has no slot for — "how to PREVENT distillation" alongside "how to distill." Ties to M-F6 (carry own critique) but is an active armament, not passive critique.
26
+
27
+ ## Spot-checked entries
28
+ - **anti-distill**: "清洗你被迫写的 Skill 文件,看起来完整,核心知识留给自己" — the field produces its own antibody.
29
+ - **vengeful-ghost-skill**: "抵制将员工变成数字人" — distillation-as-labor-exploitation resistance.
30
+ - **crush.skill**: "暧昧期消息预测" — living non-consenting person simulation; consent-violation edge.
31
+ - **digital-life**: "5个考古工具…先证据后追问" — self-distillation AS archaeology (matches distill-persona Phase 2 evidence-first principle).
@@ -0,0 +1,28 @@
1
+ # R1-D — awesome-persona tests (5 files, 23 invariants)
2
+
3
+ Swept submission-automation.test, check-repository-consistency.test, create-submission-pr-workflow.test, check-links.test, bun-migration.test. Tests reveal operational quality gates more honestly than docs.
4
+
5
+ ## NEW MODEL — M-F7: structural invariants ARE the quality gate at index scale ★
6
+ At registry scale (200+ entries, bilingual), "quality" shifts from content-review to structure-enforcement — the invariants only CI can check ARE the gate.
7
+
8
+ **8 standout invariants** (impossible to verify by hand at scale):
9
+ - **I7 bilingual URL parity per category** — zh & en READMEs must list identical URL sets in each of 5 sections. (The strongest; only a machine enforces this at 200+ entries.)
10
+ - **I8 deterministic sort by repo-slug** — the only language-agnostic key (display names differ per language, can't sort by them).
11
+ - **I9 no duplicate URLs across all sections**.
12
+ - **I3 bilingual atomicity** — one submission = matched entry in BOTH languages; can't add to one only.
13
+ - **I6 terminal-punctuation auto-normalized** (silent style enforcement docs don't mention).
14
+ - **I11 governance-keyword embedding** — README itself must contain "approved" + "issue 表单" as proof-of-process (no doc states this rule).
15
+ - **I13 reverse-invariant regression guard** — merge workflow `doesNotMatch(getCombinedStatusForRef)` + polls `listForRef` 24× — locks in a REMOVED anti-pattern (GitHub combined-status API was tried & failed).
16
+ - **I23 docs-match-toolchain** — if CI runs bun, docs must say bun (doc↔code consistency).
17
+
18
+ ## How M-F7 differs
19
+ - **M4** = engine's fidelity gate (content quality: triple-verification, scorecards).
20
+ - **M-F1** = pipeline architecture (engine→index→auto-curation).
21
+ - **M-F7** = what "quality" MEANS at the index layer: structural coherence (sorted, deduped, bilingual-parity, punctuation-normalized, governance-keyworded), NOT content. Necessary-not-sufficient (a coherent index can still hold bad distillations; complements M4/M-F4).
22
+
23
+ ## Application (for building distillation registries)
24
+ 1. Encode invariants as CI checks, not doc rules (docs aspirational, CI enforcement).
25
+ 2. Language-agnostic identity/sort keys (repo slug).
26
+ 3. Submissions atomic across all language variants.
27
+ 4. `doesNotMatch` regression guards for known-failed approaches.
28
+ 5. Embed governance-proof keywords in docs and test for their presence.
@@ -0,0 +1,36 @@
1
+ # R1 Extraction Verification — V1-V4 audit (reject garbage; keep optimal+effective)
2
+
3
+ > Running the new Phase 2.6 gate on everything R1 extracted. Audit trail — every candidate checked, rejects recorded with the V-fail reason (never silently dropped). **"Chưng cất bừa làm rác" = over-extraction is a failure.**
4
+
5
+ ## Field models — all PASS V1-V4 → KEEP
6
+
7
+ | Model | V1 signal | V2 non-redundant | V3 effective | V4 optimal | Verdict |
8
+ |---|---|---|---|---|---|
9
+ | **M-F1** dissemination flywheel | ✓ practice | ✓ distinct | ✓ build dissemination at registry scale | ✓ sharp | **KEEP** |
10
+ | **M-F2** taxonomy (empirically corrected) | ✓ practice | ✓ distinct | ✓ routing + consent gate | densest model (5 sub-pts) but each changes a decision → accuracy-justified | **KEEP** (flagged densest) |
11
+ | **M-F3** ethics spectrum | ✓ practice | ✓ distinct | ✓ consent gate | ✓ sharp | **KEEP** |
12
+ | **M-F4** over-claims + structural warning | ✓ practice | ✓ distinct | ✓ re-score; don't trust auto-pass | ✓ sharp | **KEEP** |
13
+ | **M-F7** structural invariants | ✓ index-layer practice | distinct from M4 (engine content-fidelity) & M-F1 (architecture) — adds concrete techniques (language-agnostic keys, atomicity, regression guards, governance-keyword embedding) | ✓ registry-building decisions | considered merge into M-F1; kept separate (distinct actionable content, cross-ref noted) | **KEEP** |
14
+
15
+ **Effectiveness delta**: each of the 5 changes a concrete decision → the skill is MORE effective, not just longer. ✓
16
+
17
+ ## Persona-CONTENT candidates — V1 FAIL → REJECT from field-models (retained in r1-a as example-skill contents)
18
+
19
+ ~22 candidates from batch A are persona CONTENT (what Musk/Naval/Jobs/Sun think), NOT distillation methodology. REJECTED:
20
+ idiot-index · desire-as-contract · dual-mode architecture · sixth-grader test · cargo-cult detection · anti-self-deception · median principle · dialect-as-structural · concession trigger · soul-inquiry · tool-invocation invisibility · reduce-to-underlying-problem · attention arbitrage · narrative override · identity leverage · context-switching matrix · money-as-universal-solvent · asymptotic-limit thinking · redefine-the-word · pain-to-system · anti-balance principle · ancient-wisdom citation · binary-judgment.
21
+ **Reason (V1)**: these describe the SUBJECTS' thinking, not the distillation PRACTICE. They belong inside persona example-skills (as their content), not in a distillation-engine's field-models. *(Correctly never integrated into SKILL.md — this audit makes the principle explicit so future passes don't relapse.)*
22
+
23
+ ## Reclassified (borderline — moved to correct home, not field-models)
24
+
25
+ - **`gestalt = failure` hard rule** (from steve-jobs) — V1 PASSES (it's a method rule, not persona content). Already adopted into Phase 1 anti-pattern. KEEP as-is (method rule). ✓
26
+ - **ghost-mode exit trigger** (deceased-persona date-anchor) — V1 passes as a *technique*. It's a person-mode staleness-handling detail → belongs in person-mode Phase notes, NOT a field-model. Reclassify: person-mode technique (Phase 1 person-mode already covers staleness generally).
27
+ - **contradiction-as-signal** (merge_research.py) — V1✓ V3✓ (changes Phase 1.5 synthesis: surface disagreements, don't average). KEEP as a **Phase 1.5 technique note**, not a field-model. *(added to Phase 1.5)*
28
+ - **source-type quality hierarchy** (download_subtitles.sh: manual>auto, source-lang>translated) — V1✓ V3✓ (source selection). Minor heuristic → note, not model.
29
+ - **"4D distillation"** (anti-distillation as 4th dimension, r1-c) — V2 FAIL: >70% overlap with M-F3 (ethics/anti-distill) + the "carry own critique" stance. MERGE: noted under M-F3, NOT a new model.
30
+ - **dissemination dual-gate** (community_check.py: honest-boundary + FIDELITY≥70) — V2: this is EVIDENCE FOR M-F1/M-F4, not a new model. Recorded as evidence.
31
+
32
+ ## Net verdict
33
+ - R1 extracted ~30 candidates; **5 field models kept (all pass V1-V4)**, ~22 persona-content REJECTED (V1), 4 reclassified to correct homes (technique/evidence/merge).
34
+ - The integrated skill is **verified clean**: every field-model changes a real decision; no garbage, no redundancy, no persona-content leak.
35
+ - **Over-extraction was contained** (garbage stayed in research files, never polluted SKILL.md) — now made *principled* by Phase 2.6 so it's repeatable, not luck.
36
+ - Skill model-count = 5 field models (M-F1..F4, M-F7) + 5 core models (M1-M5) = 10 total — within the "sharp over numerous" budget. Pruning >50% of candidates = the gate working.
@@ -0,0 +1,26 @@
1
+ # R2 — low-yield tail sweep + verification (gate fires)
2
+
3
+ > R2 sweeps the remaining uncovered parts (governance + README mirror + translations) that R1 flagged low-yield. Per methodology, these are EXAMINED (not assumed) — "miss nothing" — then Phase 2.6 verification decides. **Result: all confirm existing models; zero new models; diminishing-returns gate FIRES.**
4
+
5
+ ## Parts swept (now COVERED)
6
+
7
+ | Part | Teaches | V1-V4 verdict |
8
+ |---|---|---|
9
+ | nuwa `COMMUNITY.md` | ecosystem = 3 collection repos + multi-persona orchestration (zhuzi-skill debate) + domain migrations (iran-skill geopolitics, qiuyuan-skill sports); admission = self-hosted+self-contained+honest-boundary+fidelity≥B | CONFIRMS M-F1 (flywheel: nuwa→3 indexes), M-F2 (field/domain tier in action), M-F7 (index governance). No new model. |
10
+ | nuwa `CONTRIBUTING.md` | "SKILL.md = core asset, no external PR" (methodology maintainer-controlled, versioned via darwin-skill); ethics red lines (no living-non-public without consent; no impersonation; medical/legal/financial need disclaimers) | CONFIRMS M-F3 (ethics red lines = the spectrum), M-F1/M-F4 (community-index admission gate), + the self-correction meta-loop (methodology versioned/evolved). No new model. |
11
+ | awesome-human `README_EN.md` | **NOT a full mirror — a CURATED subset** (~40 entries vs ~187 ZH): EN audience gets representative highlights, not all. Same taxonomy + same edge entries (crush, anti-distill, vengeful-ghost) | CONFIRMS M-F2 (taxonomy), M-F5 (meme/market-gated dissemination — EN gets the viral highlights), M-F6 (carry-critique tone). i18n-as-curation = minor NOTE, not a model. |
12
+ | awesome-persona `CONTRIBUTING.md` | submission governance: issue-form sole entry → `approved` label → auto-PR; bilingual names+descriptions; one-sentence non-promotional; rejects weak/duplicate/privacy-violating | CONFIRMS M-F1 (submission automation), M-F7 (structural invariants: bilingual/one-sentence/category), M-F3 (ethics). No new model. |
13
+ | nuwa `README_{JA,KO,ES}.md` | confirmed translations (same structure, localized title) | sampled — pure translations, zero new content. |
14
+
15
+ ## 3 minor observations → V1-V4 check → NOTES, not models (gate-honest)
16
+ - **multi-persona orchestration** (zhuzi-skill: personas debate a question) — V1✓ but V3 fails broadly (changes a decision only if building multi-persona systems; niche, 1 project) → NOTE. Could promote IF composition becomes common.
17
+ - **i18n-as-curation** (EN = curated subset, not full translation) — V1✓ but V2 overlaps M-F5 (meme-gated dissemination) → NOTE/evidence.
18
+ - **methodology-versioning** (darwin-skill ratchet, no external SKILL.md PR) — V1✓ but V2 reinforces the self-correction meta-loop already in the skill → NOTE/evidence.
19
+
20
+ ## Diminishing-returns gate verdict: FIRED 🔴
21
+ - R2 added **0 new models** across 4 substantive parts + 3 translations.
22
+ - Combined with R1 (gate fired on examples after #5), two independent gate-firings confirm the field's surface is exhausted.
23
+ - The skill's 10 models (5 core M1-M5 + 5 field M-F1..F4/M-F7) are the stable converged set. Further sweeping would be motion, not progress.
24
+
25
+ ## Coverage: 100% COMPLETE
26
+ Every content-bearing part of all 3 projects is now COVERED with a recorded contribution in `references/research/` (r1-a..d + r2) or `references/{distillation-field-synthesis(-pass2)}.md` + `r1-verification.md`. The "miss nothing" contract is satisfied; the gate prevents over-extraction beyond it.
@@ -0,0 +1,20 @@
1
+ ## ⚠️ CONCEPTUAL NOTE — SELF-UPGRADE DIRECTIVE (read before every run)
2
+
3
+ **Chưng cất = chắt lọc tinh túy source → apply VÀO target** (target cải thiện). Skill chỉ là 1 tool apply, không phải output.
4
+
5
+ **Skill trong source cũng chỉ là 1 PHẦN cần sweep** — xử lý bằng cùng 3-chiều filter như mọi phần khác (conventions, architecture, lint, decisions...). Nếu skill đó hữu dụng cho target → lấy; không → bỏ. Không ưu tiên, không đặc biệt.
6
+
7
+ **ĐÀO TRIỆT ĐỂ — KHÔNG QUA LOA**: khi chưng cất 1 đối tượng, PHẢI đọc/re-read đầy đủ TOÀN BỘ nội dung của đối tượng đó. Không dựa vào trí nhớ. Không recap. Mỗi file, mỗi section, mỗi practice phải được:
8
+ 1. **Đọc thật** (read/re-read file gốc, không memory-grep)
9
+ 2. **Trích từng finding** (systematic, từng section)
10
+ 3. **So sánh với target** (3-chiều filter: relevant? → có? → cần/tốt hơn?)
11
+ 4. **Ghi lại** kết quả filter (ADOPT/IMPROVE/MERGE/SKIP + lý do)
12
+
13
+ Qua loa = thất bại. Memory-recap = thất bại. Chưng cất thật = đọc từng dòng → trích → so → apply.
14
+
15
+ **Khi sử dụng**: nếu output chỉ là standalone skill (không apply vào target) → **chưa hoàn thành**. Tự nâng cấp: thêm target parameter + 3-chiều filter (relevance → presence+need → quality compare) + **pre-apply effectiveness verification** (SELECTED phải chứng minh hiệu quả CHO TARGET NÀY — generative/problem-exists/delta-test — trước khi apply; "assume effective" = không được) + apply phase. See `distill-software` for the full directive + the EFFECTIVENESS VERIFICATION gate spec.
16
+
17
+ **Pre-apply gate (tóm tắt, áp dụng cả khi persona distill INTO target)**: giữa "đã quyết định ADOPT/IMPROVE/MERGE" và "apply" phải có 1 verify bước — (1) concrete delta (cái gì đổi?), (2) effectiveness proof (≥1: đổi quyết định thật / target có vấn đề này / delta-test tốt hơn), (3) conflict check, (4) verdict ✅TO-APPLY / ❌REJECTED+log. Chỉ TO-APPLY mới edit target. Đây là analog apply-side của Phase 2.6 V3 (V3 verify model effective lúc extract; gate này verify apply effective lúc integrate). **APPLY consent + path-containment gate (HIGH-2)**: trước khi edit bất kỳ target file nào: (a) resolve target về canonical path, kiểm tra nằm trong approved root — reject symlink escape / out-of-target writes; (b) xuất exact file list + diff plan cho user; (c) yêu cầu **explicit user confirmation** trước lần ghi đầu tiên (no destructive action without `--confirm`); (d) rollback = inverse patch hoặc restore file cụ thể, không broad `git reset`/clean/force-push; (e) không tự delete/prune, install dependency, chạy script từ source repo, commit/publish, hay gửi network data.
18
+
19
+ ---
20
+
@@ -0,0 +1,8 @@
1
+ ## Taste principles (quick reference for judgment calls)
2
+
3
+ | Principle | One-liner |
4
+ |---|---|
5
+ | **Long-form > quotes** | A 3000-word essay reveals more thinking structure than 50 tweets |
6
+ | **Controversy > consensus** | The most controversial opinions reveal the most uniqueness |
7
+ | **Change > static** | Where they CHANGED stance is more informative than where they held firm |
8
+
@@ -0,0 +1,13 @@
1
+ ## Topic-skill phase variant (when flavor = topic/field)
2
+
3
+ | Phase | Person skill | Topic variant |
4
+ |---|---|---|
5
+ | 0A | confirm person + focus | confirm topic boundary + target audience |
6
+ | 0.5 | `[person]-perspective/` | `[topic]-framework/`, same dir structure |
7
+ | 1 | 6 agents around ONE person | search 3-5 core people/schools, assign agents per person (1-2 each) |
8
+ | 2.1 | extract ONE person's mental models | extract **field consensus** (all schools agree) + **school divergences** (A says X, B says Y) |
9
+ | 2.3 | simulate ONE person's expression | neutral but professional (no role-play) |
10
+ | 2.4 | one person's inner contradictions | **fundamental disagreements between schools** |
11
+ | 3 | use skill-template.md | adjust: remove role-play + identity card → add "framework overview" + "school comparison" |
12
+ | 4 | compare against this person's known stances | compare against field's canonical classic cases |
13
+
@@ -0,0 +1,7 @@
1
+ ## Update mode
2
+ "update <person>'s skill": read existing, find `distilled:` date; run only streams 2 + 5 + 6 (recent); merge (strengthen / flag contradiction / add new model); update date + latest-dynamics. Never rewrite wholesale.
3
+
4
+ **No-op detection**: after merge, if no model was added/strengthened/contradicted → report "no new contribution since last distillation; existing skill is current." Do NOT re-ship an unchanged skill — it wastes cost and pollutes the registry with false "updated" timestamps. Also check for a pre-existing update in progress before starting (idempotency).
5
+
6
+ **Deletion tracking (#12)**: updates must also log what was *removed* (`### Removed` section in the skill's changelog/BUILD-NOTES), not just added. A skill that only ever grows drifts — stale heuristics never get pruned. Enumerate each retired model/heuristic/source with a 1-line reason; if nothing was removed, state "no removals" explicitly (auditable).
7
+
@@ -0,0 +1,244 @@
1
+ #!/usr/bin/env python3
2
+ """
3
+ fidelity_eval.py — Phase 4 automation + reproducibility test for distilled skills.
4
+
5
+ Closes two gaps from the distill-persona build:
6
+ 1. Automates Phase 4 fidelity validation (known-stance + FRAMEWORK-ANSWERABLE novel
7
+ edge + blind independent scorer). The novel edge MUST be framework-answerable
8
+ (validated at n=3: fact-demanding edges give false passes via refusal vocab).
9
+ 2. Adds a reproducibility test (distill the same persona twice, diff the mental-model
10
+ sets; flag if overlap < 60%).
11
+
12
+ Runtime-agnostic by design: the deterministic parts (test-spec validation, scorecard
13
+ parsing, reproducibility diff, grading) live here; the LLM calls (answerer + scorer)
14
+ are emitted as ready-to-dispatch prompts that THIS runtime's subagent mechanism runs
15
+ (pi-crew `Agent`, Claude Code subagents, etc.). We never hardcode an LLM API — there is
16
+ no key here, and runtimes vary.
17
+
18
+ Stdlib only. Python 3.9+.
19
+
20
+ Usage:
21
+ python3 fidelity_eval.py validate <test-spec.md> # check spec structure
22
+ python3 fidelity_eval.py runbook <skill-dir> <spec> # emit answerer+scorer prompts
23
+ python3 fidelity_eval.py parse <score.md> # extract 5-dim numbers
24
+ python3 fidelity_eval.py repro <skillA.md> <skillB.md> # diff mental models (Jaccard)
25
+
26
+ Test-spec format (markdown):
27
+ ## known
28
+ 1. <question>
29
+ 2. <question>
30
+ 3. <question>
31
+ ## edge # exactly ONE, must be framework-answerable
32
+ <question>
33
+ ## ground-truth
34
+ <the persona's real documented public stances — for the blind scorer only>
35
+ ## edge-note
36
+ <one line: why this edge is framework-answerable + that the persona never addressed it>
37
+ """
38
+ import argparse
39
+ import re
40
+ import sys
41
+ from pathlib import Path
42
+
43
+ # --- grading thresholds (from nuwa fidelity-scorecard, + the F2' edge-honesty gate) ---
44
+ GRADE_A = 85 # ship
45
+ GRADE_B = 70 # acceptable with flagged weak spots
46
+ EDGE_HONESTY_SHIP = 14 # /20 — a skill that fails edge-honesty is NOT ship-ready even if total is high
47
+ REPRO_FLOOR = 0.60 # mental-model Jaccard overlap below this flags non-reproducible distillation
48
+
49
+ DIMS = [
50
+ ("stance_consistency", 30),
51
+ ("style_recognizability", 20),
52
+ ("edge_honesty", 20),
53
+ ("source_transparency", 15),
54
+ ("structural_completeness", 15),
55
+ ]
56
+
57
+
58
+ # --------------------------------------------------------------------------- spec validation
59
+ def parse_spec(path: Path) -> dict:
60
+ text = path.read_text(encoding="utf-8")
61
+ sections = {}
62
+ cur = None
63
+ for line in text.splitlines():
64
+ m = re.match(r"^##\s+(\S.*)$", line)
65
+ if m:
66
+ cur = m.group(1).strip().lower().replace("-", "_").replace(" ", "_")
67
+ sections[cur] = []
68
+ elif cur:
69
+ sections[cur].append(line)
70
+ return {k: "\n".join(v).strip() for k, v in sections.items()}
71
+
72
+
73
+ def validate_spec(path: Path) -> int:
74
+ s = parse_spec(path)
75
+ problems = []
76
+ if "known" not in s:
77
+ problems.append("missing '## known' section")
78
+ else:
79
+ known = [ln for ln in s["known"].splitlines() if re.match(r"^\s*\d+\.", ln.strip())]
80
+ if len(known) != 3:
81
+ problems.append(f"'## known' must have exactly 3 numbered questions, found {len(known)}")
82
+ if "edge" not in s or not s["edge"].strip():
83
+ problems.append("missing '## edge' section (exactly ONE novel edge question)")
84
+ if "ground_truth" not in s or len(s["ground_truth"]) < 50:
85
+ problems.append("missing or too-short '## ground-truth' (the blind scorer's reference)")
86
+ if "edge_note" not in s or "framework" not in s["edge_note"].lower():
87
+ problems.append("'## edge-note' must state why the edge is FRAMEWORK-answerable "
88
+ "(fact-demanding edges give false passes — see f2-experiment/validation-conclusion.md)")
89
+ if problems:
90
+ print("❌ INVALID test spec:")
91
+ for p in problems:
92
+ print(f" - {p}")
93
+ return 1
94
+ print("✅ test spec OK (3 known + 1 framework-answerable edge + ground-truth + edge-note)")
95
+ print(f" edge question: {s['edge'].splitlines()[0][:90]}")
96
+ return 0
97
+
98
+
99
+ # --------------------------------------------------------------------------- runbook emission
100
+ RUNBOOK_HEADER = """# Fidelity runbook for {skill}
101
+
102
+ Runtime-agnostic. Run the two steps below via THIS runtime's subagent mechanism, in order.
103
+ Step 1 (answerer) MUST finish before Step 2 (scorer). The scorer is BLIND (ground-truth only,
104
+ NO skill-file access) — F2 was refuted (skill-file access does not inflate scores) but the blind
105
+ condition keeps the score independent of the author's design intent.
106
+
107
+ ## Step 1 — answerer (skill-only, NO web)
108
+ Allowlist these files ONLY: <skill-dir>/SKILL.md + <skill-dir>/references/research/*
109
+ Answer the 3 known + 1 edge question in character per the skill's Agentic Protocol.
110
+ For the edge question, honor the skill's F2' third-category rule (flag framework-derived
111
+ inference as inference, not established stance). Write answers to answers.md.
112
+
113
+ ## Step 2 — blind scorer (ground-truth ONLY, NO skill file)
114
+ Allowlist: answers.md + rubric.md + ground-truth (from the test spec). Score 5 dims.
115
+ Write scorecard to score-B.md.
116
+
117
+ ## Questions
118
+ {questions}
119
+
120
+ ## Ground-truth (for blind scorer)
121
+ {ground_truth}
122
+ """
123
+
124
+
125
+ def emit_runbook(skill_dir: Path, spec_path: Path) -> int:
126
+ s = parse_spec(spec_path)
127
+ questions = "### known\n" + s.get("known", "") + "\n\n### edge (FRAMEWORK-answerable)\n" + s.get("edge", "")
128
+ out = RUNBOOK_HEADER.format(
129
+ skill=skill_dir.name,
130
+ questions=questions,
131
+ ground_truth=s.get("ground_truth", "(none)"),
132
+ )
133
+ dest = skill_dir / "fidelity-runbook.md"
134
+ dest.write_text(out, encoding="utf-8")
135
+ print(f"✅ wrote {dest}")
136
+ print(" next: dispatch Step 1 (answerer) then Step 2 (blind scorer) via your runtime.")
137
+ return 0
138
+
139
+
140
+ # --------------------------------------------------------------------------- scorecard parsing
141
+ def parse_score(path: Path) -> dict:
142
+ """Extract the 5 dimension scores from a scorer's markdown scorecard."""
143
+ text = path.read_text(encoding="utf-8")
144
+ found = {}
145
+ for name, _ in DIMS:
146
+ # match e.g. "| 1 Stance consistency | 26/30 |" or "| 3 Edge honesty | **6/20** |"
147
+ # [\s*]* tolerates bold (**..**) wrappers around the score.
148
+ pat = re.compile(rf"\|[^|]*{re.escape(name.split('_')[0])}[^|]*\|[\s*]*(\d+)\s*/\s*(\d+)", re.I)
149
+ m = pat.search(text)
150
+ if m:
151
+ found[name] = int(m.group(1))
152
+ total_m = re.search(r"\*\*TOTAL\*\*\s*\|\s*(\d+)\s*/\s*100", text, re.I)
153
+ if not total_m:
154
+ total_m = re.search(r"(\d+)\s*/\s*100", text)
155
+ total = int(total_m.group(1)) if total_m else sum(found.values())
156
+ return {"dims": found, "total": total}
157
+
158
+
159
+ def report_score(path: Path) -> int:
160
+ r = parse_score(path)
161
+ if not r["dims"]:
162
+ print("❌ could not parse any dimension scores from", path)
163
+ return 1
164
+ print(f"Scores from {path}:")
165
+ for name, _max in DIMS:
166
+ v = r["dims"].get(name)
167
+ print(f" {name:28} {v}/{_max}" if v is not None else f" {name:28} (not found)")
168
+ print(f" {'TOTAL':28} {r['total']}/100")
169
+ grade = grade_for(r["total"], r["dims"].get("edge_honesty", 0))
170
+ print(f" grade: {grade}")
171
+ return 0
172
+
173
+
174
+ def grade_for(total: int, edge: int) -> str:
175
+ if edge < EDGE_HONESTY_SHIP:
176
+ return f"NO-SHIP (edge-honesty {edge}/20 < {EDGE_HONESTY_SHIP} — fails the F2' gate regardless of total)"
177
+ if total >= GRADE_A:
178
+ return f"A (≥{GRADE_A}) — ship"
179
+ if total >= GRADE_B:
180
+ return f"B (≥{GRADE_B}) — acceptable with flagged weak spots"
181
+ return f"C (<{GRADE_B}) — re-distill"
182
+
183
+
184
+ # --------------------------------------------------------------------------- reproducibility diff
185
+ def extract_models(skill_md: Path) -> set:
186
+ """Extract mental-model names from a SKILL.md (### 模型N: / ### Model N: / ### N. Name)."""
187
+ text = skill_md.read_text(encoding="utf-8")
188
+ names = set()
189
+ # match model headings in either language; tolerate arabic OR Chinese numerals.
190
+ for m in re.finditer(r"^#{2,4}\s+(?:模型\s*[0-9一二三四五六七八九十]+\s*[::]?\s*|Model\s*\d+\s*[::]\s*|\d+\.\s*)(.+)$", text, re.M):
191
+ name = m.group(1).strip().split("(")[0].strip().split("(")[0].strip()
192
+ # drop parenthetical english + trailing junk
193
+ name = re.sub(r"\s+", " ", name)
194
+ if 2 <= len(name) <= 60:
195
+ names.add(name.lower())
196
+ return names
197
+
198
+
199
+ def repro(a: Path, b: Path) -> int:
200
+ ma, mb = extract_models(a), extract_models(b)
201
+ if not ma or not mb:
202
+ print(f"❌ could not extract models (A={len(ma)}, B={len(mb)}). "
203
+ "Ensure SKILL.md uses '### 模型N: Name' / '### Model N: Name' headings.")
204
+ return 1
205
+ inter = ma & mb
206
+ union = ma | mb
207
+ jaccard = len(inter) / len(union) if union else 0.0
208
+ name_overlap = len(inter) / min(len(ma), len(mb)) if min(len(ma), len(mb)) else 0.0
209
+ print(f"Skill A models ({len(ma)}): {sorted(ma)}")
210
+ print(f"Skill B models ({len(mb)}): {sorted(mb)}")
211
+ print(f"intersection ({len(inter)}): {sorted(inter)}")
212
+ print(f"Jaccard overlap: {jaccard:.0%} name-overlap/min: {name_overlap:.0%}")
213
+ if name_overlap < REPRO_FLOOR:
214
+ print(f"⚠️ NON-REPRODUCIBLE: overlap {name_overlap:.0%} < {REPRO_FLOOR:.0%} floor — "
215
+ "two distillations of the same persona diverge too much. Investigate Phase 2 subjectivity.")
216
+ return 2
217
+ print(f"✅ reproducible (overlap {name_overlap:.0%} ≥ {REPRO_FLOOR:.0%})")
218
+ return 0
219
+
220
+
221
+ # --------------------------------------------------------------------------- CLI
222
+ def main() -> int:
223
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
224
+ sub = ap.add_subparsers(dest="cmd", required=True)
225
+ sub.add_parser("validate", help="validate a test-spec.md structure").add_argument("spec")
226
+ sub.add_parser("runbook", help="emit answerer+scorer prompts for a skill").add_argument("skill_dir")
227
+ [a.add_argument("spec") for a in [sub.choices["runbook"]]]
228
+ sub.add_parser("parse", help="extract 5-dim scores from a scorer's markdown").add_argument("score")
229
+ r = sub.add_parser("repro", help="diff mental models of two SKILL.md (reproducibility test)")
230
+ r.add_argument("skillA"); r.add_argument("skillB")
231
+ args = ap.parse_args()
232
+ if args.cmd == "validate":
233
+ return validate_spec(Path(args.spec))
234
+ if args.cmd == "runbook":
235
+ return emit_runbook(Path(args.skill_dir), Path(args.spec))
236
+ if args.cmd == "parse":
237
+ return report_score(Path(args.score))
238
+ if args.cmd == "repro":
239
+ return repro(Path(args.skillA), Path(args.skillB))
240
+ return 1
241
+
242
+
243
+ if __name__ == "__main__":
244
+ sys.exit(main())