paperlint 13.3.1 → 13.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/domain/repro-layout.d.ts.map +1 -1
- package/dist/domain/repro-layout.js +9 -1
- package/dist/domain/repro-layout.js.map +1 -1
- package/dist/skills/build-label-page/scripts/label-page-core.d.ts +4 -0
- package/docs/layout.md +2 -0
- package/docs/rules/paper/allowed-files-only.md +2 -0
- package/docs/rules/repro/allowed-files-only.md +2 -0
- package/package.json +1 -1
- package/skills/assess-novelty/scripts/candidate_page.test.ts +1 -32
- package/skills/build-label-page/SKILL.md +3 -2
- package/skills/build-label-page/SKILL.md.spec.ts +2 -1
- package/skills/build-label-page/references/checklist.md +4 -2
- package/skills/build-label-page/references/closed-choice-generator.md +12 -3
- package/skills/build-label-page/scripts/label-page-browser.test.ts +187 -0
- package/skills/build-label-page/scripts/label-page-core.js +39 -7
- package/skills/build-label-page/scripts/label-page-core.test.ts +65 -2
- package/skills/build-label-page/scripts/label-page-core.ts +53 -12
- package/skills/build-label-page/scripts/label-page-html.test.ts +22 -0
- package/skills/build-label-page/scripts/label-page.js +1 -0
- package/skills/build-label-page/scripts/label-page.ts +1 -0
- package/skills/build-label-page/scripts/page-template.html +70 -11
- package/src/domain/repro-layout.test.ts +2 -0
- package/src/domain/repro-layout.ts +9 -1
- package/src/repro-layout-rule.test.ts +10 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"repro-layout.d.ts","sourceRoot":"","sources":["../../src/domain/repro-layout.ts"],"names":[],"mappings":"AAmCA,OAAO,EAAoB,KAAK,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAEnE,kEAAkE;AAClE,eAAO,MAAM,SAAS,UAAU,CAAC;AAEjC,iFAAiF;AACjF,eAAO,MAAM,YAAY,iBAAiB,CAAC;AAE3C,+EAA+E;AAC/E,eAAO,MAAM,MAAM,cAAc,CAAC;AAElC,6DAA6D;AAC7D,eAAO,MAAM,gBAAgB,EAAE,SAAS;IACtC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB,
|
|
1
|
+
{"version":3,"file":"repro-layout.d.ts","sourceRoot":"","sources":["../../src/domain/repro-layout.ts"],"names":[],"mappings":"AAmCA,OAAO,EAAoB,KAAK,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAEnE,kEAAkE;AAClE,eAAO,MAAM,SAAS,UAAU,CAAC;AAEjC,iFAAiF;AACjF,eAAO,MAAM,YAAY,iBAAiB,CAAC;AAE3C,+EAA+E;AAC/E,eAAO,MAAM,MAAM,cAAc,CAAC;AAElC,6DAA6D;AAC7D,eAAO,MAAM,gBAAgB,EAAE,SAAS;IACtC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB,EAkBA,CAAC;AACF,eAAO,MAAM,WAAW,EAAE,SAAS,MAAM,EAExC,CAAC;AAEF,sEAAsE;AACtE,eAAO,MAAM,SAAS,UAAU,CAAC;AACjC,eAAO,MAAM,QAAQ,SAAS,CAAC;AAC/B,eAAO,MAAM,eAAe,EAAE,SAAS;IACrC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB,EAMA,CAAC;AACF,eAAO,MAAM,UAAU,EAAE,SAAS,MAAM,EAAuC,CAAC;AAWhF,8CAA8C;AAC9C,eAAO,MAAM,WAAW,EAAE,SAAS,QAAQ,EAI1C,CAAC;AAEF,iGAAiG;AACjG,eAAO,MAAM,iBAAiB,EAAE,MAG/B,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,gBAAgB,EAAE,SAAS,MAAM,EAU7C,CAAC;AAEF,+EAA+E;AAC/E,eAAO,MAAM,UAAU,QAAgC,CAAC;AASxD,0FAA0F;AAC1F,eAAO,MAAM,qBAAqB,GAAI,MAAM,MAAM,KAAG,OACnB,CAAC;AAmBnC,8GAA8G;AAC9G,eAAO,MAAM,WAAW,GAAI,MAAM,MAAM,KAAG,OACY,CAAC;AAExD;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAOvD;AAED,oGAAoG;AACpG,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAOlD;AAED,0EAA0E;AAC1E,eAAO,MAAM,iBAAiB,EAAE,SAAS,MAAM,EAkB9C,CAAC;AAEF,2EAA2E;AAC3E,eAAO,MAAM,UAAU,EAAE,SAAS,MAAM,EAQvC,CAAC;AAEF,6EAA6E;AAC7E,eAAO,MAAM,eAAe,EAAE,SAAS,MAAM,EAqB5C,CAAC;AAEF,gCAAgC;AAChC,eAAO,MAAM,eAAe,EAAE,SAAS,MAAM,EAAyB,CAAC;AAEvE,2BAA2B;AAC3B,eAAO,MAAM,WAAW,+KACiJ,CAAC;AAE1K,uBAAuB;AACvB,eAAO,MAAM,SAAS,4IACgH,CAAC;AAEvI,yBAAyB;AACzB,eAAO,MAAM,SAAS,yJACkI,CAAC;AAEzJ,2CAA2C;AAC3C,eAAO,MAAM,iBAAiB,2MACmJ,CAAC;AAElL,mGAAmG;AACnG,eAAO,MAAM,WAAW,EAAE,SAAS;IACjC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB,EAaA,CAAC;AAkBF,0GAA0G;AAC1G,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,CAAC;IAC9B,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;CAC7B;AAED,0DAA0D;AAC1D,MAAM,WAAW,YAAY;IAC3B,0EAA0E;IAC1E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,0FAA0F;IAC1F,QAAQ,CAAC,OAAO,EAAE,OAAO,GAAG,WAAW,CAAC;IACxC,qCAAqC;IACrC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AA0BD,0GAA0G;AAC1G,wBAAgB,UAAU,CACxB,OAAO,EAAE,SAAS,UAAU,EAAE,GAC7B,SAAS,YAAY,EAAE,CAEzB;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,CAAC,EAAE,YAAY,GAAG,MAAM,CAWpD"}
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* ├── tools/ scripts used across the paper (numbers, figures, anonymity check, harnesses)
|
|
14
14
|
* └── data/ shared data (tables, caches)
|
|
15
15
|
*
|
|
16
|
-
* plus `.gitignore`. A loose note belongs in its study's folder or in the paper's `research/`.
|
|
16
|
+
* plus `.gitignore`, and optionally the agent instruction files `CLAUDE.md` and `AGENTS.md`. A loose note belongs in its study's folder or in the paper's `research/`.
|
|
17
17
|
*
|
|
18
18
|
* ── A STUDY'S NAME ───────────────────────────────────────────────────────────────
|
|
19
19
|
* `<id>-<slug>` (`P1-S03-baseline`): the id of the study's row in `repro/studies.json` — the paper's
|
|
@@ -51,6 +51,14 @@ export const REPRO_FILE_NOTES = [
|
|
|
51
51
|
note: "the study registry: id, title, status and result of each study",
|
|
52
52
|
},
|
|
53
53
|
{ name: ".gitignore", note: "keeps the scripts' byproducts out of git" },
|
|
54
|
+
{
|
|
55
|
+
name: "CLAUDE.md",
|
|
56
|
+
note: "optional: instructions an agent loads when it works in repro/",
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
name: "AGENTS.md",
|
|
60
|
+
note: "optional: the same, for agents that read AGENTS.md",
|
|
61
|
+
},
|
|
54
62
|
];
|
|
55
63
|
export const REPRO_FILES = REPRO_FILE_NOTES.map((f) => f.name);
|
|
56
64
|
/** The shared folders `repro/` may hold besides its study folders. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"repro-layout.js","sourceRoot":"","sources":["../../src/domain/repro-layout.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,UAAU,EAAiB,MAAM,kBAAkB,CAAC;AAEnE,kEAAkE;AAClE,MAAM,CAAC,MAAM,SAAS,GAAG,OAAO,CAAC;AAEjC,iFAAiF;AACjF,MAAM,CAAC,MAAM,YAAY,GAAG,cAAc,CAAC;AAE3C,+EAA+E;AAC/E,MAAM,CAAC,MAAM,MAAM,GAAG,WAAW,CAAC;AAElC,6DAA6D;AAC7D,MAAM,CAAC,MAAM,gBAAgB,GAGvB;IACJ;QACE,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,4DAA4D;KACnE;IACD;QACE,IAAI,EAAE,YAAY;QAClB,IAAI,EAAE,gEAAgE;KACvE;IACD,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,0CAA0C,EAAE;
|
|
1
|
+
{"version":3,"file":"repro-layout.js","sourceRoot":"","sources":["../../src/domain/repro-layout.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,UAAU,EAAiB,MAAM,kBAAkB,CAAC;AAEnE,kEAAkE;AAClE,MAAM,CAAC,MAAM,SAAS,GAAG,OAAO,CAAC;AAEjC,iFAAiF;AACjF,MAAM,CAAC,MAAM,YAAY,GAAG,cAAc,CAAC;AAE3C,+EAA+E;AAC/E,MAAM,CAAC,MAAM,MAAM,GAAG,WAAW,CAAC;AAElC,6DAA6D;AAC7D,MAAM,CAAC,MAAM,gBAAgB,GAGvB;IACJ;QACE,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,4DAA4D;KACnE;IACD;QACE,IAAI,EAAE,YAAY;QAClB,IAAI,EAAE,gEAAgE;KACvE;IACD,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,0CAA0C,EAAE;IACxE;QACE,IAAI,EAAE,WAAW;QACjB,IAAI,EAAE,+DAA+D;KACtE;IACD;QACE,IAAI,EAAE,WAAW;QACjB,IAAI,EAAE,oDAAoD;KAC3D;CACF,CAAC;AACF,MAAM,CAAC,MAAM,WAAW,GAAsB,gBAAgB,CAAC,GAAG,CAChE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CACd,CAAC;AAEF,sEAAsE;AACtE,MAAM,CAAC,MAAM,SAAS,GAAG,OAAO,CAAC;AACjC,MAAM,CAAC,MAAM,QAAQ,GAAG,MAAM,CAAC;AAC/B,MAAM,CAAC,MAAM,eAAe,GAGtB;IACJ;QACE,IAAI,EAAE,SAAS;QACf,IAAI,EAAE,8EAA8E;KACrF;IACD,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,8BAA8B,EAAE;CACzD,CAAC;AACF,MAAM,CAAC,MAAM,UAAU,GAAsB,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AAEhF,iGAAiG;AACjG,MAAM,UAAU,GAAa;IAC3B,IAAI,EAAE,cAAc;IACpB,IAAI,EAAE,mFAAmF;IACzF,QAAQ,EAAE;QACR,IAAI,CAAC,MAAM,EAAE,sDAAsD,CAAC;KACrE;CACF,CAAC;AAEF,8CAA8C;AAC9C,MAAM,CAAC,MAAM,WAAW,GAAwB;IAC9C,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IACpD,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IACzD,UAAU;CACX,CAAC;AAEF,iGAAiG;AACjG,MAAM,CAAC,MAAM,iBAAiB,GAAW,UAAU,CACjD,GAAG,SAAS,GAAG,EACf,WAAW,CACZ,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAsB;IACjD,aAAa;IACb,eAAe;IACf,aAAa;IACb,aAAa;IACb,oBAAoB;IACpB,cAAc;IACd,OAAO;IACP,WAAW;IACX,WAAW;CACZ,CAAC;AAEF,+EAA+E;AAC/E,MAAM,CAAC,MAAM,UAAU,GAAG,6BAA6B,CAAC;AAExD,+FAA+F;AAC/F,MAAM,UAAU,GAAG,4DAA4D,CAAC;AAEhF,6FAA6F;AAC7F,MAAM,qBAAqB,GACzB,oDAAoD,CAAC;AAEvD,0FAA0F;AAC1F,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,IAAY,EAAW,EAAE,CAC7D,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEnC,MAAM,UAAU,GAAG,CAAC,CAAS,EAAW,EAAE,CACxC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC;AAElD,gDAAgD;AAChD,MAAM,MAAM,GAAG,CAAC,CAAS,EAAE,CAAS,EAAU,EAAE,CAC9C,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAE5E,sGAAsG;AACtG,SAAS,gBAAgB,CAAC,IAAY;IACpC,MAAM,CAAC,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC7B,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3B,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,OAAO,KAAK,IAAI,CAAC,IAAI,KAAK,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC,IAAI,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AAC7E,CAAC;AAED,8GAA8G;AAC9G,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,IAAY,EAAW,EAAE,CACnD,qBAAqB,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC,IAAI,CAAC,CAAC;AAExD;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,MAAM,IAAI,GAAG,IAAI;SACd,WAAW,EAAE;SACb,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC;SAC5B,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IAC5B,IAAI,gBAAgB,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACxC,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,aAAa,CAAC;AAC7C,CAAC;AAED,oGAAoG;AACpG,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,MAAM,IAAI,GAAG,IAAI;SACd,WAAW,EAAE;SACb,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC;SAC5B,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;SACxB,OAAO,CAAC,iCAAiC,EAAE,EAAE,CAAC,CAAC;IAClD,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC;AACvC,CAAC;AAED,0EAA0E;AAC1E,MAAM,CAAC,MAAM,iBAAiB,GAAsB;IAClD,KAAK;IACL,QAAQ;IACR,KAAK;IACL,OAAO;IACP,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,IAAI;IACJ,KAAK;IACL,KAAK;IACL,KAAK;IACL,MAAM;IACN,MAAM;CACP,CAAC;AAEF,2EAA2E;AAC3E,MAAM,CAAC,MAAM,UAAU,GAAsB;IAC3C,UAAU;IACV,YAAY;IACZ,kBAAkB;IAClB,gBAAgB;IAChB,SAAS;IACT,cAAc;IACd,mBAAmB;CACpB,CAAC;AAEF,6EAA6E;AAC7E,MAAM,CAAC,MAAM,eAAe,GAAsB;IAChD,MAAM;IACN,MAAM;IACN,OAAO;IACP,QAAQ;IACR,SAAS;IACT,OAAO;IACP,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,UAAU;IACV,SAAS;IACT,KAAK;IACL,OAAO;IACP,MAAM;IACN,MAAM;IACN,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;CACP,CAAC;AAEF,gCAAgC;AAChC,MAAM,CAAC,MAAM,eAAe,GAAsB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;AAEvE,2BAA2B;AAC3B,MAAM,CAAC,MAAM,WAAW,GACtB,uKAAuK,CAAC;AAE1K,uBAAuB;AACvB,MAAM,CAAC,MAAM,SAAS,GACpB,oIAAoI,CAAC;AAEvI,yBAAyB;AACzB,MAAM,CAAC,MAAM,SAAS,GACpB,sJAAsJ,CAAC;AAEzJ,2CAA2C;AAC3C,MAAM,CAAC,MAAM,iBAAiB,GAC5B,+KAA+K,CAAC;AAElL,mGAAmG;AACnG,MAAM,CAAC,MAAM,WAAW,GAGlB;IACJ;QACE,IAAI,EAAE,uBAAuB;QAC7B,IAAI,EAAE,0NAA0N;KACjO;IACD;QACE,IAAI,EAAE,iBAAiB;QACvB,IAAI,EAAE,oJAAoJ;KAC3J;IACD;QACE,IAAI,EAAE,kCAAkC;QACxC,IAAI,EAAE,uKAAuK;KAC9K;CACF,CAAC;AAEF,uFAAuF;AACvF,SAAS,QAAQ,CAAC,IAAY;IAC5B,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;IACvD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC,IAAI,CAAC;IAC3C,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;IACxC,IAAI,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,iBAAiB,CAAC,QAAQ,CAAC,GAAG,CAAC;QAC9D,OAAO,WAAW,CAAC;IACrB,IAAI,eAAe,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IACpD,IAAI,eAAe,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IACpD,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED,sGAAsG;AACtG,MAAM,UAAU,GAAG,CAAC,IAAY,EAAU,EAAE,CAC1C,uBAAuB,aAAa,CAAC,IAAI,CAAC,kCAAkC,YAAY,4KAA4K,kBAAkB,CAAC,IAAI,CAAC,qIAAqI,IAAI,iJAAiJ,CAAC;AAmBzjB,MAAM,SAAS,GAAG,CAAC,CAAa,EAAW,EAAE,CAC3C,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC;IACjC,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM;QAChB,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9B,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;AAE1D,gEAAgE;AAChE,SAAS,UAAU,CAAC,CAAa;IAC/B,MAAM,IAAI,GAAG,GAAG,SAAS,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IACpE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;QACf,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,OAAO;YAChB,IAAI,EAAE,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC;SAC/D,CAAC;IACJ,IAAI,CAAC,CAAC,IAAI,KAAK,KAAK,IAAI,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;QACzD,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,WAAW;YACpB,IAAI,EAAE,OAAO,IAAI,GAAG,MAAM,8FAA8F,YAAY,EAAE;SACvI,CAAC;IACJ,OAAO,IAAI,CAAC;AACd,CAAC;AAED,0GAA0G;AAC1G,MAAM,UAAU,UAAU,CACxB,OAA8B;IAE9B,OAAO,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAqB,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;AAC9E,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,CAAe;IAC1C,MAAM,IAAI,GACR,CAAC,CAAC,OAAO,KAAK,OAAO;QACnB,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,2KAA2K,CAAC,CAAC,IAAI,GAAG;QACjM,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,kCAAkC,MAAM,oDAAoD,CAAC,CAAC,IAAI,GAAG,CAAC;IACvH,OAAO;QACL,IAAI;QACJ,iIAAiI;QACjI,iBAAiB;QACjB,qHAAqH;KACtH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC"}
|
|
@@ -19,6 +19,9 @@ type SchemaNode = {
|
|
|
19
19
|
} | {
|
|
20
20
|
kind: "integer";
|
|
21
21
|
required: boolean;
|
|
22
|
+
} | {
|
|
23
|
+
kind: "boolean";
|
|
24
|
+
required: boolean;
|
|
22
25
|
} | {
|
|
23
26
|
kind: "array";
|
|
24
27
|
required: boolean;
|
|
@@ -69,6 +72,7 @@ interface Config {
|
|
|
69
72
|
readonly seed: number;
|
|
70
73
|
readonly groups: Readonly<Record<string, string>>;
|
|
71
74
|
readonly labeller_knows?: readonly string[];
|
|
75
|
+
readonly familiarity?: boolean;
|
|
72
76
|
readonly familiarity_question?: string;
|
|
73
77
|
readonly questions: readonly Question[];
|
|
74
78
|
readonly practice: readonly Practice[];
|
package/docs/layout.md
CHANGED
|
@@ -72,6 +72,8 @@ creates one.
|
|
|
72
72
|
│ ├── README.md ← the index: one line per study, generated from studies.json
|
|
73
73
|
│ ├── studies.json ← the study registry: id, title, status and result of each study
|
|
74
74
|
│ ├── .gitignore ← keeps the scripts' byproducts out of git
|
|
75
|
+
│ ├── CLAUDE.md ← optional: instructions an agent loads when it works in repro/
|
|
76
|
+
│ ├── AGENTS.md ← optional: the same, for agents that read AGENTS.md
|
|
75
77
|
│ ├── tools/ ← scripts used across the paper (numbers, figures, anonymity check, harnesses)
|
|
76
78
|
│ ├── data/ ← shared data (tables, caches)
|
|
77
79
|
│ └── <id>-<slug>/ ← one folder per study or spike, named by its row in studies.json (P1-S03-baseline)
|
|
@@ -37,6 +37,8 @@ it: [the map](../../layout.md).
|
|
|
37
37
|
│ ├── README.md ← the index: one line per study, generated from studies.json
|
|
38
38
|
│ ├── studies.json ← the study registry: id, title, status and result of each study
|
|
39
39
|
│ ├── .gitignore ← keeps the scripts' byproducts out of git
|
|
40
|
+
│ ├── CLAUDE.md ← optional: instructions an agent loads when it works in repro/
|
|
41
|
+
│ ├── AGENTS.md ← optional: the same, for agents that read AGENTS.md
|
|
40
42
|
│ ├── tools/ ← scripts used across the paper (numbers, figures, anonymity check, harnesses)
|
|
41
43
|
│ ├── data/ ← shared data (tables, caches)
|
|
42
44
|
│ └── <id>-<slug>/ ← one folder per study or spike, named by its row in studies.json (P1-S03-baseline)
|
|
@@ -19,6 +19,8 @@ repro/
|
|
|
19
19
|
├── README.md ← the index: one line per study, generated from studies.json
|
|
20
20
|
├── studies.json ← the study registry: id, title, status and result of each study
|
|
21
21
|
├── .gitignore ← keeps the scripts' byproducts out of git
|
|
22
|
+
├── CLAUDE.md ← optional: instructions an agent loads when it works in repro/
|
|
23
|
+
├── AGENTS.md ← optional: the same, for agents that read AGENTS.md
|
|
22
24
|
├── tools/ ← scripts used across the paper (numbers, figures, anonymity check, harnesses)
|
|
23
25
|
├── data/ ← shared data (tables, caches)
|
|
24
26
|
└── <id>-<slug>/ ← one folder per study or spike, named by its row in studies.json (P1-S03-baseline)
|
package/package.json
CHANGED
|
@@ -26,42 +26,11 @@ import assert from "node:assert/strict";
|
|
|
26
26
|
const here = dirname(fileURLToPath(import.meta.url));
|
|
27
27
|
const TOOL = join(here, "candidate_page.ts");
|
|
28
28
|
const FIXTURE = join(here, "fixtures", "novelty");
|
|
29
|
-
// playwright-core is what package.json declares; the browser is whichever chromiumPath() finds (CHROMIUM_PATH, a
|
|
30
|
-
// build in PLAYWRIGHT_BROWSERS_PATH, or the Chrome a GitHub runner ships), so nothing is downloaded.
|
|
31
29
|
import { chromium } from "playwright-core";
|
|
32
|
-
import {
|
|
30
|
+
import { chromiumPath } from "../../../test/chromium.ts";
|
|
33
31
|
import { test } from "vitest";
|
|
34
32
|
|
|
35
33
|
test("candidate_page", async () => {
|
|
36
|
-
const BROWSERS_DIR =
|
|
37
|
-
process.env.PLAYWRIGHT_BROWSERS_PATH || "/opt/pw-browsers";
|
|
38
|
-
const SYSTEM_CHROME = [
|
|
39
|
-
"/usr/bin/google-chrome",
|
|
40
|
-
"/usr/bin/google-chrome-stable",
|
|
41
|
-
"/usr/bin/chromium",
|
|
42
|
-
"/usr/bin/chromium-browser",
|
|
43
|
-
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", // the macOS runner's Chrome
|
|
44
|
-
];
|
|
45
|
-
const chromiumPath = (): string => {
|
|
46
|
-
const builds = existsSync(BROWSERS_DIR)
|
|
47
|
-
? readdirSync(BROWSERS_DIR)
|
|
48
|
-
.filter((d) => /^chromium-\d+$/.test(d))
|
|
49
|
-
.sort((a, b) => Number(b.split("-")[1]) - Number(a.split("-")[1]))
|
|
50
|
-
.flatMap((d) =>
|
|
51
|
-
["chrome-linux64", "chrome-linux"].map((sub) =>
|
|
52
|
-
join(BROWSERS_DIR, d, sub, "chrome"),
|
|
53
|
-
),
|
|
54
|
-
)
|
|
55
|
-
: [];
|
|
56
|
-
const found = [process.env.CHROMIUM_PATH, ...builds, ...SYSTEM_CHROME].find(
|
|
57
|
-
(p) => p && existsSync(p),
|
|
58
|
-
);
|
|
59
|
-
if (!found)
|
|
60
|
-
throw new Error(
|
|
61
|
-
`no Chromium: set CHROMIUM_PATH, or install one in ${BROWSERS_DIR} or ${SYSTEM_CHROME[0]}`,
|
|
62
|
-
);
|
|
63
|
-
return found;
|
|
64
|
-
};
|
|
65
34
|
// A 1x1 PNG, enough for an <img> to load.
|
|
66
35
|
const PNG = Buffer.from(
|
|
67
36
|
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==",
|
|
@@ -4,7 +4,7 @@ description: Build a phone labelling page published as a claude.ai Artifact with
|
|
|
4
4
|
allowed-tools: [Read, Write, Edit, Grep, Glob, Bash, Artifact, ArtifactData]
|
|
5
5
|
---
|
|
6
6
|
|
|
7
|
-
<!-- vigiles:sha256:
|
|
7
|
+
<!-- vigiles:sha256:bd616b2694831fa1 compiled from skills/build-label-page/SKILL.md.spec.ts -->
|
|
8
8
|
|
|
9
9
|
# build-label-page — a page the person labels on a phone, and the labels back in the repo
|
|
10
10
|
|
|
@@ -63,7 +63,8 @@ building. In one line each:
|
|
|
63
63
|
source labels on the page. The generator refuses to build when an item carries a field outside an
|
|
64
64
|
allow-list; a page written by hand checks its items against the same list.
|
|
65
65
|
5. **Ask only what the labeller can answer.** Only the languages and tools they know; one "how well do
|
|
66
|
-
you know it" question per group, used later as an analysis filter
|
|
66
|
+
you know it" question per group, used later as an analysis filter (off on an approval page,
|
|
67
|
+
`"familiarity": false`: the author has not seen those repositories); "clear-cut or real boundary"
|
|
67
68
|
is the labeller's judgement, not the designer's.
|
|
68
69
|
6. **Export into the repository right away**, one JSON per page into the study's `data/` with the
|
|
69
70
|
export date, commit it, and record answers given in chat with `source: "chat"`. If the page fails,
|
|
@@ -74,7 +74,8 @@ building. In one line each:
|
|
|
74
74
|
source labels on the page. The generator refuses to build when an item carries a field outside an
|
|
75
75
|
allow-list; a page written by hand checks its items against the same list.
|
|
76
76
|
5. **Ask only what the labeller can answer.** Only the languages and tools they know; one "how well do
|
|
77
|
-
you know it" question per group, used later as an analysis filter
|
|
77
|
+
you know it" question per group, used later as an analysis filter (off on an approval page,
|
|
78
|
+
\`"familiarity": false\`: the author has not seen those repositories); "clear-cut or real boundary"
|
|
78
79
|
is the labeller's judgement, not the designer's.
|
|
79
80
|
6. **Export into the repository right away**, one JSON per page into the study's \`data/\` with the
|
|
80
81
|
export date, commit it, and record answers given in chat with \`source: "chat"\`. If the page fails,
|
|
@@ -83,7 +83,9 @@ the text of an allowed field says.
|
|
|
83
83
|
other groups are not put on the page and get no id.
|
|
84
84
|
- Add one question per group, "how well do you know this language / tool" (well / somewhat / not
|
|
85
85
|
really). It is an analysis filter: the study can report the labels with and without the groups
|
|
86
|
-
the labeller does not know well.
|
|
86
|
+
the labeller does not know well. An approval page asks none (`"familiarity": false` in
|
|
87
|
+
`items.json`): its question is about a drawn unit, not about repositories the author has seen,
|
|
88
|
+
and an answer to "how well do you know the projects of this group" there means nothing.
|
|
87
89
|
- The per-item judgement "clear-cut or a real boundary case" is the labeller's. The study design
|
|
88
90
|
does not decide it in advance and does not pre-fill it.
|
|
89
91
|
|
|
@@ -92,7 +94,7 @@ answer is the only way to find it afterwards. And a boundary case is boundary be
|
|
|
92
94
|
person hesitates; a designer who tags it in advance has measured the design, not the item.
|
|
93
95
|
|
|
94
96
|
**Check.** The build prints how many items were left out; the export has a familiarity value for
|
|
95
|
-
every group.
|
|
97
|
+
every group, or the page was built with `"familiarity": false` and the export has no groups.
|
|
96
98
|
|
|
97
99
|
## 6. Export into the repository right away
|
|
98
100
|
|
|
@@ -42,7 +42,9 @@ length.
|
|
|
42
42
|
- **`seed`**: fixes the order. Rebuilding with the same seed keeps every id, and a new item gets the
|
|
43
43
|
next one; a different seed is refused once a key exists.
|
|
44
44
|
- **`ask_why`** on an option: a prompt. That answer is incomplete, on the page and in the export,
|
|
45
|
-
until a reason is saved.
|
|
45
|
+
until a reason is saved. The reason saves as it is typed (after a pause, and when the page is
|
|
46
|
+
left), so a labeller who closes the page from inside the box does not leave a drop without its
|
|
47
|
+
reason; a store echo while typing keeps the box and its focus.
|
|
46
48
|
|
|
47
49
|
## The approval page
|
|
48
50
|
|
|
@@ -73,8 +75,15 @@ drop, the drop carrying `ask_why`.
|
|
|
73
75
|
}
|
|
74
76
|
```
|
|
75
77
|
|
|
76
|
-
The
|
|
77
|
-
|
|
78
|
+
The page sets `"familiarity": false`: the author is judging drawn units, not repositories they
|
|
79
|
+
have seen, so the per-group "how well do you know it" question is left out (and a
|
|
80
|
+
`familiarity_question` beside it is refused).
|
|
81
|
+
|
|
82
|
+
The items are the drawn units verbatim, in a language the author reads. Units in other languages
|
|
83
|
+
are taken out before the draw, as a population rule the study's `plan.json` states ("English-language
|
|
84
|
+
units only") with the number excluded reported in the paper; a unit dropped at approval because the
|
|
85
|
+
author cannot read it is an exclusion made after the draw and goes unrecorded. The practice is three
|
|
86
|
+
units whose fate is obvious. Each dropped unit carries its reason in the labels file. The author's words, with the date, then go into
|
|
78
87
|
`author_check` of the plan; a dropped unit is replaced by the next one in the draw.
|
|
79
88
|
|
|
80
89
|
## The export
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The generator's page in a real Chromium, against a stand-in store whose writes come back as a
|
|
3
|
+
* snapshot a moment later (as the claude.ai store does). What the fake DOM of label-page-html.test.ts
|
|
4
|
+
* cannot show: focus, typing, and a page that is left without the box ever losing focus.
|
|
5
|
+
*/
|
|
6
|
+
import assert from "node:assert/strict";
|
|
7
|
+
import { chromium, type Page } from "playwright-core";
|
|
8
|
+
import { test } from "vitest";
|
|
9
|
+
import { z } from "zod";
|
|
10
|
+
import { chromiumPath } from "../../../test/chromium.ts";
|
|
11
|
+
import {
|
|
12
|
+
APPROVAL_PRACTICE,
|
|
13
|
+
approval,
|
|
14
|
+
build,
|
|
15
|
+
setup,
|
|
16
|
+
} from "../../../test/label-page-kit.ts";
|
|
17
|
+
|
|
18
|
+
/** A store that keeps every write and echoes it back as a snapshot after `echoMs`. */
|
|
19
|
+
const STORE = (practice: Record<string, unknown>, echoMs: number): string => `
|
|
20
|
+
window.__writes = [];
|
|
21
|
+
(function () {
|
|
22
|
+
var docs = { answers: {}, groups: {}, practice: ${JSON.stringify(practice)} };
|
|
23
|
+
var subs = {};
|
|
24
|
+
function snap(n) {
|
|
25
|
+
(subs[n] || []).forEach(function (cb) {
|
|
26
|
+
cb({ docs: Object.keys(docs[n]).map(function (id) { return { id: id, data: function () { return docs[n][id]; } }; }) });
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
var db = { collection: function (n) { return {
|
|
30
|
+
onSnapshot: function (cb) { (subs[n] = subs[n] || []).push(cb); snap(n); },
|
|
31
|
+
doc: function (id) { return { set: function (body) {
|
|
32
|
+
window.__writes.push(body);
|
|
33
|
+
docs[n][id] = body;
|
|
34
|
+
return new Promise(function (r) { setTimeout(function () { r(); snap(n); }, ${String(echoMs)}); });
|
|
35
|
+
} }; }
|
|
36
|
+
}; } };
|
|
37
|
+
window.claude = { use: function () { return Promise.resolve(db); } };
|
|
38
|
+
})();`;
|
|
39
|
+
|
|
40
|
+
const PRACTICE_DONE = Object.fromEntries(
|
|
41
|
+
APPROVAL_PRACTICE.map((p, n) => [
|
|
42
|
+
`r1-P${String(n + 1)}`,
|
|
43
|
+
{ answers: p.expected, source: "page" },
|
|
44
|
+
]),
|
|
45
|
+
);
|
|
46
|
+
|
|
47
|
+
/** Every document the page wrote, in order, read back as JSON (the stand-in store keeps them). */
|
|
48
|
+
const Writes = z.array(
|
|
49
|
+
z.object({ reasons: z.record(z.string(), z.string()) }).loose(),
|
|
50
|
+
);
|
|
51
|
+
const writes = async (page: Page) =>
|
|
52
|
+
Writes.parse(
|
|
53
|
+
JSON.parse(await page.evaluate("JSON.stringify(window.__writes)")),
|
|
54
|
+
);
|
|
55
|
+
/** The reason the store was last given for the one question. */
|
|
56
|
+
const storedReason = async (page: Page): Promise<string | undefined> =>
|
|
57
|
+
(await writes(page)).at(-1)?.reasons["keep"];
|
|
58
|
+
|
|
59
|
+
/** Wait until the store's last write carries `text` as the reason (the page saves on its own). */
|
|
60
|
+
async function reasonStored(page: Page, text: string): Promise<void> {
|
|
61
|
+
await page.waitForFunction(
|
|
62
|
+
`(window.__writes || []).at(-1)?.reasons?.keep === ${JSON.stringify(text)}`,
|
|
63
|
+
undefined,
|
|
64
|
+
{ timeout: 10_000 },
|
|
65
|
+
);
|
|
66
|
+
assert.equal(await storedReason(page), text);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** A browser test launches Chromium and types: far above vitest's 5 s default on a slow runner. */
|
|
70
|
+
const BROWSER_TEST_MS = 60_000;
|
|
71
|
+
|
|
72
|
+
async function onApprovalPage(
|
|
73
|
+
name: string,
|
|
74
|
+
echoMs: number,
|
|
75
|
+
steps: (page: Page) => Promise<void>,
|
|
76
|
+
filter: "all" | "next" = "all",
|
|
77
|
+
): Promise<void> {
|
|
78
|
+
const { page: html } = build(setup(name, approval()));
|
|
79
|
+
const browser = await chromium.launch({
|
|
80
|
+
executablePath: chromiumPath(),
|
|
81
|
+
args: ["--no-sandbox"],
|
|
82
|
+
});
|
|
83
|
+
try {
|
|
84
|
+
const page = await browser.newPage();
|
|
85
|
+
await page.addInitScript(STORE(PRACTICE_DONE, echoMs));
|
|
86
|
+
await page.goto("about:blank");
|
|
87
|
+
await page.setContent(html, { waitUntil: "domcontentloaded" });
|
|
88
|
+
if (filter === "next")
|
|
89
|
+
await page.getByRole("button", { name: "Next unanswered" }).click();
|
|
90
|
+
await page.getByRole("button", { name: /Drop/ }).first().click();
|
|
91
|
+
await steps(page);
|
|
92
|
+
} finally {
|
|
93
|
+
await browser.close();
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
test(
|
|
98
|
+
"🔴 a reason typed and never blurred is saved: after a pause in typing, and when the page is left",
|
|
99
|
+
async () => {
|
|
100
|
+
await onApprovalPage("browser-pause", 20, async (page) => {
|
|
101
|
+
await page.locator("textarea").first().click();
|
|
102
|
+
await page.keyboard.type("a process rule");
|
|
103
|
+
await reasonStored(page, "a process rule");
|
|
104
|
+
});
|
|
105
|
+
await onApprovalPage("browser-leave", 20, async (page) => {
|
|
106
|
+
await page.locator("textarea").first().click();
|
|
107
|
+
await page.keyboard.type("not a rule");
|
|
108
|
+
await page.evaluate(() => {
|
|
109
|
+
window.dispatchEvent(new Event("pagehide"));
|
|
110
|
+
});
|
|
111
|
+
await reasonStored(page, "not a rule");
|
|
112
|
+
});
|
|
113
|
+
},
|
|
114
|
+
BROWSER_TEST_MS,
|
|
115
|
+
);
|
|
116
|
+
|
|
117
|
+
test(
|
|
118
|
+
"🔴 the store's echo while typing keeps the box, its focus and every keystroke",
|
|
119
|
+
async () => {
|
|
120
|
+
await onApprovalPage("browser-echo", 300, async (page) => {
|
|
121
|
+
await page.locator("textarea").first().click();
|
|
122
|
+
await page.keyboard.type("a pro");
|
|
123
|
+
await page.waitForTimeout(450); // the Drop's own write comes back and the page renders again
|
|
124
|
+
await page.keyboard.type("cess rule");
|
|
125
|
+
assert.equal(
|
|
126
|
+
await page.evaluate(() => document.activeElement?.tagName),
|
|
127
|
+
"TEXTAREA",
|
|
128
|
+
);
|
|
129
|
+
assert.equal(
|
|
130
|
+
await page.locator("textarea").first().inputValue(),
|
|
131
|
+
"a process rule",
|
|
132
|
+
);
|
|
133
|
+
await reasonStored(page, "a process rule");
|
|
134
|
+
});
|
|
135
|
+
},
|
|
136
|
+
BROWSER_TEST_MS,
|
|
137
|
+
);
|
|
138
|
+
|
|
139
|
+
test(
|
|
140
|
+
"🔴 under 'Next unanswered', the save that completes an item keeps the box being typed in on the page",
|
|
141
|
+
async () => {
|
|
142
|
+
await onApprovalPage(
|
|
143
|
+
"browser-next",
|
|
144
|
+
20,
|
|
145
|
+
async (page) => {
|
|
146
|
+
await page.locator("textarea").first().click();
|
|
147
|
+
await page.keyboard.type("a pro");
|
|
148
|
+
await reasonStored(page, "a pro"); // the item is complete now, and the filter hides complete items
|
|
149
|
+
await page.keyboard.type("cess rule");
|
|
150
|
+
assert.equal(
|
|
151
|
+
await page.evaluate(() => document.activeElement?.tagName),
|
|
152
|
+
"TEXTAREA",
|
|
153
|
+
);
|
|
154
|
+
await reasonStored(page, "a process rule");
|
|
155
|
+
},
|
|
156
|
+
"next",
|
|
157
|
+
);
|
|
158
|
+
},
|
|
159
|
+
BROWSER_TEST_MS,
|
|
160
|
+
);
|
|
161
|
+
|
|
162
|
+
test(
|
|
163
|
+
"a Keep needs no reason and writes none: the page stays quiet",
|
|
164
|
+
async () => {
|
|
165
|
+
const { page: html } = build(setup("browser-keep", approval()));
|
|
166
|
+
const browser = await chromium.launch({
|
|
167
|
+
executablePath: chromiumPath(),
|
|
168
|
+
args: ["--no-sandbox"],
|
|
169
|
+
});
|
|
170
|
+
try {
|
|
171
|
+
const page = await browser.newPage();
|
|
172
|
+
await page.addInitScript(STORE(PRACTICE_DONE, 20));
|
|
173
|
+
await page.goto("about:blank");
|
|
174
|
+
await page.setContent(html, { waitUntil: "domcontentloaded" });
|
|
175
|
+
await page.getByRole("button", { name: /Keep/ }).first().click();
|
|
176
|
+
await page.waitForTimeout(1200); // longer than the reason's pause: nothing more may arrive
|
|
177
|
+
assert.equal(await page.locator("textarea").count(), 0);
|
|
178
|
+
assert.deepEqual(
|
|
179
|
+
(await writes(page)).map((w) => w.reasons),
|
|
180
|
+
[{}],
|
|
181
|
+
);
|
|
182
|
+
} finally {
|
|
183
|
+
await browser.close();
|
|
184
|
+
}
|
|
185
|
+
},
|
|
186
|
+
BROWSER_TEST_MS,
|
|
187
|
+
);
|
|
@@ -123,6 +123,10 @@ const integer = (required = true) => ({
|
|
|
123
123
|
kind: "integer",
|
|
124
124
|
required,
|
|
125
125
|
});
|
|
126
|
+
const boolean = (required = true) => ({
|
|
127
|
+
kind: "boolean",
|
|
128
|
+
required,
|
|
129
|
+
});
|
|
126
130
|
const array = (items, required = true, min = 0) => ({
|
|
127
131
|
kind: "array",
|
|
128
132
|
required,
|
|
@@ -173,6 +177,9 @@ export const CONFIG_SCHEMA = obj({
|
|
|
173
177
|
seed: integer(),
|
|
174
178
|
groups: mapping(string(), NAME_PATTERN, 1), // ids become store document ids
|
|
175
179
|
labeller_knows: array(string(true, NAME_PATTERN), false),
|
|
180
|
+
// false: no "how well do you know {group}" question (an approval page: the author has never seen
|
|
181
|
+
// these repositories, so its answer means nothing). Left out, every group is asked (rule 5).
|
|
182
|
+
familiarity: boolean(false),
|
|
176
183
|
familiarity_question: string(false),
|
|
177
184
|
questions: array(QUESTION_SCHEMA, true, 1),
|
|
178
185
|
practice: array(PRACTICE_SCHEMA, true, MIN_PRACTICE),
|
|
@@ -227,6 +234,10 @@ function checkObject(value, node, path, where) {
|
|
|
227
234
|
fail(`${at(path, unknown)}: not a known field (known: ${Object.keys(node.fields).join(", ")})`);
|
|
228
235
|
}
|
|
229
236
|
}
|
|
237
|
+
function checkBoolean(value, where) {
|
|
238
|
+
if (typeof value !== "boolean")
|
|
239
|
+
fail(`${where}: must be true or false, got ${repr(value)}`);
|
|
240
|
+
}
|
|
230
241
|
/** Refuse the first place `value` departs from `node`; the message starts with the field's path. */
|
|
231
242
|
function checkValue(value, node, path) {
|
|
232
243
|
const where = path || "items.json";
|
|
@@ -239,6 +250,9 @@ function checkValue(value, node, path) {
|
|
|
239
250
|
fail(`${where}: must be an integer, got ${repr(value)}`);
|
|
240
251
|
}
|
|
241
252
|
break;
|
|
253
|
+
case "boolean":
|
|
254
|
+
checkBoolean(value, where);
|
|
255
|
+
break;
|
|
242
256
|
case "array":
|
|
243
257
|
checkArray(value, node, path, where);
|
|
244
258
|
break;
|
|
@@ -258,15 +272,17 @@ function assertConfig(value) {
|
|
|
258
272
|
* The page script (`DATA.allowed`) and the exporter (`key.json`) both read this one result, so a
|
|
259
273
|
* stored value counts as an answer on the page exactly when it counts in the export.
|
|
260
274
|
*/
|
|
261
|
-
const allowedValues = (questions) => ({
|
|
275
|
+
const allowedValues = (questions, asksFamiliarity) => ({
|
|
262
276
|
questions: Object.fromEntries(questions.map((q) => [q.id, q.options.map((o) => o.value)])),
|
|
263
277
|
// The answers that need a reason, per question: the page and the export read the same list.
|
|
264
278
|
needs_why: Object.fromEntries(questions.map((q) => [
|
|
265
279
|
q.id,
|
|
266
280
|
q.options.filter((o) => o.ask_why !== undefined).map((o) => o.value),
|
|
267
281
|
])),
|
|
268
|
-
familiarity:
|
|
282
|
+
// Empty when the page asks no familiarity: the export then expects no group document at all.
|
|
283
|
+
familiarity: asksFamiliarity ? FAMILIARITY.map((o) => o.value) : [],
|
|
269
284
|
});
|
|
285
|
+
const asksFamiliarity = (cfg) => cfg.familiarity !== false;
|
|
270
286
|
/** Two values that read the same once case and surrounding space are gone. */
|
|
271
287
|
const readsAs = (s) => s.trim().toLowerCase();
|
|
272
288
|
/** The first entry whose reading repeats an earlier one, with the index of that earlier one. */
|
|
@@ -330,6 +346,9 @@ function checkPractice(cfg) {
|
|
|
330
346
|
}
|
|
331
347
|
function checkConfig(cfg) {
|
|
332
348
|
checkGroups(cfg);
|
|
349
|
+
if (!asksFamiliarity(cfg) && cfg.familiarity_question !== undefined) {
|
|
350
|
+
fail(`familiarity_question: ${repr(cfg.familiarity_question)} is never asked, because \`familiarity\` is false — remove one of the two`);
|
|
351
|
+
}
|
|
333
352
|
cfg.questions.forEach((q, qn) => {
|
|
334
353
|
checkQuestion(q, qn, cfg.questions.slice(0, qn).map((x) => x.id));
|
|
335
354
|
});
|
|
@@ -432,9 +451,11 @@ function payloadOf(cfg, page) {
|
|
|
432
451
|
title: cfg.title,
|
|
433
452
|
intro: cfg.intro ?? "",
|
|
434
453
|
groups: used.map((g) => ({ id: g, label: String(cfg.groups[g]) })),
|
|
435
|
-
familiarity_question: cfg
|
|
436
|
-
|
|
437
|
-
|
|
454
|
+
familiarity_question: asksFamiliarity(cfg)
|
|
455
|
+
? (cfg.familiarity_question ?? "How well do you know {group}?")
|
|
456
|
+
: "",
|
|
457
|
+
familiarity: asksFamiliarity(cfg) ? FAMILIARITY : [],
|
|
458
|
+
allowed: allowedValues(cfg.questions, asksFamiliarity(cfg)),
|
|
438
459
|
questions: cfg.questions,
|
|
439
460
|
practice: { round: practiceRound(cfg), items: practiceOf(cfg) },
|
|
440
461
|
items: page.items.map((i) => ({ id: page.idOf(i), ...shownFields(i) })),
|
|
@@ -478,7 +499,7 @@ function keyOf(cfg, page, earlier) {
|
|
|
478
499
|
questions: cfg.questions.map((q) => q.id),
|
|
479
500
|
// What counts as an answer: the same lists the page was given (`allowedValues`). `export`
|
|
480
501
|
// checks every stored answer and familiarity against them, so the build's options decide.
|
|
481
|
-
allowed: allowedValues(cfg.questions),
|
|
502
|
+
allowed: allowedValues(cfg.questions, asksFamiliarity(cfg)),
|
|
482
503
|
// The practice is scored against this at export; its answers never go into the labels.
|
|
483
504
|
practice: {
|
|
484
505
|
round: practiceRound(cfg),
|
|
@@ -658,6 +679,14 @@ function checkDump(dump, key) {
|
|
|
658
679
|
}
|
|
659
680
|
}
|
|
660
681
|
}
|
|
682
|
+
/** A page built with `familiarity: false` asks no group anything: a group document came from elsewhere. */
|
|
683
|
+
function checkNoFamiliarity(dump, allowed) {
|
|
684
|
+
const groups = isObject(dump["groups"]) ? dump["groups"] : {};
|
|
685
|
+
const [first] = Object.keys(groups).sort(byText);
|
|
686
|
+
if (allowed.familiarity.length === 0 && first !== undefined) {
|
|
687
|
+
fail(`groups/${first}: familiarity is off for this page (items.json \`familiarity: false\`), so no group document is expected — take it out of the dump`);
|
|
688
|
+
}
|
|
689
|
+
}
|
|
661
690
|
/** The key's `allowed` lists, or a refusal naming a key made by an older build. */
|
|
662
691
|
function assertAllowed(a, key, keyPath) {
|
|
663
692
|
const questions = isObject(a) ? a["questions"] : undefined;
|
|
@@ -818,7 +847,9 @@ function exportRound(study, round, items, at) {
|
|
|
818
847
|
function labelsOf(dump, ctx, day) {
|
|
819
848
|
const onPage = Object.entries(ctx.key.ids).filter(([, m]) => m.on_page);
|
|
820
849
|
const items = onPage.map(([nid, meta]) => labelOf(nid, meta, docOf(dump["answers"], nid), ctx));
|
|
821
|
-
const groups =
|
|
850
|
+
const groups = ctx.allowed.familiarity.length
|
|
851
|
+
? groupsOf([...new Set(onPage.map(([, m]) => m.group))], dump["groups"], ctx.allowed)
|
|
852
|
+
: {};
|
|
822
853
|
return {
|
|
823
854
|
study: ctx.key.study,
|
|
824
855
|
exported: day,
|
|
@@ -839,6 +870,7 @@ export function exportLabels(dumpPath, keyPath, outDir, day) {
|
|
|
839
870
|
const allowed = key.allowed;
|
|
840
871
|
assertAllowed(allowed, key, keyPath);
|
|
841
872
|
checkDump(dump, key);
|
|
873
|
+
checkNoFamiliarity(dump, allowed);
|
|
842
874
|
const out = labelsOf(dump, { key, allowed }, day);
|
|
843
875
|
mkdirSync(outDir, { recursive: true });
|
|
844
876
|
const path = join(outDir, `${key.study}-labels-${day}.json`);
|
|
@@ -14,6 +14,7 @@ import {
|
|
|
14
14
|
ITEMS,
|
|
15
15
|
PRACTICE,
|
|
16
16
|
QUESTIONS,
|
|
17
|
+
WORK,
|
|
17
18
|
approval,
|
|
18
19
|
build,
|
|
19
20
|
config,
|
|
@@ -26,7 +27,7 @@ import {
|
|
|
26
27
|
/** A node of the declared config schema, as `example-generator.py schema` prints it. */
|
|
27
28
|
const Node: z.ZodType<SchemaNode> = z.lazy(() =>
|
|
28
29
|
z.object({
|
|
29
|
-
kind: z.enum(["string", "integer", "map", "array", "object"]),
|
|
30
|
+
kind: z.enum(["string", "integer", "boolean", "map", "array", "object"]),
|
|
30
31
|
required: z.boolean(),
|
|
31
32
|
pattern: z.string().optional(),
|
|
32
33
|
key_pattern: z.string().optional(),
|
|
@@ -39,7 +40,7 @@ const Node: z.ZodType<SchemaNode> = z.lazy(() =>
|
|
|
39
40
|
);
|
|
40
41
|
|
|
41
42
|
interface SchemaNode {
|
|
42
|
-
readonly kind: "string" | "integer" | "map" | "array" | "object";
|
|
43
|
+
readonly kind: "string" | "integer" | "boolean" | "map" | "array" | "object";
|
|
43
44
|
readonly required: boolean;
|
|
44
45
|
readonly pattern?: string | undefined;
|
|
45
46
|
readonly key_pattern?: string | undefined;
|
|
@@ -77,6 +78,7 @@ const FULL: DocObject = (() => {
|
|
|
77
78
|
QUESTIONS[1],
|
|
78
79
|
],
|
|
79
80
|
labeller_knows: ["python", "go", "rust"],
|
|
81
|
+
familiarity: true,
|
|
80
82
|
familiarity_question: "How well do you know {group}?",
|
|
81
83
|
items: ITEMS.map((i, n) =>
|
|
82
84
|
n === 0 ? { ...i, title: "T", code: "c = 1", context: "ctx" } : i,
|
|
@@ -175,6 +177,7 @@ function spoil(path: string, bad: Doc | undefined): DocObject {
|
|
|
175
177
|
const WRONG_TYPES: Readonly<Record<SchemaNode["kind"], readonly Doc[]>> = {
|
|
176
178
|
string: [null, 1, true, ["x"], {}, "", " "],
|
|
177
179
|
integer: [null, "7", true, 7.5, []],
|
|
180
|
+
boolean: [null, "false", 0, [], {}],
|
|
178
181
|
map: [null, "x", ["x"], 1],
|
|
179
182
|
array: [null, "x", {}, 1],
|
|
180
183
|
object: [null, "x", ["x"], 1],
|
|
@@ -238,6 +241,7 @@ const DOCUMENTED_FIELDS = [
|
|
|
238
241
|
"seed",
|
|
239
242
|
"groups",
|
|
240
243
|
"labeller_knows",
|
|
244
|
+
"familiarity",
|
|
241
245
|
"familiarity_question",
|
|
242
246
|
"questions",
|
|
243
247
|
"items",
|
|
@@ -1162,3 +1166,62 @@ test("🔴 a question dropped between practice rounds: the earlier round's answe
|
|
|
1162
1166
|
assert.equal(r.status, 0, r.out);
|
|
1163
1167
|
assert.match(r.out, /practice round 1: 3 of 3 matched/);
|
|
1164
1168
|
});
|
|
1169
|
+
|
|
1170
|
+
test("🔴 familiarity off: the key asks for none, the export reports no missing familiarity, and a stored one is refused", () => {
|
|
1171
|
+
const dir = setup("familiarity-off", approval({ familiarity: false }));
|
|
1172
|
+
const { key } = build(dir);
|
|
1173
|
+
assert.deepEqual(key.allowed.familiarity, []);
|
|
1174
|
+
const [a] = Object.keys(key.ids);
|
|
1175
|
+
assert.ok(a !== undefined);
|
|
1176
|
+
writeFileSync(
|
|
1177
|
+
join(dir, "dump.json"),
|
|
1178
|
+
JSON.stringify({
|
|
1179
|
+
answers: { [a]: { answers: { keep: "keep" }, source: "page" } },
|
|
1180
|
+
groups: {},
|
|
1181
|
+
}),
|
|
1182
|
+
);
|
|
1183
|
+
const r = exportAt(dir);
|
|
1184
|
+
assert.equal(r.status, 0, r.out);
|
|
1185
|
+
assert.doesNotMatch(r.out, /no familiarity answer|invalid group/);
|
|
1186
|
+
assert.deepEqual(Exported.parse(JSON.parse(labelsFile(dir))).groups, {});
|
|
1187
|
+
writeFileSync(
|
|
1188
|
+
join(dir, "dump.json"),
|
|
1189
|
+
JSON.stringify({
|
|
1190
|
+
answers: {},
|
|
1191
|
+
groups: { python: { familiarity: "well", source: "chat" } },
|
|
1192
|
+
}),
|
|
1193
|
+
);
|
|
1194
|
+
const refused = exportAt(dir);
|
|
1195
|
+
assert.notEqual(refused.status, 0);
|
|
1196
|
+
assert.match(refused.out, /groups\/python: .*familiarity is off/);
|
|
1197
|
+
});
|
|
1198
|
+
|
|
1199
|
+
test("🔴 familiarity off with a familiarity_question is a contradiction the build refuses", () => {
|
|
1200
|
+
const r = run([
|
|
1201
|
+
"build",
|
|
1202
|
+
join(
|
|
1203
|
+
setup(
|
|
1204
|
+
"familiarity-contradiction",
|
|
1205
|
+
approval({ familiarity: false, familiarity_question: "How well?" }),
|
|
1206
|
+
),
|
|
1207
|
+
"items.json",
|
|
1208
|
+
),
|
|
1209
|
+
join(WORK, "contradiction-out"),
|
|
1210
|
+
]);
|
|
1211
|
+
assert.notEqual(r.status, 0);
|
|
1212
|
+
assert.match(
|
|
1213
|
+
r.out,
|
|
1214
|
+
/REFUSED: familiarity_question: .*`familiarity` is false/,
|
|
1215
|
+
);
|
|
1216
|
+
});
|
|
1217
|
+
|
|
1218
|
+
test("familiarity left out: every group is still asked (rule 5), and a missing one is reported", () => {
|
|
1219
|
+
const dir = setup("familiarity-default", approval());
|
|
1220
|
+
const { key } = build(dir);
|
|
1221
|
+
assert.deepEqual(key.allowed.familiarity, ["well", "somewhat", "not_really"]);
|
|
1222
|
+
writeFileSync(
|
|
1223
|
+
join(dir, "dump.json"),
|
|
1224
|
+
JSON.stringify({ answers: {}, groups: {} }),
|
|
1225
|
+
);
|
|
1226
|
+
assert.match(exportAt(dir).out, /group python: no familiarity answer/);
|
|
1227
|
+
});
|
|
@@ -132,6 +132,7 @@ const SOURCES_SHOWN = `(${SOURCES.map(quoted).join(", ")})`;
|
|
|
132
132
|
type SchemaNode =
|
|
133
133
|
| { kind: "string"; required: boolean; pattern?: string }
|
|
134
134
|
| { kind: "integer"; required: boolean }
|
|
135
|
+
| { kind: "boolean"; required: boolean }
|
|
135
136
|
| { kind: "array"; required: boolean; min: number; items: SchemaNode }
|
|
136
137
|
| {
|
|
137
138
|
kind: "map";
|
|
@@ -155,6 +156,10 @@ const integer = (required = true): SchemaNode => ({
|
|
|
155
156
|
kind: "integer",
|
|
156
157
|
required,
|
|
157
158
|
});
|
|
159
|
+
const boolean = (required = true): SchemaNode => ({
|
|
160
|
+
kind: "boolean",
|
|
161
|
+
required,
|
|
162
|
+
});
|
|
158
163
|
const array = (items: SchemaNode, required = true, min = 0): SchemaNode => ({
|
|
159
164
|
kind: "array",
|
|
160
165
|
required,
|
|
@@ -220,6 +225,9 @@ export const CONFIG_SCHEMA = obj({
|
|
|
220
225
|
seed: integer(),
|
|
221
226
|
groups: mapping(string(), NAME_PATTERN, 1), // ids become store document ids
|
|
222
227
|
labeller_knows: array(string(true, NAME_PATTERN), false),
|
|
228
|
+
// false: no "how well do you know {group}" question (an approval page: the author has never seen
|
|
229
|
+
// these repositories, so its answer means nothing). Left out, every group is asked (rule 5).
|
|
230
|
+
familiarity: boolean(false),
|
|
223
231
|
familiarity_question: string(false),
|
|
224
232
|
questions: array(QUESTION_SCHEMA, true, 1),
|
|
225
233
|
practice: array(PRACTICE_SCHEMA, true, MIN_PRACTICE),
|
|
@@ -296,6 +304,10 @@ function checkObject(
|
|
|
296
304
|
);
|
|
297
305
|
}
|
|
298
306
|
}
|
|
307
|
+
function checkBoolean(value: Json, where: string): void {
|
|
308
|
+
if (typeof value !== "boolean")
|
|
309
|
+
fail(`${where}: must be true or false, got ${repr(value)}`);
|
|
310
|
+
}
|
|
299
311
|
/** Refuse the first place `value` departs from `node`; the message starts with the field's path. */
|
|
300
312
|
function checkValue(value: Json, node: SchemaNode, path: string): void {
|
|
301
313
|
const where = path || "items.json";
|
|
@@ -308,6 +320,9 @@ function checkValue(value: Json, node: SchemaNode, path: string): void {
|
|
|
308
320
|
fail(`${where}: must be an integer, got ${repr(value)}`);
|
|
309
321
|
}
|
|
310
322
|
break;
|
|
323
|
+
case "boolean":
|
|
324
|
+
checkBoolean(value, where);
|
|
325
|
+
break;
|
|
311
326
|
case "array":
|
|
312
327
|
checkArray(value, node, path, where);
|
|
313
328
|
break;
|
|
@@ -352,6 +367,7 @@ interface Config {
|
|
|
352
367
|
readonly seed: number;
|
|
353
368
|
readonly groups: Readonly<Record<string, string>>;
|
|
354
369
|
readonly labeller_knows?: readonly string[];
|
|
370
|
+
readonly familiarity?: boolean;
|
|
355
371
|
readonly familiarity_question?: string;
|
|
356
372
|
readonly questions: readonly Question[];
|
|
357
373
|
readonly practice: readonly Practice[];
|
|
@@ -368,7 +384,10 @@ function assertConfig(value: Json): asserts value is Json & Config {
|
|
|
368
384
|
* The page script (`DATA.allowed`) and the exporter (`key.json`) both read this one result, so a
|
|
369
385
|
* stored value counts as an answer on the page exactly when it counts in the export.
|
|
370
386
|
*/
|
|
371
|
-
const allowedValues = (
|
|
387
|
+
const allowedValues = (
|
|
388
|
+
questions: readonly Question[],
|
|
389
|
+
asksFamiliarity: boolean,
|
|
390
|
+
): JsonObject => ({
|
|
372
391
|
questions: Object.fromEntries(
|
|
373
392
|
questions.map((q) => [q.id, q.options.map((o) => o.value)]),
|
|
374
393
|
),
|
|
@@ -379,8 +398,10 @@ const allowedValues = (questions: readonly Question[]): JsonObject => ({
|
|
|
379
398
|
q.options.filter((o) => o.ask_why !== undefined).map((o) => o.value),
|
|
380
399
|
]),
|
|
381
400
|
),
|
|
382
|
-
familiarity:
|
|
401
|
+
// Empty when the page asks no familiarity: the export then expects no group document at all.
|
|
402
|
+
familiarity: asksFamiliarity ? FAMILIARITY.map((o) => o.value) : [],
|
|
383
403
|
});
|
|
404
|
+
const asksFamiliarity = (cfg: Config): boolean => cfg.familiarity !== false;
|
|
384
405
|
|
|
385
406
|
/** Two values that read the same once case and surrounding space are gone. */
|
|
386
407
|
const readsAs = (s: string): string => s.trim().toLowerCase();
|
|
@@ -464,6 +485,11 @@ function checkPractice(cfg: Config): void {
|
|
|
464
485
|
}
|
|
465
486
|
function checkConfig(cfg: Config): void {
|
|
466
487
|
checkGroups(cfg);
|
|
488
|
+
if (!asksFamiliarity(cfg) && cfg.familiarity_question !== undefined) {
|
|
489
|
+
fail(
|
|
490
|
+
`familiarity_question: ${repr(cfg.familiarity_question)} is never asked, because \`familiarity\` is false — remove one of the two`,
|
|
491
|
+
);
|
|
492
|
+
}
|
|
467
493
|
cfg.questions.forEach((q, qn) => {
|
|
468
494
|
checkQuestion(
|
|
469
495
|
q,
|
|
@@ -618,10 +644,11 @@ function payloadOf(cfg: Config, page: ReturnType<typeof itemsOnPage>) {
|
|
|
618
644
|
title: cfg.title,
|
|
619
645
|
intro: cfg.intro ?? "",
|
|
620
646
|
groups: used.map((g) => ({ id: g, label: String(cfg.groups[g]) })),
|
|
621
|
-
familiarity_question:
|
|
622
|
-
cfg.familiarity_question ?? "How well do you know {group}?"
|
|
623
|
-
|
|
624
|
-
|
|
647
|
+
familiarity_question: asksFamiliarity(cfg)
|
|
648
|
+
? (cfg.familiarity_question ?? "How well do you know {group}?")
|
|
649
|
+
: "",
|
|
650
|
+
familiarity: asksFamiliarity(cfg) ? FAMILIARITY : [],
|
|
651
|
+
allowed: allowedValues(cfg.questions, asksFamiliarity(cfg)),
|
|
625
652
|
questions: cfg.questions,
|
|
626
653
|
practice: { round: practiceRound(cfg), items: practiceOf(cfg) },
|
|
627
654
|
items: page.items.map((i) => ({ id: page.idOf(i), ...shownFields(i) })),
|
|
@@ -675,7 +702,7 @@ function keyOf(
|
|
|
675
702
|
questions: cfg.questions.map((q) => q.id),
|
|
676
703
|
// What counts as an answer: the same lists the page was given (`allowedValues`). `export`
|
|
677
704
|
// checks every stored answer and familiarity against them, so the build's options decide.
|
|
678
|
-
allowed: allowedValues(cfg.questions),
|
|
705
|
+
allowed: allowedValues(cfg.questions, asksFamiliarity(cfg)),
|
|
679
706
|
// The practice is scored against this at export; its answers never go into the labels.
|
|
680
707
|
practice: {
|
|
681
708
|
round: practiceRound(cfg),
|
|
@@ -931,6 +958,17 @@ function checkDump(dump: Json, key: Key): asserts dump is JsonObject {
|
|
|
931
958
|
}
|
|
932
959
|
}
|
|
933
960
|
|
|
961
|
+
/** A page built with `familiarity: false` asks no group anything: a group document came from elsewhere. */
|
|
962
|
+
function checkNoFamiliarity(dump: JsonObject, allowed: Allowed): void {
|
|
963
|
+
const groups = isObject(dump["groups"]) ? dump["groups"] : {};
|
|
964
|
+
const [first] = Object.keys(groups).sort(byText);
|
|
965
|
+
if (allowed.familiarity.length === 0 && first !== undefined) {
|
|
966
|
+
fail(
|
|
967
|
+
`groups/${first}: familiarity is off for this page (items.json \`familiarity: false\`), so no group document is expected — take it out of the dump`,
|
|
968
|
+
);
|
|
969
|
+
}
|
|
970
|
+
}
|
|
971
|
+
|
|
934
972
|
// ── export ────────────────────────────────────────────────────────────────────────────────────
|
|
935
973
|
interface Allowed {
|
|
936
974
|
readonly questions: Readonly<Record<string, readonly string[]>>;
|
|
@@ -1165,11 +1203,13 @@ function labelsOf(dump: JsonObject, ctx: Ctx, day: string) {
|
|
|
1165
1203
|
const items = onPage.map(([nid, meta]) =>
|
|
1166
1204
|
labelOf(nid, meta, docOf(dump["answers"], nid), ctx),
|
|
1167
1205
|
);
|
|
1168
|
-
const groups =
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1206
|
+
const groups = ctx.allowed.familiarity.length
|
|
1207
|
+
? groupsOf(
|
|
1208
|
+
[...new Set(onPage.map(([, m]) => m.group))],
|
|
1209
|
+
dump["groups"],
|
|
1210
|
+
ctx.allowed,
|
|
1211
|
+
)
|
|
1212
|
+
: {};
|
|
1173
1213
|
return {
|
|
1174
1214
|
study: ctx.key.study,
|
|
1175
1215
|
exported: day,
|
|
@@ -1197,6 +1237,7 @@ export function exportLabels(
|
|
|
1197
1237
|
const allowed = key.allowed;
|
|
1198
1238
|
assertAllowed(allowed, key, keyPath);
|
|
1199
1239
|
checkDump(dump, key);
|
|
1240
|
+
checkNoFamiliarity(dump, allowed);
|
|
1200
1241
|
const out = labelsOf(dump, { key, allowed }, day);
|
|
1201
1242
|
mkdirSync(outDir, { recursive: true });
|
|
1202
1243
|
const path = join(outDir, `${key.study}-labels-${day}.json`);
|
|
@@ -411,3 +411,25 @@ test("🔴 without the store the page is read-only, and the real items stay read
|
|
|
411
411
|
assert.match(shown, /cannot be saved in this view/);
|
|
412
412
|
assert.match(shown, /G001/, "a real item is shown read-only");
|
|
413
413
|
});
|
|
414
|
+
|
|
415
|
+
test("🔴 familiarity off: no group asks how well it is known, and a group is done when its items are", async () => {
|
|
416
|
+
const { page, key } = build(
|
|
417
|
+
setup("familiarity-off-page", approval({ familiarity: false })),
|
|
418
|
+
);
|
|
419
|
+
const practice = Object.fromEntries(
|
|
420
|
+
APPROVAL_PRACTICE.map((p, n) => [
|
|
421
|
+
`r1-P${String(n + 1)}`,
|
|
422
|
+
{ answers: p.expected, source: "page" },
|
|
423
|
+
]),
|
|
424
|
+
);
|
|
425
|
+
const answers = Object.fromEntries(
|
|
426
|
+
Object.keys(key.ids).map((id) => [
|
|
427
|
+
id,
|
|
428
|
+
{ answers: { keep: "keep" }, source: "page" },
|
|
429
|
+
]),
|
|
430
|
+
);
|
|
431
|
+
const shown = await showPage(page, answers, {}, practice);
|
|
432
|
+
assert.doesNotMatch(shown, /How well/);
|
|
433
|
+
assert.doesNotMatch(shown, /how well you know it/);
|
|
434
|
+
assert.match(shown, /Nothing left/, shown);
|
|
435
|
+
});
|
|
@@ -40,6 +40,7 @@
|
|
|
40
40
|
* "seed": 7, fixes the order; change it and every id changes
|
|
41
41
|
* "groups": {"python": "Python"},
|
|
42
42
|
* "labeller_knows": ["python"], optional: items of other groups are not put on the page
|
|
43
|
+
* "familiarity": false, optional: no "how well do you know" question (approval page)
|
|
43
44
|
* "familiarity_question": "How well do you know {group}?", optional
|
|
44
45
|
* "questions": [{"id": "label", "text": "Is this correct?", "short": "correct?",
|
|
45
46
|
* "options": [{"value": "yes", "label": "Yes", "example": "one real case",
|
|
@@ -40,6 +40,7 @@
|
|
|
40
40
|
* "seed": 7, fixes the order; change it and every id changes
|
|
41
41
|
* "groups": {"python": "Python"},
|
|
42
42
|
* "labeller_knows": ["python"], optional: items of other groups are not put on the page
|
|
43
|
+
* "familiarity": false, optional: no "how well do you know" question (approval page)
|
|
43
44
|
* "familiarity_question": "How well do you know {group}?", optional
|
|
44
45
|
* "questions": [{"id": "label", "text": "Is this correct?", "short": "correct?",
|
|
45
46
|
* "options": [{"value": "yes", "label": "Yes", "example": "one real case",
|
|
@@ -90,6 +90,9 @@ pre { overflow-x: auto; background: var(--surface); border: 1px solid var(--line
|
|
|
90
90
|
editing: {}, // item id -> the question id the labeller tapped "Change" on
|
|
91
91
|
mineWhy: {}, // item id -> the reasons this page has just written (ahead of the store)
|
|
92
92
|
draftWhy: {}, // item id -> {question id: text typed and not yet saved}
|
|
93
|
+
whyTimers: {}, // "<item id>|<question id>" -> the pending save of a reason being typed
|
|
94
|
+
whyBoxes: {}, // "<item id>|<question id>" -> the reason box of the last render
|
|
95
|
+
typingIn: null, // the item whose reason box has the keyboard: shown even when complete
|
|
93
96
|
mine: {}, // item/group key -> the answers this page has just written (ahead of the store)
|
|
94
97
|
inflight: {}, // item/group key -> writes not yet confirmed
|
|
95
98
|
failed: {}, // item/group key -> error code of the last refused write
|
|
@@ -99,6 +102,7 @@ pre { overflow-x: auto; background: var(--surface); border: 1px solid var(--line
|
|
|
99
102
|
};
|
|
100
103
|
var db = null;
|
|
101
104
|
var chains = {};
|
|
105
|
+
var WHY_PAUSE_MS = 800;
|
|
102
106
|
|
|
103
107
|
try { if (localStorage.getItem("label-filter") === "next") state.filter = "next"; } catch (e) {}
|
|
104
108
|
|
|
@@ -142,7 +146,11 @@ pre { overflow-x: auto; background: var(--surface); border: 1px solid var(--line
|
|
|
142
146
|
}).filter(function (q) { return q !== null; });
|
|
143
147
|
}
|
|
144
148
|
function itemDone(id) { return missingFor(id).length === 0; }
|
|
149
|
+
// A page built with "familiarity": false asks no group anything (an approval page): every group
|
|
150
|
+
// counts as rated, and neither the question nor "how well you know it" is ever shown.
|
|
151
|
+
var ASKS_FAMILIARITY = DATA.familiarity.length > 0;
|
|
145
152
|
function familiarity(g) {
|
|
153
|
+
if (!ASKS_FAMILIARITY) return null;
|
|
146
154
|
var d = state.mine["group:" + g] || state.groups[g];
|
|
147
155
|
return d && valid(DATA.allowed.familiarity, d.familiarity) ? d.familiarity : undefined;
|
|
148
156
|
}
|
|
@@ -197,6 +205,8 @@ pre { overflow-x: auto; background: var(--surface); border: 1px solid var(--line
|
|
|
197
205
|
}
|
|
198
206
|
// A reason saves when the labeller leaves the box (the text equivalent of a tap).
|
|
199
207
|
function setReason(id, qid, text) {
|
|
208
|
+
clearTimeout(state.whyTimers[id + "|" + qid]);
|
|
209
|
+
delete state.whyTimers[id + "|" + qid];
|
|
200
210
|
if (!canWrite()) return;
|
|
201
211
|
var why = Object.assign({}, reasonsOf(id));
|
|
202
212
|
why[qid] = text;
|
|
@@ -315,8 +325,16 @@ pre { overflow-x: auto; background: var(--surface); border: 1px solid var(--line
|
|
|
315
325
|
var draft = draftOf(item.id, q.id);
|
|
316
326
|
box.value = draft !== undefined ? draft : reasonsOf(item.id)[q.id] || "";
|
|
317
327
|
box.disabled = off;
|
|
318
|
-
// Every keystroke is kept as a draft, so a render from the store never erases unsaved text
|
|
319
|
-
|
|
328
|
+
// Every keystroke is kept as a draft, so a render from the store never erases unsaved text,
|
|
329
|
+
// and a pause in typing saves it: a reason that waits for the box to lose focus is lost when
|
|
330
|
+
// the labeller leaves the page from inside the box (measured: the store kept the drop alone).
|
|
331
|
+
box.addEventListener("input", function () {
|
|
332
|
+
keepDraft(item.id, q.id, box.value);
|
|
333
|
+
var k = item.id + "|" + q.id;
|
|
334
|
+
clearTimeout(state.whyTimers[k]);
|
|
335
|
+
state.whyTimers[k] = setTimeout(function () { setReason(item.id, q.id, box.value); }, WHY_PAUSE_MS);
|
|
336
|
+
});
|
|
337
|
+
state.whyBoxes[item.id + "|" + q.id] = box;
|
|
320
338
|
box.addEventListener("change", function () { setReason(item.id, q.id, box.value); });
|
|
321
339
|
card.appendChild(box);
|
|
322
340
|
});
|
|
@@ -407,20 +425,56 @@ pre { overflow-x: auto; background: var(--surface); border: 1px solid var(--line
|
|
|
407
425
|
function groupBlock(g, off, onlyOpen) {
|
|
408
426
|
var sec = el("section");
|
|
409
427
|
sec.appendChild(el("h2", "", g.label));
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
428
|
+
if (ASKS_FAMILIARITY) {
|
|
429
|
+
var fam = el("div", "ask");
|
|
430
|
+
fam.appendChild(el("p", "q", DATA.familiarity_question.replace("{group}", g.label)));
|
|
431
|
+
fam.appendChild(options(DATA.familiarity, familiarity(g.id), function (v) { setFamiliarity(g.id, v); }, off));
|
|
432
|
+
var gs = state.groups[g.id];
|
|
433
|
+
fam.appendChild(statusLine("group:" + g.id,
|
|
434
|
+
familiarity(g.id) === undefined ? [{ text: "an answer" }] : [], gs && gs.savedAt));
|
|
435
|
+
sec.appendChild(fam);
|
|
436
|
+
}
|
|
417
437
|
groupItems(g.id).forEach(function (i) {
|
|
418
|
-
if (onlyOpen && itemDone(i.id)) return;
|
|
438
|
+
if (onlyOpen && itemDone(i.id) && i.id !== state.typingIn) return;
|
|
419
439
|
sec.appendChild(itemCard(i, off));
|
|
420
440
|
});
|
|
421
441
|
return sec;
|
|
422
442
|
}
|
|
443
|
+
// Every reason typed and not yet saved goes to the store now: the page is being left.
|
|
444
|
+
function flushReasons() {
|
|
445
|
+
Object.keys(state.whyTimers).forEach(function (k) {
|
|
446
|
+
var at = k.indexOf("|"), id = k.slice(0, at), qid = k.slice(at + 1);
|
|
447
|
+
var draft = draftOf(id, qid);
|
|
448
|
+
if (draft === undefined) { clearTimeout(state.whyTimers[k]); delete state.whyTimers[k]; return; }
|
|
449
|
+
setReason(id, qid, draft);
|
|
450
|
+
});
|
|
451
|
+
}
|
|
452
|
+
// The page is rebuilt on every store snapshot; the reason box being typed in gets the keyboard back,
|
|
453
|
+
// with its caret where it was, or the rest of the sentence goes nowhere.
|
|
454
|
+
// Chrome fires "change" on a focused box while a render is removing it, and that change saves and
|
|
455
|
+
// renders: a render started inside a render would paint with no box focused and hide the item being
|
|
456
|
+
// typed in. It is skipped; the write it started renders again when the store confirms it.
|
|
457
|
+
var painting = false;
|
|
423
458
|
function render() {
|
|
459
|
+
if (painting) return;
|
|
460
|
+
painting = true;
|
|
461
|
+
try { renderOnce(); } finally { painting = false; }
|
|
462
|
+
}
|
|
463
|
+
function renderOnce() {
|
|
464
|
+
var typing = null;
|
|
465
|
+
Object.keys(state.whyBoxes).forEach(function (k) {
|
|
466
|
+
var b = state.whyBoxes[k];
|
|
467
|
+
if (document.activeElement === b) typing = { key: k, start: b.selectionStart, end: b.selectionEnd };
|
|
468
|
+
});
|
|
469
|
+
state.whyBoxes = {};
|
|
470
|
+
// Under "Next unanswered" the save that completes an item would hide it, box and all, mid-sentence:
|
|
471
|
+
// the item being typed in stays on the page until the labeller leaves its box.
|
|
472
|
+
state.typingIn = typing ? typing.key.slice(0, typing.key.indexOf("|")) : null;
|
|
473
|
+
paint();
|
|
474
|
+
var again = typing && state.whyBoxes[typing.key];
|
|
475
|
+
if (again) { again.focus(); again.setSelectionRange(typing.start, typing.end); }
|
|
476
|
+
}
|
|
477
|
+
function paint() {
|
|
424
478
|
var off = !canWrite();
|
|
425
479
|
var done = DATA.items.filter(function (i) { return itemDone(i.id); }).length;
|
|
426
480
|
var total = DATA.items.length;
|
|
@@ -482,7 +536,8 @@ pre { overflow-x: auto; background: var(--surface); border: 1px solid var(--line
|
|
|
482
536
|
progress.appendChild(note);
|
|
483
537
|
}
|
|
484
538
|
DATA.groups.forEach(function (g) {
|
|
485
|
-
|
|
539
|
+
var typingHere = groupItems(g.id).some(function (i) { return i.id === state.typingIn; });
|
|
540
|
+
if (onlyOpen && groupDone(g.id) && !typingHere) return;
|
|
486
541
|
app.appendChild(groupBlock(g, off, onlyOpen));
|
|
487
542
|
});
|
|
488
543
|
}
|
|
@@ -499,6 +554,10 @@ pre { overflow-x: auto; background: var(--surface); border: 1px solid var(--line
|
|
|
499
554
|
}
|
|
500
555
|
function broken() { state.store = "none"; render(); }
|
|
501
556
|
|
|
557
|
+
document.addEventListener("visibilitychange", function () {
|
|
558
|
+
if (document.visibilityState !== "visible") flushReasons();
|
|
559
|
+
});
|
|
560
|
+
window.addEventListener("pagehide", flushReasons);
|
|
502
561
|
render();
|
|
503
562
|
if (!window.claude || typeof window.claude.use !== "function") { broken(); return; }
|
|
504
563
|
window.claude.use("db").then(function (store) {
|
|
@@ -208,6 +208,8 @@ describe("reproMessage — stands alone", () => {
|
|
|
208
208
|
"├── README.md ← the index: one line per study, generated from studies.json",
|
|
209
209
|
"├── studies.json ← the study registry: id, title, status and result of each study",
|
|
210
210
|
"├── .gitignore ← keeps the scripts' byproducts out of git",
|
|
211
|
+
"├── CLAUDE.md ← optional: instructions an agent loads when it works in repro/",
|
|
212
|
+
"├── AGENTS.md ← optional: the same, for agents that read AGENTS.md",
|
|
211
213
|
"├── tools/ ← scripts used across the paper (numbers, figures, anonymity check, harnesses)",
|
|
212
214
|
"├── data/ ← shared data (tables, caches)",
|
|
213
215
|
"└── <id>-<slug>/ ← one folder per study or spike, named by its row in studies.json (P1-S03-baseline)",
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* ├── tools/ scripts used across the paper (numbers, figures, anonymity check, harnesses)
|
|
14
14
|
* └── data/ shared data (tables, caches)
|
|
15
15
|
*
|
|
16
|
-
* plus `.gitignore`. A loose note belongs in its study's folder or in the paper's `research/`.
|
|
16
|
+
* plus `.gitignore`, and optionally the agent instruction files `CLAUDE.md` and `AGENTS.md`. A loose note belongs in its study's folder or in the paper's `research/`.
|
|
17
17
|
*
|
|
18
18
|
* ── A STUDY'S NAME ───────────────────────────────────────────────────────────────
|
|
19
19
|
* `<id>-<slug>` (`P1-S03-baseline`): the id of the study's row in `repro/studies.json` — the paper's
|
|
@@ -58,6 +58,14 @@ export const REPRO_FILE_NOTES: readonly {
|
|
|
58
58
|
note: "the study registry: id, title, status and result of each study",
|
|
59
59
|
},
|
|
60
60
|
{ name: ".gitignore", note: "keeps the scripts' byproducts out of git" },
|
|
61
|
+
{
|
|
62
|
+
name: "CLAUDE.md",
|
|
63
|
+
note: "optional: instructions an agent loads when it works in repro/",
|
|
64
|
+
},
|
|
65
|
+
{
|
|
66
|
+
name: "AGENTS.md",
|
|
67
|
+
note: "optional: the same, for agents that read AGENTS.md",
|
|
68
|
+
},
|
|
61
69
|
];
|
|
62
70
|
export const REPRO_FILES: readonly string[] = REPRO_FILE_NOTES.map(
|
|
63
71
|
(f) => f.name,
|
|
@@ -94,6 +94,16 @@ describe("repro/allowed-files-only — silent", () => {
|
|
|
94
94
|
).toEqual([]);
|
|
95
95
|
});
|
|
96
96
|
|
|
97
|
+
it("on agent instruction files at the top (CLAUDE.md, AGENTS.md)", () => {
|
|
98
|
+
expect(
|
|
99
|
+
lint({
|
|
100
|
+
...SCAFFOLD,
|
|
101
|
+
[`${DIR}/repro/CLAUDE.md`]: "x",
|
|
102
|
+
[`${DIR}/repro/AGENTS.md`]: "x",
|
|
103
|
+
}),
|
|
104
|
+
).toEqual([]);
|
|
105
|
+
});
|
|
106
|
+
|
|
97
107
|
it("on a paper with no repro/", () => {
|
|
98
108
|
expect(lint(SCAFFOLD)).toEqual([]);
|
|
99
109
|
});
|