quillbench 1.4.1__tar.gz

quillbench-1.4.1/LICENSE

@@ -0,0 +1,26 @@
+QuillBench Academic Non-Commercial License
+
+Copyright (c) 2026 A.C. Pogaku. All rights reserved.
+
+The copyright in this software and its documentation (the "Software") belongs
+solely to A.C. Pogaku. All rights not expressly granted here are reserved.
+
+Permission is granted, free of charge, to students, teachers, researchers and
+academic or other non-profit research institutions to use, copy and modify the
+Software, and to share copies or modified versions with others, for academic
+teaching, study and research only, on these conditions:
+
+  1. No commercial use. You may not sell the Software, offer it as a paid
+     service, or use it in or for any commercial product, service or activity,
+     or for the benefit of a commercial organisation. No commercial license is
+     offered under these terms.
+  2. Keep this notice. Every copy or modified version must include this license
+     and the copyright notice above, and must carry these same terms.
+  3. Mark your changes. Modified versions must state that they were changed.
+
+Results and data you produce with the Software belong to you. Publishing
+academic work that used the Software is permitted.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY ARISING FROM THE SOFTWARE OR ITS USE.

quillbench-1.4.1/PKG-INFO

@@ -0,0 +1,205 @@
+Metadata-Version: 2.4
+Name: quillbench
+Version: 1.4.1
+Summary: Build benchmark datasets with human annotators: a config-driven annotation platform for research.
+Author: A.C. Pogaku
+License: QuillBench Academic Non-Commercial License
+        
+        Copyright (c) 2026 A.C. Pogaku. All rights reserved.
+        
+        The copyright in this software and its documentation (the "Software") belongs
+        solely to A.C. Pogaku. All rights not expressly granted here are reserved.
+        
+        Permission is granted, free of charge, to students, teachers, researchers and
+        academic or other non-profit research institutions to use, copy and modify the
+        Software, and to share copies or modified versions with others, for academic
+        teaching, study and research only, on these conditions:
+        
+          1. No commercial use. You may not sell the Software, offer it as a paid
+             service, or use it in or for any commercial product, service or activity,
+             or for the benefit of a commercial organisation. No commercial license is
+             offered under these terms.
+          2. Keep this notice. Every copy or modified version must include this license
+             and the copyright notice above, and must carry these same terms.
+          3. Mark your changes. Modified versions must state that they were changed.
+        
+        Results and data you produce with the Software belong to you. Publishing
+        academic work that used the Software is permitted.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM,
+        DAMAGES OR OTHER LIABILITY ARISING FROM THE SOFTWARE OR ITS USE.
+        
+Project-URL: Homepage, https://github.com/quillbench/quillbench
+Project-URL: Source, https://github.com/quillbench/quillbench
+Project-URL: Changelog, https://github.com/quillbench/quillbench/blob/main/CHANGELOG.md
+Keywords: annotation,labeling,benchmark,dataset,human evaluation,streamlit,research
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: Other/Proprietary License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: streamlit<2.0,>=1.50
+Requires-Dist: pandas<3.0,>=2.0
+Requires-Dist: PyYAML<7.0,>=6.0
+Requires-Dist: neo4j<7.0,>=5.0
+Provides-Extra: test
+Requires-Dist: pytest>=7; extra == "test"
+Provides-Extra: ldap
+Requires-Dist: ldap3<3,>=2.9; extra == "ldap"
+Dynamic: license-file
+
+<p align="center"><img src="https://raw.githubusercontent.com/quillbench/quillbench/main/docs/logo.svg" width="120" alt="QuillBench logo"></p>
+
+# QuillBench
+
+Build benchmark datasets with human annotators. A config-driven annotation platform for research, by **A.C. Pogaku**.
+
+Each paper keeps its own annotation data in its own
+folder, a **workspace**, described by one `quillbench_workspace.yaml` file. The studio reads
+that file and writes answers straight into the paper's folders, so there is nothing to
+copy and the two are always in step.
+
+```
+~/.config/quillbench/  private studio state: registry, accounts (hashed), settings
+<paper>/quillbench_workspace.yaml           one paper's projects
+<paper>/batches/*.csv|*.jsonl        items to annotate, one file per batch
+<paper>/progress/<user>__<batch>.csv answers, written by the studio
+<paper>/quillbench_assignments.json         who works on which batch (no secrets)
+<paper>/quillbench_exports/<time>/          exports: answers.csv/.jsonl, agreement.csv, DATASHEET.md
+```
+
+## Install
+
+```
+pip install quillbench
+quillbench serve                       # this computer only: http://localhost:8560/quillbench/
+quillbench serve --address 0.0.0.0     # let others on the network reach it
+```
+
+Python 3.10 or newer. Answers and settings stay on the machine that runs `quillbench serve`.
+Free for academic, non-commercial use only; see [LICENSE](https://github.com/quillbench/quillbench/blob/main/LICENSE).
+
+## Getting started in the browser
+
+Run `quillbench serve` and open the page. The first person to sign up becomes the admin, so do
+this yourself before sharing the address. Then, with no files to edit, the admin can use
+**Admin → Projects** to create a project, upload CSVs at any time, set label buttons or a slider, split
+the items into batches and assign them. Add people under **Admin → People**, and follow the
+work in the **Progress** tab.
+
+## Task types
+
+| task | what the annotator does |
+|---|---|
+| `rating` | moves a slider on a scale (e.g. 0–1 or 1–5), optional "can't judge" |
+| `choice` | picks one of the labels the manifest lists |
+| `text` | writes a short answer |
+| `verdict`, `gate_review`, `gate_calibration`, `item_question`, `item_query`, `item_disposition` | reviews a question + reference query + precomputed results |
+| `adjudication` | judges a predicted query against a reference (results withheld) |
+| `pair_semantics` | blinded A/B comparison of two queries |
+
+Per project the manifest can set `instructions`, `note_label`, `note_placeholder`,
+`note_required_for`, `display` (which item columns to show and how), `checks`
+(hidden attention-check items with expected answers) and `also_accept` (labels
+from an earlier version of the task that still count as answers).
+
+## Everyday commands
+
+```bash
+quillbench init-workspace ~/my_paper/annotation      # start a new paper from a template
+quillbench workspace add ~/my_paper/annotation        # register it
+quillbench validate                                   # check every workspace
+quillbench user add alice                             # asks for a password
+quillbench assign alice my_paper rate_01 rate_02
+quillbench status my_paper                            # progress per person and batch
+quillbench quality my_paper                           # Krippendorff's alpha, Cohen's kappa, checks, time
+quillbench export my_paper                            # anonymised answers + datasheet
+quillbench backup                                     # also runs nightly
+quillbench serve                                      # http://127.0.0.1:8560/quillbench/
+```
+
+(`python -m quillbench …` works the same without installing; `pip install -e .` adds the `quillbench` command.)
+
+In the app: **1–9** answer, **S** skip, **J/→** next, **K/←** previous, **Enter** saves a rating.
+
+Useful manifest options for rating studies:
+
+```yaml
+scale: {min: 0, max: 1, step: 0.05, unsure_label: "Can't judge",
+        marks: {0: false, 0.5: half true, 1: true}}   # words along the slider
+display:
+  - {field: statement, label: Statement}             # the first text field is shown large
+  - {field: hourly, as: chart, chart: {kind: bar, x_label: hour, lines: [15.5, 35.5]}}
+practice: true      # show each item's 'expected' and 'explanation' columns after answering
+```
+
+## For annotators: connect to a server
+
+If someone runs QuillBench for your group, you only need the address they give you:
+
+```
+pip install quillbench
+quillbench connect annotate.example.edu/quillbench
+```
+
+This checks the server, remembers it and opens the sign-in page. Next time, run
+`quillbench open`. Sign in with the login you were given (or your university login, if the
+server uses LDAP). Your answers are saved on the server, not on your computer.
+
+## Signing in with university (LDAP) accounts
+
+QuillBench has its own accounts. A server can also accept the logins people already have,
+from LDAP or Active Directory. Install the extra package and add an `ldap` block to
+`settings.yaml` in the studio home:
+
+```
+pip install "quillbench[ldap]"
+```
+
+```yaml
+ldap:
+  url: ldaps://ldap.example.edu                         # or ldap:// with start_tls: true
+  user_dn: "uid={user},ou=people,dc=example,dc=edu"     # Active Directory: "{user}@example.edu"
+  auto_create: true     # first sign-in makes an annotator account; false = admins add people first
+```
+
+Passwords are checked by the LDAP server and never stored by QuillBench. Accounts with their
+own QuillBench password keep using it. Admins still assign batches and can disable anyone in
+Admin → People. Use `ldaps://` or `start_tls`, so passwords are not sent in plain text.
+
+## Running as a service
+
+```bash
+cp deploy/quillbench.service deploy/quillbench-backup.service deploy/quillbench-backup.timer ~/.config/systemd/user/
+systemctl --user daemon-reload
+systemctl --user enable --now quillbench quillbench-backup.timer
+sudo loginctl enable-linger $USER     # keep it running after logout
+```
+
+nginx: add `deploy/nginx-quillbench.conf` to the server block.
+
+## Quality measures
+
+* **Krippendorff's alpha** (nominal for labels; interval or ordinal for ratings) over all
+  items answered by two or more people; skips and "can't judge" left out. Verified against
+  the worked example in Krippendorff (2011): 0.743 / 0.815 / 0.849.
+* **Cohen's kappa** per pair of annotators (labels), or mean absolute difference and
+  Pearson r (ratings).
+* **Check items** passed per annotator, and **median seconds per item** (QuillBench records
+  `updated_at` and `seconds` for every answer).
+
+## Security
+
+Passwords are stored as salted PBKDF2-SHA256 hashes in the studio home (mode 600).
+Sessions use an HMAC-signed cookie. Exports replace annotator ids with codes
+(A01, …); the code book stays in the studio home.
+
+## Origin
+
+© 2026 A.C. Pogaku. Free for academic, non-commercial use; commercial use is not permitted. See [LICENSE](LICENSE).

quillbench-1.4.1/README.md

@@ -0,0 +1,149 @@
+<p align="center"><img src="https://raw.githubusercontent.com/quillbench/quillbench/main/docs/logo.svg" width="120" alt="QuillBench logo"></p>
+
+# QuillBench
+
+Build benchmark datasets with human annotators. A config-driven annotation platform for research, by **A.C. Pogaku**.
+
+Each paper keeps its own annotation data in its own
+folder, a **workspace**, described by one `quillbench_workspace.yaml` file. The studio reads
+that file and writes answers straight into the paper's folders, so there is nothing to
+copy and the two are always in step.
+
+```
+~/.config/quillbench/  private studio state: registry, accounts (hashed), settings
+<paper>/quillbench_workspace.yaml           one paper's projects
+<paper>/batches/*.csv|*.jsonl        items to annotate, one file per batch
+<paper>/progress/<user>__<batch>.csv answers, written by the studio
+<paper>/quillbench_assignments.json         who works on which batch (no secrets)
+<paper>/quillbench_exports/<time>/          exports: answers.csv/.jsonl, agreement.csv, DATASHEET.md
+```
+
+## Install
+
+```
+pip install quillbench
+quillbench serve                       # this computer only: http://localhost:8560/quillbench/
+quillbench serve --address 0.0.0.0     # let others on the network reach it
+```
+
+Python 3.10 or newer. Answers and settings stay on the machine that runs `quillbench serve`.
+Free for academic, non-commercial use only; see [LICENSE](https://github.com/quillbench/quillbench/blob/main/LICENSE).
+
+## Getting started in the browser
+
+Run `quillbench serve` and open the page. The first person to sign up becomes the admin, so do
+this yourself before sharing the address. Then, with no files to edit, the admin can use
+**Admin → Projects** to create a project, upload CSVs at any time, set label buttons or a slider, split
+the items into batches and assign them. Add people under **Admin → People**, and follow the
+work in the **Progress** tab.
+
+## Task types
+
+| task | what the annotator does |
+|---|---|
+| `rating` | moves a slider on a scale (e.g. 0–1 or 1–5), optional "can't judge" |
+| `choice` | picks one of the labels the manifest lists |
+| `text` | writes a short answer |
+| `verdict`, `gate_review`, `gate_calibration`, `item_question`, `item_query`, `item_disposition` | reviews a question + reference query + precomputed results |
+| `adjudication` | judges a predicted query against a reference (results withheld) |
+| `pair_semantics` | blinded A/B comparison of two queries |
+
+Per project the manifest can set `instructions`, `note_label`, `note_placeholder`,
+`note_required_for`, `display` (which item columns to show and how), `checks`
+(hidden attention-check items with expected answers) and `also_accept` (labels
+from an earlier version of the task that still count as answers).
+
+## Everyday commands
+
+```bash
+quillbench init-workspace ~/my_paper/annotation      # start a new paper from a template
+quillbench workspace add ~/my_paper/annotation        # register it
+quillbench validate                                   # check every workspace
+quillbench user add alice                             # asks for a password
+quillbench assign alice my_paper rate_01 rate_02
+quillbench status my_paper                            # progress per person and batch
+quillbench quality my_paper                           # Krippendorff's alpha, Cohen's kappa, checks, time
+quillbench export my_paper                            # anonymised answers + datasheet
+quillbench backup                                     # also runs nightly
+quillbench serve                                      # http://127.0.0.1:8560/quillbench/
+```
+
+(`python -m quillbench …` works the same without installing; `pip install -e .` adds the `quillbench` command.)
+
+In the app: **1–9** answer, **S** skip, **J/→** next, **K/←** previous, **Enter** saves a rating.
+
+Useful manifest options for rating studies:
+
+```yaml
+scale: {min: 0, max: 1, step: 0.05, unsure_label: "Can't judge",
+        marks: {0: false, 0.5: half true, 1: true}}   # words along the slider
+display:
+  - {field: statement, label: Statement}             # the first text field is shown large
+  - {field: hourly, as: chart, chart: {kind: bar, x_label: hour, lines: [15.5, 35.5]}}
+practice: true      # show each item's 'expected' and 'explanation' columns after answering
+```
+
+## For annotators: connect to a server
+
+If someone runs QuillBench for your group, you only need the address they give you:
+
+```
+pip install quillbench
+quillbench connect annotate.example.edu/quillbench
+```
+
+This checks the server, remembers it and opens the sign-in page. Next time, run
+`quillbench open`. Sign in with the login you were given (or your university login, if the
+server uses LDAP). Your answers are saved on the server, not on your computer.
+
+## Signing in with university (LDAP) accounts
+
+QuillBench has its own accounts. A server can also accept the logins people already have,
+from LDAP or Active Directory. Install the extra package and add an `ldap` block to
+`settings.yaml` in the studio home:
+
+```
+pip install "quillbench[ldap]"
+```
+
+```yaml
+ldap:
+  url: ldaps://ldap.example.edu                         # or ldap:// with start_tls: true
+  user_dn: "uid={user},ou=people,dc=example,dc=edu"     # Active Directory: "{user}@example.edu"
+  auto_create: true     # first sign-in makes an annotator account; false = admins add people first
+```
+
+Passwords are checked by the LDAP server and never stored by QuillBench. Accounts with their
+own QuillBench password keep using it. Admins still assign batches and can disable anyone in
+Admin → People. Use `ldaps://` or `start_tls`, so passwords are not sent in plain text.
+
+## Running as a service
+
+```bash
+cp deploy/quillbench.service deploy/quillbench-backup.service deploy/quillbench-backup.timer ~/.config/systemd/user/
+systemctl --user daemon-reload
+systemctl --user enable --now quillbench quillbench-backup.timer
+sudo loginctl enable-linger $USER     # keep it running after logout
+```
+
+nginx: add `deploy/nginx-quillbench.conf` to the server block.
+
+## Quality measures
+
+* **Krippendorff's alpha** (nominal for labels; interval or ordinal for ratings) over all
+  items answered by two or more people; skips and "can't judge" left out. Verified against
+  the worked example in Krippendorff (2011): 0.743 / 0.815 / 0.849.
+* **Cohen's kappa** per pair of annotators (labels), or mean absolute difference and
+  Pearson r (ratings).
+* **Check items** passed per annotator, and **median seconds per item** (QuillBench records
+  `updated_at` and `seconds` for every answer).
+
+## Security
+
+Passwords are stored as salted PBKDF2-SHA256 hashes in the studio home (mode 600).
+Sessions use an HMAC-signed cookie. Exports replace annotator ids with codes
+(A01, …); the code book stays in the studio home.
+
+## Origin
+
+© 2026 A.C. Pogaku. Free for academic, non-commercial use; commercial use is not permitted. See [LICENSE](LICENSE).

quillbench-1.4.1/pyproject.toml

@@ -0,0 +1,44 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "quillbench"
+version = "1.4.1"
+description = "Build benchmark datasets with human annotators: a config-driven annotation platform for research."
+readme = "README.md"
+keywords = ["annotation", "labeling", "benchmark", "dataset", "human evaluation", "streamlit", "research"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Education",
+    "Intended Audience :: Science/Research",
+    "License :: Other/Proprietary License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+authors = [{ name = "A.C. Pogaku" }]
+license = { file = "LICENSE" }
+requires-python = ">=3.10"
+dependencies = ["streamlit>=1.50,<2.0", "pandas>=2.0,<3.0", "PyYAML>=6.0,<7.0", "neo4j>=5.0,<7.0"]
+
+[project.urls]
+Homepage = "https://github.com/quillbench/quillbench"
+Source = "https://github.com/quillbench/quillbench"
+Changelog = "https://github.com/quillbench/quillbench/blob/main/CHANGELOG.md"
+
+[project.optional-dependencies]
+test = ["pytest>=7"]
+ldap = ["ldap3>=2.9,<3"]
+
+[project.scripts]
+quillbench = "quillbench.cli:main"
+
+[tool.setuptools.packages.find]
+include = ["quillbench*"]
+
+[tool.setuptools.package-data]
+"quillbench.ui" = ["assets/*"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

quillbench-1.4.1/quillbench/__init__.py

@@ -0,0 +1,11 @@
+"""QuillBench.
+
+A config-driven human-annotation platform for research datasets. The software lives
+in its own folder; every paper keeps its annotation data in its own folder (a
+*workspace*), described by a ``quillbench_workspace.yaml`` manifest that the studio reads.
+"""
+
+__version__ = "1.4.1"
+NAME = "QuillBench"
+SHORT_NAME = "QuillBench"
+AUTHOR = "A.C. Pogaku"

quillbench-1.4.1/quillbench/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+main()