quillbench 1.4.1__py3-none-any.whl

quillbench/__init__.py

@@ -0,0 +1,11 @@
+"""QuillBench.
+
+A config-driven human-annotation platform for research datasets. The software lives
+in its own folder; every paper keeps its annotation data in its own folder (a
+*workspace*), described by a ``quillbench_workspace.yaml`` manifest that the studio reads.
+"""
+
+__version__ = "1.4.1"
+NAME = "QuillBench"
+SHORT_NAME = "QuillBench"
+AUTHOR = "A.C. Pogaku"

quillbench/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+main()

quillbench/accounts.py

@@ -0,0 +1,344 @@
+"""
+Accounts, assignments and session persistence.
+
+Two files, two owners:
+
+* **Identities** -- who a person is -- live once, in the studio home
+  (``accounts.json``, mode 600)::
+
+      {"users": {"alice": {"password_hash": "pbkdf2_sha256$...", "role": "annotator",
+                           "display_name": "Alice", "active": true}}}
+
+  Passwords are stored as salted PBKDF2-SHA256 hashes (standard library, no
+  extra dependency).
+
+* **Assignments** -- what a person works on in a paper -- live in that paper's
+  workspace (``quillbench_assignments.json``), so they travel with the paper's data and
+  contain no secrets::
+
+      {"alice": {"batches": ["demo_001", "demo_002"]}}
+
+Session persistence uses an HMAC-signed cookie rather than session state alone:
+annotators work over flaky mobile and SSH-tunnelled connections, and every
+WebSocket reconnect wipes Streamlit's ``session_state``. The cookie carries
+``name:issued:token`` where ``token`` is an HMAC of the name, the issue time and
+the user's ``session_epoch`` under a server-side secret, so it cannot be forged
+client-side. The server rejects a cookie older than the session lifetime, and
+bumping ``session_epoch`` (password change, disabling, signing out) cancels every
+cookie already handed out. It is a cookie rather than a URL parameter because
+URLs leak through history, screen-sharing and proxy logs.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import json
+import os
+import secrets
+import tempfile
+import threading
+import time
+from pathlib import Path
+
+from . import ldap_auth, paths
+from .config import WorkspaceConfig
+
+ROLES = ("annotator", "admin")
+_ITERATIONS = 310_000
+
+
+# ──────────────────────────────────────────────────────────────────────────
+# Password hashing
+# ──────────────────────────────────────────────────────────────────────────
+def hash_password(password: str, *, iterations: int = _ITERATIONS) -> str:
+    salt = secrets.token_hex(16)
+    digest = hashlib.pbkdf2_hmac("sha256", password.encode(), salt.encode(), iterations).hex()
+    return f"pbkdf2_sha256${iterations}${salt}${digest}"
+
+
+def verify_password(password: str, stored: str) -> bool:
+    try:
+        algo, iterations, salt, digest = stored.split("$", 3)
+    except ValueError:
+        return False
+    if algo != "pbkdf2_sha256":
+        return False
+    check = hashlib.pbkdf2_hmac("sha256", password.encode(), salt.encode(), int(iterations)).hex()
+    return hmac.compare_digest(check, digest)
+
+
+# ──────────────────────────────────────────────────────────────────────────
+# JSON files written atomically with private permissions
+# ──────────────────────────────────────────────────────────────────────────
+def _read_json(path: Path) -> dict:
+    if not path.exists():
+        return {}
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError:
+        return {}
+    return data if isinstance(data, dict) else {}
+
+
+def write_json_atomic(path: Path, data: dict, mode: int = 0o600) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.", suffix=".tmp")
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as fh:
+            json.dump(data, fh, indent=2, ensure_ascii=False)
+        os.chmod(tmp, mode)
+        os.replace(tmp, path)
+    except BaseException:
+        Path(tmp).unlink(missing_ok=True)
+        raise
+
+
+# ──────────────────────────────────────────────────────────────────────────
+# Identities
+# ──────────────────────────────────────────────────────────────────────────
+def load_users(path: Path | None = None) -> dict:
+    return _read_json(path or paths.accounts_file()).get("users", {})
+
+
+def save_users(users: dict, path: Path | None = None) -> None:
+    write_json_atomic(path or paths.accounts_file(), {"users": users})
+
+
+def add_user(name: str, password: str, role: str = "annotator", display_name: str = "",
+             path: Path | None = None) -> None:
+    name = name.strip()
+    if not name or ":" in name or "/" in name or "__" in name:
+        raise ValueError("A user id must be non-empty and contain no ':', '/' or '__'.")
+    if role not in ROLES:
+        raise ValueError(f"role must be one of {ROLES}")
+    if len(password) < 8:
+        raise ValueError("Use a password of at least 8 characters.")
+    users = load_users(path)
+    if name in users:
+        raise ValueError(f"User '{name}' already exists.")
+    users[name] = {"password_hash": hash_password(password), "role": role,
+                   "display_name": display_name or name, "active": True}
+    save_users(users, path)
+
+
+def create_first_admin(name: str, password: str, display_name: str = "") -> None:
+    """Make the first account an admin. Only ever works while no account exists."""
+    claim = paths.studio_home() / "first_admin.claim"
+    try:
+        fd = os.open(claim, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
+    except FileExistsError:
+        raise ValueError("Setup is already being completed.") from None
+    os.close(fd)
+    try:
+        if load_users():
+            raise ValueError("An admin already exists.")
+        add_user(name, password, "admin", display_name)
+    finally:
+        claim.unlink(missing_ok=True)
+
+
+def set_password(name: str, password: str, path: Path | None = None) -> None:
+    if len(password) < 8:
+        raise ValueError("Use a password of at least 8 characters.")
+    users = load_users(path)
+    if name not in users:
+        raise ValueError(f"No user '{name}'.")
+    users[name]["password_hash"] = hash_password(password)
+    users[name]["session_epoch"] = int(users[name].get("session_epoch", 0)) + 1
+    save_users(users, path)
+
+
+def update_user(name: str, path: Path | None = None, **fields) -> None:
+    users = load_users(path)
+    if name not in users:
+        raise ValueError(f"No user '{name}'.")
+    if "role" in fields and fields["role"] not in ROLES:
+        raise ValueError(f"role must be one of {ROLES}")
+    was_active = users[name].get("active", True)
+    users[name].update(fields)
+    if was_active and not users[name].get("active", True):
+        users[name]["session_epoch"] = int(users[name].get("session_epoch", 0)) + 1
+    save_users(users, path)
+
+
+def revoke_sessions(name: str, path: Path | None = None) -> None:
+    """Cancel every sign-in cookie this person holds, on every device."""
+    users = load_users(path)
+    if name in users:
+        users[name]["session_epoch"] = int(users[name].get("session_epoch", 0)) + 1
+        save_users(users, path)
+
+
+# Verified against when the user id is unknown, so a wrong id takes as long as a
+# wrong password and response time does not reveal which ids exist.
+_DUMMY_HASH = hash_password(secrets.token_hex(16))
+
+
+def check_login(name: str, password: str, users: dict) -> bool:
+    record = users.get(name)
+    if not isinstance(record, dict):
+        verify_password(password, _DUMMY_HASH)
+        return False
+    ok = verify_password(password, str(record.get("password_hash", "")))
+    return ok and record.get("active", True)
+
+
+def sign_in(name: str, password: str, users: dict, ldap: dict | None = None) -> bool:
+    """Check a sign-in against QuillBench's own accounts, then LDAP when it is set up.
+
+    An account with its own password always uses it, so LDAP can never take over a
+    local account. A person LDAP accepts for the first time gets an annotator
+    account (no password stored) when ``auto_create`` is on, or must already have
+    been added by an admin.
+    """
+    record = users.get(name)
+    if not ldap_auth.enabled(ldap) or (isinstance(record, dict) and record.get("password_hash")):
+        return check_login(name, password, users)
+    if isinstance(record, dict) and not record.get("active", True):
+        return False
+    if record is None and not ldap.get("auto_create", True):
+        verify_password(password, _DUMMY_HASH)
+        return False
+    if not ldap_auth.check(name, password, ldap):
+        return False
+    if record is None:
+        users = load_users()
+        if name not in users:
+            users[name] = {"role": "annotator", "display_name": name, "active": True, "source": "ldap"}
+            save_users(users)
+    return True
+
+
+# ──────────────────────────────────────────────────────────────────────────
+# Sign-in throttling (in-process: the studio runs as one server process)
+# ──────────────────────────────────────────────────────────────────────────
+MAX_FAILURES = 5
+LOCKOUT_SECONDS = 15 * 60
+_failures: dict[str, list[float]] = {}
+_failures_lock = threading.Lock()
+
+
+def _recent_failures(name: str, now: float) -> list[float]:
+    recent = [t for t in _failures.get(name, []) if now - t < LOCKOUT_SECONDS]
+    if recent:
+        _failures[name] = recent
+    else:
+        _failures.pop(name, None)
+    return recent
+
+
+def login_locked(name: str) -> bool:
+    """True after MAX_FAILURES wrong passwords for this id in the last LOCKOUT_SECONDS."""
+    with _failures_lock:
+        return len(_recent_failures(name, time.time())) >= MAX_FAILURES
+
+
+def record_login(name: str, ok: bool) -> None:
+    with _failures_lock:
+        if ok:
+            _failures.pop(name, None)
+        else:
+            _recent_failures(name, time.time())
+            _failures.setdefault(name, []).append(time.time())
+
+
+def session_epoch(name: str, users: dict) -> int:
+    return int(users.get(name, {}).get("session_epoch", 0))
+
+
+def is_admin(name: str, users: dict) -> bool:
+    return users.get(name, {}).get("role", "") == "admin"
+
+
+# ──────────────────────────────────────────────────────────────────────────
+# Per-workspace assignments
+# ──────────────────────────────────────────────────────────────────────────
+def load_assignments(ws: WorkspaceConfig) -> dict:
+    return _read_json(ws.assignments_file)
+
+
+def save_assignments(ws: WorkspaceConfig, data: dict) -> None:
+    write_json_atomic(ws.assignments_file, data, mode=0o640)
+
+
+def assigned_batches(ws: WorkspaceConfig, name: str, assignments: dict | None = None) -> list[str]:
+    assignments = load_assignments(ws) if assignments is None else assignments
+    batches = assignments.get(name, {}).get("batches", [])
+    return [str(b) for b in batches] if isinstance(batches, list) else []
+
+
+def legacy_view(ws: WorkspaceConfig, users: dict, assignments: dict | None = None) -> dict:
+    """``{name: {"role": ..., "batches": [...]}}`` for one workspace.
+
+    The combined shape the query-review tasks use (role and batches in one
+    record); views that need both -- e.g. the adjudicator's live peer labels --
+    read this.
+    """
+    assignments = load_assignments(ws) if assignments is None else assignments
+    return {name: {"role": users.get(name, {}).get("role", "annotator"),
+                   "batches": assigned_batches(ws, name, assignments)}
+            for name in assignments}
+
+
+# ──────────────────────────────────────────────────────────────────────────
+# Signed session tokens
+# ──────────────────────────────────────────────────────────────────────────
+def app_secret(path: Path | None = None) -> str:
+    path = path or paths.secret_file()
+    if not path.exists():
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(secrets.token_hex(32), encoding="utf-8")
+        try:
+            path.chmod(0o600)
+        except OSError:
+            pass
+    return path.read_text(encoding="utf-8").strip()
+
+
+def make_token(name: str, issued: int, epoch: int, secret: str | None = None) -> str:
+    secret = secret or app_secret()
+    msg = f"{name}:{issued}:{epoch}".encode()
+    return hmac.new(secret.encode(), msg, hashlib.sha256).hexdigest()[:32]
+
+
+def cookie_value(name: str, issued: int, users: dict, secret: str | None = None) -> str:
+    return f"{name}:{issued}:{make_token(name, issued, session_epoch(name, users), secret)}"
+
+
+def session_from_cookie(raw: str, users: dict, secret: str | None = None,
+                        max_age: int = 30 * 24 * 3600) -> tuple[str, int] | None:
+    """Return ``(user, issued)`` for a valid, unexpired cookie value, else None.
+
+    DECODE FIRST, THEN SPLIT: the cookie is written through JS
+    ``encodeURIComponent()``, which encodes the ':' separators as '%3A', so a
+    check against the raw value would reject every valid cookie. ``rsplit``
+    because the issue time and token can never contain ':', while a user id is
+    parsed unambiguously from the right.
+    """
+    from urllib.parse import unquote
+
+    if not raw:
+        return None
+    parts = unquote(raw).rsplit(":", 2)
+    if len(parts) != 3:
+        return None
+    name, issued_s, token = parts
+    try:
+        issued = int(issued_s)
+    except ValueError:
+        return None
+    if not 0 <= time.time() - issued <= max_age:
+        return None
+    record = users.get(name)
+    if not record or not record.get("active", True):
+        return None
+    if hmac.compare_digest(token, make_token(name, issued, session_epoch(name, users), secret)):
+        return name, issued
+    return None
+
+
+def name_from_cookie(raw: str, users: dict, secret: str | None = None,
+                     max_age: int = 30 * 24 * 3600) -> str | None:
+    found = session_from_cookie(raw, users, secret, max_age)
+    return found[0] if found else None

quillbench/backup.py

@@ -0,0 +1,48 @@
+"""
+Backups: every workspace's answers and assignments, plus the studio home.
+
+One ``.tar.gz`` per run in ``settings.backup_dir`` (mode 600), the newest
+``backup_keep`` kept. Batch and benchmark files are not included -- they are
+inputs that already live in the papers' own folders and version control; the
+answers are what cannot be recreated.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import tarfile
+from datetime import datetime
+from pathlib import Path
+
+from . import paths, registry
+from .settings import load_settings
+
+
+def run(dest: Path | None = None, keep: int | None = None) -> Path:
+    settings = load_settings()
+    dest = Path(dest or settings.backup_dir)
+    keep = settings.backup_keep if keep is None else keep
+    dest.mkdir(parents=True, exist_ok=True)
+    stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    target = dest / f"quillbench_backup_{stamp}.tar.gz"
+    workspaces, _ = registry.load_all(settings)
+    tmp = target.with_suffix(".tmp")
+    with tarfile.open(tmp, "w:gz") as tar:
+        tar.add(paths.studio_home(), arcname="studio_home")
+        for ws in workspaces:
+            base = f"workspaces/{ws.id}"
+            tar.add(ws.manifest, arcname=f"{base}/{ws.manifest.name}")
+            if ws.assignments_file.exists():
+                tar.add(ws.assignments_file, arcname=f"{base}/{ws.assignments_file.name}")
+            for i, d in enumerate(ws.all_progress_dirs):
+                if d.exists():
+                    tar.add(d, arcname=f"{base}/progress_{i}_{d.parent.name}")
+    tmp.chmod(0o600)
+    tmp.rename(target)
+    digest = hashlib.sha256(target.read_bytes()).hexdigest()
+    target.with_name(target.name + ".sha256").write_text(f"{digest}  {target.name}\n")
+    old = sorted(dest.glob("quillbench_backup_*.tar.gz")) + sorted(dest.glob("labelwise_backup_*.tar.gz"))
+    for stale in old[:-keep] if keep > 0 else []:
+        stale.unlink(missing_ok=True)
+        stale.with_name(stale.name + ".sha256").unlink(missing_ok=True)
+    return target

quillbench/builder.py

@@ -0,0 +1,190 @@
+"""Create projects from the web interface.
+
+Each project made here is its own workspace folder under the studio home, so it
+never touches a workspace that already exists: nothing here edits another
+paper's manifest, batches, progress or assignments.
+"""
+
+from __future__ import annotations
+
+import re
+import shutil
+from pathlib import Path
+
+import pandas as pd
+import yaml
+
+from . import accounts, paths, registry
+from .config import MANIFEST_NAME, ConfigError, WorkspaceConfig, load_workspace
+
+BATCH_PREFIX = "batch_"
+TASKS = {"choice": "Label buttons", "rating": "Slider rating", "text": "Short written answer"}
+
+
+def projects_root() -> Path:
+    return paths.studio_home() / "projects"
+
+
+def is_app_project(ws: WorkspaceConfig) -> bool:
+    return projects_root().resolve() in ws.root.parents
+
+
+def clean_labels(text: str) -> list[str]:
+    """One label per comma or line, duplicates dropped, order kept."""
+    return list(dict.fromkeys(s.strip() for s in re.split(r"[,\n]", text) if s.strip()))
+
+
+def read_table(source) -> pd.DataFrame:
+    df = pd.read_csv(source, dtype=str, keep_default_na=False)
+    if df.empty:
+        raise ValueError("The CSV has no rows.")
+    return df
+
+
+def with_ids(df: pd.DataFrame, id_column: str | None, start: int = 1) -> pd.DataFrame:
+    """Give the table the ``id`` column batches need, unique within it."""
+    df = df.copy()
+    if id_column and id_column != "id":
+        df = df.drop(columns=["id"], errors="ignore").rename(columns={id_column: "id"})
+    elif not id_column:
+        df = df.drop(columns=["id"], errors="ignore")
+        df.insert(0, "id", [str(i) for i in range(start, start + len(df))])
+    if df["id"].astype(str).str.strip().eq("").any():
+        raise ValueError("Some rows have an empty id.")
+    repeats = df["id"][df["id"].duplicated()].unique().tolist()
+    if repeats:
+        raise ValueError(f"These ids appear more than once: {repeats[:5]}")
+    return df
+
+
+def split(df: pd.DataFrame, size: int, shuffle: bool = False, seed: int = 42) -> list[pd.DataFrame]:
+    """Cut a table into batches of ``size`` rows; 0 keeps it whole."""
+    if shuffle:
+        df = df.sample(frac=1, random_state=seed).reset_index(drop=True)
+    size = size if size > 0 else len(df)
+    return [df.iloc[i:i + size] for i in range(0, len(df), size)]
+
+
+def _existing(ws_root: Path) -> list[Path]:
+    return sorted((ws_root / "batches").glob(f"{BATCH_PREFIX}*.csv"))
+
+
+def _write_batches(root: Path, parts: list[pd.DataFrame]) -> list[str]:
+    taken = [int(m.group(1)) for p in _existing(root)
+             if (m := re.fullmatch(rf"{BATCH_PREFIX}(\d+)", p.stem))]
+    number = max(taken, default=0)
+    names = []
+    for part in parts:
+        number += 1
+        name = f"{BATCH_PREFIX}{number:03d}"
+        part.to_csv(root / "batches" / f"{name}.csv", index=False)
+        names.append(name)
+    return names
+
+
+def assign(ws: WorkspaceConfig, batches: list[str], people: list[str], per_batch: int = 1) -> None:
+    """Deal batches out in turn, each to ``per_batch`` different people."""
+    if not people:
+        return
+    data = accounts.load_assignments(ws)
+    per_batch = min(max(per_batch, 1), len(people))
+    for i, batch in enumerate(batches):
+        for k in range(per_batch):
+            person = people[(i * per_batch + k) % len(people)]
+            queue = data.setdefault(person, {"batches": []})["batches"]
+            if batch not in queue:
+                queue.append(batch)
+    accounts.save_assignments(ws, data)
+
+
+def _new_id(name: str) -> str:
+    base = re.sub(r"[^a-z0-9]+", "_", name.lower()).strip("_") or "project"
+    taken = {p.name for p in projects_root().glob("*")}
+    for manifest in registry.manifest_paths():
+        taken.add(registry._peek_id(manifest))
+    found, n = base, 1
+    while found in taken:
+        n += 1
+        found = f"{base}_{n}"
+    return found
+
+
+def create_project(name: str, task: str, instructions: str, table: pd.DataFrame | None,
+                   display: list[str], owner: str, batch_size: int = 0, shuffle: bool = False,
+                   labels: list[str] | None = None, scale: dict | None = None,
+                   people: list[str] | None = None, per_batch: int = 1) -> WorkspaceConfig:
+    name = name.strip()
+    if not name:
+        raise ValueError("Give the project a name.")
+    if task not in TASKS:
+        raise ValueError(f"Unknown task '{task}'.")
+    if task == "choice" and len(labels or []) < 2:
+        raise ValueError("Add at least two labels.")
+    if task == "rating" and not (scale and scale["min"] < scale["max"] and scale["step"] > 0):
+        raise ValueError("The slider needs a minimum below its maximum and a step above 0.")
+    if table is not None and not display:
+        raise ValueError("Choose at least one column to show.")
+
+    ws_id = _new_id(name)
+    project = {"id": ws_id, "name": name, "task": task, "default": True,
+               "batch_prefix": BATCH_PREFIX, "display": list(display or []),
+               "instructions": instructions.strip()}
+    if task == "choice":
+        project["labels"] = list(labels)
+    if task == "rating":
+        project["scale"] = scale
+    manifest = {"workspace": {"id": ws_id, "name": name, "owner": owner,
+                              "batches_dir": "batches", "progress_dir": "progress"},
+                "projects": [project]}
+    root = projects_root() / ws_id
+    (root / "batches").mkdir(parents=True)
+    try:
+        (root / MANIFEST_NAME).write_text(yaml.safe_dump(manifest, sort_keys=False, allow_unicode=True),
+                                          encoding="utf-8")
+        batches = [] if table is None else _write_batches(root, split(table, batch_size, shuffle))
+        registry.add(root)
+    except (ConfigError, OSError, ValueError):
+        shutil.rmtree(root, ignore_errors=True)
+        raise
+    ws = load_workspace(root)
+    assign(ws, batches, people or [], per_batch)
+    return ws
+
+
+def _update_project(ws: WorkspaceConfig, **fields) -> None:
+    raw = yaml.safe_load(ws.manifest.read_text(encoding="utf-8"))
+    raw["projects"][0].update(fields)
+    ws.manifest.write_text(yaml.safe_dump(raw, sort_keys=False, allow_unicode=True), encoding="utf-8")
+
+
+def add_data(ws: WorkspaceConfig, table: pd.DataFrame, id_column: str | None, batch_size: int = 0,
+             shuffle: bool = False, people: list[str] | None = None, per_batch: int = 1,
+             display: list[str] | None = None) -> list[str]:
+    """Add rows to a project made here, as new batches; existing batches are untouched.
+
+    ``display`` picks the columns to show when the project has none chosen yet.
+    """
+    seen: list[str] = []
+    for path in _existing(ws.root):
+        seen += pd.read_csv(path, dtype=str, keep_default_na=False)["id"].tolist()
+    new = with_ids(table, id_column, start=len(seen) + 1)
+    clash = sorted(set(new["id"]) & set(seen))
+    if clash:
+        raise ValueError(f"These ids are already in the project: {clash[:5]}")
+    shown = [f.column for f in ws.projects[0].display]
+    missing = [c for c in shown if c not in new.columns]
+    if missing:
+        raise ValueError(f"The CSV is missing the column(s) annotators see: {missing}")
+    if display and not shown:
+        _update_project(ws, display=list(display))
+    batches = _write_batches(ws.root, split(new, batch_size, shuffle))
+    assign(ws, batches, people or [], per_batch)
+    return batches
+
+
+def add_labels(ws: WorkspaceConfig, new: list[str]) -> None:
+    """Add label buttons to a project made here. Labels are never removed: saved answers use them."""
+    if ws.projects[0].task != "choice":
+        raise ValueError("Only label-button projects have labels.")
+    current = yaml.safe_load(ws.manifest.read_text(encoding="utf-8"))["projects"][0]["labels"]
+    _update_project(ws, labels=list(dict.fromkeys(current + new)))