goldlabel 0.2.1__py3-none-any.whl

goldlabel/__init__.py

@@ -0,0 +1,69 @@
+"""Content analysis over a corpus (seeds, keywords or the whole corpus, then an LLM classifier), as a library.
+
+Importing reads, writes and networks nothing. Two exported names shadow submodules of the package
+(`goldlabel.cost` is the function, `goldlabel.settings` the object): inside the package and in tests
+import by name, `from goldlabel.cost import cache_key`, `from goldlabel.settings import resolve`.
+"""
+
+from importlib.metadata import PackageNotFoundError, version
+
+from goldlabel._gold_rule import GOLD_RULE_FULL, GOLD_RULE_SHORT, GoldError
+from goldlabel._label import label
+from goldlabel.candidates import (
+    Candidates,
+    aget_candidates,
+    get_candidates,
+    read_seeds,
+)
+from goldlabel.classifier import Classifier, CostLimit, Estimate, Predictions
+from goldlabel.cost import Cost, cost
+from goldlabel.evaluation import (
+    BandReport,
+    EvalLog,
+    Evaluation,
+    aevaluate,
+    band_sample,
+    eval_log,
+    evaluate,
+)
+from goldlabel.gold import Gold, GoldSet
+from goldlabel.search import PgCorpus, S3Corpus, Table
+from goldlabel.settings import settings
+from goldlabel.taxonomy import Taxonomy
+
+try:
+    __version__ = version("goldlabel")
+except PackageNotFoundError:  # running from a source tree that was never installed
+    __version__ = "0+unknown"
+
+__all__ = [
+    "GOLD_RULE_FULL",
+    "GOLD_RULE_SHORT",
+    "BandReport",
+    "Candidates",
+    "Classifier",
+    "Cost",
+    "CostLimit",
+    "Estimate",
+    "EvalLog",
+    "Evaluation",
+    "Gold",
+    "GoldError",
+    "GoldSet",
+    "PgCorpus",
+    "Predictions",
+    "S3Corpus",
+    "Table",
+    "Taxonomy",
+    "__version__",
+    "aevaluate",
+    "aget_candidates",
+    "band_sample",
+    "cost",
+    "eval_log",
+    "evaluate",
+    "get_candidates",
+    "label",
+    "read_seeds",
+    "settings",
+]

goldlabel/_frame.py

@@ -0,0 +1,33 @@
+"""One place that turns a list of row dicts into a polars or a pandas table, so a result can never export one without the other."""
+
+
+def _columns(rows: list[dict]) -> dict[str, list]:
+    names = list(
+        dict.fromkeys(k for r in rows for k in r)
+    )  # all keys, first-seen order
+    return {name: [r.get(name) for r in rows] for name in names}
+
+
+def to_polars(rows: list[dict]):
+    try:
+        import polars
+    except ImportError:
+        raise ImportError("polars is not installed: pip install polars") from None
+    # strict=False lets int and float share a column; the schema is from every row, not the first 100
+    return polars.DataFrame(_columns(rows), strict=False)
+
+
+def to_pandas(rows: list[dict]):
+    try:
+        import pandas
+    except ImportError:
+        raise ImportError("pandas is not installed: pip install pandas") from None
+    # a column with a missing value stays `object` so it holds None, as polars does (not NaN)
+    return pandas.DataFrame(
+        {
+            name: pandas.Series(values, dtype=object)
+            if any(v is None for v in values)
+            else values
+            for name, values in _columns(rows).items()
+        }
+    )

goldlabel/_gold_rule.py

@@ -0,0 +1,31 @@
+"""The gold rule text (`gold_rule.md`, package data) and the error type that carries it."""
+
+from importlib import resources
+
+_text = (
+    resources.files("goldlabel").joinpath("gold_rule.md").read_text(encoding="utf-8")
+)
+_short, _, _full = _text.partition("## Full")
+GOLD_RULE_SHORT = _short.removeprefix("## Short").strip()
+GOLD_RULE_FULL = _full.strip()
+
+
+class GoldError(ValueError):
+    """An error about gold. `str()` is the message, two newlines and the Short gold rule;
+    `.text` is the message alone (for the labelling page's HTTP errors, which omit the rule).
+
+    The `TypeError` and `RuntimeError` cases raised from `gold.py`/`_label.py` use the two
+    subclasses below, so `except TypeError` / `except RuntimeError` keep working.
+    """
+
+    def __init__(self, text: str):
+        self.text = text
+        super().__init__(f"{text}\n\n{GOLD_RULE_SHORT}")
+
+
+class GoldTypeError(GoldError, TypeError):
+    pass
+
+
+class GoldRuntimeError(GoldError, RuntimeError):
+    pass

goldlabel/_html.py

@@ -0,0 +1,56 @@
+"""Tiny helpers for the notebook reprs. All text is escaped; no colours are set, so the notebook theme shows through."""
+
+import html
+from collections.abc import Sequence
+
+CAP = 50
+_RULE = "border-bottom:1px solid currentColor"
+
+
+def esc(value) -> str:
+    return html.escape("" if value is None else str(value))
+
+
+def _cell(value) -> str:
+    if isinstance(value, (list, tuple)):
+        value = ", ".join(str(v) for v in value)
+    return esc(value)
+
+
+def table(columns: Sequence[str], rows: Sequence[Sequence], cap: int = CAP) -> str:
+    """A table of at most `cap` rows, then a "N more" line."""
+    head = "".join(
+        f'<th style="text-align:left;{_RULE}">{esc(c)}</th>' for c in columns
+    )
+    body = "".join(
+        "<tr>"
+        + "".join(f'<td style="vertical-align:top">{_cell(v)}</td>' for v in row)
+        + "</tr>"
+        for row in rows[:cap]
+    )
+    more = f"<p>{len(rows) - cap} more rows</p>" if len(rows) > cap else ""
+    return f"<table><tr>{head}</tr>{body}</table>{more}"
+
+
+def bars(values: Sequence[tuple[str, int]]) -> str:
+    """One line per (label, count): a bar (shape) and the count as text."""
+    top = max([n for _, n in values] + [1])
+    lines = "".join(
+        f"<div>{esc(name)} "
+        f'<span style="display:inline-block;height:0;width:{100 * n / top:.0f}px;'
+        f'border-top:0.8em solid currentColor;opacity:0.35"></span> {n}</div>'
+        for name, n in values
+    )
+    return f"<div>{lines}</div>"
+
+
+def warning(text: str) -> str:
+    return f'<p style="font-weight:bold;{_RULE}">WARNING: {esc(text)}</p>'
+
+
+def details(summary: str, body: str, open: bool = False) -> str:
+    return f"<details{' open' if open else ''}><summary>{esc(summary)}</summary>{body}</details>"
+
+
+def line(text: str) -> str:
+    return f"<p>{esc(text)}</p>"

goldlabel/_label.py

@@ -0,0 +1,329 @@
+"""The labelling server: the one place (besides `Gold.import_rows`) where gold labels are written.
+
+A label is written only because a person's HTTP request says so, stamped `labelled_by: "goldlabel-ui"`.
+Nothing here calls a model or returns a prediction. The server decides which item is next, so the
+page stays thin. The page is `web/index.html` + `app.js` + `app.css` from the package; the three are served only with
+the token and Host checks like everything else, and the HTML carries the token in its asset URLs.
+"""
+
+import atexit
+import json
+import os
+import secrets
+import sys
+import threading
+import webbrowser
+from datetime import UTC, datetime
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from importlib import resources
+from urllib.parse import parse_qs, urlparse
+
+from goldlabel._gold_rule import GOLD_RULE_SHORT, GoldError, GoldRuntimeError
+from goldlabel.gold import SPLITS, Gold, GoldSet
+from goldlabel.taxonomy import OFF_TOPIC, Taxonomy, validate_labels
+
+OFF_TOPIC_DESCRIPTION = "the item matches none of the labels above"
+MAX_BODY = 1_000_000
+PAGES = {  # the only static paths: URL path -> (file in goldlabel/web, content type)
+    "/": ("index.html", "text/html"),
+    "/app.js": ("app.js", "text/javascript"),
+    "/app.css": ("app.css", "text/css"),
+}
+
+
+def _alive(pid: int) -> bool:
+    if sys.platform == "win32":  # os.kill(pid, 0) would terminate the process there
+        import ctypes
+
+        handle = ctypes.windll.kernel32.OpenProcess(0x1000, False, pid)
+        if handle:
+            ctypes.windll.kernel32.CloseHandle(handle)
+        return bool(handle)
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:  # it exists, it is somebody else's
+        return True
+    return True
+
+
+def _acquire(lock) -> None:
+    lock.parent.mkdir(parents=True, exist_ok=True)
+    for _ in range(3):
+        try:
+            fd = os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
+        except FileExistsError:
+            try:
+                info = json.loads(lock.read_text())
+                pid, started = int(info["pid"]), info.get("started")
+            except (OSError, ValueError, KeyError, TypeError):
+                pid, started = None, None  # unreadable: stale
+            if pid is not None and _alive(pid):
+                raise GoldRuntimeError(
+                    f"{lock.with_suffix('')} is already being labelled (pid {pid}, started "
+                    f"{started}); finish or stop that session first"
+                ) from None
+            lock.unlink(missing_ok=True)
+            continue
+        with os.fdopen(fd, "w") as f:
+            json.dump({"pid": os.getpid(), "started": datetime.now(UTC).isoformat()}, f)
+        return
+    raise GoldRuntimeError(f"could not take the lock {lock}")
+
+
+def _release(lock) -> None:
+    try:
+        if json.loads(lock.read_text()).get("pid") == os.getpid():
+            lock.unlink(missing_ok=True)
+    except (OSError, ValueError, AttributeError):
+        pass
+
+
+class Handle:
+    def __init__(self, gold: Gold, taxonomy: Taxonomy, split: str | None):
+        self.gold, self.taxonomy, self.split = gold, taxonomy, split
+        self.token = secrets.token_urlsafe(32)
+        self.lock = gold.path.with_name(gold.path.name + ".lock")
+        self._mutex = threading.Lock()
+        self._stopping = False
+        self._stopped = threading.Event()  # set once the lock is released
+        self._history: list[str] = []
+        self.current: str | None = None
+        self.server: ThreadingHTTPServer
+
+    @property
+    def port(self) -> int:
+        return self.server.server_address[1]
+
+    @property
+    def url(self) -> str:
+        return f"http://127.0.0.1:{self.port}/?token={self.token}"
+
+    def wait(self) -> None:
+        self._stopped.wait()
+
+    def stop(self) -> None:
+        with self._mutex:
+            if self._stopping:
+                return
+            self._stopping = True
+        atexit.unregister(self.stop)
+        self.server.shutdown()
+        self.server.server_close()
+        _release(self.lock)
+        self._stopped.set()
+
+    # --- state; the caller holds self._mutex ------------------------------------------------
+
+    def _scope(self):
+        return [r for r in self.gold._read() if self.split in (None, r.split)]
+
+    def _after(self, rows, id_):
+        """The next pending row after `id_` in file order, else the first pending one."""
+        ids = [r.id for r in rows]
+        start = ids.index(id_) + 1 if id_ in ids else 0
+        for r in rows[start:] + rows[:start]:
+            if not r.labels:
+                return r.id
+        return None
+
+    def _state(self) -> dict:
+        rows = self._scope()
+        by_id = {r.id: r for r in rows}
+        row = by_id.get(self.current) if self.current else None
+        done = sum(1 for r in rows if r.labels)
+        position = [r.id for r in rows].index(row.id) + 1 if row else len(rows)
+        descriptions = [
+            (label.name, label.description) for label in self.taxonomy.labels
+        ]
+        descriptions.append((OFF_TOPIC, OFF_TOPIC_DESCRIPTION))
+        return {
+            "item": {"id": row.id, "text": row.text, "labels": row.labels or None}
+            if row
+            else None,
+            "position": position,
+            "done": done,
+            "total": len(rows),
+            "labels": [{"name": n, "description": d} for n, d in descriptions],
+            "mode": self.taxonomy.mode,
+            "counts": GoldSet(rows).counts(self.taxonomy),
+            "finished": row is None,
+        }
+
+    def _move(self, rows, after) -> None:
+        if self.current is not None:
+            self._history.append(self.current)
+        self.current = self._after(rows, after)
+
+    def state(self) -> dict:
+        with self._mutex:
+            return self._state()
+
+    def label(self, id_: str, labels: list[str]) -> dict:
+        validate_labels(labels, self.taxonomy)
+        with self._mutex:
+            all_rows = self.gold._read()
+            rows = [r for r in all_rows if self.split in (None, r.split)]
+            target = next((r for r in all_rows if r.id == id_ and r in rows), None)
+            if target is None:
+                raise KeyError(id_)
+            stamp = {
+                "labels": labels,
+                "labelled_by": "goldlabel-ui",
+                "labelled_at": datetime.now(UTC).isoformat(),
+            }
+            self.gold._write(
+                [r.model_copy(update=stamp) if r is target else r for r in all_rows]
+            )
+            self._move(self._scope(), id_)
+            return self._state()
+
+    def back(self) -> dict:
+        with self._mutex:
+            if self._history:
+                self.current = self._history.pop()
+            return self._state()
+
+    def skip(self) -> dict:
+        with self._mutex:
+            if self.current is not None:
+                self._move(self._scope(), self.current)
+            return self._state()
+
+
+class _Request(BaseHTTPRequestHandler):
+    session: Handle  # set on a subclass made per server in `serve`
+
+    def log_message(self, format, *args):  # keep the notebook output clean
+        pass
+
+    def _send(self, status: int, body, content_type="application/json") -> None:
+        data = (
+            json.dumps(body) if content_type == "application/json" else body
+        ).encode()
+        self.send_response(status)
+        self.send_header("Content-Type", content_type + "; charset=utf-8")
+        self.send_header("Content-Length", str(len(data)))
+        self.send_header("Cache-Control", "no-store")
+        self.end_headers()
+        self.wfile.write(data)
+
+    def _allowed(self, handle: Handle) -> bool:
+        # The Host check stops DNS rebinding; the token stops any other local process or page.
+        if self.headers.get("Host") not in (
+            f"127.0.0.1:{handle.port}",
+            f"localhost:{handle.port}",
+        ):
+            return False
+        given = (
+            self.headers.get("X-Goldlabel-Token")
+            or (parse_qs(urlparse(self.path).query).get("token", [""])[0])
+        )
+        return secrets.compare_digest(given.encode(), handle.token.encode())
+
+    def _body(self) -> dict | None:
+        try:
+            n = int(self.headers.get("Content-Length") or 0)
+            body = json.loads(self.rfile.read(n)) if 0 < n <= MAX_BODY else {}
+        except ValueError:
+            return None
+        return body if isinstance(body, dict) else None
+
+    def do_GET(self):
+        handle = self.session
+        if not self._allowed(handle):
+            return self._send(403, {"error": "forbidden"})
+        path = urlparse(self.path).path
+        if path in PAGES:
+            name, kind = PAGES[path]
+            page = resources.files("goldlabel").joinpath("web", name).read_text("utf-8")
+            return self._send(200, page.replace("__TOKEN__", handle.token), kind)
+        if path == "/api/state":
+            return self._send(200, handle.state())
+        self._send(404, {"error": "not found"})
+
+    def do_POST(self):
+        handle = self.session
+        if not self._allowed(handle):
+            return self._send(403, {"error": "forbidden"})
+        path = urlparse(self.path).path
+        body = self._body()
+        if body is None:
+            return self._send(400, {"error": "the body must be a JSON object"})
+        if path == "/api/label":
+            id_, labels = body.get("id"), body.get("labels")
+            if not isinstance(id_, str) or not (
+                isinstance(labels, list) and all(isinstance(x, str) for x in labels)
+            ):
+                return self._send(400, {"error": "send {id: str, labels: [str]}"})
+            try:
+                return self._send(200, handle.label(id_, labels))
+            except KeyError:
+                return self._send(404, {"error": f"no row {id_!r} in this split"})
+            except ValueError as e:
+                return self._send(400, {"error": getattr(e, "text", str(e))})
+        if path == "/api/back":
+            return self._send(200, handle.back())
+        if path == "/api/skip":
+            return self._send(200, handle.skip())
+        if path == "/api/done":
+            self._send(200, handle.state())
+            threading.Thread(target=handle.stop, daemon=True).start()
+            return
+        self._send(404, {"error": "not found"})
+
+
+def serve(gold: Gold, taxonomy: Taxonomy, *, split=None, port=None) -> Handle:
+    """Start the labelling server on 127.0.0.1 and return at once; it resumes at the first
+    unlabelled row (file order) of `split` (`dev`, `test` or both)."""
+    if split is not None and split not in SPLITS:
+        raise GoldError(f"split must be 'dev', 'test' or None, not {split!r}")
+    handle = Handle(gold, taxonomy, split)
+    _acquire(handle.lock)
+    try:
+        handle.current = handle._after(handle._scope(), "")
+        request = type("Request", (_Request,), {"session": handle})
+        handle.server = ThreadingHTTPServer(("127.0.0.1", port or 0), request)
+        threading.Thread(target=handle.server.serve_forever, daemon=True).start()
+    except BaseException:
+        _release(handle.lock)
+        raise
+    atexit.register(handle.stop)
+    return handle
+
+
+def label(
+    gold: Gold,
+    taxonomy: Taxonomy,
+    *,
+    split: str | None = None,
+    block: bool = True,
+    open_browser: bool = True,
+    port: int | None = None,
+):
+    """Open the labelling page. Returns the labelled rows after **Done**, or the `Handle` when
+    `block=False`. Resumes at the first unlabelled row; every click is saved at once."""
+    handle = serve(gold, taxonomy, split=split, port=port)
+    try:
+        orphans = len(gold.orphaned(taxonomy))
+        if orphans:
+            print(
+                f"{orphans} labelled row(s) in {gold.path} use labels that are not in this "
+                "taxonomy; continuing."
+            )
+        print(
+            f"Labelling page: {handle.url}\n"
+            "Label the rows in your browser, then press Done on the page.\n"
+            f"{GOLD_RULE_SHORT}"
+        )
+        if open_browser:
+            webbrowser.open(handle.url)
+        if not block:
+            return handle
+        handle.wait()
+    except BaseException:
+        handle.stop()
+        raise
+    rows = [r for r in gold._read() if r.labels and split in (None, r.split)]
+    return GoldSet(rows, split or "all", gold.path)

goldlabel/_sync.py

@@ -0,0 +1,55 @@
+import asyncio
+import threading
+
+
+def run(coro):
+    """Run a coroutine to completion from sync code, also inside a running event loop."""
+    try:
+        asyncio.get_running_loop()
+    except RuntimeError:
+        return asyncio.run(coro)
+    box: dict = {}
+    done = threading.Event()
+
+    def work():
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        task = loop.create_task(coro)
+        box["loop"], box["task"] = loop, task
+        try:
+            box["value"] = loop.run_until_complete(task)
+        except BaseException as exc:  # noqa: BLE001 - re-raised in the caller
+            box["error"] = exc
+        finally:
+            try:
+                rest = [t for t in asyncio.all_tasks(loop) if not t.done()]
+                for t in rest:
+                    t.cancel()
+                if rest:
+                    loop.run_until_complete(
+                        asyncio.gather(*rest, return_exceptions=True)
+                    )
+                loop.run_until_complete(loop.shutdown_asyncgens())
+            finally:
+                loop.close()
+                done.set()
+
+    thread = threading.Thread(target=work, daemon=True)
+    thread.start()
+    try:
+        done.wait()
+    except KeyboardInterrupt:
+        while thread.is_alive() and "task" not in box:
+            thread.join(0.01)
+        if not box["loop"].is_closed():
+            try:
+                box["loop"].call_soon_threadsafe(box["task"].cancel)
+            except RuntimeError:  # loop already closed: the work finished
+                pass
+        done.wait()
+        thread.join()
+        raise
+    thread.join()
+    if "error" in box:
+        raise box["error"]
+    return box["value"]