spanmark 0.1.0__py3-none-any.whl

spanmark/_source.py

@@ -0,0 +1,179 @@
+"""Private document sources for in-memory and indexed JSONL inputs."""
+
+from __future__ import annotations
+
+import json
+import os
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any, cast
+
+from spanmark._model import Document, normalize_document
+
+
+class InMemoryDocumentSource:
+    """A normalized in-memory document source that preserves original records."""
+
+    def __init__(self, documents: Sequence[Mapping[str, Any]]) -> None:
+        records: list[dict[str, Any]] = []
+        for index, document in enumerate(documents):
+            if not isinstance(document, Mapping):
+                raise TypeError(f"Document at index {index} must be a mapping")
+            records.append(dict(document))
+
+        normalized = tuple(
+            normalize_document(record, index) for index, record in enumerate(records)
+        )
+        if not normalized:
+            raise ValueError("documents must contain at least one document")
+
+        ids = tuple(document.id for document in normalized)
+        if len(set(ids)) != len(ids):
+            raise ValueError("document ids must be unique")
+
+        self._records = tuple(records)
+        self._documents = normalized
+        self.ids = ids
+        self._index_by_id = {
+            document_id: index for index, document_id in enumerate(ids)
+        }
+
+    def __len__(self) -> int:
+        return len(self._documents)
+
+    def __getitem__(self, index: int) -> Document:
+        return self._documents[index]
+
+    def record(self, index: int) -> Mapping[str, Any]:
+        """Return the original input mapping for *index*."""
+        return self._records[index]
+
+    def index_of(self, document_id: str) -> int:
+        return self._index_by_id[document_id]
+
+
+class JsonlDocumentSource:
+    """A JSONL source indexed by byte offset instead of retained document text."""
+
+    def __init__(self, path: os.PathLike[str] | Path) -> None:
+        self.path = Path(path)
+        self._offsets: tuple[int, ...] = ()
+        self._signature = (0, 0)
+        self.ids: tuple[str, ...] = ()
+        self._index_by_id: dict[str, int] = {}
+        self.refresh()
+
+    def __len__(self) -> int:
+        return len(self._offsets)
+
+    def __getitem__(self, index: int) -> Document:
+        return normalize_document(self.record(index), index)
+
+    def record(self, index: int) -> Mapping[str, Any]:
+        """Load one original JSONL record by byte offset."""
+        if index < 0:
+            index += len(self)
+        if not 0 <= index < len(self):
+            raise IndexError(index)
+
+        self.check_unchanged()
+
+        with self.path.open("rb") as file:
+            file.seek(self._offsets[index])
+            raw_line = file.readline()
+
+        raw = _decode_json_line(
+            raw_line,
+            path=self.path,
+            line_number=None,
+        )
+        document = normalize_document(raw, index)
+
+        expected_id = self.ids[index]
+        if document.id != expected_id:
+            raise RuntimeError(
+                f"JSONL source {self.path} changed while the session was open: "
+                f"expected document {expected_id!r}, found {document.id!r}"
+            )
+
+        return raw
+
+    def index_of(self, document_id: str) -> int:
+        return self._index_by_id[document_id]
+
+    def check_unchanged(self) -> None:
+        """Raise if the indexed JSONL file changed since the last refresh."""
+        if _file_signature(self.path) != self._signature:
+            raise RuntimeError(
+                f"JSONL source {self.path} changed while the session was open"
+            )
+
+    def refresh(self) -> None:
+        """Rebuild byte offsets after spanmark atomically rewrites this file."""
+        offsets: list[int] = []
+        ids: list[str] = []
+        index_by_id: dict[str, int] = {}
+
+        with self.path.open("rb") as file:
+            line_number = 0
+            document_index = 0
+            while True:
+                offset = file.tell()
+                raw_line = file.readline()
+                if not raw_line:
+                    break
+                line_number += 1
+
+                if not raw_line.strip():
+                    continue
+
+                raw = _decode_json_line(
+                    raw_line,
+                    path=self.path,
+                    line_number=line_number,
+                )
+                document = normalize_document(raw, document_index)
+
+                if document.id in index_by_id:
+                    raise ValueError(f"document ids must be unique: {document.id!r}")
+
+                index_by_id[document.id] = document_index
+                offsets.append(offset)
+                ids.append(document.id)
+                document_index += 1
+
+        if not offsets:
+            raise ValueError("documents must contain at least one document")
+
+        self._offsets = tuple(offsets)
+        self.ids = tuple(ids)
+        self._index_by_id = index_by_id
+        self._signature = _file_signature(self.path)
+
+
+def _decode_json_line(
+    raw_line: bytes,
+    *,
+    path: Path,
+    line_number: int | None,
+) -> Mapping[str, Any]:
+    location = f"{path} line {line_number}" if line_number is not None else str(path)
+
+    try:
+        text = raw_line.decode("utf-8")
+    except UnicodeDecodeError as exc:
+        raise ValueError(f"Invalid UTF-8 in {location}") from exc
+
+    try:
+        raw: object = json.loads(text)
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"Invalid JSON in {location}") from exc
+
+    if isinstance(raw, Mapping):
+        return cast(Mapping[str, Any], raw)
+    raise TypeError(f"Each JSONL record in {location} must be an object")
+
+
+def _file_signature(path: Path) -> tuple[int, int]:
+    stat = path.stat()
+    return stat.st_size, stat.st_mtime_ns

spanmark/_storage.py

@@ -0,0 +1,97 @@
+"""Small, boring JSONL persistence helpers."""
+
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from hashlib import sha256
+from pathlib import Path
+from typing import Any
+
+
+@dataclass(frozen=True, slots=True)
+class FileFingerprint:
+    """Stable identity for one complete file snapshot."""
+
+    size: int
+    sha256: str
+
+
+def fingerprint_file(path: Path) -> FileFingerprint:
+    """Return a content fingerprint for *path*."""
+    digest = sha256()
+    size = 0
+
+    with path.open("rb") as file:
+        while chunk := file.read(1024 * 1024):
+            digest.update(chunk)
+            size += len(chunk)
+
+    return FileFingerprint(size=size, sha256=digest.hexdigest())
+
+
+def atomic_write_jsonl(
+    path: Path,
+    records: Iterable[Mapping[str, Any]],
+) -> Path:
+    """Atomically replace *path* with the supplied complete JSONL dataset."""
+    result, _ = atomic_write_jsonl_with_fingerprint(path, records)
+    return result
+
+
+def atomic_write_jsonl_with_fingerprint(
+    path: Path,
+    records: Iterable[Mapping[str, Any]],
+) -> tuple[Path, FileFingerprint]:
+    """Atomically replace *path* and return the written file fingerprint."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp_path: Path | None = None
+    digest = sha256()
+    size = 0
+
+    try:
+        with tempfile.NamedTemporaryFile(
+            "wb",
+            dir=path.parent,
+            prefix=f".{path.name}.",
+            suffix=".tmp",
+            delete=False,
+        ) as file:
+            tmp_path = Path(file.name)
+            for record in records:
+                line = (json.dumps(record, ensure_ascii=False) + "\n").encode("utf-8")
+                file.write(line)
+                digest.update(line)
+                size += len(line)
+            file.flush()
+            os.fsync(file.fileno())
+
+        os.replace(tmp_path, path)
+        tmp_path = None
+        _fsync_directory(path.parent)
+    finally:
+        if tmp_path is not None:
+            tmp_path.unlink(missing_ok=True)
+
+    return path, FileFingerprint(size=size, sha256=digest.hexdigest())
+
+
+def _fsync_directory(path: Path) -> None:
+    """Best-effort directory fsync after an atomic replacement on POSIX."""
+    if os.name != "posix":
+        return
+
+    try:
+        file_descriptor = os.open(path, os.O_RDONLY)
+    except OSError:
+        return
+
+    try:
+        os.fsync(file_descriptor)
+    except OSError:
+        pass
+    finally:
+        os.close(file_descriptor)

spanmark/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)
+
+__commit_id__ = commit_id = None

spanmark/_widget.py

@@ -0,0 +1,41 @@
+"""The thin anywidget bridge between Python state and the browser UI."""
+
+from pathlib import Path
+
+import anywidget
+import traitlets
+
+_STATIC = Path(__file__).parent / "static"
+
+
+class SpanmarkWidget(anywidget.AnyWidget):
+    """Private browser widget used by `AnnotationSession`."""
+
+    _esm = _STATIC / "widget.js"
+    _css = _STATIC / "widget.css"
+
+    text = traitlets.Unicode("").tag(sync=True)
+    labels = traitlets.List(traitlets.Unicode(), default_value=[]).tag(sync=True)
+    spans = traitlets.List(traitlets.Dict(), default_value=[]).tag(sync=True)
+
+    doc_id = traitlets.Unicode("").tag(sync=True)
+    index = traitlets.Int(0).tag(sync=True)
+    total = traitlets.Int(0).tag(sync=True)
+
+    answer = traitlets.Unicode("").tag(sync=True)
+    flagged = traitlets.Bool(False).tag(sync=True)
+    allow_overlaps = traitlets.Bool(False).tag(sync=True)
+    can_undo = traitlets.Bool(False).tag(sync=True)
+    complete = traitlets.Bool(False).tag(sync=True)
+    reviewing_flagged = traitlets.Bool(False).tag(sync=True)
+
+    decided_count = traitlets.Int(0).tag(sync=True)
+    accept_count = traitlets.Int(0).tag(sync=True)
+    reject_count = traitlets.Int(0).tag(sync=True)
+    ignore_count = traitlets.Int(0).tag(sync=True)
+    flagged_count = traitlets.Int(0).tag(sync=True)
+
+    trim_whitespace = traitlets.Bool(True).tag(sync=True)
+
+    event = traitlets.Dict().tag(sync=True)
+    status = traitlets.Unicode("").tag(sync=True)

spanmark/static/widget.css

@@ -0,0 +1,308 @@
+.so-root {
+    /* Theme contract: component rules below consume only these --so-* tokens. */
+    --so-background: #ffffff;
+    --so-surface: #ffffff;
+    --so-text: #202124;
+    --so-muted-text: #5f6368;
+    --so-border: #dadce0;
+    --so-border-strong: #bdc1c6;
+    --so-hover: #f1f3f4;
+    --so-brand: #3b73b9;
+    --so-code-font: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
+
+    color-scheme: light;
+    font-family: system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
+    max-width: 1000px;
+    border: 1px solid var(--so-border);
+    border-radius: 9px;
+    padding: 13px;
+    background: var(--so-background);
+    color: var(--so-text);
+}
+
+.so-head {
+    display: flex;
+    justify-content: space-between;
+    gap: 12px;
+    align-items: center;
+    flex-wrap: wrap;
+    margin-bottom: 10px;
+}
+
+.so-doc {
+    font-weight: 650;
+}
+
+.so-progress {
+    color: var(--so-muted-text);
+    font-size: 0.9em;
+}
+
+.so-labelbar,
+.so-actions {
+    display: flex;
+    align-items: center;
+    gap: 6px;
+    flex-wrap: wrap;
+}
+
+.so-labelbar {
+    margin-bottom: 10px;
+}
+
+.so-actions {
+    margin-top: 12px;
+}
+
+.so-root button {
+    border: 1px solid var(--so-border-strong);
+    border-radius: 6px;
+    padding: 6px 10px;
+    background: var(--so-surface);
+    color: var(--so-text);
+    cursor: pointer;
+    font-size: 0.9em;
+}
+
+.so-root button:hover {
+    background: var(--so-hover);
+}
+
+.so-root button:disabled {
+    cursor: default;
+    opacity: 0.45;
+}
+
+.so-root button:focus-visible {
+    outline: 2px solid var(--so-brand);
+    outline-offset: 2px;
+}
+
+.so-labelbar button.so-label-button {
+    border-color: var(--so-label-color, var(--so-border-strong));
+    background: color-mix(in srgb,
+            var(--so-label-color, var(--so-brand)) 10%,
+            var(--so-surface));
+}
+
+.so-labelbar button.so-label-button.active {
+    background: var(--so-label-color, var(--so-brand));
+    color: var(--so-label-foreground, white);
+    border-color: var(--so-label-color, var(--so-brand));
+}
+
+.so-actions button {
+    min-width: 92px;
+    font-weight: 700;
+    padding: 8px 13px;
+}
+
+.so-actions button[data-action="accept"].active {
+    background: #287a43;
+    color: white;
+    border-color: #287a43;
+}
+
+.so-actions button[data-action="reject"].active {
+    background: #a53b36;
+    color: white;
+    border-color: #a53b36;
+}
+
+.so-actions button[data-action="ignore"].active {
+    background: var(--so-muted-text);
+    color: var(--so-surface);
+    border-color: var(--so-muted-text);
+}
+
+.so-actions button[data-action="flag"].active {
+    background: #9a6b00;
+    color: white;
+    border-color: #9a6b00;
+}
+
+.so-text {
+    position: relative;
+    height: 400px;
+    box-sizing: border-box;
+    overflow-y: auto;
+    overflow-x: hidden;
+    white-space: pre-wrap;
+    overflow-wrap: anywhere;
+    line-height: calc(1.6em + var(--so-rail-extra, 0px));
+    padding: 16px;
+    border: 1px solid var(--so-border);
+    border-radius: 7px;
+    background: var(--so-surface);
+    color: var(--so-text);
+    cursor: text;
+    user-select: text;
+    --so-rail-extra: 0px;
+}
+
+.so-textlayer {
+    position: relative;
+    z-index: 1;
+    white-space: pre-wrap;
+    overflow-wrap: anywhere;
+    user-select: text;
+}
+
+.so-rail-layer {
+    position: absolute;
+    inset: 0;
+    z-index: 2;
+    pointer-events: none;
+}
+
+/* Classic non-overlap mode: inline labels participate in normal text layout. */
+.so-textlayer mark.so-inline-span {
+    padding: 0 3px;
+    border-radius: 3px;
+    cursor: pointer;
+    background: color-mix(in srgb,
+            var(--so-span-color, var(--so-brand)) 22%,
+            transparent);
+    color: inherit;
+    border: 1px solid color-mix(in srgb,
+            var(--so-span-color, var(--so-brand)) 42%,
+            transparent);
+    line-height: 1.3;
+    box-decoration-break: clone;
+    -webkit-box-decoration-break: clone;
+}
+
+.so-textlayer mark.so-inline-span.so-model {
+    border-style: dashed;
+    border-color: var(--so-span-color, var(--so-brand));
+}
+
+.so-textlayer mark.so-inline-span.so-selected {
+    outline: 2px solid var(--so-span-color, var(--so-brand));
+    outline-offset: 1px;
+}
+
+.so-inline-label {
+    display: inline-block;
+    margin-left: 5px;
+    color: var(--so-muted-text);
+    font-size: 0.6em;
+    font-weight: 750;
+    line-height: 1;
+    vertical-align: baseline;
+    white-space: nowrap;
+    user-select: none;
+}
+
+.so-rail-seg,
+.so-rail-badge {
+    position: absolute;
+    pointer-events: auto;
+}
+
+.so-rail-seg {
+    height: 0;
+    border-top: 3px solid var(--so-span-color, var(--so-brand));
+    cursor: pointer;
+}
+
+.so-rail-seg.so-model {
+    border-top-style: dashed;
+}
+
+.so-rail-badge {
+    font-size: 0.72em;
+    line-height: 1.1;
+    padding: 1px 5px;
+    border-radius: 999px;
+    border: 1px solid var(--so-span-color, var(--so-brand));
+    background: color-mix(in srgb,
+            var(--so-span-color, var(--so-brand)) 12%,
+            var(--so-surface));
+    color: var(--so-text);
+    white-space: nowrap;
+    cursor: pointer;
+    user-select: none;
+}
+
+.so-rail-badge.so-model {
+    border-style: dashed;
+}
+
+.so-rail-seg.so-selected {
+    border-top-width: 5px;
+    filter: saturate(1.2);
+}
+
+.so-rail-badge.so-selected {
+    box-shadow: 0 0 0 2px var(--so-span-color, var(--so-brand));
+    filter: saturate(1.2);
+}
+
+.so-completion {
+    display: none;
+    margin-top: 8px;
+    padding: 24px 18px;
+    border: 1px solid var(--so-border);
+    border-radius: 8px;
+    text-align: center;
+    background: var(--so-surface);
+}
+
+.so-completion.show {
+    display: block;
+}
+
+.so-completion-title {
+    font-size: 1.35em;
+    font-weight: 750;
+    margin-bottom: 6px;
+}
+
+.so-completion-counts {
+    display: flex;
+    justify-content: center;
+    gap: 12px;
+    flex-wrap: wrap;
+    margin: 14px 0;
+    color: var(--so-muted-text);
+}
+
+.so-completion-count {
+    padding: 5px 8px;
+    border: 1px solid var(--so-border);
+    border-radius: 999px;
+}
+
+.so-completion-actions {
+    display: flex;
+    justify-content: center;
+    gap: 8px;
+    flex-wrap: wrap;
+}
+
+.so-completion button {
+    font-weight: 700;
+    padding: 8px 13px;
+}
+
+.so-help,
+.so-status {
+    margin-top: 8px;
+    color: var(--so-muted-text);
+    font-size: 0.82em;
+}
+
+.so-status {
+    min-height: 1.2em;
+    font-weight: 550;
+}
+
+.so-key {
+    font-family: var(--so-code-font);
+    border: 1px solid var(--so-border-strong);
+    border-bottom-width: 2px;
+    border-radius: 4px;
+    padding: 0 4px;
+    font-size: 0.92em;
+}