open-interpretability 0.0.20__py3-none-any.whl

openinterp/syntax.py

@@ -0,0 +1,399 @@
+"""Label the tokens of source code from its syntax tree, for the scan's syntax label sets.
+
+A document with a known programming language (a corpus's ``code_language`` field) goes
+through tree-sitter. Tree-sitter tolerates the broken code that a cut at ``max_chars``
+leaves. Every character of the document then has one label in each of these label sets.
+
+* ``syntax node``: identifier, keyword, string, comment, number, operator, bracket,
+  punctuation, whitespace or other. A string or a comment counts as one node, so its
+  contents are not read as code.
+* ``bracket depth``: how many round, square and curly brackets are open. A bracket has
+  the depth of what it holds. Angle brackets don't count, because most grammars do not
+  tell them from comparison operators. Depths of ``MAX_DEPTH`` or more share one label.
+* ``round depth``: the same count for round brackets only. GPT-2's L2 N676+ follows this
+  count, and the count of all brackets hides it (papers/syntax-positive-controls/README.md).
+* ``indent depth``: the indentation of the character's line, in levels. A blank line has
+  the level of the line before it.
+* ``syntax construct``: the innermost function, type definition, parameter list, call's
+  arguments, import, loop or conditional around the character, or ``top level``.
+
+The grammars name their nodes differently. So this module sorts node names into shared
+labels by pattern. The sort is approximate and is not a language's own specification.
+A language with no grammar in :data:`GRAMMARS` has no spans, so its tokens are unlabelled.
+Tree-sitter gives offsets in UTF-8 bytes, and this module converts them to characters.
+"""
+
+from __future__ import annotations
+
+import importlib
+import re
+from collections import Counter
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+from functools import cache
+from importlib.metadata import version
+from itertools import accumulate, groupby
+
+from openinterp.core.confounds import (
+    BRACKET_DEPTH,
+    INDENT_DEPTH,
+    ROUND_DEPTH,
+    SYNTAX_CONSTRUCT,
+    SYNTAX_NODE,
+)
+from openinterp.harness.annotations import Span
+
+# The github-code preset's `language` values and their grammars: (package, module function).
+# These are its most frequent programming languages. Markup and build files have no entry.
+GRAMMARS = {
+    "C": ("tree-sitter-c", "language"),
+    "C#": ("tree-sitter-c-sharp", "language"),
+    "C++": ("tree-sitter-cpp", "language"),
+    "GO": ("tree-sitter-go", "language"),
+    "Haskell": ("tree-sitter-haskell", "language"),
+    "Java": ("tree-sitter-java", "language"),
+    "JavaScript": ("tree-sitter-javascript", "language"),
+    "Lua": ("tree-sitter-lua", "language"),
+    "PHP": ("tree-sitter-php", "language_php"),
+    "Python": ("tree-sitter-python", "language"),
+    "Ruby": ("tree-sitter-ruby", "language"),
+    "Rust": ("tree-sitter-rust", "language"),
+    "Scala": ("tree-sitter-scala", "language"),
+    "Shell": ("tree-sitter-bash", "language"),
+    "TypeScript": ("tree-sitter-typescript", "language_typescript"),
+}
+
+IDENTIFIER = "identifier"
+KEYWORD = "keyword"
+STRING = "string"
+COMMENT = "comment"
+NUMBER = "number"
+OPERATOR = "operator"
+BRACKET = "bracket"
+PUNCTUATION = "punctuation"
+WHITESPACE = "whitespace"
+OTHER = "other"
+
+TOP_LEVEL = "top level"
+MAX_DEPTH = 8  # depths from here share a label: few tokens are deeper
+DEFAULT_INDENT = 4  # the columns of one level, in a document whose lines don't show it
+
+CLOSER = {"(": ")", "[": "]", "{": "}"}
+OPENING, CLOSING = set(CLOSER), set(CLOSER.values())
+# What can end a comment or a string that a cut leaves open: a line comment, a block
+# comment, then the quotes. The first entry adds nothing.
+_STARTS = ("", "\n", "*/", '"', "'", "`")
+END_KEYWORD_LANGUAGES = {"Lua", "Ruby"}  # a block closes with `end`
+MAX_END_KEYWORDS = 12  # more blocks than a cut document leaves open
+_SEPARATORS = {",", ";", ":", "::", "."}
+
+_ATOMS = (
+    (COMMENT, re.compile(r"comment")),
+    (STRING, re.compile(r"(^|_)(string|char|character|heredoc|regex)(_|$)")),
+)
+_NUMBER = re.compile(r"(^|_)(number|integer|int|float|floating|decimal|hex|octal|binary|real)(_|$)")
+_NAMED_KEYWORD = re.compile(
+    r"^(true|false|null|nil|none|this|self|super|primitive_type|predefined_type|void_type"
+    r"|boolean_type|boolean_literal|null_literal|integral_type|floating_point_type)$"
+)
+_NAME = re.compile(r"identifier|name|word|variable|constant|symbol|field|type")
+# Innermost wins, because the walk paints a child's characters after its parent's.
+_CONSTRUCTS = (
+    ("import", re.compile(r"import|include|using_directive|use_declaration|namespace_use")),
+    ("call arguments", re.compile(r"^(argument_list|arguments|actual_parameters)$")),
+    ("parameters", re.compile(r"parameter")),
+    (
+        "function",
+        re.compile(
+            r"^(function|method|lambda|closure_expression|func_literal|arrow_function"
+            r"|generator_function|anonymous_function)"
+            r"(_definition|_declaration|_item|_expression|_creation_expression)?$"
+            r"|^constructor_declaration$"
+        ),
+    ),
+    (
+        "type definition",
+        re.compile(
+            r"^(class|struct|interface|enum|trait|union|impl|object|module|namespace|type)"
+            r"_(definition|declaration|specifier|item)$|^(class|module)$"
+        ),
+    ),
+    (
+        "loop",
+        re.compile(
+            r"^(for|while|do|foreach|enhanced_for|for_in|for_range|repeat|loop|until)"
+            r"(_statement|_expression)?$"
+        ),
+    ),
+    ("conditional", re.compile(r"^(if|unless|match|case)(_statement|_expression)?$|switch_")),
+)
+
+
+def detector() -> str:
+    """Name the parser and grammar versions, which fix what the syntax label sets say."""
+    packages = ("tree-sitter", *sorted(package for package, _ in GRAMMARS.values()))
+    return f"openinterp.syntax ({', '.join(f'{p} {version(p)}' for p in packages)})"
+
+
+@cache
+def _parser(language: str):
+    """Return the parser of ``language``, a key of :data:`GRAMMARS`."""
+    import tree_sitter
+
+    package, function = GRAMMARS[language]
+    module = importlib.import_module(package.replace("-", "_"))
+    return tree_sitter.Parser(tree_sitter.Language(getattr(module, function)()))
+
+
+def _char_offsets(text: str) -> list[int]:
+    """Return the map from each UTF-8 byte offset of ``text`` to its character offset."""
+    if text.isascii():
+        return list(range(len(text) + 1))
+    widths = (1 if c < "\x80" else 2 if c < "ࠀ" else 3 if c < "\U00010000" else 4 for c in text)
+    out = []
+    for i, width in enumerate(widths):
+        out += [i] * width
+    return [*out, len(text)]
+
+
+def _encode(text: str) -> bytes:
+    """UTF-8 of ``text``. A corpus's JSON can hold a lone surrogate, so this keeps it."""
+    return text.encode("utf-8", "surrogatepass")
+
+
+def _node_kind(node_type: str, named: bool, text: str) -> str:
+    """Return the ``syntax node`` label of a tree leaf (or of a string or comment)."""
+    if not named:
+        if text in OPENING or text in CLOSING:
+            return BRACKET
+        if text in _SEPARATORS:
+            return PUNCTUATION
+        return KEYWORD if text[:1].isalpha() or text[:1] == "_" else OPERATOR
+    for label, pattern in _ATOMS:
+        if pattern.search(node_type):
+            return label
+    if _NUMBER.search(node_type):
+        return NUMBER
+    if _NAMED_KEYWORD.match(node_type):
+        return KEYWORD
+    return IDENTIFIER if _NAME.search(node_type) else OTHER
+
+
+def _atom(node) -> bool:
+    """Is this node a string or a comment, whose contents don't count as code?"""
+    return node.is_named and any(pattern.search(node.type) for _, pattern in _ATOMS)
+
+
+def _construct(node) -> str | None:
+    """Return the ``syntax construct`` label of a node, or ``None`` if it opens none."""
+    if node.is_named:
+        for label, pattern in _CONSTRUCTS:
+            if pattern.search(node.type):
+                return label
+    return None
+
+
+def _paint(text: str, language: str) -> tuple[list[str], list[str]]:
+    """Return each character's ``syntax node`` and ``syntax construct`` labels, from one parse."""
+    chars = _char_offsets(text)
+    nodes: list[str | None] = [None] * len(text)
+    constructs = [TOP_LEVEL] * len(text)
+    root = _parser(language).parse(_encode(text)).root_node
+    todo = [root]
+    while todo:  # not recursive, because deep code would pass Python's recursion limit
+        node = todo.pop()
+        start, end = chars[node.start_byte], chars[node.end_byte]
+        # The root is the file. Python calls it `module`, which is not a Ruby `module`.
+        if node != root and (construct := _construct(node)) is not None:
+            constructs[start:end] = [construct] * (end - start)
+        if _atom(node) or not node.children:
+            kind = _node_kind(node.type, node.is_named, text[start:end])
+            nodes[start:end] = [kind] * (end - start)
+        else:
+            todo.extend(reversed(node.children))
+    return [_gap_kind(c) if n is None else n for n, c in zip(nodes, text, strict=True)], constructs
+
+
+def _open_brackets(text: str, nodes: Sequence[str]) -> str:
+    """Return the closing brackets of the brackets still open at the end of ``text``."""
+    stack: list[str] = []
+    for char, node in zip(text, nodes, strict=True):
+        if node == BRACKET and char in OPENING:
+            stack.append(CLOSER[char])
+        elif node == BRACKET and char in CLOSING and stack:
+            stack.pop()
+    return "".join(reversed(stack))
+
+
+def _missing_closers(text: str, nodes: Sequence[str]) -> str:
+    """Return what closes the brackets still open at the end of ``text``."""
+    # A cut also leaves a statement unfinished, and a block needs its statements to end.
+    return _open_brackets(text, nodes).replace("}", ";}")
+
+
+def _error_chars(text: str, limit: int, language: str) -> int:
+    """Count the characters before ``limit`` that the parser puts in an error node.
+
+    A node that the parser had to make up (a missing token) counts as one character more.
+    A parse with no error gives 0.
+    """
+    root = _parser(language).parse(_encode(text)).root_node
+    if not root.has_error:
+        return 0
+    chars, count, todo = _char_offsets(text), 0, [root]
+    while todo:
+        node = todo.pop()
+        if node.type == "ERROR" or node.is_missing:
+            start, end = chars[node.start_byte], chars[node.end_byte]
+            count += max(0, min(end, limit) - min(start, limit)) + node.is_missing
+        elif node.has_error:
+            todo.extend(node.children)
+    return count
+
+
+def _endings(text: str, nodes: Sequence[str], language: str) -> list[str]:
+    """Return the texts that can end a cut ``text``, shortest first.
+
+    A cut can stop inside a comment or a string. So each ending starts with nothing, a line
+    break, the end of a block comment or a quote. Then come the closing brackets that are
+    open after that start, with and without a ``;`` before each ``}``. In
+    :data:`END_KEYWORD_LANGUAGES`, a block closes with ``end`` and not with a bracket. So
+    there, endings also add up to ``MAX_END_KEYWORDS`` of them, before or after the brackets.
+    """
+    out: set[str] = set()
+    for start in _STARTS:
+        start_nodes = nodes if not start else _paint(text + start, language)[0][: len(text)]
+        brackets = _open_brackets(text, start_nodes)
+        tails = {brackets, brackets.replace("}", ";}")}
+        if language in END_KEYWORD_LANGUAGES:
+            for n in range(1, MAX_END_KEYWORDS + 1):
+                tails |= {brackets + "\nend" * n, "\nend" * n + brackets}
+        out |= {start + tail for tail in tails}
+    return sorted(out - {""}, key=lambda ending: (len(ending), ending))
+
+
+def parse_labels(text: str, language: str) -> tuple[list[str], list[str]]:
+    """Return each character's ``syntax node`` and ``syntax construct`` labels.
+
+    The corpus cuts a document at ``max_chars``, usually inside a class or a function. The
+    parser then has no node for that class or function, so its contents read as top level.
+    So when the parse has errors, this parses again with an ending added, and labels only
+    ``text``. The ending is the one of :func:`_endings` that leaves the fewest characters
+    of ``text`` in error nodes, and the shortest of those. If no ending lowers that count,
+    the ending closes the open brackets as the first parse reads them.
+    """
+    nodes, constructs = _paint(text, language)
+    errors = _error_chars(text, len(text), language)
+    if not errors:
+        return nodes, constructs
+    scored = (
+        (_error_chars(text + e, len(text), language), e) for e in _endings(text, nodes, language)
+    )
+    fewest, ending = min(scored, key=lambda pair: pair[0], default=(errors, ""))
+    if fewest >= errors:
+        ending = _missing_closers(text, nodes)
+    if not ending:
+        return nodes, constructs
+    nodes, constructs = _paint(text + ending, language)
+    return nodes[: len(text)], constructs[: len(text)]
+
+
+def _gap_kind(char: str) -> str:
+    """Return the ``syntax node`` label of a character that no tree leaf covers."""
+    return WHITESPACE if char.isspace() else OTHER
+
+
+def _depth_label(depth: int) -> str:
+    return f"{MAX_DEPTH}+" if depth >= MAX_DEPTH else str(depth)
+
+
+def bracket_depths(text: str, nodes: Sequence[str], kinds: str = "([{") -> list[str]:
+    """Return each character's depth of the brackets that ``kinds`` opens.
+
+    ``nodes`` holds the characters' ``syntax node`` labels. So a bracket in a string or
+    a comment doesn't count. A closing bracket with no opening bracket leaves the depth at 0.
+    """
+    openers, closers = set(kinds), {CLOSER[k] for k in kinds}
+    out, depth = [], 0
+    for char, node in zip(text, nodes, strict=True):
+        if node == BRACKET and char in openers:
+            depth += 1
+        out.append(_depth_label(depth))
+        if node == BRACKET and char in closers:
+            depth = max(0, depth - 1)
+    return out
+
+
+def _indent_unit(widths: Sequence[int]) -> int:
+    """Return the columns of one level: the most frequent step between indented lines.
+
+    Lines' first indents can mislead. A `` * `` comment line has one space, and is not a
+    level. So this takes the step that the lines repeat most. ``DEFAULT_INDENT`` if there
+    is no step.
+    """
+    steps = Counter(b - a for a, b in zip(widths, widths[1:], strict=False) if b > a)
+    return steps.most_common(1)[0][0] if steps else DEFAULT_INDENT
+
+
+def indent_depths(text: str) -> list[str]:
+    """Return each character's ``indent depth`` label, from its line's leading whitespace."""
+    lines = text.splitlines(keepends=True)
+    tabbed = [len(line) - len(line.lstrip(" \t")) for line in lines]
+    blank = [not line.strip() for line in lines]
+    tabs = [line[:n].count("\t") for line, n in zip(lines, tabbed, strict=True)]
+    spaces = [n - t for n, t in zip(tabbed, tabs, strict=True)]
+    solid = [s for s, b, t in zip(spaces, blank, tabs, strict=True) if not b and not t]
+    unit = _indent_unit(solid)
+    out, level = [], 0
+    for line, is_blank, tab, space in zip(lines, blank, tabs, spaces, strict=True):
+        if not is_blank:
+            level = tab + space // unit
+        out += [_depth_label(level)] * len(line)
+    return out
+
+
+def _runs(labels: Sequence[str]) -> list[Span]:
+    """Join neighbouring characters of one label into spans."""
+    lengths = [(label, len(list(run))) for label, run in groupby(labels)]
+    ends = accumulate(n for _, n in lengths)
+    return [Span(end - n, end, label) for (label, n), end in zip(lengths, ends, strict=True)]
+
+
+@dataclass(frozen=True)
+class SyntaxTagger:
+    """Label a document's characters with ``read``, for documents that have a grammar."""
+
+    name: str
+    read: Callable[[int, str], list[Span]]
+
+    def spans(self, doc: int, text: str) -> list[Span]:
+        """The document's spans in this label set. A document without a grammar has none."""
+        return self.read(doc, text)
+
+
+def syntax_taggers(languages: Sequence[str | None]) -> list[SyntaxTagger]:
+    """Return one tagger per syntax label set, over documents of these ``languages``.
+
+    The taggers share one parse of each document. ``languages`` holds one value per
+    document, or ``None``. A document with a language outside :data:`GRAMMARS` has no spans.
+    """
+
+    @cache
+    def parsed(doc: int, text: str) -> tuple[list[str], list[str]] | None:
+        language = languages[doc]
+        return parse_labels(text, language) if language in GRAMMARS else None
+
+    def tagger(name: str, labels: Callable[[str, list[str], list[str]], Sequence[str]]):
+        def read(doc: int, text: str) -> list[Span]:
+            found = parsed(doc, text)
+            return [] if found is None else _runs(labels(text, *found))
+
+        return SyntaxTagger(name, read)
+
+    return [
+        tagger(SYNTAX_NODE, lambda _, nodes, __: nodes),
+        tagger(BRACKET_DEPTH, lambda text, nodes, _: bracket_depths(text, nodes)),
+        tagger(ROUND_DEPTH, lambda text, nodes, _: bracket_depths(text, nodes, "(")),
+        tagger(INDENT_DEPTH, lambda text, _, __: indent_depths(text)),
+        tagger(SYNTAX_CONSTRUCT, lambda _, __, constructs: constructs),
+    ]

openinterp/takedown_checks.py

@@ -0,0 +1,76 @@
+"""Part of check 5 of ``open-interpretability check``: the takedown file.
+
+The file's format is in :mod:`openinterp.core.takedowns`.
+
+The file parses and has unique ids. A key in the allowed signers (from ``base_ref`` if
+given) signed it, so a branch can't add a takedown of its own. With ``base_ref`` the file
+is append-only, and must not be deleted.
+
+:func:`page_redaction_ok` is the one exception to write-once experiment pages. A page may
+change outside its Discussion section when the change only removes what the takedowns
+cover.
+"""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from openinterp.core.takedowns import all_hashes, load_takedowns, parse_takedowns, redact
+from openinterp.core.takedowns import takedowns_path as path_of
+from openinterp.errata_checks import changed_file, check_signature
+from openinterp.signing import allowed_signers_text, signature_path
+
+if TYPE_CHECKING:
+    from openinterp.checks import CheckReport
+
+WHAT = "takedown file"
+
+
+def check_takedowns(
+    wiki_dir: Path,
+    report: CheckReport,
+    base_ref: str | None,
+    changes: list[tuple[str, Path]],
+    allowed_signers: Path,
+) -> None:
+    """Add a problem to ``report`` for each way the takedown file is not valid."""
+    path = path_of(wiki_dir)
+    if any(p == path and status == "D" for status, p in changes):
+        report.add(path, f"the {WHAT} is append-only and must not be deleted")
+    if not path.exists():
+        if signature_path(path).exists():
+            report.add(signature_path(path), f"signature of a {WHAT} that does not exist")
+        return
+    text = path.read_text(encoding="utf-8")
+    try:
+        takedowns = parse_takedowns(text)
+    except ValueError as exc:
+        report.add(path, f"invalid takedown: {exc}")
+        return
+    ids = [t.id for t in takedowns]
+    for dup in sorted({i for i in ids if ids.count(i) > 1}):
+        report.add(path, f"takedown id {dup!r} is used more than once")
+    changed = changed_file(path, base_ref, changes)
+    if base_ref and changed:
+        shown = subprocess.run(
+            ["git", "show", f"{base_ref}:{path.as_posix()}"], capture_output=True, text=True
+        )
+        if shown.returncode == 0 and not text.startswith(shown.stdout):
+            report.add(path, f"the {WHAT} is append-only: add lines at the end, change none")
+    signers = allowed_signers_text(allowed_signers, base_ref)
+    check_signature(path, report, signers, changed, WHAT)
+
+
+def page_redaction_ok(before: str, after: str, wiki_dir: Path) -> bool:
+    """Whether ``after`` is ``before`` with exactly the text that the takedowns cover removed.
+
+    Both are a page's text before its Discussion section. The check that the takedown
+    file is signed is :func:`check_takedowns`, which runs in the same
+    ``open-interpretability check``.
+    """
+    from openinterp.builder.pii import REMOVED_TEXT
+
+    hashes = all_hashes(load_takedowns(wiki_dir))
+    return before != after and redact(before, hashes, REMOVED_TEXT) == after

openinterp/toxicity.py

@@ -0,0 +1,158 @@
+"""Find toxic text and hate speech in a document, for the scan's label sets.
+
+Both labels are models' guesses, for experiments only. Nothing removes text because of
+them. The site's spoiler guard uses a word list (``sensitive.is_offensive``), so that a
+site build needs no model.
+
+A model reads text, not tokens. So each sentence is read alone, and all of its tokens take
+the sentence's label. A label therefore says where toxic text is, not which word made it so.
+
+The two models do different work. We compared them on HateCheck (Röttger et al., 2021),
+which neither trained on, at a score of at least 0.5:
+
+- ``unitary/unbiased-toxic-roberta`` (Apache 2.0) gives ``toxicity``. It alone finds
+  obscenity, insults, threats and sexual text. Its identity outputs (``female``, ``black``)
+  say that a text mentions a group, so we leave them out. Built from them, a target
+  category found 0% to 76% of hateful cases, and 0% for immigrants, trans people and
+  disabled people.
+- ``cardiffnlp/twitter-roberta-base-hate-multiclass-latest`` gives ``hate speech``. It
+  found 40% of hateful cases, and called 9% of non-hateful cases hate. It names the right
+  target in 13% to 62% of a group's cases, but in 1% for disabled people. Its model card
+  states no licence. It trained on tweets. We leave out its ``other`` output, because that
+  output called lines of Ruby code hate.
+
+Both models are English models. Scoring runs on the CPU, at about 10 ms a sentence for each
+model. Each document's result stays on disk. The models (about 500 MB each) are downloaded
+once, pinned to a commit.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+from collections.abc import Sequence
+from dataclasses import dataclass
+from functools import cache
+from pathlib import Path
+
+THRESHOLD = 0.5  # the score at which a sentence takes a label
+_BATCH = 16
+_MAX_CHARS = 1000  # a longer "sentence" (code, a table) is read in pieces of this length
+_MAX_TOKENS = 512  # what the models accept
+# On our corpus, the models called one-word lines hate or toxic ("Fluker.", "F.eks.").
+_MIN_WORDS = 2
+# Up to a stop followed by a space, else up to the end of the line.
+_SENTENCE = re.compile(r"\S(?:[^\n]*?[.!?](?=\s)|[^\n]*)")
+
+
+@dataclass(frozen=True)
+class _Classifier:
+    repo: str
+    revision: str  # a commit, so a later scan reads the same weights
+    name: str  # part of the cache's path
+    labels: tuple[str, ...]  # the outputs that can label a sentence, most specific first
+    softmax: bool = False  # the outputs compete; else each output is its own yes or no
+    fallback: str | None = None  # the label of a sentence with no other label
+
+
+_TOXICITY = _Classifier(
+    "unitary/unbiased-toxic-roberta",
+    "36295dd80b422dc49f40052021430dae76241adc",
+    "toxicity",
+    ("threat", "identity_attack", "sexual_explicit", "obscene", "insult"),
+    fallback="toxicity",
+)
+_HATE = _Classifier(
+    "cardiffnlp/twitter-roberta-base-hate-multiclass-latest",
+    "b9a303f920f8527ac4151e65953c04505fdf0587",
+    "hate",
+    ("racism", "sexism", "religion", "sexual_orientation", "disability"),
+    softmax=True,
+)
+DETECTOR = "; ".join(f"{c.repo}@{c.revision[:12]}" for c in (_TOXICITY, _HATE))
+DETECTOR = f"{DETECTOR} (score at least {THRESHOLD}, per sentence of {_MIN_WORDS}+ words)"
+
+
+def sentences(text: str) -> list[tuple[int, int]]:
+    """Return the ``(start, end)`` of each sentence in ``text``, split to ``_MAX_CHARS``."""
+    return [
+        (start, min(start + _MAX_CHARS, match.end()))
+        for match in _SENTENCE.finditer(text)
+        for start in range(match.start(), match.end(), _MAX_CHARS)
+    ]
+
+
+def _label(classifier: _Classifier, scores: dict[str, float]) -> str | None:
+    """The strongest of a sentence's labels, else the fallback when its score passes."""
+    passed = [label for label in classifier.labels if scores[label] >= THRESHOLD]
+    if passed:
+        return max(passed, key=scores.__getitem__)
+    fallback = classifier.fallback
+    return fallback if fallback and scores[fallback] >= THRESHOLD else None
+
+
+@cache
+def _model(classifier: _Classifier):
+    from transformers import AutoModelForSequenceClassification, AutoTokenizer
+
+    tokenizer = AutoTokenizer.from_pretrained(classifier.repo, revision=classifier.revision)
+    model = AutoModelForSequenceClassification.from_pretrained(
+        classifier.repo, revision=classifier.revision
+    )
+    return tokenizer, model.eval()
+
+
+def _score(classifier: _Classifier, texts: Sequence[str]) -> list[dict[str, float]]:
+    """Return each text's score per output of the model."""
+    import torch
+
+    tokenizer, model = _model(classifier)
+    outputs = [model.config.id2label[i] for i in range(model.config.num_labels)]
+    found: list[dict[str, float]] = []
+    for first in range(0, len(texts), _BATCH):
+        batch = tokenizer(
+            list(texts[first : first + _BATCH]),
+            truncation=True,
+            max_length=_MAX_TOKENS,
+            padding=True,
+            return_tensors="pt",
+        )
+        with torch.no_grad():
+            logits = model(**batch).logits
+        scores = logits.softmax(-1) if classifier.softmax else logits.sigmoid()
+        found += [dict(zip(outputs, row, strict=True)) for row in scores.tolist()]
+    return found
+
+
+def _cache_dir() -> Path:
+    return Path(os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache") / "openinterp"
+
+
+def _matches(classifier: _Classifier, text: str) -> tuple[tuple[int, int, str], ...]:
+    """Return ``(start, end, label)`` of each labelled sentence, kept on disk by the text."""
+    key = hashlib.sha256(text.encode()).hexdigest()
+    version = f"{classifier.revision[:12]}-{THRESHOLD}-{_MIN_WORDS}"
+    path = _cache_dir() / "toxicity" / classifier.name / version / f"{key}.json"
+    if path.exists():
+        return tuple((start, end, label) for start, end, label in json.loads(path.read_text()))
+    spans = [(a, b) for a, b in sentences(text) if len(text[a:b].split()) >= _MIN_WORDS]
+    scores = _score(classifier, [text[start:end] for start, end in spans]) if spans else []
+    labels = [_label(classifier, s) for s in scores]
+    found = [(*span, label) for span, label in zip(spans, labels, strict=True) if label]
+    path.parent.mkdir(parents=True, exist_ok=True)
+    partial = path.with_suffix(f".{os.getpid()}.part")  # scans can run side by side
+    partial.write_text(json.dumps(found))
+    partial.replace(path)
+    return tuple(found)
+
+
+def toxicity_matches(text: str) -> tuple[tuple[int, int, str], ...]:
+    """Return ``(start, end, label)`` of each toxic sentence in ``text``, sorted."""
+    return _matches(_TOXICITY, text)
+
+
+def hate_matches(text: str) -> tuple[tuple[int, int, str], ...]:
+    """Return ``(start, end, label)`` of each sentence of hate speech in ``text``, sorted."""
+    return _matches(_HATE, text)