thread-sdk 0.2.0__cp311-abi3-win_amd64.whl

thread_sdk/__init__.py

@@ -0,0 +1,143 @@
+"""thread-sdk: query and update a thread blob-native search index (dense vectors, BM25 text, or both fused by rank)
+from Python. The index lives as immutable objects in R2, any S3-compatible store or a local directory; the numeric
+kernels are the same Rust code (`thread-core`) the Worker and the TypeScript SDK run, as a native extension.
+
+    from thread_sdk import S3BlobStore, open_index
+
+    store = S3BlobStore.r2(account_id, "my-bucket", key_id, secret)      # or open_index("/data/idx"), a directory,
+    with open_index(store) as index:                                      # or open_index("idx.threadpack"), one file
+        r = index.search(text="deploy failed", vector=embedding, top_k=10, docs=True)
+        hits = index.query(embedding, top_k=10).hits      # dense only
+"""
+
+from . import _native
+from .attrs import compile_filter, define_attrs, filter_matches, pack_attrs, unpack_attrs, validate_schema
+from .chunk import (CHUNK_KEY_SEPARATOR, DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_WORDS, Chunk, ChunkOptions, chunk_index_of,
+                    chunk_key, chunk_text, resolve_chunk_options)
+from .index import KeyRecord, Rank, SearchHit, SearchResult, ThreadIndex, head_revision, load_rev_state, open_index, resolve_target
+from .lex import DocRecord, KeyPage, LexHit, LexIndex, LexSearchResult, LexSegment
+from .pack import PackedBlobStore, PackResult, PackWriter, copy_objects, pack_index, pack_to_bytes, store_at
+from .query import MULTI_COALESCE, QUERY_INFLIGHT, Hit, QueryResult, RevState, coalesce_runs
+from .revision import snapshot_keys
+from .s3 import S3_INFLIGHT, S3BlobStore, SigningKeyCache, sign_v4
+from .store import (
+    BlobCache,
+    BlobHead,
+    BlobRange,
+    BlobStore,
+    BlobTrace,
+    CachedBlobStore,
+    DiskCache,
+    FileBlobStore,
+    MemoryBlobStore,
+    MISS_POLICIES,
+    MemoryCache,
+    PrefixedBlobStore,
+)
+from .warm import WarmPrefixStore, build_warm_pack, parse_warm_pack, seed_warm_pack, write_warm_pack
+from .embed import (EMBED_BATCH_DEFAULTS, EMBEDDING_PROVIDERS, embedding_batches, embedding_inputs, normalize_rows,
+                    resolve_embedder, split_rows, validate_embedding_config)
+from .wire import COMPACTION_PROFILES, CREATE_DEFAULTS, ResolvedLexConfig, ThreadError, keys
+from .write import (
+    MAX_UPSERT,
+    CompactSegmentsResult,
+    ConfigureAttrsResult,
+    ConfigureLexicalResult,
+    DeleteResult,
+    UpsertResult,
+    create_generation,
+    create_lexical_generation,
+)
+
+MissingRanges = _native.MissingRanges
+__version__ = _native.__version__
+
+__all__ = [
+    "open_index",
+    "create_generation",
+    "EMBED_BATCH_DEFAULTS",
+    "EMBEDDING_PROVIDERS",
+    "COMPACTION_PROFILES",
+    "CREATE_DEFAULTS",
+    "embedding_batches",
+    "embedding_inputs",
+    "normalize_rows",
+    "resolve_embedder",
+    "split_rows",
+    "validate_embedding_config",
+    "ThreadIndex",
+    "ThreadError",
+    "MissingRanges",
+    "QueryResult",
+    "Hit",
+    "SearchResult",
+    "SearchHit",
+    "Rank",
+    "LexSearchResult",
+    "LexHit",
+    "DocRecord",
+    "KeyRecord",
+    "KeyPage",
+    "UpsertResult",
+    "DeleteResult",
+    "ConfigureLexicalResult",
+    "ConfigureAttrsResult",
+    "CompactSegmentsResult",
+    "create_lexical_generation",
+    "define_attrs",
+    "validate_schema",
+    "pack_attrs",
+    "unpack_attrs",
+    "compile_filter",
+    "filter_matches",
+    "MAX_UPSERT",
+    "Chunk",
+    "ChunkOptions",
+    "chunk_text",
+    "chunk_key",
+    "chunk_index_of",
+    "resolve_chunk_options",
+    "DEFAULT_CHUNK_WORDS",
+    "DEFAULT_CHUNK_OVERLAP",
+    "CHUNK_KEY_SEPARATOR",
+    "QUERY_INFLIGHT",
+    "BlobStore",
+    "BlobRange",
+    "BlobHead",
+    "BlobTrace",
+    "MemoryBlobStore",
+    "FileBlobStore",
+    "S3BlobStore",
+    "PrefixedBlobStore",
+    "PackedBlobStore",
+    "PackWriter",
+    "PackResult",
+    "pack_index",
+    "pack_to_bytes",
+    "copy_objects",
+    "store_at",
+    "WarmPrefixStore",
+    "build_warm_pack",
+    "write_warm_pack",
+    "parse_warm_pack",
+    "seed_warm_pack",
+    "snapshot_keys",
+    "CachedBlobStore",
+    "BlobCache",
+    "MemoryCache",
+    "DiskCache",
+    "ResolvedLexConfig",
+    "RevState",
+    "LexIndex",
+    "LexSegment",
+    "keys",
+    "sign_v4",
+    "SigningKeyCache",
+    "S3_INFLIGHT",
+    "MULTI_COALESCE",
+    "MISS_POLICIES",
+    "coalesce_runs",
+    "resolve_target",
+    "load_rev_state",
+    "head_revision",
+]

thread_sdk/_native.pyi

@@ -0,0 +1,202 @@
+"""Type stubs for the native extension (crates/thread-py): thread-core's kernels, without I/O."""
+
+from collections.abc import Sequence
+
+from typing_extensions import Buffer
+from typing import Any, Literal
+
+__version__: str
+
+FloatsLike = Buffer | Sequence[float]
+IdsLike = Buffer | Sequence[int]
+
+class MissingRanges(Exception):
+    ranges: list[tuple[int, int]]
+
+class IndexFormatError(ValueError): ...
+class IndexCorruptError(ValueError): ...
+class IndexStateError(RuntimeError): ...
+
+class PqCodebook:
+    def __init__(self, codebook: Buffer) -> None: ...
+    @property
+    def dim(self) -> int: ...
+    @property
+    def m(self) -> int: ...
+
+class Query:
+    @property
+    def dim(self) -> int: ...
+    @property
+    def has_dots(self) -> bool: ...
+    @property
+    def has_table(self) -> bool: ...
+    def vector(self) -> list[float]: ...
+
+class Searcher:
+    def __init__(self, dim: int, k1: int, k2: int, coarse_means: FloatsLike, fine_means: FloatsLike) -> None: ...
+    @staticmethod
+    def from_table(dim: int, k1: int, k2: int, coarse_means: FloatsLike, table: Buffer, elem: str) -> Searcher: ...
+    @property
+    def dim(self) -> int: ...
+    @property
+    def cells(self) -> int: ...
+    def query(self, q: FloatsLike, dots: bool = False, codebook: PqCodebook | None = None) -> Query: ...
+    def route(self, query: Query, probe_coarse: int, nprobe: int) -> list[int]: ...
+
+class ColdRouter:
+    def __init__(self, tree: Buffer, elem: str, head: Buffer | None = None) -> None: ...
+    @property
+    def needs_head(self) -> bool: ...
+    def set_head(self, head: Buffer) -> None: ...
+    @property
+    def dim(self) -> int: ...
+    @property
+    def k(self) -> int: ...
+    @property
+    def groups(self) -> int: ...
+    @property
+    def row_bytes(self) -> int: ...
+    @property
+    def loaded_cells(self) -> int: ...
+    @property
+    def loaded_groups(self) -> int: ...
+    def group_rows(self, g: int) -> tuple[int, int]: ...
+    def cell_row(self, c: int) -> int | None: ...
+    def missing_cells(self, cells: IdsLike) -> list[int]: ...
+    def missing_groups(self, query: Query, beam: int) -> list[int]: ...
+    def add_rows(self, first_row: int, rows: Buffer) -> int: ...
+    def query(self, q: FloatsLike, dot_cells: IdsLike | None = None, codebook: PqCodebook | None = None) -> Query: ...
+    def route_beam(self, query: Query, beam: int, nprobe: int) -> list[int]: ...
+
+def routing_table_encode(means: FloatsLike, dim: int, elem: str) -> bytes: ...
+def routing_table_decode(table: Buffer, dim: int, elem: str) -> bytes: ...
+def grouped_table_encode(means: FloatsLike, dim: int, tree: Buffer, elem: str) -> bytes: ...
+def grouped_table_info(table: Buffer) -> list[int]: ...
+def build_centroid_tree(means: FloatsLike, dim: int, groups: int, iters: int, seed: int) -> bytes: ...
+
+FilterLike = Buffer | Sequence[int] | None
+
+class QueryRun:
+    def __init__(self, query: Query, t: int, do_rerank: bool, return_payload: bool,
+                 filter: FilterLike = None) -> None: ...
+    def candidates(self) -> int: ...
+    def add_slab(self, cell: int, prefix: Buffer, full: Buffer | None = None,
+                 tombstones: Buffer | None = None) -> tuple[bool, int]: ...
+    def speculation(self) -> list[tuple[int, int, int, list[int]]]: ...
+    def add_delta(self, delta: Buffer, tombstones: Buffer | None = None) -> None: ...
+    def add_records(self, cell: int, first_idx: int, data: Buffer, idxs: Sequence[int], spec: bool = False) -> None: ...
+    def plan(self) -> list[tuple[int, int, int, list[int]]]: ...
+    def finish(self, k: int) -> tuple[list[tuple[int, float, int, Literal["slab", "delta"], bytes | None]],
+                                      list[tuple[int, int, int, int]]]: ...
+    def stats(self) -> dict[str, int]: ...
+
+def scan(slab: Buffer, query: Query, k: int, tombstones: Buffer | None = None,
+         filter: FilterLike = None) -> tuple[bytes, int, int, int]: ...
+def scan_delta(delta: Buffer, query: Query, k: int, tombstones: Buffer | None = None,
+               filter: FilterLike = None) -> tuple[bytes, int, int, int]: ...
+def filter_check(filter: FilterLike = None) -> int: ...
+def filter_matches(filter: FilterLike, attrs: IdsLike) -> list[bool]: ...
+def merge(packed: Buffer, k: int) -> bytes: ...
+def rerank(query: Query, centroid: FloatsLike, records: Buffer, n: int) -> list[float]: ...
+def normalize_rows(vectors: FloatsLike, dim: int) -> tuple[bytes, int]: ...
+
+class DeltaJob:
+    def __init__(self, dim: int, k: int, means: FloatsLike, seq: int) -> None: ...
+    def set_pq_codebook(self, codebook: PqCodebook) -> None: ...
+    def add(self, vectors: FloatsLike, ids: IdsLike, attrs: IdsLike | None = None,
+            payloads: Sequence[Buffer] | None = None) -> list[int]: ...
+    def __len__(self) -> int: ...
+    @property
+    def byte_len(self) -> int: ...
+    @property
+    def pq_m(self) -> int: ...
+    def finish(self) -> bytes: ...
+
+def delta_info(delta: Buffer) -> list[int]: ...
+def delta_cells(delta: Buffer) -> list[tuple[int, int]]: ...
+def delta_records_for_cell(delta: Buffer, cell: int, tombstones: Buffer | None = None) -> bytes: ...
+def tombstones_merge(existing: Buffer, add: IdsLike) -> bytes: ...
+
+class AssignJob:
+    def __init__(self, dim: int, k1: int, k2: int, coarse_means: FloatsLike, fine_means: FloatsLike,
+                 probe_coarse: int, m: int) -> None: ...
+    def set_codebook(self, codebook: Buffer) -> None: ...
+    def add_batch(self, vectors: FloatsLike, first_id: int) -> int: ...
+    def take_part(self, g: int) -> bytes: ...
+
+class SlabJob:
+    def __init__(self, dim: int, cell: int, centroid_mean: FloatsLike, format: int = 2) -> None: ...
+    def add_records(self, records: Buffer) -> int: ...
+    def finish(self) -> bytes: ...
+
+def part_directory(part: Buffer) -> tuple[int, int, list[tuple[int, int, int]]]: ...
+def slab_info(slab: Buffer) -> list[int]: ...
+def residuals_for(vectors: FloatsLike, means: FloatsLike, assign: IdsLike, dim: int) -> bytes: ...
+def pq_init_codebook(residuals: FloatsLike, dim: int, m: int, seed: int) -> bytes: ...
+
+class LexTokenizer:
+    def __init__(self, config_json: str | None = None) -> None: ...
+    def tokenize(self, text: str) -> list[str]: ...
+    def config(self) -> str: ...
+
+def tokenizer_config_json(config_json: str | None = None) -> str: ...
+
+class SegmentJob:
+    def __init__(self, tokenizer_json: str | None, k1: float, b: float, lengths: str, bucket_terms: int, docs: bool,
+                 text: bool, bucket_docs: int, keys: bool = False, bucket_keys: int = 0) -> None: ...
+    def add(self, id: int, text: str | None = None, meta: str | None = None, attrs: int = 0,
+            key: str | None = None) -> int: ...
+    def __len__(self) -> int: ...
+    @property
+    def tokens(self) -> int: ...
+    def finish(self) -> bytes: ...
+
+def segment_meta(segment: Buffer) -> str: ...
+def merge_segments(segments: Sequence[Buffer], tombstones: Sequence[Buffer | None], tokenizer_json: str | None,
+                   k1: float, b: float, lengths: str, bucket_terms: int, docs: bool, text: bool,
+                   bucket_docs: int, keys: bool = False, bucket_keys: int = 0) -> bytes: ...
+
+class SegmentReader:
+    def __init__(self, total_len: int) -> None: ...
+    @property
+    def total_len(self) -> int: ...
+    @property
+    def held_bytes(self) -> int: ...
+    def feed(self, offset: int, data: Buffer) -> None: ...
+    def evict_ranges(self, ranges: Sequence[tuple[int, int]]) -> None: ...
+    def evict_all(self) -> None: ...
+    def missing(self, ranges: Sequence[tuple[int, int]]) -> list[tuple[int, int]]: ...
+    def plan_open(self) -> list[tuple[int, int]]: ...
+    def open(self) -> dict[str, Any]: ...
+    def plan_headers(self) -> list[tuple[int, int]]: ...
+    def info(self) -> dict[str, Any]: ...
+    def plan_directories(self) -> list[tuple[int, int]]: ...
+    def lookup(self, terms: Sequence[str]) -> list[tuple[int, int, int]]: ...
+    def search(self, infos: Sequence[tuple[int, int, int]], dfs: Sequence[float], weights: Sequence[float],
+               num_docs: int, total_len: int, k1: float, b: float, k: int, tombstones: Buffer | None = None) -> bytes: ...
+    def docs_get(self, ids: IdsLike, with_text: bool) -> list[tuple[int, str, str | None, int] | None]: ...
+    def keys_get(self, keys: Sequence[str]) -> list[int | None]: ...
+    def keys_list(self, prefix: str, after: str | None = None, limit: int = 100) -> dict[str, Any]: ...
+
+class SearchPlan:
+    @property
+    def mode(self) -> Literal["dense", "lexical", "hybrid"]: ...
+    @property
+    def top_k(self) -> int: ...
+    @property
+    def depth(self) -> int: ...
+    @property
+    def rrf_k(self) -> float: ...
+    @property
+    def dense_weight(self) -> float: ...
+    @property
+    def lexical_weight(self) -> float: ...
+
+FusionParams = tuple[float | None, int | None, float | None, float | None]
+
+def plan_search(top_k: float | None, has_text: bool, has_vector: bool, has_lexical: bool, has_dense: bool,
+                request: FusionParams | None = None, manifest: FusionParams | None = None) -> SearchPlan: ...
+def fuse_search(plan: SearchPlan, dense: tuple[Sequence[int], Sequence[float]] | None = None,
+                lexical: tuple[Sequence[int], Sequence[float]] | None = None,
+                ) -> list[tuple[int, float, tuple[int, float] | None, tuple[int, float] | None]]: ...

thread_sdk/attrs.py

@@ -0,0 +1,331 @@
+"""Attributes and filters, the Python twin of the TypeScript SDK's `attrs.ts`.
+
+Every document carries one 32-bit `attrs` word, stored in its dense entry (slabs and delta segments, inside the scan
+prefix every query reads anyway) and in its `docs` entry. A generation may declare a schema (`manifest["attrs"]`)
+that packs named fields into that word; `pack_attrs` turns `{"tenant": 5, "lang": "en"}` into the word and
+`compile_filter` turns `{"tenant": 5, "lang": {"$in": ["en", "fr"]}}` into the clause list the kernels apply inside
+every scan, before top-k insertion (so a filtered-out entry never takes a candidate slot). Without a schema the
+whole word is one numeric field named `attrs`.
+
+Compiled form (`u32` words, `thread-core/src/filter.rs`): `[n, (mask, op, a, b, n_values, values...) x n]`, the
+clauses ANDed; every clause tests `x = attrs & mask`. Both SDKs compile the same expressions to the same words
+(`sdk/ts/test/fixtures/filters.json` holds the shared vectors).
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from array import array
+from typing import Any, Mapping, Sequence
+
+from .wire import ThreadError
+
+__all__ = ["define_attrs", "validate_schema", "schema_extends", "pack_attrs", "unpack_attrs", "compile_filter",
+           "filter_matches", "AttrSchema", "AttrFilterExpr"]
+
+#: `{"fields": [{"name", "type": "number" | "enum" | "bool", "offset", "bits", "min"?, "values"?}, ...]}`.
+AttrSchema = dict[str, Any]
+#: `{field: value | {"$eq" | "$ne" | "$in" | "$nin" | "$lt" | "$lte" | "$gt" | "$gte" | "$any" | "$all" | "$none": ...}}`.
+AttrFilterExpr = Mapping[str, Any]
+
+OP_EQ, OP_NE, OP_IN, OP_NIN, OP_RANGE, OP_BITS_ANY, OP_BITS_ALL, OP_BITS_NONE = range(8)
+
+_RAW_FIELD: dict[str, Any] = {"name": "attrs", "type": "number", "offset": 0, "bits": 32}
+_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+_U32 = 0xFFFFFFFF
+
+
+def _is_int(v: Any) -> bool:
+    return isinstance(v, int) and not isinstance(v, bool)
+
+
+def _mask(bits: int) -> int:
+    return _U32 if bits >= 32 else (1 << bits) - 1
+
+
+def define_attrs(spec: Mapping[str, Any], prev: AttrSchema | None = None) -> AttrSchema:
+    """Lays out named fields in declaration order into the 32-bit word. Each field is `{"bits": n, "min"?: m}` (an
+    unsigned integer), `{"values": [...]}` (an enum, stored as 1..n) or `"bool"`. With `prev` (the schema in
+    force), fields that already exist keep their offset and width (an enum may append values that fit its width;
+    a number keeps its bits and min), fields `prev` has that `spec` omits are kept, and new fields go after them:
+    stored words stay valid."""
+    fields: list[dict[str, Any]] = [dict(f, **({"values": list(f["values"])} if "values" in f else {}))
+                                    for f in (prev["fields"] if prev else [])]
+    offset = max((f["offset"] + f["bits"] for f in fields), default=0)
+    for name, s in spec.items():
+        if not isinstance(name, str) or not _NAME.match(name):
+            raise ThreadError(f"attrs: bad field name {name!r}", 400)
+        if s == "bool" or (isinstance(s, Mapping) and s.get("bool")):
+            f: dict[str, Any] = {"name": name, "type": "bool", "offset": offset, "bits": 1}
+        elif isinstance(s, Mapping) and "values" in s:
+            values = s["values"]
+            if (not isinstance(values, (list, tuple)) or not values or any(not isinstance(v, str) for v in values)
+                    or len(set(values)) != len(values)):
+                raise ThreadError(f"attrs: field {name}: values must be distinct strings", 400)
+            f = {"name": name, "type": "enum", "offset": offset, "bits": max(1, math.ceil(math.log2(len(values) + 1))),
+                 "values": list(values)}
+        elif isinstance(s, Mapping) and "bits" in s:
+            bits, mn = s["bits"], s.get("min")
+            if not _is_int(bits) or not 1 <= bits <= 32:
+                raise ThreadError(f"attrs: field {name}: bits must be 1..32", 400)
+            if mn is not None and not _is_int(mn):
+                raise ThreadError(f"attrs: field {name}: min must be an integer", 400)
+            f = {"name": name, "type": "number", "offset": offset, "bits": bits}
+            if mn:
+                f["min"] = mn
+        else:
+            raise ThreadError(f"attrs: field {name}: expected {{bits}}, {{values}} or \"bool\"", 400)
+        at = next((i for i, p in enumerate(fields) if p["name"] == name), -1)
+        if at >= 0:
+            # an existing field keeps its layout
+            p = fields[at]
+            same = p["type"] == f["type"] and (p["type"] != "number" or (p["bits"] == f["bits"] and (p.get("min") or 0) == (f.get("min") or 0)))
+            enum_ok = p["type"] != "enum" or (f["values"][: len(p["values"])] == p["values"] and len(f["values"]) <= _mask(p["bits"]))
+            if not same or not enum_ok:
+                raise ThreadError(f"attrs: field {name} exists with another layout (a schema only adds fields, or enum "
+                                  f"values that fit the field's {p['bits']} bits)", 400)
+            if "values" in f:
+                fields[at] = {**p, "values": f["values"]}
+            continue
+        offset += f["bits"]
+        fields.append(f)
+    if offset > 32:
+        raise ThreadError(f"attrs: the fields need {offset} bits, the word has 32", 400)
+    return validate_schema({"fields": fields})
+
+
+def validate_schema(schema: Any) -> AttrSchema:
+    """Checks a schema read from a manifest (fields disjoint and within the word, names unique)."""
+    if not isinstance(schema, Mapping) or not isinstance(schema.get("fields"), list):
+        raise ThreadError("attrs: schema needs a fields array", 400)
+    names: set[str] = set()
+    used = 0
+    for f in schema["fields"]:
+        name = f.get("name") if isinstance(f, Mapping) else None
+        if not isinstance(name, str) or not name or name in names:
+            raise ThreadError(f"attrs: bad or duplicate field name {name!r}", 400)
+        names.add(name)
+        if f.get("type") not in ("number", "enum", "bool"):
+            raise ThreadError(f"attrs: field {name}: unknown type {f.get('type')!r}", 400)
+        off, bits = f.get("offset"), f.get("bits")
+        if not _is_int(off) or not _is_int(bits) or off < 0 or bits < 1 or off + bits > 32:
+            raise ThreadError(f"attrs: field {name}: bad offset/bits", 400)
+        if f["type"] == "bool" and bits != 1:
+            raise ThreadError(f"attrs: field {name}: a bool is one bit", 400)
+        if f["type"] == "enum" and (not isinstance(f.get("values"), list) or len(f["values"]) > _mask(bits)):
+            raise ThreadError(f"attrs: field {name}: enum values do not fit {bits} bits", 400)
+        m = _mask(bits) << off
+        if used & m:
+            raise ThreadError(f"attrs: field {name} overlaps another field", 400)
+        used |= m
+    return dict(schema)
+
+
+def schema_extends(prev: AttrSchema, nxt: AttrSchema) -> bool:
+    """Whether `nxt` keeps every field of `prev` as it is (an enum may append values): the rule for changing a
+    schema."""
+    by_name = {f["name"]: f for f in nxt["fields"]}
+    for p in prev["fields"]:
+        n = by_name.get(p["name"])
+        if (n is None or n["type"] != p["type"] or n["offset"] != p["offset"] or n["bits"] != p["bits"]
+                or (n.get("min") or 0) != (p.get("min") or 0)):
+            return False
+        if p["type"] == "enum" and n.get("values", [])[: len(p.get("values", []))] != p.get("values", []):
+            return False
+    return True
+
+
+def _field(schema: AttrSchema | None, name: str) -> dict[str, Any]:
+    if schema:
+        for f in schema["fields"]:
+            if f["name"] == name:
+                return f
+    if not (schema and schema["fields"]) and name == _RAW_FIELD["name"]:
+        return _RAW_FIELD
+    if schema:
+        raise ThreadError(f"attrs: unknown field {name!r}", 400)
+    raise ThreadError('attrs: this index has no attribute schema; the only field is "attrs"', 400)
+
+
+def _encode(f: dict[str, Any], v: Any) -> int | None:
+    """A field value as the unsigned integer it is stored as, or None when it cannot be represented."""
+    if f["type"] == "bool":
+        if v is True or (v == 1 and _is_int(v)):
+            return 1
+        if v is False or (v == 0 and _is_int(v)):
+            return 0
+        return None
+    if f["type"] == "enum":
+        if not isinstance(v, str):
+            return None
+        try:
+            return f["values"].index(v) + 1
+        except ValueError:
+            return None
+    if not _is_int(v):
+        return None
+    x = v - (f.get("min") or 0)
+    return x if 0 <= x <= _mask(f["bits"]) else None
+
+
+def pack_attrs(schema: AttrSchema | None, values: Any) -> int:
+    """Packs field values into the word (unset fields are 0). An int is taken as the raw word."""
+    if _is_int(values):
+        if not 0 <= values <= _U32:
+            raise ThreadError("attrs: a raw word must be a u32", 400)
+        return values
+    if not isinstance(values, Mapping):
+        raise ThreadError("attrs: expected a mapping of field values or a u32", 400)
+    word = 0
+    for name, v in values.items():
+        if v is None:
+            continue
+        f = _field(schema, name)
+        x = _encode(f, v)
+        if x is None:
+            extra = f" ({f['bits']} bits from {f.get('min') or 0})" if f["type"] == "number" else ""
+            raise ThreadError(f"attrs: {name}: {v!r} is not a valid {f['type']}{extra}", 400)
+        word |= x << f["offset"]
+    return word
+
+
+def unpack_attrs(schema: AttrSchema | None, word: int) -> dict[str, Any]:
+    """The field values of a word under the schema (`{"attrs": word}` without one)."""
+    if not (schema and schema["fields"]):
+        return {"attrs": word & _U32}
+    out: dict[str, Any] = {}
+    for f in schema["fields"]:
+        x = (word >> f["offset"]) & _mask(f["bits"])
+        if f["type"] == "bool":
+            out[f["name"]] = x == 1
+        elif f["type"] == "enum":
+            out[f["name"]] = "" if x == 0 or x > len(f["values"]) else f["values"][x - 1]
+        else:
+            out[f["name"]] = x + (f.get("min") or 0)
+    return out
+
+
+_Clause = tuple[int, int, int, int, list[int]]
+_NOTHING: _Clause = (0, OP_RANGE, 1, 0, [])
+
+
+def compile_filter(schema: AttrSchema | None, expr: AttrFilterExpr | None) -> array:
+    """Compiles a filter expression against the schema into the kernel's words (`array('I')`, empty for no
+    filter). Conditions on different fields are ANDed; within a field each operator adds a clause (`$gt`/`$gte`/
+    `$lt`/`$lte` combine into one range). A value a field cannot hold makes `$eq` match nothing and `$ne` match
+    everything; such values are dropped from `$in` / `$nin`."""
+    if not expr:
+        return array("I")
+    if not isinstance(expr, Mapping):
+        raise ThreadError("filter: expected a mapping of field conditions", 400)
+    clauses: list[_Clause] = []
+    for name, cond in expr.items():
+        if not isinstance(name, str) or name.startswith("$"):
+            raise ThreadError(f"filter: unsupported operator {name} at the top level (conditions are ANDed; nest "
+                              "operators under a field)", 400)
+        f = _field(schema, name)
+        shift = f["offset"]
+        fmask = _mask(f["bits"])
+        mask = fmask << shift
+        c = cond if isinstance(cond, Mapping) else {"$eq": cond}
+
+        def enc(v: Any, op: str) -> int | None:
+            if v is None or not isinstance(v, (int, str, bool)):
+                raise ThreadError(f"filter: {name} {op}: bad value {v!r}", 400)
+            x = _encode(f, v)
+            return None if x is None else x << shift
+
+        lo, hi, ranged = 0, fmask, False
+        for op, v in c.items():
+            if op == "$eq":
+                x = enc(v, op)
+                clauses.append(_NOTHING if x is None else (mask, OP_EQ, x, 0, []))
+            elif op == "$ne":
+                x = enc(v, op)
+                if x is not None:
+                    clauses.append((mask, OP_NE, x, 0, []))
+            elif op in ("$in", "$nin"):
+                if not isinstance(v, (list, tuple)):
+                    raise ThreadError(f"filter: {name} {op}: expected a list", 400)
+                xs = sorted({x for x in (enc(e, op) for e in v) if x is not None})
+                if op == "$in":
+                    clauses.append((mask, OP_IN, 0, 0, xs) if xs else _NOTHING)
+                elif xs:
+                    clauses.append((mask, OP_NIN, 0, 0, xs))
+            elif op in ("$lt", "$lte", "$gt", "$gte"):
+                if f["type"] != "number" or not _is_int(v):
+                    raise ThreadError(f"filter: {name} {op}: needs an integer on a number field", 400)
+                x = v - (f.get("min") or 0)
+                ranged = True
+                if op == "$lt":
+                    hi = min(hi, x - 1)
+                elif op == "$lte":
+                    hi = min(hi, x)
+                elif op == "$gt":
+                    lo = max(lo, x + 1)
+                else:
+                    lo = max(lo, x)
+            elif op in ("$any", "$all", "$none"):
+                if f["type"] != "number" or not _is_int(v) or not 0 <= v <= _U32:
+                    raise ThreadError(f"filter: {name} {op}: needs a u32 bit mask on a number field", 400)
+                m = (v & fmask) << shift
+                if m == 0:
+                    if op == "$any":
+                        clauses.append(_NOTHING)
+                    continue
+                clauses.append((m, {"$any": OP_BITS_ANY, "$all": OP_BITS_ALL, "$none": OP_BITS_NONE}[op], 0, 0, []))
+            else:
+                raise ThreadError(f"filter: {name}: unknown operator {op}", 400)
+        if ranged:
+            lo, hi = max(lo, 0), min(hi, fmask)
+            if lo > hi:
+                clauses.append(_NOTHING)
+            elif lo > 0 or hi < fmask:
+                clauses.append((mask, OP_RANGE, lo << shift, hi << shift, []))
+    return clauses_to_words(clauses)
+
+
+def clauses_to_words(clauses: Sequence[tuple[int, int, int, int, Sequence[int]]]) -> array:
+    out = array("I")
+    if not clauses:
+        return out
+    out.append(len(clauses))
+    for mask, op, a, b, values in clauses:
+        out.extend((mask & _U32, op, a & _U32, b & _U32, len(values)))
+        out.extend(v & _U32 for v in values)
+    return out
+
+
+def filter_matches(words: Sequence[int] | None, attrs: int) -> bool:
+    """Evaluates compiled words in Python (the kernel's `AttrFilter::matches` rule)."""
+    if not words:
+        return True
+    n, pos = words[0], 1
+    for _ in range(n):
+        mask, op, a, b, nv = words[pos : pos + 5]
+        pos += 5
+        values = words[pos : pos + nv]
+        pos += nv
+        x = attrs & mask
+        if op == OP_EQ:
+            ok = x == a
+        elif op == OP_NE:
+            ok = x != a
+        elif op == OP_IN:
+            ok = x in values
+        elif op == OP_NIN:
+            ok = x not in values
+        elif op == OP_RANGE:
+            ok = a <= x <= b
+        elif op == OP_BITS_ANY:
+            ok = x != 0
+        elif op == OP_BITS_ALL:
+            ok = x == mask
+        elif op == OP_BITS_NONE:
+            ok = x == 0
+        else:
+            raise ThreadError(f"filter: unknown op {op}", 400)
+        if not ok:
+            return False
+    return True