matchlab 0.1.0.dev1__py3-none-any.whl

matchlab/core/logging.py

@@ -0,0 +1,190 @@
+"""The logger and console everything in matchlab reports through.
+
+Two objects, one each for the two ways output leaves the library:
+
+* `logger` — the `matchlab` logger, wrapped so any record can carry a prefix saying
+  which part of a plan produced it. Like any library logger it has no handler of its
+  own, and is silent until the application configures logging.
+* `console` — the Rich console. `matchlab.progress` draws the collection tree on it,
+  and tests swap it for a quiet one.
+
+Both are module attributes rather than values passed around, so `matchlab.progress`
+reads them through the module (`mlog.console`) instead of importing them directly.
+That indirection is what lets a test patch one and have the library pick it up.
+
+**A prefix is usually ambient, not passed.** The thing worth quoting is a step's
+position in the plan being collected, and the code that knows a message worth logging
+is rarely the code that knows the position — a `Linker` counting matches is three
+layers below the walk that numbered its step. `collect` therefore publishes the
+position in `prefix`, a `ContextVar`, and every record emitted while that step runs
+picks it up. Call sites stay ignorant, which is what keeps positions a property of the
+walk rather than of the step (see `matchlab.steps.Step.__init__`).
+
+The `prefix=` keyword remains for callers with something to say outside a collection,
+where there is no position to quote. It is a fallback, not an override: an ambient
+position wins, so records read consistently within the run that printed the tree.
+"""
+
+import logging
+import sys
+from collections.abc import Generator
+from contextlib import contextmanager
+from contextvars import ContextVar
+from typing import IO, Any, Final
+
+from rich.console import Console
+from rich.file_proxy import FileProxy
+
+prefix: Final[ContextVar[str | None]] = ContextVar("matchlab_prefix", default=None)
+"""The prefix records carry right now, or `None` outside any collection.
+
+Set by `matchlab.progress.Progress` for the duration of each step, so that anything
+logged while that step runs is tied to the plan the collection drew. Being a
+`ContextVar` is what makes it restore correctly when a collection runs inside another:
+the inner one holds the outer one's token and gives it back on the way out.
+
+A handler or filter that wants to know which step is running can read it directly.
+Setting it is `Progress`'s business — a record of your own takes `prefix=` instead.
+"""
+
+
+class PrefixedLoggerAdapter(logging.LoggerAdapter):
+    """A logger adapter that tags records with the plan position they came from.
+
+    The prefix is normally ambient, read from `prefix`. A `prefix=` keyword on the call
+    supplies one for records emitted outside a collection.
+    """
+
+    def process(
+        self,
+        msg: Any,  # noqa: ANN401
+        kwargs: dict[str, Any],
+    ) -> tuple[Any, dict[str, Any]]:
+        """Add the ambient or explicit prefix to `msg`, if either is set."""
+        # Popped either way: `prefix` is ours, and `Logger.log` would reject it.
+        explicit = kwargs.pop("prefix", None)
+        # Ambient first: inside a collection the position is the prefix that matches
+        # the tree, and it is the one a reader can look up. A caller's own idea of
+        # what it is only applies where there is no position to quote.
+        tag = prefix.get() or explicit
+
+        if tag:
+            msg = f"[{tag}] {msg}"
+
+        return msg, kwargs
+
+
+logger: Final[PrefixedLoggerAdapter] = PrefixedLoggerAdapter(
+    logging.getLogger("matchlab"), {}
+)
+"""Logger for matchlab.
+
+Used for all logging in the matchlab library.
+
+Records logged while a collection is running are prefixed with the position of the step
+that produced them, which is what ties a record to the plan `collect` drew. Nothing at
+the call site arranges that; see `prefix`.
+
+Examples:
+    ```python
+    logger.info(f"Round {round_num}: Found {len(matches):,} matches")
+    ```
+"""
+
+console: Final[Console] = Console()
+"""Console for matchlab.
+
+Where `matchlab.progress` draws a collection's plan, and where any CLI utility writes.
+"""
+
+
+@contextmanager
+def through_console() -> Generator[None]:
+    """Route handlers already writing to this terminal through the console instead.
+
+    Rich redirects `sys.stdout` and `sys.stderr` while a live display is running, so a
+    bare `print` lands above the frame rather than through it. Logging misses out:
+    `logging.StreamHandler` reads `sys.stderr` **once**, when it is constructed, and
+    `logging.basicConfig()` constructs one at import time — long before there is a
+    frame to avoid. That handler then writes straight past Rich for the rest of the
+    process.
+
+    That is the ordinary case, not an exotic one. Rather than asking every caller to
+    work around it, `matchlab.progress` closes the gap itself: for as long as it owns
+    the terminal, it points any handler bound to that terminal at the same proxy Rich
+    installs, and puts it back afterwards. Handlers writing anywhere else — a file, a
+    socket, another stream — are left alone, having nothing to collide with.
+
+    It restores the original streams on the way out, however the block ends. It's
+    reentrant only in the sense that a handler already proxied is skipped, so a nested
+    display can't double-wrap one.
+
+    It does nothing in a notebook. The problem it solves is a cursor collision, and a
+    notebook has no cursor: Rich draws a live display into an `ipywidgets.Output` that
+    redraws itself in isolation, so nothing can smear it. Routing records through the
+    console there would replace each one with its own `display()` call — a separate
+    HTML block, boxed and spaced apart. Left alone, the stream they already write to
+    renders them as ordinary contiguous text instead.
+    """
+    if console.is_jupyter:
+        yield
+        return
+
+    # Read before Rich swaps them: afterwards `sys.stdout` is a proxy, and a handler
+    # holding the real stream would no longer look like it points at the terminal.
+    terminal = {sys.stdout, sys.stderr, console.file}
+    patched: list[tuple[logging.StreamHandler, IO[str]]] = []
+
+    target: logging.Logger | None = logging.getLogger("matchlab")
+    while target is not None:
+        for handler in target.handlers:
+            stream = getattr(handler, "stream", None)
+            if (
+                isinstance(handler, logging.StreamHandler)
+                and not isinstance(stream, FileProxy)
+                and any(stream is candidate for candidate in terminal)
+            ):
+                patched.append((handler, stream))
+                handler.setStream(FileProxy(console, stream))
+        target = target.parent if target.propagate else None
+
+    try:
+        yield
+    finally:
+        for handler, stream in patched:
+            handler.setStream(stream)
+
+
+class ConsoleHandler(logging.Handler):
+    """A logging handler that writes through matchlab's own console.
+
+    Sharing the console is what lets logging coexist with the live tree
+    `matchlab.progress` draws: Rich prints through the live region, so records land
+    above the frame instead of over it. A handler holding a raw stream — which is what
+    `logging.basicConfig()` gives you, bound at the moment it was called — writes
+    straight past Rich and smears whatever is being redrawn.
+
+    Deliberately plainer than `rich.logging.RichHandler`, which lays records out in a
+    table and would wrap a logged plan inside its narrow message column. What matchlab
+    logs is mostly trees, and a tree that wraps is not a tree. Records are written at
+    the console's full width, unstyled, and formatted however the application asked.
+
+    You do not need this to run a collection at a terminal: `through_console` already
+    borrows a handler bound to one for as long as a live tree is up. This is for saying
+    so up front, and for the handler that outlives any one collection.
+
+    Examples:
+        ```python
+        logging.basicConfig(level=logging.INFO, handlers=[ConsoleHandler()])
+        ```
+    """
+
+    def emit(self, record: logging.LogRecord) -> None:
+        """Write `record` to the console."""
+        try:
+            # `markup=False` matters: a drawn tree is full of `[2]`, which Rich would
+            # otherwise try to parse as a style tag. `highlight=False` likewise — its
+            # guesses at numbers and brackets colour the tree into confetti.
+            console.print(self.format(record), markup=False, highlight=False)
+        except Exception:  # noqa: BLE001 - a handler must never raise into the caller
+            self.handleError(record)

matchlab/core/resolver_output.py

@@ -0,0 +1,173 @@
+"""Each resolver materialises its complete, merge-forward output once.
+
+Every resolver, when it runs, computes that output and stores it. Everything
+downstream (`entities`, `lookup_key`, models querying through it) just reads the
+stored table.
+
+This module holds the pure functions that produce that output:
+
+* `leaf_id` / `root_id` — content-addressed cluster IDs. A leaf is a hash of a
+  record's content. That is the correct anchor for evaluation, not merely a
+  reproducibility trick — see `leaf_id` for the full reasoning.
+* `materialise_resolver_output` — turns a resolver's clusters and upstream output
+  into the complete result table. See its own docstring for the shape of each.
+
+**Merge-forward** means a leaf grouped upstream but untouched by this resolver
+inherits its upstream cluster rather than collapsing to a singleton.
+`materialise_resolver_output` implements this by giving every *untouched* reachable ID
+its own component ("fall-through"), so the upstream grouping survives.
+
+This module assumes a resolver's inputs share a consistent lineage, so each
+reachable leaf maps to exactly one upstream ID. DAG construction enforces that, so
+nothing here needs to reconcile a leaf that maps to more than one.
+"""
+
+from collections.abc import Iterable
+
+import polars as pl
+import polars_hash  # noqa: F401 - registers the `nchash` expression namespace
+
+from matchlab.core.schemas import SCHEMA_RESOLVER_OUTPUT
+
+# Every reachable query-space ID, mapped to a source row and its leaf. This is the
+# shape `materialise_resolver_output` expects for its `upstream` argument.
+UPSTREAM_COLUMNS = ("id", "source", "key", "leaf")
+
+
+def leaf_id(row_hash: pl.Expr) -> pl.Expr:
+    """Map a column of row content hashes to stable 64-bit leaf cluster IDs.
+
+    A record's identity is a hash of its **content**, not of its key. This is the
+    load-bearing decision, and it is deliberate, not an accident of implementation.
+    The reason is evaluation.
+
+    A judgement is a person or a model saying "looking at *this*, these records are the
+    same entity." The only input to that decision is the content shown — a name, a
+    postcode. The key plays no part. The judge never sees it. A judgement must
+    therefore be anchored to the content it was actually made against:
+
+    * If you anchor to the **key**, a judgement outlives a content change. Say a
+      judgement was made when the evidence read "acme / london = acme / london". If
+      that evidence later becomes "acme / manchester", the judgement still stands —
+      attributed to evidence the judge never saw. It looks inexplicable later, or
+      quietly validates a match nobody made.
+    * If you anchor to the **content** instead (this module's choice), the leaf ID
+      changes whenever the content does, so the old judgement stops applying. That is
+      correct. Nobody has judged the new content, so there should be no judgement
+      about it. Judgements decay exactly when their evidence does.
+
+    The same logic explains why identical rows share a leaf. To any judge they are
+    indistinguishable, so there is no decision to make. Differing keys don't separate
+    them, because a key is not evidence.
+
+    **What gets hashed must match what gets shown.** If a column feeds the hash but
+    the sampler doesn't display it, a judgement decays for a reason the judge never
+    saw. If a column appears in the sampler but doesn't feed the hash, a judgement
+    survives a change the judge did see, which is wrong.
+
+    The two stay in sync today because both read the same definition. `leaf_id`
+    hashes every non-key column (see `Source._read_warehouse`), and `get_samples`
+    displays every non-key column. Selecting a column in the extract is what makes it
+    both hashed and shown — there is no second list to keep in step by hand (see
+    `Source`'s "the extract is the whole declaration").
+
+    Content-addressing also makes runs reproducible and IDs stable across re-collect,
+    but that is a consequence, not the reason.
+
+    This is vectorised deliberately. Calling a Python function once per row, as
+    `map_elements` would, dominates collection time.
+    """
+    return row_hash.bin.slice(0, 8).bin.reinterpret(dtype=pl.UInt64, endianness="big")
+
+
+def root_id(leaves: pl.Expr) -> pl.Expr:
+    """Deterministic 64-bit root cluster ID for a column of leaf lists.
+
+    This is invariant to leaf order (callers sort) and to the arbitrary component
+    label, so two runs that produce the same clustering produce the same root IDs.
+
+    This is vectorised for the same reason as `leaf_id`. Calling this once per
+    cluster, in Python, would mean as many calls as there are entities.
+    """
+    # `polars_hash` registers the `nchash` namespace on polars expressions at import.
+    joined = leaves.cast(pl.List(pl.Utf8)).list.join(",")
+    return joined.nchash.xxh3_64().cast(pl.UInt64)
+
+
+def root_id_of(leaves: Iterable[int]) -> int:
+    """`root_id` for a single leaf set. For tests and one-off checks, not for loops."""
+    leaves = sorted(leaves)
+    if not leaves:
+        raise ValueError("A cluster must contain at least one leaf")
+    frame = pl.DataFrame({"leaves": [leaves]}).with_columns(
+        pl.col("leaves").list.eval(pl.element().cast(pl.UInt64))
+    )
+    return frame.select(root_id(pl.col("leaves")).alias("root"))["root"][0]
+
+
+def materialise_resolver_output(
+    clusters: pl.DataFrame,
+    upstream: pl.DataFrame,
+) -> pl.DataFrame:
+    """Build a resolver's complete, merge-forward output.
+
+    Args:
+        clusters: `SCHEMA_CLUSTERS` `(parent_id, child_id)` from the resolver's
+            methodology. `child_id`s are query-space IDs the models formed edges over.
+            IDs not present here are untouched by this resolver.
+        upstream: The union of the resolver's input queries as `(id, source, key,
+            leaf)` rows, covering *every* leaf reachable by the resolver, whether or
+            not an edge formed over it. `id` is the query-space ID the model
+            referenced.
+
+    Returns:
+        One row per reachable source record, shaped as `SCHEMA_RESOLVER_OUTPUT`
+        `(root, leaf, key, source)`, with `root` the cluster it resolves to. Complete
+        and merge-forward.
+    """
+    missing = set(UPSTREAM_COLUMNS) - set(upstream.columns)
+    if missing:
+        raise ValueError(f"upstream is missing columns: {sorted(missing)}")
+
+    upstream = upstream.select(
+        pl.col("id").cast(pl.UInt64),
+        pl.col("source").cast(pl.Utf8),
+        pl.col("key").cast(pl.Utf8),
+        pl.col("leaf").cast(pl.UInt64),
+    )
+
+    if upstream.height == 0:
+        return pl.from_arrow(SCHEMA_RESOLVER_OUTPUT.empty_table())
+
+    child_to_parent = clusters.select(
+        pl.col("child_id").cast(pl.UInt64),
+        pl.col("parent_id").cast(pl.UInt64),
+    )
+
+    # Component label per reachable row:
+    #   * touched IDs (in a cluster)  -> "c{parent_id}"  (this resolver's new cluster)
+    #   * untouched IDs               -> "s{id}"         (fall-through: keep upstream)
+    # Prefixes keep the two integer spaces from colliding.
+    labelled = upstream.join(
+        child_to_parent, left_on="id", right_on="child_id", how="left"
+    ).with_columns(
+        pl.when(pl.col("parent_id").is_not_null())
+        .then("c" + pl.col("parent_id").cast(pl.Utf8))
+        .otherwise("s" + pl.col("id").cast(pl.Utf8))
+        .alias("_component")
+    )
+
+    # Content-addressed root per component, derived from its full leaf set.
+    component_roots = (
+        labelled.group_by("_component")
+        .agg(pl.col("leaf").unique().sort().alias("_leaves"))
+        .with_columns(root_id(pl.col("_leaves")).alias("root"))
+        .select("_component", "root")
+    )
+
+    return (
+        labelled.join(component_roots, on="_component")
+        .select("root", "leaf", "key", "source")
+        .unique()
+        .sort("root", "leaf", "key")
+    )

matchlab/core/schemas.py

@@ -0,0 +1,82 @@
+"""Arrow schemas for the artifacts an adapter persists.
+
+These are the shapes that cross the boundary between a step and its storage: what a
+`Model` or `Resolver` hands the adapter, and what the adapter hands back. They live here
+rather than with their producers because both sides need them, and `core` is the only
+layer both a producer and an adapter can depend on without inverting.
+
+The membership rule is exactly that boundary — **the adapter persists it or returns
+it**. A shape that never leaves the process belongs with the code that produces it:
+`SCHEMA_CLUSTERS` is a resolver methodology's output, read by the `Resolver` that called
+it and by nothing else, so it lives in `matchlab.resolvers.base`. Keeping those out is
+what stops this module from becoming a catch-all for anything that happens to be an
+Arrow schema.
+"""
+
+from typing import Final
+
+import pyarrow as pa
+from pyarrow import Schema
+
+from matchlab.core.exceptions import SchemaMismatch
+
+SCHEMA_MODEL_EDGES: Final[pa.Schema] = pa.schema(
+    [
+        ("left_id", pa.uint64()),
+        ("right_id", pa.uint64()),
+        ("score", pa.float32()),
+    ]
+)
+"""A deduper's or linker's output: a scored edge list."""
+
+SCHEMA_RESOLVER_OUTPUT: Final[pa.Schema] = pa.schema(
+    [
+        ("root", pa.uint64()),
+        ("leaf", pa.uint64()),
+        ("key", pa.large_string()),
+        ("source", pa.large_string()),
+    ]
+)
+"""A resolver's complete flat output: one row per reachable source record.
+
+Evaluation sampling returns a row-subset of this, which is why it is named for the
+resolver's output rather than for the sample.
+"""
+
+SCHEMA_JUDGEMENTS: Final[pa.Schema] = pa.schema(
+    [
+        ("user_name", pa.large_string()),
+        ("endorsed", pa.uint64()),
+        ("shown", pa.uint64()),
+    ]
+)
+"""Stored user judgements, as returned by `Adapter.read_eval_data`."""
+
+SCHEMA_CLUSTER_EXPANSION: Final[pa.Schema] = pa.schema(
+    [
+        ("root", pa.uint64()),
+        ("leaves", pa.list_(pa.uint64())),
+    ]
+)
+"""A cluster ID mapped to all its source cluster IDs. 
+
+Used alongside `SCHEMA_JUDGEMENTS` in evaluation scoring."""
+
+
+def check_schema_subset(expected: Schema, actual: Schema) -> None:
+    """Check presence of Arrow fields, ignoring field order, extras and metadata.
+
+    Args:
+        expected: Schema with fields that must be present
+        actual: Schema to check
+    """
+    actual = pa.schema(
+        [
+            (name, actual.field(name).type)
+            for name in expected.names
+            if name in actual.names
+        ]
+    )
+
+    if not actual.equals(expected):
+        raise SchemaMismatch(expected=expected, actual=actual)

matchlab/core/sql.py

@@ -0,0 +1,12 @@
+"""SQL fragment string types shared across matchlab."""
+
+from typing import TypeAlias
+
+SQLExpression: TypeAlias = str
+"""Any SQL text `sqlglot` can parse, with no shape guaranteed beyond that."""
+
+SQLCondition: TypeAlias = SQLExpression
+"""A boolean WHERE-clause predicate, qualified with `l`/`r` per `models.comparison`."""
+
+SQLQuery: TypeAlias = SQLExpression
+"""A full SQL statement, such as a `SELECT`, not just a predicate."""