se-interpretive-kernel 0.1.0__py3-none-any.whl

se_interpretive_kernel/lifting.py

@@ -0,0 +1,374 @@
+"""lifting.py - Comparison liftings from output regimes to object regimes.
+
+Interpretation returns structured outputs, and comparing two outputs requires
+an equivalence on the atomic values they are built from.
+A *comparison lifting* carries an output regime up to the structures:
+
+    C : Eq(O) -> Eq(X)
+
+where ``O`` is the carrier of atomic outputs and ``X`` is the carrier of the
+structured objects being compared.
+
+The kernel construction requires ``C`` to preserve arbitrary meets.
+Complete meet homomorphisms of this shape are exactly the ``Eq(O)``-valued
+generalized ultrametrics on ``X``: every one of them is given by a cost
+
+    delta : X x X -> Eq(O)
+
+assigning to each pair the least output identification that would make the
+two objects the same, with
+
+    C(rho) = { (x, y) : delta(x, y) refines rho }.
+
+The cost must satisfy three laws:
+
+1. ``delta(x, x)`` is the discrete partition,
+2. ``delta(x, y) == delta(y, x)``, and
+3. ``delta(x, z)`` refines ``delta(x, y) join delta(y, z)``.
+
+A lifting that satisfies them supports the adjunction.
+A comparison that only claims to be a lifting does not, and
+:func:`check_lifting_laws` is the finite test that separates the two cases.
+
+Two comparisons in this module are present because they fail:
+
+- :class:`QuotientImage` compares unordered outputs by equality of their
+  quotient images. It is monotone but does not preserve meets, so no cost
+  function represents it.
+- :class:`OptimalMatching` looks like the natural cost for unordered outputs
+  but violates the triangle law, because the meet over alternative bijections
+  cannot be moved through the join in a lattice that is not distributive.
+
+:class:`IndexedPointwise` is the comparison this package selects.
+When the outputs are genuinely unordered, a declared canonicalization
+supplies the index, and the price of that choice is a cost no finer than the
+best matching would have required.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+from itertools import permutations
+from typing import Protocol
+
+from .partition import Item, Pair, Partition, meet_all
+
+type Form = Callable[[Item], Sequence[Item]]
+"""Map an object to the indexed sequence of atomic outputs it presents."""
+
+type Members = Callable[[Item], frozenset[Item]]
+"""Map an object to the unordered atomic outputs it contains."""
+
+
+class Comparison(Protocol):
+    """Carry an output regime up to a regime on structured objects."""
+
+    @property
+    def objects(self) -> tuple[Item, ...]:
+        """Return the carrier of structured objects being compared."""
+        ...
+
+    def induced(self, rho: Partition) -> Partition:
+        """Return the object regime induced by an output regime."""
+        ...
+
+
+class Lifting(Protocol):
+    """Assign each pair of objects the least output regime identifying them."""
+
+    @property
+    def objects(self) -> tuple[Item, ...]:
+        """Return the carrier of structured objects being compared."""
+        ...
+
+    @property
+    def output_carrier(self) -> tuple[Item, ...]:
+        """Return the carrier of atomic output values."""
+        ...
+
+    def cost(self, x: Item, y: Item) -> Partition:
+        """Return the least output regime under which two objects agree."""
+        ...
+
+    def induced(self, rho: Partition) -> Partition:
+        """Return the object regime induced by an output regime."""
+        ...
+
+
+@dataclass(frozen=True)
+class _CostBased:
+    """Provide the induced comparison shared by every cost-based lifting.
+
+    Two objects are related under an output regime exactly when the cost of
+    identifying them refines that regime.
+    """
+
+    def _induced_from_cost(
+        self,
+        objects: tuple[Item, ...],
+        cost: Callable[[Item, Item], Partition],
+        rho: Partition,
+    ) -> Partition:
+        """Return the object regime generated by pairs whose cost fits ``rho``.
+
+        Only pairs are collected here.
+        Their equivalence closure is taken by :meth:`Partition.from_pairs`,
+        which is sound because the triangle law makes the relation transitive
+        already when the laws hold, and because closing a relation that is
+        already transitive changes nothing.
+        """
+        pairs: list[Pair] = [
+            (x, y)
+            for i, x in enumerate(objects)
+            for y in objects[i + 1 :]
+            if cost(x, y).refines(rho)
+        ]
+        return Partition.from_pairs(objects, pairs)
+
+
+@dataclass(frozen=True)
+class IndexedPointwise(_CostBased):
+    """Compare objects position by position over a declared index.
+
+    ``form`` presents each object as a sequence of atomic outputs.
+    Two objects are compared by requiring agreement at every position, so the
+    cost of identifying them is generated by the pairs of values standing at
+    corresponding positions.
+
+    This comparison preserves arbitrary meets, because agreement at every
+    position is a conjunction of atomic comparisons and universal
+    quantification commutes with meet.
+
+    When the underlying outputs are unordered, ``form`` is a declared
+    canonicalization and is part of the declaration rather than an
+    implementation detail.
+    A canonicalization that is not injective identifies distinct objects at
+    zero cost, which declares an identity without ever appearing as a declared
+    identity regime.
+
+    Sequences compared here must already have equal length.
+    Length mismatch is a declared comparison outcome and belongs to the
+    tagged outcome domain, not to this module.
+    """
+
+    output_carrier: tuple[Item, ...]
+    objects: tuple[Item, ...]
+    form: Form
+
+    def cost(self, x: Item, y: Item) -> Partition:
+        """Return the closure of the pairs standing at corresponding positions."""
+        left, right = self.form(x), self.form(y)
+        if len(left) != len(right):
+            msg = (
+                "indexed comparison requires equal-length forms; "
+                "length mismatch is a tagged outcome, not a comparison"
+            )
+            raise ValueError(msg)
+        return Partition.from_pairs(self.output_carrier, zip(left, right, strict=True))
+
+    def induced(self, rho: Partition) -> Partition:
+        """Return the object regime induced by an output regime."""
+        return self._induced_from_cost(self.objects, self.cost, rho)
+
+
+@dataclass(frozen=True)
+class OptimalMatching(_CostBased):
+    """Compare unordered objects by the cheapest bijection between them.
+
+    The cost is the meet, over all bijections between the two member sets, of
+    the identification each bijection would require.
+    This is the natural reading of "there is a matching under which the two
+    agree", and it is the comparison a practitioner would reach for first.
+
+    It is not a lifting.
+    The triangle law fails, because moving the meet over alternative
+    bijections through the join requires a distributivity the partition
+    lattice does not have.
+    :func:`check_lifting_laws` returns the counterexample.
+
+    The class is retained because the failure is a result: optimal matching
+    and meet preservation cannot be had together, and a declared
+    canonicalization is the price of keeping the second.
+    """
+
+    output_carrier: tuple[Item, ...]
+    objects: tuple[Item, ...]
+    members: Members
+
+    def cost(self, x: Item, y: Item) -> Partition:
+        """Return the meet over all bijections of the cost each would require.
+
+        Member sets of different sizes admit no bijection.
+        The meet of an empty family is the top element, which reports that
+        only the total identification makes the two objects agree.
+        """
+        left, right = (
+            sorted(self.members(x), key=repr),
+            sorted(self.members(y), key=repr),
+        )
+        if len(left) != len(right):
+            return Partition.indiscrete(self.output_carrier)
+        candidates = [
+            Partition.from_pairs(self.output_carrier, zip(left, perm, strict=True))
+            for perm in permutations(right)
+        ]
+        return meet_all(self.output_carrier, candidates)
+
+    def induced(self, rho: Partition) -> Partition:
+        """Return the object regime induced by an output regime."""
+        return self._induced_from_cost(self.objects, self.cost, rho)
+
+
+@dataclass(frozen=True)
+class QuotientImage:
+    """Compare unordered objects by equality of their quotient images.
+
+    Two objects are related under an output regime when the sets of output
+    classes they occupy are equal.
+
+    This comparison is monotone: collapsing output classes further cannot
+    separate objects whose images already agreed.
+    It does not preserve meets, so it is not represented by any cost function
+    and does not support the adjunction.
+    :func:`check_meet_preservation` returns the counterexample.
+
+    The obstruction is the existential matching hidden in comparing unordered
+    images, not the fact that the outputs are sets.
+    """
+
+    output_carrier: tuple[Item, ...]
+    objects: tuple[Item, ...]
+    members: Members
+
+    def image(self, x: Item, rho: Partition) -> frozenset[int]:
+        """Return the set of output classes the object's members occupy."""
+        return frozenset(rho.label_of(m) for m in self.members(x))
+
+    def induced(self, rho: Partition) -> Partition:
+        """Return the object regime relating objects with equal images."""
+        return Partition.from_labels(
+            self.objects, {x: self.image(x, rho) for x in self.objects}
+        )
+
+
+@dataclass(frozen=True)
+class LawViolation:
+    """Report one failed law together with the values that witness it.
+
+    ``law`` names the law that failed.
+
+    ``witness`` contains the objects or output regimes the failure was found
+    on, in the order the law names them.
+
+    ``detail`` states what was expected and what held instead.
+    """
+
+    law: str
+    witness: tuple[Item, ...]
+    detail: str
+
+
+def check_lifting_laws(lifting: Lifting) -> tuple[LawViolation, ...]:
+    """Return every violation of the three cost laws, with witnesses.
+
+    Identity, symmetry, and the triangle law are checked over every ordered
+    triple of objects.
+    The carriers are finite and small by construction, so the check is
+    exhaustive rather than sampled.
+
+    An empty result means the cost is an ``Eq(O)``-valued generalized
+    ultrametric, so the comparison it induces preserves arbitrary meets and
+    the threshold construction applies to it.
+    """
+    objects = lifting.objects
+    bottom = Partition.discrete(lifting.output_carrier)
+    found: list[LawViolation] = []
+
+    for x in objects:
+        if lifting.cost(x, x) != bottom:
+            found.append(
+                LawViolation(
+                    "identity",
+                    (x,),
+                    "cost of an object against itself is not the discrete partition",
+                )
+            )
+
+    for i, x in enumerate(objects):
+        for y in objects[i + 1 :]:
+            if lifting.cost(x, y) != lifting.cost(y, x):
+                found.append(
+                    LawViolation("symmetry", (x, y), "cost depends on argument order")
+                )
+
+    for x in objects:
+        for y in objects:
+            for z in objects:
+                direct = lifting.cost(x, z)
+                detour = lifting.cost(x, y).join(lifting.cost(y, z))
+                if not direct.refines(detour):
+                    found.append(
+                        LawViolation(
+                            "triangle",
+                            (x, y, z),
+                            "direct cost is not refined by the cost through the "
+                            "intermediate object",
+                        )
+                    )
+
+    return tuple(found)
+
+
+def check_meet_preservation(
+    comparison: Comparison, samples: Sequence[Partition]
+) -> tuple[LawViolation, ...]:
+    """Return every failure of meet preservation over the sampled regimes.
+
+    A comparison preserves meets when the regime induced by a meet of output
+    regimes equals the meet of the regimes each induces separately.
+    Failure means no cost function represents the comparison, so the
+    threshold construction and the adjunction are both unavailable for it.
+    """
+    found: list[LawViolation] = []
+    for i, rho in enumerate(samples):
+        for sigma in samples[i + 1 :]:
+            together = comparison.induced(rho.meet(sigma))
+            apart = comparison.induced(rho).meet(comparison.induced(sigma))
+            if together != apart:
+                found.append(
+                    LawViolation(
+                        "meet-preservation",
+                        (rho, sigma),
+                        "the regime induced by the meet differs from the meet of "
+                        "the induced regimes",
+                    )
+                )
+    return tuple(found)
+
+
+def check_monotonicity(
+    comparison: Comparison, samples: Sequence[Partition]
+) -> tuple[LawViolation, ...]:
+    """Return every failure of monotonicity over the sampled regimes.
+
+    Coarsening an output regime may only coarsen the regime it induces.
+    Monotonicity is strictly weaker than meet preservation, and a comparison
+    may satisfy it while still admitting no cost function.
+    Reporting the two separately keeps that distinction visible.
+    """
+    found: list[LawViolation] = []
+    for rho in samples:
+        for sigma in samples:
+            if rho.refines(sigma) and not comparison.induced(rho).refines(
+                comparison.induced(sigma)
+            ):
+                found.append(
+                    LawViolation(
+                        "monotonicity",
+                        (rho, sigma),
+                        "a coarser output regime induced a strictly finer object "
+                        "regime",
+                    )
+                )
+    return tuple(found)

se_interpretive_kernel/literal.py

@@ -0,0 +1,87 @@
+"""literal.py - Independent literal definitions, used as an oracle.
+
+Every function here works on explicit sets of ordered pairs and uses a naive
+fixed-point closure. Nothing here uses :class:`Partition` internals, labels, or
+the union-find, so agreement with the partition-based modules is a real
+cross-check rather than a restatement.
+
+The functions are slow and intended only for small finite cases.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+
+from .artifact import Artifact
+from .partition import Item, Pair, Partition
+
+type PairSet = frozenset[Pair]
+
+
+def pairs_of(partition: Partition) -> PairSet:
+    """Return a partition as the explicit set of related ordered pairs."""
+    return frozenset(
+        (x, y)
+        for x in partition.carrier
+        for y in partition.carrier
+        if partition.same(x, y)
+    )
+
+
+def equivalence_closure(carrier: Sequence[Item], pairs: Iterable[Pair]) -> PairSet:
+    """Return the least equivalence relation containing ``pairs``.
+
+    Starts from the identity, adds the reverse of every pair, and repeats
+    composition until nothing new appears.
+    """
+    relation: set[Pair] = {(x, x) for x in carrier}
+    for x, y in pairs:
+        relation.add((x, y))
+        relation.add((y, x))
+    changed = True
+    while changed:
+        changed = False
+        for x, y in list(relation):
+            for y2, z in list(relation):
+                if y == y2 and (x, z) not in relation:
+                    relation.add((x, z))
+                    changed = True
+    return frozenset(relation)
+
+
+def kernel_pairs(artifact: Artifact, selected: Sequence[Item], rho: PairSet) -> PairSet:
+    """Return {(F, F') | for every selected p, (I(p,F), I(p,F')) in rho}."""
+    return frozenset(
+        (f1, f2)
+        for f1 in artifact.frames
+        for f2 in artifact.frames
+        if all(
+            (artifact.interpret(p, f1), artifact.interpret(p, f2)) in rho
+            for p in selected
+        )
+    )
+
+
+def generator_pairs(
+    artifact: Artifact, selected: Sequence[Item], phi: PairSet
+) -> PairSet:
+    """Return {(I(p,F), I(p,F')) | p selected and (F, F') in phi}."""
+    return frozenset(
+        (artifact.interpret(p, f1), artifact.interpret(p, f2))
+        for (f1, f2) in phi
+        for p in selected
+    )
+
+
+def generated_pairs(
+    artifact: Artifact, selected: Sequence[Item], phi: PairSet
+) -> PairSet:
+    """Return the equivalence closure of the generators of phi."""
+    return equivalence_closure(
+        artifact.outputs, generator_pairs(artifact, selected, phi)
+    )
+
+
+def is_subset(smaller: PairSet, larger: PairSet) -> bool:
+    """Return whether one relation is contained in another."""
+    return smaller <= larger

se_interpretive_kernel/loader.py

@@ -0,0 +1,238 @@
+"""loader.py - Build an Artifact and a frame regime from tabular data.
+
+The theory needs a total interpretation: every probe must produce an output for
+every frame. Real tables have empty cells, so this module makes the answer to
+"what does an empty cell produce" an explicit, reported choice instead of a
+silent default.
+
+Probe columns are supplied already computed. This module does not normalize,
+clean, or interpret values. Normalization is the consumer's declared probe.
+
+Missing values. An empty cell becomes the output :class:`Absent`. Two policies
+decide whether different empty cells are the same output:
+
+- ``MissingPolicy.DISTINCT`` (default): each empty cell is its own output. An
+  empty cell never agrees with anything, so an unknown value is not treated as
+  evidence of sameness.
+- ``MissingPolicy.SHARED``: all empty cells of a probe are one output. Records
+  with no value then agree with each other. In the generated regime this one
+  output can also join every value of every record that the frame regime groups
+  with an empty-cell record, so it can chain unrelated values together.
+
+Neither policy is correct in general. Run both and compare the results.
+
+Output space. By default an output carries the name of its probe column, so
+equal text in two different columns is two different outputs. With
+``shared_outputs=True`` equal text is one output across columns, which is
+appropriate only when the probe columns are alternative readings of one field.
+
+This module reports what was loaded. It does not judge whether a table, a
+probe, or a clustering is right.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping, Sequence
+import csv
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+
+from .artifact import Artifact
+from .partition import Item, Partition
+
+
+class MissingPolicy(Enum):
+    """Decide whether different empty cells are the same output."""
+
+    DISTINCT = "distinct"
+    SHARED = "shared"
+
+
+@dataclass(frozen=True)
+class Observed:
+    """Contain an output that was read from a non-empty cell.
+
+    ``scope`` is the probe column, or None when outputs are shared across
+    columns.
+    """
+
+    scope: str | None
+    text: str
+
+
+@dataclass(frozen=True)
+class Absent:
+    """Contain the output of an empty cell.
+
+    ``frame`` identifies the cell under ``MissingPolicy.DISTINCT`` and is None
+    under ``MissingPolicy.SHARED``.
+    """
+
+    scope: str | None
+    frame: Item | None
+
+
+@dataclass(frozen=True)
+class Loaded:
+    """Contain a loaded artifact and what was decided while loading it."""
+
+    artifact: Artifact
+    policy: MissingPolicy
+    shared_outputs: bool
+    missing_counts: Mapping[str, int]
+
+
+def load_artifact(
+    rows: Iterable[Mapping[str, str | None]],
+    id_column: str,
+    probe_columns: Sequence[str],
+    *,
+    missing_values: frozenset[str] = frozenset({""}),
+    policy: MissingPolicy = MissingPolicy.DISTINCT,
+    shared_outputs: bool = False,
+    label: str = "",
+) -> Loaded:
+    """Return an artifact whose frames are the row ids and probes are columns.
+
+    A cell is empty when it is None or exactly one of ``missing_values``.
+    Values are compared exactly: no trimming or case folding is applied.
+    Frame order is row order. A duplicate or empty id is an error.
+    """
+    columns = tuple(probe_columns)
+    if not columns:
+        msg = "at least one probe column is required"
+        raise ValueError(msg)
+    if len(set(columns)) != len(columns):
+        msg = "probe columns must be distinct"
+        raise ValueError(msg)
+    if id_column in columns:
+        msg = "the id column cannot also be a probe column"
+        raise ValueError(msg)
+
+    frames: list[Item] = []
+    seen: set[str] = set()
+    table: dict[tuple[Item, Item], Item] = {}
+    missing_counts = {column: 0 for column in columns}
+    outputs: dict[Item, None] = {}
+
+    for number, row in enumerate(rows, start=1):
+        if id_column not in row:
+            msg = f"row {number}: no column {id_column!r}"
+            raise ValueError(msg)
+        frame = row[id_column]
+        if frame is None or frame == "":
+            msg = f"row {number}: empty id"
+            raise ValueError(msg)
+        if frame in seen:
+            msg = f"row {number}: duplicate id {frame!r}"
+            raise ValueError(msg)
+        seen.add(frame)
+        frames.append(frame)
+        for column in columns:
+            if column not in row:
+                msg = f"row {number}: no column {column!r}"
+                raise ValueError(msg)
+            text = row[column]
+            scope = None if shared_outputs else column
+            output: Item
+            if text is None or text in missing_values:
+                missing_counts[column] += 1
+                cell = frame if policy is MissingPolicy.DISTINCT else None
+                output = Absent(scope, cell)
+            else:
+                output = Observed(scope, text)
+            table[(column, frame)] = output
+            outputs[output] = None
+
+    if not frames:
+        msg = "no rows"
+        raise ValueError(msg)
+    artifact = Artifact(tuple(frames), tuple(outputs), columns, table, label)
+    return Loaded(artifact, policy, shared_outputs, missing_counts)
+
+
+def _read_csv(path: str | Path, required: Sequence[str]) -> list[dict[str, str | None]]:
+    """Return the rows of a UTF-8 CSV file, checking that columns exist."""
+    with Path(path).open(newline="", encoding="utf-8") as handle:
+        reader = csv.DictReader(handle)
+        header = reader.fieldnames or []
+        for column in required:
+            if column not in header:
+                msg = f"column {column!r} not found in {path}"
+                raise ValueError(msg)
+        return [dict(row) for row in reader]
+
+
+def load_artifact_csv(
+    path: str | Path,
+    id_column: str,
+    probe_columns: Sequence[str],
+    *,
+    missing_values: frozenset[str] = frozenset({""}),
+    policy: MissingPolicy = MissingPolicy.DISTINCT,
+    shared_outputs: bool = False,
+    label: str = "",
+) -> Loaded:
+    """Return :func:`load_artifact` applied to the rows of a UTF-8 CSV file."""
+    rows = _read_csv(path, [id_column, *probe_columns])
+    return load_artifact(
+        rows,
+        id_column,
+        probe_columns,
+        missing_values=missing_values,
+        policy=policy,
+        shared_outputs=shared_outputs,
+        label=label,
+    )
+
+
+def regime_from_rows(
+    rows: Iterable[Mapping[str, str | None]],
+    frames: Sequence[Item],
+    id_column: str,
+    cluster_column: str,
+) -> Partition:
+    """Return the frame regime declared by a clustering column.
+
+    Records with the same cluster value are in one block. A record with an empty
+    cluster value is alone in its own block. The clustering is taken as declared
+    by whoever produced it; nothing here judges it.
+
+    Every frame must appear exactly once, and no other ids may appear.
+    """
+    labels: dict[Item, object] = {}
+    for number, row in enumerate(rows, start=1):
+        frame = row.get(id_column)
+        if frame is None or frame == "":
+            msg = f"row {number}: empty id"
+            raise ValueError(msg)
+        if frame in labels:
+            msg = f"row {number}: duplicate id {frame!r}"
+            raise ValueError(msg)
+        cluster = row.get(cluster_column)
+        if cluster is None or cluster == "":
+            labels[frame] = ("unclustered", frame)
+        else:
+            labels[frame] = ("cluster", cluster)
+    known = set(frames)
+    extra = [frame for frame in labels if frame not in known]
+    if extra:
+        msg = f"ids not in the artifact: {extra[:3]!r}"
+        raise ValueError(msg)
+    absent = [frame for frame in frames if frame not in labels]
+    if absent:
+        msg = f"artifact frames with no clustering row: {absent[:3]!r}"
+        raise ValueError(msg)
+    return Partition.from_labels(frames, labels)
+
+
+def load_regime_csv(
+    path: str | Path,
+    frames: Sequence[Item],
+    id_column: str,
+    cluster_column: str,
+) -> Partition:
+    """Return :func:`regime_from_rows` applied to the rows of a UTF-8 CSV file."""
+    rows = _read_csv(path, [id_column, cluster_column])
+    return regime_from_rows(rows, frames, id_column, cluster_column)