lp2graph 0.3.0__py3-none-any.whl

lp2graph/metrics/result.py

@@ -0,0 +1,26 @@
+"""Result type returned by every metric."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(frozen=True, slots=True)
+class MetricResult:
+    """The output of a single metric.
+
+    ``name`` is a stable machine identifier; ``value`` is whatever the
+    metric produces (int, float, bool, dict). ``explanation`` is a short
+    one-line human-readable description. ``data`` carries optional
+    auxiliary fields (e.g. the diameter path along with the diameter
+    length).
+    """
+
+    name: str
+    value: Any
+    explanation: str = ""
+    data: dict[str, Any] = field(default_factory=dict)
+
+
+__all__ = ["MetricResult"]

lp2graph/metrics/structural.py

@@ -0,0 +1,236 @@
+"""Structural metrics over typed graphs.
+
+Adapted from joernmht/raiLPminerExperimentation
+(railpminer/analysis/metrics.py and railpminer/visualization/diameter.py),
+MIT License. The algorithms are the same; the surface targets the new
+internal :class:`~lp2graph.core.graph.Graph` rather than NetworkX
+directly.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict, deque
+
+from lp2graph.core.graph import Graph
+from lp2graph.core.model import Formulation
+from lp2graph.metrics.result import MetricResult
+
+
+def node_counts_by_class(g: Graph) -> MetricResult:
+    """Count nodes by their high-level class.
+
+    Returns a dict like ``{"variable": 4, "constraint": 6, ...}``.
+    """
+    counts: dict[str, int] = defaultdict(int)
+    for n in g.nodes:
+        counts[n.cls] += 1
+    return MetricResult(
+        name="node_counts_by_class",
+        value=dict(counts),
+        explanation="Count of nodes grouped by class.",
+    )
+
+
+def edge_density(g: Graph) -> MetricResult:
+    """Edge density: |E| / max(1, |V| * (|V| - 1))."""
+    n = len(g)
+    e = len(g.edges)
+    density = 0.0 if n <= 1 else e / (n * (n - 1))
+    return MetricResult(
+        name="edge_density",
+        value=density,
+        explanation="Directed-graph edge density.",
+        data={"nodes": n, "edges": e},
+    )
+
+
+def constraint_variable_ratio(g: Graph) -> MetricResult:
+    """Number of constraint nodes divided by number of variable nodes.
+
+    Operates on the schema or hybrid view. For ground views, instance
+    nodes are counted instead.
+    """
+    constraints = len([n for n in g.nodes if n.cls in ("constraint", "instance_constraint")])
+    variables = len([n for n in g.nodes if n.cls in ("variable", "instance_variable")])
+    if variables == 0:
+        ratio = float("inf") if constraints > 0 else 0.0
+    else:
+        ratio = constraints / variables
+    return MetricResult(
+        name="constraint_variable_ratio",
+        value=ratio,
+        explanation="Constraint-to-variable count ratio.",
+        data={"constraints": constraints, "variables": variables},
+    )
+
+
+def minimal_size(g: Graph) -> MetricResult:
+    """The product |constraints| · |variables| (the dense incidence-matrix size).
+
+    A weak proxy for problem size; useful for relative comparison across
+    formulations.
+    """
+    constraints = len([n for n in g.nodes if n.cls in ("constraint", "instance_constraint")])
+    variables = len([n for n in g.nodes if n.cls in ("variable", "instance_variable")])
+    cv = max(constraints, 1) * max(variables, 1)
+    return MetricResult(
+        name="minimal_size",
+        value=cv,
+        explanation="Product of constraint count and variable count.",
+        data={"constraints": constraints, "variables": variables},
+    )
+
+
+def model_coherence(g: Graph) -> MetricResult:
+    """1 if the underlying undirected graph is connected, else 0.
+
+    A coherent model has every variable reached by some path through
+    constraints. Disconnected components usually signal a missing
+    coupling.
+    """
+    if len(g) <= 1:
+        return MetricResult(name="model_coherence", value=1, explanation="Trivially connected.")
+    seen: set[str] = set()
+    start = g.nodes[0].id
+    queue = deque([start])
+    seen.add(start)
+    adj: dict[str, set[str]] = defaultdict(set)
+    for e in g.edges:
+        adj[e.src].add(e.dst)
+        adj[e.dst].add(e.src)
+    while queue:
+        n = queue.popleft()
+        for m in adj[n]:
+            if m not in seen:
+                seen.add(m)
+                queue.append(m)
+    coherent = 1 if len(seen) == len(g) else 0
+    return MetricResult(
+        name="model_coherence",
+        value=coherent,
+        explanation="1 if undirected graph is connected, else 0.",
+        data={"reached": len(seen), "total": len(g)},
+    )
+
+
+def model_completeness(f: Formulation) -> MetricResult:
+    """1 if the formulation is a recoverable model, else 0.
+
+    A *complete* model declares an objective together with at least one
+    variable and at least one constraint -- the minimal evidence that a
+    whole model, rather than a fragment, was extracted. This is the
+    companion well-formedness indicator to :func:`model_coherence`; unlike
+    coherence (a graph property) it reads the canonical
+    :class:`~lp2graph.core.model.Formulation` directly.
+    """
+    complete = (
+        1 if f.objective is not None and len(f.variables) >= 1 and len(f.constraints) >= 1 else 0
+    )
+    return MetricResult(
+        name="model_completeness",
+        value=complete,
+        explanation="1 if the model declares an objective, >=1 variable and >=1 constraint, else 0.",
+        data={
+            "has_objective": f.objective is not None,
+            "n_variables": len(f.variables),
+            "n_constraints": len(f.constraints),
+        },
+    )
+
+
+def graph_diameter(g: Graph) -> MetricResult:
+    """Diameter of the largest connected component (undirected, unweighted).
+
+    Returns the diameter and one diameter-realizing path. If the graph
+    has fewer than two nodes, returns 0.
+    """
+    if len(g) < 2:
+        return MetricResult(name="graph_diameter", value=0, explanation="Too small.")
+    adj: dict[str, set[str]] = defaultdict(set)
+    for e in g.edges:
+        adj[e.src].add(e.dst)
+        adj[e.dst].add(e.src)
+
+    # Find the largest connected component.
+    seen_global: set[str] = set()
+    components: list[set[str]] = []
+    for n in g.nodes:
+        if n.id in seen_global:
+            continue
+        comp: set[str] = set()
+        queue = deque([n.id])
+        seen_global.add(n.id)
+        comp.add(n.id)
+        while queue:
+            x = queue.popleft()
+            for y in adj[x]:
+                if y not in seen_global:
+                    seen_global.add(y)
+                    comp.add(y)
+                    queue.append(y)
+        components.append(comp)
+    largest = max(components, key=len)
+    if len(largest) < 2:
+        return MetricResult(name="graph_diameter", value=0, explanation="LCC has < 2 nodes.")
+
+    # Two-pass BFS approximation gives the exact diameter for trees and
+    # an exact answer for arbitrary graphs only when followed by full
+    # BFS from every node. Here we do the full BFS for correctness; the
+    # graphs we care about are small (template-scale).
+    best = 0
+    best_path: list[str] = []
+    nodes = sorted(largest)  # deterministic
+    for src in nodes:
+        dist = {src: 0}
+        prev: dict[str, str | None] = {src: None}
+        queue = deque([src])
+        while queue:
+            x = queue.popleft()
+            for y in adj[x]:
+                if y not in dist:
+                    dist[y] = dist[x] + 1
+                    prev[y] = x
+                    queue.append(y)
+        target = max(dist, key=lambda k: (dist[k], k))
+        if dist[target] > best:
+            best = dist[target]
+            # Reconstruct path.
+            path: list[str] = []
+            cur: str | None = target
+            while cur is not None:
+                path.append(cur)
+                cur = prev[cur]
+            best_path = list(reversed(path))
+    return MetricResult(
+        name="graph_diameter",
+        value=best,
+        explanation="Longest shortest path in the largest connected component.",
+        data={"path": best_path},
+    )
+
+
+def structural_summary(g: Graph) -> dict[str, MetricResult]:
+    """Convenience: compute every structural metric at once."""
+    return {
+        m.name: m
+        for m in (
+            node_counts_by_class(g),
+            edge_density(g),
+            constraint_variable_ratio(g),
+            minimal_size(g),
+            model_coherence(g),
+            graph_diameter(g),
+        )
+    }
+
+
+__all__ = [
+    "constraint_variable_ratio",
+    "edge_density",
+    "graph_diameter",
+    "minimal_size",
+    "model_coherence",
+    "model_completeness",
+    "node_counts_by_class",
+    "structural_summary",
+]

lp2graph/mining/__init__.py

@@ -0,0 +1,47 @@
+"""LP mining extensions for lp2graph.
+
+This subpackage implements the modules the *LP Mining with LP2Graph*
+method needs on top of the deterministic core library:
+
+- :mod:`lp2graph.mining.ingest` (M1) — heterogeneous ingestion front-end
+  turning solver code and non-canonical LaTeX into validated
+  ``Formulation`` objects, with source-span provenance.
+- :mod:`lp2graph.mining.homologize` (M2) — lexical homologizer and
+  TF-IDF concept vectorizer.
+- :mod:`lp2graph.mining.cluster` (M3) — the cluster-and-name operator
+  ``CN`` and the bottom-up multi-level taxonomy induction.
+- :mod:`lp2graph.mining.label` (M4) — the two-stage labeling service with
+  a versioned, self-growing closed-loop store.
+- :mod:`lp2graph.mining.corpusmgr` (M5) — corpus & provenance manager.
+- :mod:`lp2graph.mining.isomorphism` (M6) — intra-cluster schema-graph
+  isomorphism reporting.
+
+Every module is deterministic: frozen resources are versioned in
+:mod:`lp2graph.mining.versions` and stamped into emitted records.
+"""
+
+from __future__ import annotations
+
+from lp2graph.mining.provenance import ProvenanceMap, Rewrite, SourceSpan
+from lp2graph.mining.versions import (
+    CLUSTERING_VERSION,
+    LABEL_LEXICON_VERSION,
+    LEXICON_VERSION,
+    REWRITE_RULES_VERSION,
+    THESAURUS_VERSION,
+    VOCABULARY_VERSION,
+    WORDNET_VERSION,
+)
+
+__all__ = [
+    "CLUSTERING_VERSION",
+    "LABEL_LEXICON_VERSION",
+    "LEXICON_VERSION",
+    "REWRITE_RULES_VERSION",
+    "THESAURUS_VERSION",
+    "VOCABULARY_VERSION",
+    "WORDNET_VERSION",
+    "ProvenanceMap",
+    "Rewrite",
+    "SourceSpan",
+]

lp2graph/mining/cluster/__init__.py

@@ -0,0 +1,65 @@
+"""M3 — cluster-and-name operator and multi-level taxonomy induction.
+
+Public surface:
+
+- :func:`CN` / :class:`ClusterConfig` / :class:`NamedClustering` — the core
+  cluster-and-name operator.
+- :func:`induce` / :class:`Taxonomy` / :class:`LevelResult` — the bottom-up
+  Level V → C → M passes plus the text-only domain / solution-approach
+  clusterings.
+- :func:`stability_report` / :class:`StabilityReport` — silhouette, bootstrap
+  ARI, and sensitivity diagnostics.
+
+Everything is deterministic given the versioned :class:`ClusterConfig`.
+"""
+
+from __future__ import annotations
+
+from lp2graph.mining.cluster.distance import (
+    cosine_distance,
+    cosine_similarity,
+    distance_matrix,
+)
+from lp2graph.mining.cluster.operator import (
+    CN,
+    UNASSIGNED,
+    Algorithm,
+    ClusterConfig,
+    NamedClustering,
+)
+from lp2graph.mining.cluster.silhouette import silhouette_samples, silhouette_score
+from lp2graph.mining.cluster.stability import (
+    StabilityReport,
+    adjusted_rand_index,
+    bootstrap_ari,
+    sensitivity,
+    stability_report,
+)
+from lp2graph.mining.cluster.taxonomy import (
+    LevelResult,
+    Taxonomy,
+    induce,
+    model_feature_document,
+)
+
+__all__ = [
+    "CN",
+    "UNASSIGNED",
+    "Algorithm",
+    "ClusterConfig",
+    "LevelResult",
+    "NamedClustering",
+    "StabilityReport",
+    "Taxonomy",
+    "adjusted_rand_index",
+    "bootstrap_ari",
+    "cosine_distance",
+    "cosine_similarity",
+    "distance_matrix",
+    "induce",
+    "model_feature_document",
+    "sensitivity",
+    "silhouette_samples",
+    "silhouette_score",
+    "stability_report",
+]

lp2graph/mining/cluster/agglomerative.py

@@ -0,0 +1,82 @@
+"""Deterministic average-linkage agglomerative clustering (M3).
+
+A small, dependency-free clustering core used as the default backend for the
+``CN`` operator. Two stopping modes:
+
+- **threshold** — keep merging the two closest clusters until the closest
+  average-linkage distance exceeds ``distance_threshold``; this discovers the
+  number of clusters (and naturally leaves singletons), the HDBSCAN-style
+  behavior the method asks for without the dependency.
+- **fixed K** — merge until exactly ``k`` clusters remain.
+
+Determinism is the whole point: ties (equal merge distances) are broken by
+the lexicographically smallest cluster-index pair, so the dendrogram — and
+every partition cut from it — is reproducible across runs and machines.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+
+def _labels_from_groups(groups: list[list[int]], n: int) -> tuple[int, ...]:
+    """Assign each point a cluster id, ids ordered by smallest member index."""
+    ordered = sorted(groups, key=min)
+    labels = [-1] * n
+    for cid, group in enumerate(ordered):
+        for idx in group:
+            labels[idx] = cid
+    return tuple(labels)
+
+
+def _average_linkage(dist: Sequence[Sequence[float]], a: Sequence[int], b: Sequence[int]) -> float:
+    total = 0.0
+    for i in a:
+        for j in b:
+            total += dist[i][j]
+    return total / (len(a) * len(b))
+
+
+def agglomerative(
+    dist: Sequence[Sequence[float]],
+    *,
+    distance_threshold: float | None = 0.7,
+    k: int | None = None,
+) -> tuple[int, ...]:
+    """Cluster points described by precomputed distance matrix ``dist``.
+
+    Provide exactly one stopping rule: ``k`` (fixed cluster count) or
+    ``distance_threshold`` (merge while the closest pair is within it).
+    Returns a tuple of cluster ids aligned to the input order.
+    """
+    if (k is None) == (distance_threshold is None):
+        raise ValueError("provide exactly one of 'k' or 'distance_threshold'")
+    n = len(dist)
+    if n == 0:
+        return ()
+    if n == 1:
+        return (0,)
+
+    groups: list[list[int]] = [[i] for i in range(n)]
+
+    while len(groups) > 1:
+        if k is not None and len(groups) <= k:
+            break
+        # Find the closest pair of groups; deterministic tie-break by (i, j).
+        best: tuple[float, int, int] | None = None
+        for i in range(len(groups)):
+            for j in range(i + 1, len(groups)):
+                d = _average_linkage(dist, groups[i], groups[j])
+                if best is None or d < best[0]:
+                    best = (d, i, j)
+        assert best is not None
+        merge_d, gi, gj = best
+        if k is None and distance_threshold is not None and merge_d > distance_threshold:
+            break
+        groups[gi] = groups[gi] + groups[gj]
+        del groups[gj]
+
+    return _labels_from_groups(groups, n)
+
+
+__all__ = ["agglomerative"]

lp2graph/mining/cluster/distance.py

@@ -0,0 +1,65 @@
+"""Cosine distance over TF-IDF concept vectors (M3).
+
+The clustering operator works in cosine-distance space. Because the M2
+vectorizer already L2-normalizes its output, cosine similarity is just the
+dot product and cosine distance is ``1 - dot``. Zero vectors (entities whose
+text yielded no in-vocabulary concept) are at distance 1 from everything and
+are routed to the explicit *unassigned* part upstream.
+
+Pure Python, no numpy dependency, fully order-deterministic.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+Vector = Sequence[float]
+
+
+def dot(a: Vector, b: Vector) -> float:
+    """Dot product of two equal-length vectors."""
+    if len(a) != len(b):
+        raise ValueError(f"vector length mismatch: {len(a)} vs {len(b)}")
+    return sum(x * y for x, y in zip(a, b, strict=True))
+
+
+def cosine_similarity(a: Vector, b: Vector) -> float:
+    """Cosine similarity assuming ``a``/``b`` are L2-normalized (or zero)."""
+    return dot(a, b)
+
+
+def cosine_distance(a: Vector, b: Vector) -> float:
+    """Cosine distance ``1 - cosine_similarity`` clamped to ``[0, 2]``."""
+    d = 1.0 - cosine_similarity(a, b)
+    if d < 0.0:
+        return 0.0
+    if d > 2.0:
+        return 2.0
+    return d
+
+
+def is_zero(v: Vector) -> bool:
+    """True if every coordinate is exactly zero (no concept signal)."""
+    return all(x == 0.0 for x in v)
+
+
+def distance_matrix(vectors: Sequence[Vector]) -> list[list[float]]:
+    """Symmetric pairwise cosine-distance matrix with a zero diagonal."""
+    n = len(vectors)
+    mat = [[0.0] * n for _ in range(n)]
+    for i in range(n):
+        for j in range(i + 1, n):
+            d = cosine_distance(vectors[i], vectors[j])
+            mat[i][j] = d
+            mat[j][i] = d
+    return mat
+
+
+__all__ = [
+    "Vector",
+    "cosine_distance",
+    "cosine_similarity",
+    "distance_matrix",
+    "dot",
+    "is_zero",
+]