lp2graph 0.3.0__py3-none-any.whl

lp2graph/mining/corpusmgr/record.py

@@ -0,0 +1,101 @@
+"""Provenance records for corpus entries (M5).
+
+A :class:`ProvenanceRecord` is the bibliographic + categorization sidecar
+for one extracted :class:`~lp2graph.core.model.Formulation`. It captures the
+fields the corpus & provenance manager needs to deduplicate and to select a
+representative per cluster: *where* a formulation came from (``source_id``,
+``venue``, ``year``), *how strong* the citation signal is at the freeze date
+(``citation_count``, ``quality_tier``), and *where it sits* in the priority
+matrix (``domain_shell``, ``activity``, ``priority_cell``).
+
+Like the other mining records, this is a dependency-free frozen dataclass:
+it describes provenance *about* a formulation, not the formulation itself.
+Every field is an explicit input — ``citation_count`` is the count *at the
+freeze date*, never fetched at runtime — so a record is a reproducible,
+diffable artifact.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Final, Literal
+
+#: Quality tier of a publication venue, used to break selection ties.
+QualityTier = Literal["A_star", "A", "B", "C", "preprint", "unranked"]
+
+#: Quality tiers ordered best -> worst. The index of a tier in this tuple is
+#: its rank; lower is better. Used by dedup/selection tie-breaks so that the
+#: ordering is explicit and deterministic.
+QUALITY_TIERS: Final[tuple[QualityTier, ...]] = (
+    "A_star",
+    "A",
+    "B",
+    "C",
+    "preprint",
+    "unranked",
+)
+
+#: Priority cell of the sampling matrix (``P1`` highest priority .. ``P5``).
+PriorityCell = Literal["P1", "P2", "P3", "P4", "P5"]
+
+#: The valid priority cells, highest -> lowest.
+PRIORITY_CELLS: Final[tuple[PriorityCell, ...]] = ("P1", "P2", "P3", "P4", "P5")
+
+
+def quality_rank(tier: QualityTier) -> int:
+    """Return the rank of ``tier`` (0 = best). Unknown tiers sort last."""
+    try:
+        return QUALITY_TIERS.index(tier)
+    except ValueError:
+        return len(QUALITY_TIERS)
+
+
+@dataclass(frozen=True, slots=True)
+class ProvenanceRecord:
+    """Bibliographic + categorization provenance for one corpus entry.
+
+    ``source_id`` is a stable identifier for the source artifact (typically
+    the repository or paper id, matching the canonical formulation ``id``).
+    ``venue`` is the publication / repository venue; ``quality_tier`` ranks
+    it (see :data:`QUALITY_TIERS`). ``year`` is the publication year (or
+    ``None`` when unknown). ``citation_count`` is the citation count *as
+    measured at the corpus freeze date* — it is an input, never fetched.
+
+    ``domain_shell`` and ``activity`` place the entry in the domain
+    taxonomy; ``priority_cell`` is its cell in the sampling-priority matrix
+    (one of :data:`PRIORITY_CELLS`).
+    """
+
+    source_id: str
+    venue: str
+    quality_tier: QualityTier
+    year: int | None
+    citation_count: int
+    domain_shell: str
+    activity: str
+    priority_cell: PriorityCell
+
+    def __post_init__(self) -> None:
+        if self.priority_cell not in PRIORITY_CELLS:
+            raise ValueError(
+                f"priority_cell {self.priority_cell!r} is not one of {PRIORITY_CELLS!r}"
+            )
+        if self.quality_tier not in QUALITY_TIERS:
+            raise ValueError(f"quality_tier {self.quality_tier!r} is not one of {QUALITY_TIERS!r}")
+        if self.citation_count < 0:
+            raise ValueError("citation_count must be non-negative")
+
+    @property
+    def quality_rank(self) -> int:
+        """Rank of this record's quality tier (0 = best)."""
+        return quality_rank(self.quality_tier)
+
+
+__all__ = [
+    "PRIORITY_CELLS",
+    "QUALITY_TIERS",
+    "PriorityCell",
+    "ProvenanceRecord",
+    "QualityTier",
+    "quality_rank",
+]

lp2graph/mining/corpusmgr/select.py

@@ -0,0 +1,128 @@
+"""Reproducible representative selection per cluster (M5).
+
+Given named clusters (cluster name -> member indices) and the aligned
+provenance records, :func:`select_representatives` picks one member per
+cluster to carry forward into validation. The rule is a pure, documented,
+deterministic function:
+
+1. Pick the member with the highest ``citation_count``
+   (``reason="highest_citation"``).
+2. If *every* member has ``citation_count == 0`` (no citation signal at all),
+   fall back to the next-highest member that *does* have a non-zero count;
+   only when none exists is the cluster considered to have no signal
+   (``reason="next_highest_fallback"`` once a non-zero member is found).
+3. If the cluster is empty or still has no usable member, fall back to the
+   index supplied in ``benchmark_fallback[cluster]``
+   (``reason="benchmark_fallback"``).
+
+Ties at every step break by best quality tier, then lowest index, so the
+ranking is total and reproducible.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from typing import Literal
+
+from lp2graph.mining.corpusmgr.record import ProvenanceRecord
+
+SelectionReason = Literal[
+    "highest_citation",
+    "next_highest_fallback",
+    "benchmark_fallback",
+]
+
+
+@dataclass(frozen=True, slots=True)
+class RepresentativeChoice:
+    """The selection outcome for one cluster.
+
+    ``chosen_index`` is the selected member (or the benchmark-fallback
+    index); ``reason`` records which rule fired; ``ranked_candidates`` is the
+    full member ranking (best first) under the documented tie-break, for
+    auditing and reproducibility.
+    """
+
+    chosen_index: int | None
+    reason: SelectionReason
+    ranked_candidates: tuple[int, ...]
+
+
+def _rank_key(record: ProvenanceRecord, index: int) -> tuple[int, int, int]:
+    """Sort key: higher citation first, then best tier, then lowest index."""
+    return (-record.citation_count, record.quality_rank, index)
+
+
+def _rank_members(members: Sequence[int], records: Sequence[ProvenanceRecord]) -> tuple[int, ...]:
+    return tuple(sorted(members, key=lambda i: _rank_key(records[i], i)))
+
+
+def select_representatives(
+    clusters: Mapping[str, Sequence[int]],
+    records: Sequence[ProvenanceRecord],
+    *,
+    benchmark_fallback: Mapping[str, int] | None = None,
+) -> dict[str, RepresentativeChoice]:
+    """Pick a reproducible representative for each named cluster.
+
+    See the module docstring for the exact, deterministic rule. Returns a
+    mapping from cluster name to :class:`RepresentativeChoice`.
+    """
+    fallback = benchmark_fallback or {}
+    out: dict[str, RepresentativeChoice] = {}
+
+    for name in sorted(clusters):
+        members = list(clusters[name])
+        ranked = _rank_members(members, records)
+
+        if not ranked:
+            out[name] = RepresentativeChoice(
+                chosen_index=fallback.get(name),
+                reason="benchmark_fallback",
+                ranked_candidates=(),
+            )
+            continue
+
+        top = ranked[0]
+        if records[top].citation_count > 0:
+            out[name] = RepresentativeChoice(
+                chosen_index=top,
+                reason="highest_citation",
+                ranked_candidates=ranked,
+            )
+            continue
+
+        # All members have zero citation signal -> look for any non-zero
+        # member (none exist here, but the search is explicit and ordered),
+        # else fall back to a benchmark index if provided.
+        non_zero = next((i for i in ranked if records[i].citation_count > 0), None)
+        if non_zero is not None:
+            out[name] = RepresentativeChoice(
+                chosen_index=non_zero,
+                reason="next_highest_fallback",
+                ranked_candidates=ranked,
+            )
+        elif name in fallback:
+            out[name] = RepresentativeChoice(
+                chosen_index=fallback[name],
+                reason="benchmark_fallback",
+                ranked_candidates=ranked,
+            )
+        else:
+            # No citation signal and no benchmark fallback: still pick the
+            # documented top-of-ranking, but record it as a fallback.
+            out[name] = RepresentativeChoice(
+                chosen_index=top,
+                reason="next_highest_fallback",
+                ranked_candidates=ranked,
+            )
+
+    return out
+
+
+__all__ = [
+    "RepresentativeChoice",
+    "SelectionReason",
+    "select_representatives",
+]

lp2graph/mining/homologize/__init__.py

@@ -0,0 +1,82 @@
+"""M2 — lexical homologizer and concept vectorizer.
+
+Reduces entity names and descriptions to comparable *concepts*, emits frozen
+TF-IDF concept vectors, and exposes the structural type signature ``τ(s)``
+read from the canonical model. Everything here is deterministic and pinned by
+the versions in :mod:`lp2graph.mining.versions`.
+
+Typical use::
+
+    from lp2graph.mining.homologize import corpus_entities, concept_bag, ConceptVectorizer
+    ents = corpus_entities(formulations, "C")
+    docs = [concept_bag(e.text) for e in ents]
+    vec, vectors = ConceptVectorizer.fit_transform(docs)
+"""
+
+from __future__ import annotations
+
+from lp2graph.mining.homologize.concept import (
+    concept_backend_versions,
+    concept_bag,
+    concept_of,
+    concepts,
+)
+from lp2graph.mining.homologize.entity import (
+    Entity,
+    EntityLevel,
+    corpus_entities,
+    entities,
+    level_c_entities,
+    level_m_entity,
+    level_v_entities,
+    signature_documents,
+    text_dimension,
+)
+from lp2graph.mining.homologize.lemmatize import lemmatize
+from lp2graph.mining.homologize.signature import (
+    TypeSignature,
+    constraint_signature,
+    model_signature,
+    objective_signature,
+    parameter_signature,
+    variable_signature,
+)
+from lp2graph.mining.homologize.thesaurus import PHRASE_TO_CONCEPT
+from lp2graph.mining.homologize.tokenize import STOPLIST, split_compounds, tokenize
+from lp2graph.mining.homologize.vectorize import (
+    ConceptCounts,
+    ConceptVectorizer,
+    Vocabulary,
+    build_vocabulary,
+)
+
+__all__ = [
+    "PHRASE_TO_CONCEPT",
+    "STOPLIST",
+    "ConceptCounts",
+    "ConceptVectorizer",
+    "Entity",
+    "EntityLevel",
+    "TypeSignature",
+    "Vocabulary",
+    "build_vocabulary",
+    "concept_backend_versions",
+    "concept_bag",
+    "concept_of",
+    "concepts",
+    "constraint_signature",
+    "corpus_entities",
+    "entities",
+    "lemmatize",
+    "level_c_entities",
+    "level_m_entity",
+    "level_v_entities",
+    "model_signature",
+    "objective_signature",
+    "parameter_signature",
+    "signature_documents",
+    "split_compounds",
+    "text_dimension",
+    "tokenize",
+    "variable_signature",
+]

lp2graph/mining/homologize/concept.py

@@ -0,0 +1,134 @@
+"""Concept mapping ``g: token → concept`` (M2).
+
+Surface tokens are folded to concepts in three deterministic stages:
+
+1. **Morphology** — :func:`~lp2graph.mining.homologize.lemmatize.lemmatize`
+   already collapsed inflections upstream.
+2. **Domain synonymy** — the frozen thesaurus collapses curated synonym
+   families (matched greedily as bigrams then unigrams, so ``rolling stock``
+   wins over ``rolling`` + ``stock``).
+3. **Optional WordNet** — when the ``nltk`` WordNet corpus is importable, an
+   out-of-thesaurus token is folded onto the lemma name of its first synset;
+   the pinned database version is recorded. When WordNet is absent the
+   concept is the lemma itself (identity) — still fully deterministic.
+
+The function ``g`` is :func:`concept_of`; :func:`concept_bag` applies it
+across a piece of text and returns a multiset of concepts.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from functools import lru_cache
+
+from lp2graph.mining.homologize.lemmatize import lemmatize
+from lp2graph.mining.homologize.thesaurus import PHRASE_TO_CONCEPT
+from lp2graph.mining.homologize.tokenize import tokenize
+from lp2graph.mining.versions import THESAURUS_VERSION, WORDNET_VERSION
+
+
+def _lemmatize_phrase(phrase: str) -> str:
+    return " ".join(lemmatize(w) for w in phrase.split())
+
+
+# The thesaurus, re-keyed into the lemmatized space the tokenizer produces, so
+# phrase lookups line up with lemmatized token n-grams.
+_LEMMA_PHRASE_INDEX: dict[str, str] = {}
+for _phrase, _concept in PHRASE_TO_CONCEPT.items():
+    _LEMMA_PHRASE_INDEX.setdefault(_lemmatize_phrase(_phrase), _concept)
+
+_MAX_PHRASE_WORDS: int = max((len(p.split()) for p in _LEMMA_PHRASE_INDEX), default=1)
+
+
+@lru_cache(maxsize=4096)
+def _wordnet_concept(lemma: str) -> str | None:
+    """Fold ``lemma`` onto a WordNet synset lemma name, if WordNet is present.
+
+    Deterministic: always the first synset's first lemma name. Returns
+    ``None`` when WordNet is unavailable so callers fall back to identity.
+    """
+    try:
+        from nltk.corpus import wordnet
+
+        synsets = wordnet.synsets(lemma)
+    except Exception:
+        return None
+    if not synsets:
+        return None
+    lemmas = synsets[0].lemmas()
+    if not lemmas:
+        return None
+    name: str = lemmas[0].name().lower()
+    return name
+
+
+def concept_of(token: str, *, use_wordnet: bool = False) -> str:
+    """Map a single surface ``token`` to its concept.
+
+    Lemmatize, then prefer the thesaurus; optionally consult WordNet; else
+    fall back to the lemma itself. Only unigram thesaurus entries are
+    consulted here — phrase (bigram) folding lives in :func:`concept_bag`.
+    """
+    lemma = lemmatize(token.lower())
+    if lemma in _LEMMA_PHRASE_INDEX:
+        return _LEMMA_PHRASE_INDEX[lemma]
+    if use_wordnet:
+        wn = _wordnet_concept(lemma)
+        if wn is not None:
+            return wn
+    return lemma
+
+
+def concepts(text: str, *, use_wordnet: bool = False) -> list[str]:
+    """Return the ordered list of concepts for ``text``.
+
+    Tokenizes, lemmatizes, then performs a greedy longest-first n-gram scan
+    against the thesaurus so multi-word concepts are recovered before their
+    parts.
+    """
+    lemmas = [lemmatize(t) for t in tokenize(text)]
+    out: list[str] = []
+    i = 0
+    n = len(lemmas)
+    while i < n:
+        matched = False
+        upper = min(_MAX_PHRASE_WORDS, n - i)
+        for width in range(upper, 1, -1):
+            phrase = " ".join(lemmas[i : i + width])
+            concept = _LEMMA_PHRASE_INDEX.get(phrase)
+            if concept is not None:
+                out.append(concept)
+                i += width
+                matched = True
+                break
+        if matched:
+            continue
+        out.append(concept_of(lemmas[i], use_wordnet=use_wordnet))
+        i += 1
+    return out
+
+
+def concept_bag(text: str, *, use_wordnet: bool = False) -> Counter[str]:
+    """Return the concept multiset (bag) for ``text``."""
+    return Counter(concepts(text, use_wordnet=use_wordnet))
+
+
+def concept_backend_versions(*, use_wordnet: bool = False) -> dict[str, str | None]:
+    """Report the frozen resource versions behind concept mapping.
+
+    Stamped into emitted records so a vectorization is reproducible. The
+    WordNet version is only reported when the WordNet backend is actually
+    engaged and importable.
+    """
+    wn_version: str | None = None
+    if use_wordnet and _wordnet_concept("test") is not None:
+        wn_version = WORDNET_VERSION
+    return {"thesaurus": THESAURUS_VERSION, "wordnet": wn_version}
+
+
+__all__ = [
+    "concept_backend_versions",
+    "concept_bag",
+    "concept_of",
+    "concepts",
+]

lp2graph/mining/homologize/entity.py

@@ -0,0 +1,217 @@
+"""The mineable *entity* abstraction (M2, shared with M3/M4).
+
+The taxonomy passes operate on *entities*: at Level V the decision variables
+and parameters, at Level C the constraints and the objective, at Level M the
+whole model. Each entity carries both channels the method needs:
+
+- ``text`` — the name and description, fed to the lexical homologizer to
+  produce a concept bag / TF-IDF vector;
+- ``signature`` — the structural :class:`~lp2graph.mining.homologize.signature.TypeSignature`
+  read from the canonical model.
+
+This module is the single place that walks a ``Formulation`` and emits
+entities, so clustering and labeling consume one stable, ordered view.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass, field
+from typing import Literal
+
+from lp2graph.core.model import Formulation
+from lp2graph.mining.homologize.signature import (
+    TypeSignature,
+    constraint_signature,
+    model_signature,
+    objective_signature,
+    parameter_signature,
+    variable_signature,
+)
+
+EntityLevel = Literal["V", "C", "M"]
+
+
+@dataclass(frozen=True, slots=True)
+class Entity:
+    """One mineable unit at some taxonomy level.
+
+    ``id`` is unique within a corpus (``"<formulation>::<dimension>::<name>"``).
+    ``dimension`` records which kind of entity this is (``variable``,
+    ``parameter``, ``constraint``, ``objective``, ``model``). ``text`` is the
+    lexical channel; ``signature`` is the structural channel; ``tags`` carries
+    any extra free text used by the one-dimensional text clusterings.
+    """
+
+    id: str
+    formulation_id: str
+    level: EntityLevel
+    dimension: str
+    name: str
+    text: str
+    signature: TypeSignature
+    tags: tuple[str, ...] = field(default_factory=tuple)
+
+
+def _entity_id(formulation_id: str, dimension: str, name: str) -> str:
+    return f"{formulation_id}::{dimension}::{name}"
+
+
+def level_v_entities(f: Formulation) -> list[Entity]:
+    """Level-V entities: decision variables then parameters (declaration order)."""
+    out: list[Entity] = []
+    for v in f.variables:
+        out.append(
+            Entity(
+                id=_entity_id(f.id, "variable", v.name),
+                formulation_id=f.id,
+                level="V",
+                dimension="variable",
+                name=v.name,
+                text=f"{v.name} {v.description}".strip(),
+                signature=variable_signature(v),
+            )
+        )
+    for p in f.parameters:
+        out.append(
+            Entity(
+                id=_entity_id(f.id, "parameter", p.name),
+                formulation_id=f.id,
+                level="V",
+                dimension="parameter",
+                name=p.name,
+                text=f"{p.name} {p.description}".strip(),
+                signature=parameter_signature(p),
+            )
+        )
+    return out
+
+
+def level_c_entities(f: Formulation) -> list[Entity]:
+    """Level-C entities: constraints then the objective (if present)."""
+    out: list[Entity] = []
+    for c in f.constraints:
+        out.append(
+            Entity(
+                id=_entity_id(f.id, "constraint", c.name),
+                formulation_id=f.id,
+                level="C",
+                dimension="constraint",
+                name=c.name,
+                text=f"{c.name} {c.description}".strip(),
+                signature=constraint_signature(c),
+            )
+        )
+    if f.objective is not None:
+        out.append(
+            Entity(
+                id=_entity_id(f.id, "objective", f.objective.name),
+                formulation_id=f.id,
+                level="C",
+                dimension="objective",
+                name=f.objective.name,
+                text=f"{f.objective.name} {f.objective.description}".strip(),
+                signature=objective_signature(f.objective),
+            )
+        )
+    return out
+
+
+def level_m_entity(f: Formulation) -> Entity:
+    """The single Level-M entity for the whole model.
+
+    Its ``text`` pools the model name, description, and tags (the lexical
+    signal for the one-dimensional *domain* / *solution approach* text
+    clusterings); ``tags`` is preserved separately for those dimensions.
+    """
+    text = " ".join(part for part in (f.name, f.description, *f.tags) if part).strip()
+    return Entity(
+        id=_entity_id(f.id, "model", f.id),
+        formulation_id=f.id,
+        level="M",
+        dimension="model",
+        name=f.name,
+        text=text,
+        signature=model_signature(f),
+        tags=tuple(f.tags),
+    )
+
+
+def entities(f: Formulation, level: EntityLevel) -> list[Entity]:
+    """Return the entities of ``f`` at ``level`` (``"V"``, ``"C"``, or ``"M"``)."""
+    if level == "V":
+        return level_v_entities(f)
+    if level == "C":
+        return level_c_entities(f)
+    return [level_m_entity(f)]
+
+
+def corpus_entities(formulations: Iterable[Formulation], level: EntityLevel) -> list[Entity]:
+    """Flatten the entities of many formulations at one ``level``.
+
+    Order is deterministic: formulations in the given order, entities in
+    declaration order within each.
+    """
+    out: list[Entity] = []
+    for f in formulations:
+        out.extend(entities(f, level))
+    return out
+
+
+def text_dimension(entity: Entity, dimension: Literal["domain", "solution_approach"]) -> str:
+    """Return the text for a one-dimensional model-level text clustering.
+
+    Both dimensions are derived from the Level-M entity's available free text;
+    ``domain`` favors the tags + description, ``solution_approach`` favors the
+    description prose. The canonical model does not separate these fields, so
+    the split is a documented heuristic over the same source text.
+    """
+    if entity.level != "M":  # pragma: no cover - guarded by callers
+        raise ValueError("text dimensions are only defined for Level-M entities")
+    if dimension == "domain":
+        return " ".join((*entity.tags, entity.text)).strip()
+    return entity.text
+
+
+def signature_documents(items: Sequence[Entity]) -> list[dict[str, int]]:
+    """Turn each entity's signature into a categorical concept-count document.
+
+    Lets the structural channel be vectorized by the same TF-IDF machinery as
+    the lexical channel: each signature facet becomes a namespaced concept
+    token (``role:auxiliary``, ``kind:binary``, ``domain:timing``,
+    ``cmp:le``, ``quant:i:I:ne_other``, ``ref:variable``, ...).
+    """
+    docs: list[dict[str, int]] = []
+    for e in items:
+        sig = e.signature
+        tokens: list[str] = []
+        if sig.domain:
+            tokens.append(f"domain:{sig.domain}")
+        tokens.append(f"role:{sig.role}")
+        tokens.append(f"kind:{sig.kind}")
+        tokens.append(f"arity:{sig.arity}")
+        tokens.extend(f"over:{s}" for s in sig.shape)
+        # Constraint τ extras (paper Eq. homologization): comparator,
+        # quantifier/restriction pattern, and the referent-kind multiset.
+        if sig.comparator:
+            tokens.append(f"cmp:{sig.comparator}")
+        tokens.extend(f"quant:{q}" for q in sig.quantifiers)
+        tokens.extend(f"ref:{r}" for r in sig.referents)
+        doc: dict[str, int] = {}
+        for tok in tokens:
+            doc[tok] = doc.get(tok, 0) + 1
+        docs.append(doc)
+    return docs
+
+
+__all__ = [
+    "Entity",
+    "EntityLevel",
+    "corpus_entities",
+    "entities",
+    "level_c_entities",
+    "level_m_entity",
+    "level_v_entities",
+    "signature_documents",
+    "text_dimension",
+]