lexograph 0.1.0__py3-none-any.whl

lexograph/__init__.py

@@ -0,0 +1,65 @@
+"""lexograph — spatialize linear text into pictures you can read.
+
+lexograph turns a linear text into a picture through one four-step spine:
+**segment** the text into ordered units (characters, tokens, or sentences),
+**lay them out** in 2-D or 3-D space, **encode** per-unit attributes onto
+visual channels (size, colour, glyph), and **render** the result as a
+matplotlib :class:`~matplotlib.figure.Figure` that displays inline in Jupyter
+and saves cleanly with ``fig.savefig(...)``.
+"""
+
+__version__ = "0.1.0"
+
+from lexograph.datasets import load_demo_text
+from lexograph.encode import (
+    Channels,
+    categorical_colors,
+    continuous_colors,
+    normalize_size,
+)
+from lexograph.layout import (
+    kwic,
+    linear_layout,
+    rendered_widths,
+    spiral_layout,
+    term_offsets,
+    walk3d_layout,
+    walk_layout,
+)
+from lexograph.presets import (
+    concordance,
+    punctuation_spiral,
+    recurrence_plot,
+    text_walk,
+)
+from lexograph.render import frame_axes, render_path, render_path_3d, render_points
+from lexograph.scalars import frequencies, lengths, positions
+from lexograph.segment import segment
+
+__all__ = [
+    "segment",
+    "lengths",
+    "positions",
+    "frequencies",
+    "linear_layout",
+    "walk_layout",
+    "walk3d_layout",
+    "spiral_layout",
+    "rendered_widths",
+    "term_offsets",
+    "kwic",
+    "normalize_size",
+    "categorical_colors",
+    "continuous_colors",
+    "Channels",
+    "render_points",
+    "render_path",
+    "render_path_3d",
+    "frame_axes",
+    "punctuation_spiral",
+    "text_walk",
+    "recurrence_plot",
+    "concordance",
+    "load_demo_text",
+    "__version__",
+]

lexograph/_types.py

@@ -0,0 +1,26 @@
+"""Shared type aliases for the lexograph package.
+
+The visual-channel arrays named here are the package's public data contract:
+``encode`` accepts plain per-unit arrays, and ``analyze``/``integrations`` only
+ever *produce* arrays that satisfy these aliases. Nothing in the core needs to
+know where the numbers came from.
+"""
+
+from __future__ import annotations
+
+from typing import Literal, TypeAlias
+
+import numpy as np
+import numpy.typing as npt
+
+Unit: TypeAlias = str
+"""A single segmented unit of text: a character, token, or sentence."""
+
+UnitKind: TypeAlias = Literal["chars", "tokens", "sentences"]
+"""Which kind of unit a segmenter emits."""
+
+FloatArray: TypeAlias = npt.NDArray[np.float64]
+"""A 1-D array of floats: a per-unit scalar channel (e.g. ``size``)."""
+
+Coords: TypeAlias = npt.NDArray[np.float64]
+"""An ``(N, 2)`` or ``(N, 3)`` array of layout coordinates, one row per unit."""

lexograph/analyze/__init__.py

@@ -0,0 +1,139 @@
+"""Optional analysis layer: turn a text into per-sentence channel arrays.
+
+Behind the ``lexograph[graph]`` extra. It runs the Wittgenstein pipeline —
+sentence embeddings → cosine kNN graph → (optional disparity backbone) → PageRank
+and community detection — and hands back plain arrays that satisfy the
+``encode`` data contract: a ``size`` array (PageRank centrality), a ``community``
+array (colour labels), and a cosine ``distances`` matrix (for a semantic
+recurrence dotplot). The core never imports this; the arrow points inward.
+
+The core dependency-free scalars (length, position, frequency) live in
+:mod:`lexograph.scalars`.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from lexograph.analyze.backbone import extract_backbone
+from lexograph.analyze.embeddings import embed_sentences
+from lexograph.analyze.graph import (
+    community_labels,
+    embedding_distances,
+    knn_graph,
+    pagerank_scores,
+)
+from lexograph.segment.units import sentences as split_sentences
+
+if TYPE_CHECKING:
+    import numpy as np
+
+    from lexograph._types import FloatArray
+    from lexograph.analyze.graph import CommunityMethod
+
+__all__ = [
+    "Analysis",
+    "analyze_text",
+    "embed_sentences",
+    "knn_graph",
+    "embedding_distances",
+    "pagerank_scores",
+    "community_labels",
+    "extract_backbone",
+]
+
+
+@dataclass(frozen=True, slots=True)
+class Analysis:
+    """The per-sentence channel arrays derived from a text.
+
+    Attributes:
+        sentences: The segmented sentences (length ``N``).
+        embeddings: The ``(N, D)`` sentence embeddings.
+        size: PageRank centrality per sentence — the size channel.
+        community: Community id per sentence — the colour channel.
+        distances: The ``(N, N)`` cosine distance matrix — pass it to
+            ``recurrence_plot(distances=...)`` for a semantic dotplot.
+    """
+
+    sentences: list[str]
+    embeddings: FloatArray
+    size: FloatArray
+    community: np.ndarray
+    distances: FloatArray
+
+
+def analyze_text(
+    text: str,
+    *,
+    embeddings: FloatArray | None = None,
+    k: int = 5,
+    community: CommunityMethod = "louvain",
+    n_clusters: int = 10,
+    backbone: bool = False,
+    min_alpha_ptile: float = 0.5,
+    seed: int = 42,
+) -> Analysis:
+    """Run the analysis pipeline and return per-sentence channel arrays.
+
+    Args:
+        text: The source text.
+        embeddings: Precomputed ``(N, D)`` embeddings to use. If ``None``, the
+            sentences are embedded with the default model (a model download).
+        k: Neighbours per node in the kNN graph.
+        community: Community method — ``"louvain"`` (default) or ``"kmeans"``.
+        n_clusters: Number of clusters for the ``"kmeans"`` method.
+        backbone: If ``True``, sparsify the kNN graph with the disparity filter
+            before PageRank and community detection.
+        min_alpha_ptile: Disparity-filter threshold (used when ``backbone``).
+        seed: Random seed for reproducibility.
+
+    Returns:
+        An :class:`Analysis` whose arrays align to the segmented sentences.
+
+    Raises:
+        ValueError: If ``embeddings`` is given but its length does not match the
+            sentence count.
+
+    Example:
+        Drive a walk and a semantic dotplot from the analysis (not run as a
+        doctest — embedding downloads a model)::
+
+            from lexograph import text_walk, recurrence_plot, load_demo_text
+            from lexograph.analyze import analyze_text
+
+            a = analyze_text(load_demo_text())
+            walk = text_walk(load_demo_text(), colour=a.community,
+                             colour_kind="categorical", size=a.size)
+            dots = recurrence_plot(load_demo_text(), distances=a.distances,
+                                   threshold=0.4)
+    """
+    units = split_sentences(text)
+    n = len(units)
+    if embeddings is None:
+        embeddings = embed_sentences(units)
+    elif len(embeddings) != n:
+        msg = f"embeddings must have one row per sentence ({n}), got {len(embeddings)}"
+        raise ValueError(msg)
+
+    graph = knn_graph(embeddings, k=k)
+    if backbone:
+        graph = extract_backbone(graph, min_alpha_ptile=min_alpha_ptile)
+    size = pagerank_scores(graph, n)
+    labels = community_labels(
+        graph,
+        n,
+        method=community,
+        embeddings=embeddings,
+        n_clusters=n_clusters,
+        seed=seed,
+    )
+    distances = embedding_distances(embeddings)
+    return Analysis(
+        sentences=units,
+        embeddings=embeddings,
+        size=size,
+        community=labels,
+        distances=distances,
+    )

lexograph/analyze/backbone.py

@@ -0,0 +1,170 @@
+"""Disparity-filter backbone extraction for weighted graphs.
+
+The multiscale backbone of Serrano, Boguñá & Vespignani (2009),
+https://arxiv.org/pdf/0904.2389.pdf — it keeps the statistically significant
+edges of a weighted graph and discards the rest. Ported from the sibling
+``kenon`` package's ``backbone`` module (which in turn follows DerwenAI's
+``disparity_filter``); used here to sparsify the kNN sentence graph before
+PageRank and community detection.
+
+Part of the optional ``lexograph[graph]`` extra.
+"""
+
+from __future__ import annotations
+
+import copy
+
+import networkx as nx
+import numpy as np
+
+__all__ = [
+    "disparity_integral",
+    "edge_alpha",
+    "apply_disparity_filter",
+    "extract_backbone",
+]
+
+
+def disparity_integral(x: float, degree: float) -> float:
+    """Evaluate the disparity-filter PDF integral ``(1-x)^k / ((k-1)(x-1))``.
+
+    Args:
+        x: A normalised edge weight (must not be exactly 1.0).
+        degree: A node degree ``k`` (must not be exactly 1.0).
+
+    Returns:
+        The value of the integral at ``x`` for degree ``degree``.
+
+    Examples:
+        >>> disparity_integral(0.5, 3.0) != disparity_integral(0.0, 3.0)
+        True
+    """
+    return ((1.0 - x) ** degree) / ((degree - 1.0) * (x - 1.0))
+
+
+def edge_alpha(norm_weight: float, degree: float) -> float:
+    """Return the disparity significance ``alpha`` of one edge endpoint.
+
+    Args:
+        norm_weight: The edge weight divided by the node's strength.
+        degree: The node's degree.
+
+    Returns:
+        ``alpha`` in ``[0, 1]``; lower means more significant. A node of degree
+        ``<= 1`` yields ``0.0``.
+
+    Examples:
+        >>> 0.0 <= edge_alpha(0.5, 3.0) <= 1.0
+        True
+        >>> edge_alpha(0.9, 1.0)
+        0.0
+    """
+    if degree <= 1.0:
+        return 0.0
+    return 1.0 - (degree - 1.0) * (
+        disparity_integral(norm_weight, degree) - disparity_integral(0.0, degree)
+    )
+
+
+def apply_disparity_filter(graph: nx.Graph) -> list[float]:
+    """Attach disparity statistics to every edge and node, in place.
+
+    Each node gains a ``strength`` (sum of incident weights); each edge gains
+    ``norm_weight``, ``alpha`` (the more significant of its two endpoints), and
+    ``alpha_ptile`` (the percentile of its alpha among all edges).
+
+    Args:
+        graph: A weighted networkx graph (edges carry a ``weight``).
+
+    Returns:
+        The list of all edge alpha values.
+
+    Examples:
+        >>> import networkx as nx
+        >>> g = nx.Graph()
+        >>> g.add_edge("a", "b", weight=0.8)
+        >>> g.add_edge("b", "c", weight=0.3)
+        >>> g.add_edge("a", "c", weight=0.5)
+        >>> len(apply_disparity_filter(g)) == g.number_of_edges()
+        True
+    """
+    if graph.number_of_edges() == 0:
+        return []
+
+    for node in graph.nodes():
+        graph.nodes[node]["strength"] = sum(
+            data.get("weight", 1.0) for _, _, data in graph.edges(node, data=True)
+        )
+
+    alphas: list[float] = []
+    for u, v, data in graph.edges(data=True):
+        weight = data.get("weight", 1.0)
+        alpha_u = _endpoint_alpha(graph, u, weight)
+        alpha_v = _endpoint_alpha(graph, v, weight)
+        data["alpha"] = min(alpha_u, alpha_v)
+        alphas.append(data["alpha"])
+
+    sorted_alphas = np.array(sorted(alphas))
+    for _u, _v, data in graph.edges(data=True):
+        data["alpha_ptile"] = float(
+            np.searchsorted(sorted_alphas, data["alpha"])
+        ) / len(sorted_alphas)
+    return alphas
+
+
+def _endpoint_alpha(graph: nx.Graph, node: object, weight: float) -> float:
+    """Disparity alpha of ``weight`` as seen from one endpoint ``node``."""
+    strength = graph.nodes[node]["strength"]
+    degree = float(graph.degree(node))
+    norm = weight / strength if strength > 0 else 0.0
+    return edge_alpha(norm, degree)
+
+
+def extract_backbone(
+    graph: nx.Graph,
+    *,
+    min_alpha_ptile: float = 0.5,
+    min_degree: int = 1,
+) -> nx.Graph:
+    """Return the disparity-filter backbone of a weighted graph.
+
+    The input is copied (never mutated): edges below ``min_alpha_ptile`` are
+    dropped, then nodes whose degree falls below ``min_degree`` are pruned
+    iteratively until stable.
+
+    Args:
+        graph: A weighted networkx graph.
+        min_alpha_ptile: Edges with an alpha percentile below this are removed.
+        min_degree: Nodes left with a degree below this are pruned (``1`` keeps
+            any node that still has an edge).
+
+    Returns:
+        A new graph containing only the backbone.
+
+    Examples:
+        >>> import networkx as nx
+        >>> g = nx.path_graph(5)
+        >>> for u, v in g.edges():
+        ...     g[u][v]["weight"] = float(v + 1)
+        >>> bb = extract_backbone(g, min_alpha_ptile=0.3)
+        >>> bb.number_of_nodes() <= g.number_of_nodes()
+        True
+    """
+    if graph.number_of_edges() == 0:
+        return nx.Graph()
+
+    result = copy.deepcopy(graph)
+    apply_disparity_filter(result)
+    result.remove_edges_from(
+        [
+            (u, v)
+            for u, v, data in result.edges(data=True)
+            if data.get("alpha_ptile", 0.0) < min_alpha_ptile
+        ]
+    )
+    changed = True
+    while changed:
+        prune = [n for n in list(result.nodes()) if result.degree(n) < min_degree]
+        changed = bool(prune)
+        result.remove_nodes_from(prune)
+    return result

lexograph/analyze/embeddings.py

@@ -0,0 +1,68 @@
+"""Embed sentences with a sentence-transformers model.
+
+This is the only step that pulls in the heavy neural dependency, so the import is
+deferred to call time: importing this module (and the rest of ``analyze``) is
+cheap, and the model is loaded only when you actually embed. Every other
+``analyze`` function takes the embedding array, so the whole graph pipeline can
+be exercised with your own vectors and no model at all.
+
+Part of the optional ``lexograph[graph]`` extra.
+"""
+
+from __future__ import annotations
+
+import importlib
+from collections.abc import Sequence
+from typing import Any
+
+import numpy as np
+
+from lexograph._types import FloatArray
+
+__all__ = ["embed_sentences", "DEFAULT_MODEL"]
+
+DEFAULT_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
+"""The default embedding model: small, CPU-fast, 384-dimensional."""
+
+
+def embed_sentences(
+    sentences: Sequence[str],
+    *,
+    model_name: str = DEFAULT_MODEL,
+    batch_size: int = 64,
+) -> FloatArray:
+    """Embed sentences into L2-normalised vectors.
+
+    The model is downloaded on first use and cached by ``sentence-transformers``.
+    Because the embeddings are L2-normalised, a dot product equals cosine
+    similarity.
+
+    Args:
+        sentences: The sentences to embed.
+        model_name: A sentence-transformers model id.
+        batch_size: Encoding batch size.
+
+    Returns:
+        An ``(N, D)`` float array of unit-norm sentence embeddings.
+
+    Example:
+        Not run as a doctest (it would download the model)::
+
+            from lexograph import segment, load_demo_text
+            from lexograph.analyze.embeddings import embed_sentences
+
+            sentences = segment(load_demo_text())
+            embeddings = embed_sentences(sentences)
+    """
+    # Imported by name so the heavy optional dependency is neither required to
+    # import this module nor statically type-checked when it is absent.
+    st: Any = importlib.import_module("sentence_transformers")
+
+    model = st.SentenceTransformer(model_name)
+    vectors = model.encode(
+        list(sentences),
+        batch_size=batch_size,
+        convert_to_numpy=True,
+        normalize_embeddings=True,
+    )
+    return np.asarray(vectors, dtype=float)

lexograph/analyze/graph.py

@@ -0,0 +1,190 @@
+"""Build a sentence graph from embeddings and read channels off it.
+
+The pipeline ported from the Wittgenstein piece: sentence embeddings → a cosine
+k-nearest-neighbour graph → PageRank centrality (the **size** channel) and
+community detection (the **colour** channel). All functions take and return plain
+arrays / a networkx graph, so the results drop straight into ``encode``.
+
+Part of the optional ``lexograph[graph]`` extra.
+"""
+
+from __future__ import annotations
+
+from typing import Literal, cast
+
+import networkx as nx
+import numpy as np
+from sklearn.metrics.pairwise import cosine_distances
+from sklearn.neighbors import kneighbors_graph
+
+from lexograph._types import FloatArray
+
+__all__ = [
+    "knn_graph",
+    "embedding_distances",
+    "pagerank_scores",
+    "community_labels",
+]
+
+CommunityMethod = Literal["louvain", "kmeans"]
+
+
+def knn_graph(embeddings: FloatArray, *, k: int = 5) -> nx.Graph:
+    """Build a weighted cosine k-nearest-neighbour graph over the embeddings.
+
+    Node ``i`` is sentence ``i``; an edge carries the cosine **similarity**
+    (``1 - cosine distance``) as its ``weight``. The graph is undirected: the
+    mutual edge keeps the stronger of the two directed similarities.
+
+    Args:
+        embeddings: An ``(N, D)`` array of sentence embeddings.
+        k: Neighbours per node (capped at ``N - 1``).
+
+    Returns:
+        A networkx graph with nodes ``0 .. N-1`` and weighted edges.
+
+    Examples:
+        >>> import numpy as np
+        >>> emb = np.eye(4)
+        >>> g = knn_graph(emb, k=1)
+        >>> g.number_of_nodes()
+        4
+    """
+    n = len(embeddings)
+    graph = nx.Graph()
+    graph.add_nodes_from(range(n))
+    if n < 2:
+        return graph
+    k_eff = min(k, n - 1)
+    adjacency = kneighbors_graph(
+        np.asarray(embeddings, dtype=float),
+        n_neighbors=k_eff,
+        mode="distance",
+        metric="cosine",
+        include_self=False,
+    ).tocoo()
+    for i, j, distance in zip(
+        adjacency.row, adjacency.col, adjacency.data, strict=True
+    ):
+        weight = 1.0 - float(distance)
+        if weight <= 0.0:
+            continue
+        if graph.has_edge(int(i), int(j)):
+            graph[int(i)][int(j)]["weight"] = max(
+                graph[int(i)][int(j)]["weight"], weight
+            )
+        else:
+            graph.add_edge(int(i), int(j), weight=weight)
+    return graph
+
+
+def embedding_distances(embeddings: FloatArray) -> FloatArray:
+    """Return the pairwise cosine distance matrix of the embeddings.
+
+    Ready to pass as ``distances`` to
+    :func:`lexograph.presets.recurrence.recurrence_plot` for a semantic
+    recurrence dotplot.
+
+    Args:
+        embeddings: An ``(N, D)`` array of sentence embeddings.
+
+    Returns:
+        An ``(N, N)`` cosine distance matrix with a zero diagonal.
+
+    Examples:
+        >>> import numpy as np
+        >>> d = embedding_distances(np.eye(3))
+        >>> d.shape
+        (3, 3)
+    """
+    return np.asarray(
+        cosine_distances(np.asarray(embeddings, dtype=float)), dtype=float
+    )
+
+
+def pagerank_scores(graph: nx.Graph, n: int) -> FloatArray:
+    """Return weighted PageRank as a per-sentence array (the size channel).
+
+    Args:
+        graph: A weighted sentence graph (e.g. from :func:`knn_graph`).
+        n: The total sentence count, so the result aligns to every sentence even
+            if the graph has dropped some nodes.
+
+    Returns:
+        A length-``n`` float array; entry ``i`` is the PageRank of node ``i`` (or
+        ``0.0`` if the node is absent or the graph has no edges).
+
+    Examples:
+        >>> import networkx as nx
+        >>> g = nx.path_graph(3)
+        >>> for u, v in g.edges():
+        ...     g[u][v]["weight"] = 1.0
+        >>> pagerank_scores(g, 3).shape
+        (3,)
+    """
+    if graph.number_of_edges() == 0:
+        return np.zeros(n, dtype=float)
+    ranks = nx.pagerank(graph, weight="weight")
+    return np.asarray([ranks.get(i, 0.0) for i in range(n)], dtype=float)
+
+
+def community_labels(
+    graph: nx.Graph,
+    n: int,
+    *,
+    method: CommunityMethod = "louvain",
+    embeddings: FloatArray | None = None,
+    n_clusters: int = 10,
+    seed: int = 42,
+) -> np.ndarray:
+    """Return a per-sentence community label (the colour channel).
+
+    Args:
+        graph: A weighted sentence graph (used by the Louvain method).
+        n: The total sentence count, so labels align to every sentence.
+        method: ``"louvain"`` (graph communities, the default) or ``"kmeans"``
+            (clusters the embeddings directly).
+        embeddings: Required for ``"kmeans"``; the ``(N, D)`` embedding array.
+        n_clusters: Number of clusters for ``"kmeans"`` (capped at ``N``).
+        seed: Random seed for reproducibility.
+
+    Returns:
+        A length-``n`` integer array of community ids. For Louvain, id ``0`` is
+        the largest community; a node absent from the graph gets ``-1``.
+
+    Raises:
+        ValueError: If ``method`` is ``"kmeans"`` and ``embeddings`` is ``None``,
+            or ``method`` is unrecognised.
+
+    Examples:
+        >>> import networkx as nx
+        >>> g = nx.path_graph(4)
+        >>> for u, v in g.edges():
+        ...     g[u][v]["weight"] = 1.0
+        >>> labels = community_labels(g, 4)
+        >>> labels.shape
+        (4,)
+    """
+    if method == "kmeans":
+        if embeddings is None:
+            msg = "kmeans community detection requires embeddings"
+            raise ValueError(msg)
+        from sklearn.cluster import KMeans
+
+        k = min(n_clusters, n)
+        labels = KMeans(n_clusters=k, random_state=seed, n_init=10).fit_predict(
+            np.asarray(embeddings, dtype=float)
+        )
+        return labels.astype(int)
+    if method == "louvain":
+        raw = cast(
+            "list[set[int]]",
+            nx.community.louvain_communities(graph, weight="weight", seed=seed),
+        )
+        communities = sorted(raw, key=len, reverse=True)
+        label_of = {
+            node: cid for cid, members in enumerate(communities) for node in members
+        }
+        return np.asarray([label_of.get(i, -1) for i in range(n)], dtype=int)
+    msg = f"method must be 'louvain' or 'kmeans', got {method!r}"
+    raise ValueError(msg)

lexograph/datasets/__init__.py

@@ -0,0 +1,30 @@
+"""Tiny bundled text for docs, tests, and every preset's worked example.
+
+The demo text is the first chapter of Jane Austen's *Pride and Prejudice*
+(public domain; see :mod:`lexograph.datasets._pride_and_prejudice`). It is large
+enough to make a walk, spiral, dotplot, or concordance legible, and small enough
+to keep the wheel tiny and the doctests fast.
+"""
+
+from __future__ import annotations
+
+from lexograph.datasets._pride_and_prejudice import PRIDE_AND_PREJUDICE_CH1
+
+
+def load_demo_text() -> str:
+    """Return the bundled demo text: Chapter 1 of *Pride and Prejudice*.
+
+    Returns:
+        The chapter as a single string, paragraphs separated by blank lines.
+
+    Examples:
+        >>> text = load_demo_text()
+        >>> text.startswith("It is a truth universally acknowledged")
+        True
+        >>> "Bingley" in text
+        True
+    """
+    return PRIDE_AND_PREJUDICE_CH1
+
+
+__all__ = ["load_demo_text"]