sum-engine 0.1.0__py3-none-any.whl

internal/ensemble/tome_generator.py

@@ -0,0 +1,286 @@
+"""
+Autoregressive Tome Generator — Lossless Semantic Rehydration
+
+Unpacks a Gödel Integer into structured knowledge with two operating modes:
+
+**Proof Mode (Canonical):**
+    Deterministic, template-based rendering of every active axiom grouped
+    by subject entity.  This is the layer on which round-trip conservation
+    is mathematically verified.  Works without any LLM.
+
+**Narrative Mode (Optional):**
+    If an ``extrapolator`` (QuantumExtrapolator) is available, each
+    chapter's axioms are expanded into readable prose via the epistemic
+    loop.  The canonical appendix is preserved so the proof path is
+    never lost.
+
+Phase 14: The Ouroboros Protocol.
+
+Author: ototao
+License: Apache License 2.0
+"""
+
+import logging
+from typing import Dict, List, Optional
+
+from internal.algorithms.semantic_arithmetic import GodelStateAlgebra
+
+logger = logging.getLogger(__name__)
+
+# The canonical format version — treat this as an ABI contract.
+# Bump when the template grammar changes.
+CANONICAL_FORMAT_VERSION = "1.0.0"
+
+
+class AutoregressiveTomeGenerator:
+    """
+    Unpacks a Gödel Integer into a complete, structured Tome.
+
+    Supports two rendering modes:
+        * ``proof``  — deterministic canonical output (always available)
+        * ``narrative`` — LLM-enhanced prose (requires extrapolator)
+    """
+
+    def __init__(
+        self,
+        algebra: GodelStateAlgebra,
+        extrapolator=None,
+    ):
+        self.algebra = algebra
+        self.extrapolator = extrapolator  # None in math-only mode
+
+    # ------------------------------------------------------------------
+    # Core: cluster active axioms by subject
+    # ------------------------------------------------------------------
+
+    def extract_active_axioms(self, target_state: int) -> List[str]:
+        """Return all axiom keys whose primes divide the target state."""
+        active = []
+        for prime, axiom in self.algebra.prime_to_axiom.items():
+            if target_state % prime == 0:
+                active.append(axiom)
+        return active
+
+    def cluster_by_subject(
+        self, target_state: int
+    ) -> Dict[str, List[str]]:
+        """Group active axioms by their subject entity."""
+        chapters: Dict[str, List[str]] = {}
+        for axiom in self.extract_active_axioms(target_state):
+            parts = axiom.split("||")
+            if len(parts) == 3:
+                subject = parts[0].strip()
+                if subject not in chapters:
+                    chapters[subject] = []
+                chapters[subject].append(axiom)
+        return chapters
+
+    # ------------------------------------------------------------------
+    # Proof Mode: deterministic canonical rendering
+    # ------------------------------------------------------------------
+
+    def generate_canonical(
+        self, target_state: int, title: str = "Canonical Tome"
+    ) -> str:
+        """
+        Deterministic, template-based rendering.
+
+        Every active axiom is emitted in a canonical ``subject PREDICATE
+        object`` format, grouped by subject and sorted lexicographically.
+        This output is guaranteed to round-trip through the Sieve and
+        produce the same Gödel Integer.
+
+        Args:
+            target_state: The Gödel integer to unpack.
+            title:        Human-readable title for the Tome.
+
+        Returns:
+            A structured text representation of the integer's knowledge.
+        """
+        chapters = self.cluster_by_subject(target_state)
+
+        if not chapters:
+            return f"@canonical_version: {CANONICAL_FORMAT_VERSION}\n# {title}\n\nThe Gödel State is empty (= 1). No axioms to rehydrate."
+
+        lines = [f"@canonical_version: {CANONICAL_FORMAT_VERSION}", f"# {title}", ""]
+
+        # Sort subjects lexicographically for determinism
+        for subject in sorted(chapters.keys()):
+            axioms = sorted(chapters[subject])  # Sort axioms too
+            lines.append(f"## {subject.title()}")
+            lines.append("")
+            for axiom in axioms:
+                parts = axiom.split("||")
+                if len(parts) == 3:
+                    s, p, o = parts
+                    # Canonical sentence: "The {subject} {predicate} {object}."
+                    lines.append(f"The {s} {p} {o}.")
+            lines.append("")
+
+        return "\n".join(lines)
+
+    # ------------------------------------------------------------------
+    # Narrative Mode: LLM-enhanced prose with canonical fallback
+    # ------------------------------------------------------------------
+
+    async def generate_narrative(
+        self, target_state: int, title: str = "The Quantum Tome"
+    ) -> str:
+        """
+        LLM-enhanced rendering with canonical fallback.
+
+        If an extrapolator is available, each chapter's axioms are
+        expanded into readable prose.  If the extrapolator is absent
+        or fails, falls back to canonical rendering.
+
+        Args:
+            target_state: The Gödel integer to unpack.
+            title:        Human-readable title for the Tome.
+
+        Returns:
+            A narrative text representation.
+        """
+        chapters = self.cluster_by_subject(target_state)
+
+        if not chapters:
+            return f"# {title}\n\nThe Gödel State is empty (= 1). No axioms to rehydrate."
+
+        lines = [f"# {title}", ""]
+
+        for subject in sorted(chapters.keys()):
+            axioms = sorted(chapters[subject])
+            lines.append(f"## {subject.title()}")
+            lines.append("")
+
+            if self.extrapolator is not None:
+                try:
+                    narrative = await self.extrapolator.extrapolate_with_proof(
+                        target_state, axioms
+                    )
+                    lines.append(narrative)
+                except RuntimeError:
+                    # Fallback: canonical rendering
+                    for axiom in axioms:
+                        parts = axiom.split("||")
+                        if len(parts) == 3:
+                            lines.append(f"The {parts[0]} {parts[1]} {parts[2]}.")
+            else:
+                # No LLM — canonical rendering
+                for axiom in axioms:
+                    parts = axiom.split("||")
+                    if len(parts) == 3:
+                        lines.append(f"The {parts[0]} {parts[1]} {parts[2]}.")
+
+            lines.append("")
+
+        return "\n".join(lines)
+
+    # ------------------------------------------------------------------
+    # Controlled rendering — TomeSliders (founder's dream surface)
+    # ------------------------------------------------------------------
+
+    def generate_controlled(
+        self,
+        target_state: int,
+        sliders: "Optional[object]" = None,
+        title: str = "Controlled Tome",
+    ) -> str:
+        """Parameterized canonical rendering under TomeSliders control.
+
+        The density slider is actioned here: axioms are deterministically
+        subsetted by lexicographic order before canonical rendering. The
+        remaining sliders (length, formality, audience, perspective) are
+        LLM-gated; their values are captured in the output header so a
+        downstream narrative generator can honour them while the
+        deterministic path remains unchanged.
+
+        Args:
+            target_state: The Gödel integer to unpack.
+            sliders:      A TomeSliders instance. Defaults to all-balanced
+                          (equivalent to generate_canonical).
+            title:        Human-readable title.
+
+        Returns:
+            A canonical-format tome reflecting the density slider. Output
+            includes the slider header so the rendering is reproducible.
+        """
+        from internal.ensemble.tome_sliders import (
+            TomeSliders,
+            apply_density,
+        )
+
+        cfg = sliders if sliders is not None else TomeSliders()
+        if not isinstance(cfg, TomeSliders):
+            raise TypeError(
+                "sliders must be a TomeSliders instance or None"
+            )
+
+        active_axioms = self.extract_active_axioms(target_state)
+        selected = apply_density(active_axioms, cfg.density)
+
+        header = [
+            f"@canonical_version: {CANONICAL_FORMAT_VERSION}",
+            cfg.header_line(),
+            f"# {title}",
+            "",
+        ]
+
+        if not selected:
+            header.append(
+                f"No axioms survive at density={cfg.density:.3f} "
+                f"(source had {len(active_axioms)})."
+            )
+            return "\n".join(header)
+
+        chapters: Dict[str, List[str]] = {}
+        for axiom in selected:
+            parts = axiom.split("||")
+            if len(parts) == 3:
+                subject = parts[0].strip()
+                chapters.setdefault(subject, []).append(axiom)
+
+        lines = list(header)
+        for subject in sorted(chapters.keys()):
+            axioms = sorted(chapters[subject])
+            lines.append(f"## {subject.title()}")
+            lines.append("")
+            for axiom in axioms:
+                parts = axiom.split("||")
+                if len(parts) == 3:
+                    s, p, o = parts
+                    lines.append(f"The {s} {p} {o}.")
+            lines.append("")
+
+        if cfg.requires_extrapolator() and self.extrapolator is None:
+            logger.info(
+                "TomeSliders non-density axes set but no extrapolator present; "
+                "non-density sliders are recorded in the header as metadata only."
+            )
+
+        return "\n".join(lines)
+
+    # ------------------------------------------------------------------
+    # Unified entry point
+    # ------------------------------------------------------------------
+
+    async def generate_tome(
+        self,
+        target_state: int,
+        title: str = "The Quantum Tome",
+        mode: str = "proof",
+    ) -> str:
+        """
+        Generate a Tome from a Gödel Integer.
+
+        Args:
+            target_state: The integer to unpack.
+            title:        Tome title.
+            mode:         ``"proof"`` for deterministic canonical output,
+                          ``"narrative"`` for LLM-enhanced prose.
+
+        Returns:
+            The generated Tome text.
+        """
+        if mode == "narrative":
+            return await self.generate_narrative(target_state, title)
+        return self.generate_canonical(target_state, title)

internal/ensemble/tome_sliders.py

@@ -0,0 +1,104 @@
+"""
+Tome Sliders — Control Parameters for Bidirectional Knowledge Rendering
+
+The founder's core dream made concrete: tunable knobs for the tag→tome
+direction. Every SUM rendering takes a TomeSliders record in [0.0, 1.0]^5
+and produces output conformant to those controls.
+
+Implementation status (as of module authoring):
+    density    — implemented on the deterministic canonical path
+                 (axiom subsetting via lexicographic ordering)
+    length     — LLM-gated; no-op without an extrapolator
+    formality  — LLM-gated
+    audience   — LLM-gated
+    perspective — LLM-gated
+
+Slider values are captured in the output artefact's header so the same
+narrative can be regenerated with adjusted parameters and the difference
+audited.
+
+Author: ototao
+License: Apache License 2.0
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Sequence
+
+
+@dataclass(frozen=True)
+class TomeSliders:
+    """Slider configuration for controlled tome rendering.
+
+    All values in [0.0, 1.0]. Defaults preserve lossless / balanced behavior.
+
+    - density:     0.0 = empty, 1.0 = full axiom coverage
+    - length:      0.0 = telegraphic, 1.0 = expansive
+    - formality:   0.0 = casual, 1.0 = academic
+    - audience:    0.0 = novice, 1.0 = expert / jargon-dense
+    - perspective: 0.0 = first-person, 1.0 = omniscient / third-person
+    """
+
+    density: float = 1.0
+    length: float = 0.5
+    formality: float = 0.5
+    audience: float = 0.5
+    perspective: float = 0.5
+
+    def __post_init__(self) -> None:
+        for name in (
+            "density",
+            "length",
+            "formality",
+            "audience",
+            "perspective",
+        ):
+            v = getattr(self, name)
+            if not (0.0 <= v <= 1.0):
+                raise ValueError(f"{name} out of [0, 1]: {v}")
+
+    def requires_extrapolator(self) -> bool:
+        """True if any slider besides density deviates from its balanced
+        default (0.5), meaning an LLM extrapolator is needed to honour it.
+        The canonical deterministic path can only action the density slider.
+        """
+        return not (
+            self.length == 0.5
+            and self.formality == 0.5
+            and self.audience == 0.5
+            and self.perspective == 0.5
+        )
+
+    def header_line(self) -> str:
+        """Single-line serialization for canonical tome headers."""
+        return (
+            f"@sliders: density={self.density:.3f} "
+            f"length={self.length:.3f} "
+            f"formality={self.formality:.3f} "
+            f"audience={self.audience:.3f} "
+            f"perspective={self.perspective:.3f}"
+        )
+
+
+def apply_density(
+    axiom_keys: Sequence[str], density: float
+) -> list[str]:
+    """Deterministic axiom subsetting by density.
+
+    Sorts axiom keys lexicographically and keeps the first floor(N * density)
+    entries. Deterministic across runs and machines. Empty list when density
+    rounds to zero; full sorted list when density >= 1.0.
+
+    The lexicographic ordering ensures stability: running the same state
+    through the same density on two hosts produces identical subsets.
+    """
+    if not axiom_keys:
+        return []
+    if density >= 1.0:
+        return sorted(axiom_keys)
+    if density <= 0.0:
+        return []
+    sorted_keys = sorted(axiom_keys)
+    n = len(sorted_keys)
+    k = int(n * density)
+    return sorted_keys[:k]

internal/ensemble/vector_bridge.py

@@ -0,0 +1,195 @@
+"""
+Continuous-Discrete Bridge — Vector Space ↔ Gödel Space
+
+Maps the discrete world of Gödel prime integers to the continuous world
+of LLM embedding vectors, enabling semantic (fuzzy) search over a
+mathematically exact state.
+
+Architecture:
+    - Every minted prime gets a vector embedding of its natural-language
+      axiom text.
+    - ``semantic_search_godel_state`` filters to primes *alive* in the
+      current state (``global_state % prime == 0``) before ranking by
+      cosine similarity.
+    - Deleted axioms are automatically excluded because their primes no
+      longer divide the state.
+
+Horizon III — Universal Vector Alignment:
+    - Supports optional affine transformation matrices (W*, b*) from
+      Gorbett & Jana (2026) for cross-model linear alignment.
+    - Heterogeneous P2P nodes (Llama, Qwen, Mistral, etc.) can perfectly
+      align their embeddings into a single Canonical Geometry before
+      discrete prime extraction via O(1) linear affine maps.
+
+Author: ototao
+License: Apache License 2.0
+"""
+
+import logging
+from typing import Callable, Awaitable, List, Dict, Optional, Tuple
+
+import numpy as np
+
+from internal.algorithms.semantic_arithmetic import GodelStateAlgebra
+
+logger = logging.getLogger(__name__)
+
+
+class ContinuousDiscreteBridge:
+    """
+    Connects the absolute mathematical certainty of Gödel integers
+    with the fuzzy, semantic search capabilities of LLM Embeddings.
+
+    The bridge maintains a mapping from each Semantic Prime to its
+    vector embedding.  Queries are projected into the same space and
+    ranked by cosine similarity — but *only* against primes that are
+    currently alive in the global state.
+
+    Horizon III:
+        When ``affine_map`` (W*) and optional ``bias_map`` (b*) are
+        provided, all embeddings are translated into the Canonical
+        Geometry via an O(1) linear affine transformation before
+        indexing and search.  This allows heterogeneous LLM nodes
+        in a P2P swarm to perfectly align their latent spaces.
+    """
+
+    def __init__(
+        self,
+        algebra: GodelStateAlgebra,
+        embedding_model: Callable[[str], Awaitable[List[float]]],
+        affine_map: Optional[np.ndarray] = None,
+        bias_map: Optional[np.ndarray] = None,
+    ):
+        """
+        Args:
+            algebra:         A GodelStateAlgebra instance with minted primes.
+            embedding_model: Async callable (text) → List[float] vector.
+            affine_map:      Optional W* matrix (d×d) for cross-model alignment.
+            bias_map:        Optional b* bias vector (d,) for cross-model alignment.
+        """
+        self.algebra = algebra
+        self.get_embedding = embedding_model
+        self.prime_embeddings: Dict[int, np.ndarray] = {}
+
+        # W* and b* matrices from Gorbett & Jana (2026) for cross-model
+        # linear alignment into the Canonical Geometry.
+        self.affine_map = affine_map
+        self.bias_map = bias_map
+
+    # ------------------------------------------------------------------
+    # Affine alignment
+    # ------------------------------------------------------------------
+
+    def _align_vector(self, vector: np.ndarray) -> np.ndarray:
+        """
+        Applies O(1) Linear Alignment to translate heterogeneous latent
+        spaces into the Canonical Geometry.
+
+        If no affine map is configured, returns the input unchanged.
+
+        Args:
+            vector: Raw embedding vector (np.ndarray, float32).
+
+        Returns:
+            Aligned (and re-normalised) vector.
+        """
+        if self.affine_map is None:
+            return vector
+
+        v = np.dot(vector, self.affine_map)
+        if self.bias_map is not None:
+            v = v + self.bias_map
+
+        # Re-normalise to unit length for cosine similarity
+        norm = np.linalg.norm(v)
+        if norm > 0:
+            v = v / norm
+
+        return v.astype(np.float32)
+
+    # ------------------------------------------------------------------
+    # Indexing
+    # ------------------------------------------------------------------
+
+    async def index_new_primes(self) -> int:
+        """
+        Generate embeddings for any un-indexed Semantic Primes.
+
+        Returns:
+            Number of newly indexed primes.
+        """
+        newly_indexed = 0
+
+        for prime, axiom in self.algebra.prime_to_axiom.items():
+            if prime not in self.prime_embeddings:
+                # "alice||age||30" → "alice age 30"
+                natural_text = " ".join(axiom.split("||"))
+                raw_vector = await self.get_embedding(natural_text)
+                aligned = self._align_vector(
+                    np.array(raw_vector, dtype=np.float32)
+                )
+                self.prime_embeddings[prime] = aligned
+                newly_indexed += 1
+
+        if newly_indexed:
+            logger.info("Indexed %d new primes into vector space.", newly_indexed)
+
+        return newly_indexed
+
+    # ------------------------------------------------------------------
+    # Semantic search
+    # ------------------------------------------------------------------
+
+    async def semantic_search_godel_state(
+        self,
+        global_state: int,
+        query: str,
+        top_k: int = 5,
+    ) -> List[Tuple[str, float]]:
+        """
+        Query the exact Gödel state using fuzzy natural language.
+
+        Only primes that are currently *alive* in the global state are
+        considered.  Deleted axioms are automatically excluded because
+        their primes no longer divide the state.
+
+        Args:
+            global_state: The current global Gödel integer.
+            query:        Natural language query string.
+            top_k:        Maximum number of results.
+
+        Returns:
+            List of (axiom_key, similarity_score) tuples, descending.
+        """
+        raw_query = np.array(
+            await self.get_embedding(query), dtype=np.float32
+        )
+        query_vector = self._align_vector(raw_query)
+        query_norm = np.linalg.norm(query_vector)
+        if query_norm == 0:
+            return []
+
+        active_primes: List[int] = []
+        similarities: List[float] = []
+
+        for prime, vector in self.prime_embeddings.items():
+            # Only search facts alive in the current state
+            if global_state % prime == 0:
+                vec_norm = np.linalg.norm(vector)
+                if vec_norm == 0:
+                    continue
+                sim = float(
+                    np.dot(query_vector, vector) / (query_norm * vec_norm)
+                )
+                active_primes.append(prime)
+                similarities.append(sim)
+
+        if not active_primes:
+            return []
+
+        # Sort by similarity descending
+        top_indices = np.argsort(similarities)[::-1][:top_k]
+        return [
+            (self.algebra.prime_to_axiom[active_primes[i]], similarities[i])
+            for i in top_indices
+        ]