spell-exploder 0.1.0__py3-none-any.whl

spell_exploder/extractors/noun_dependencies.py

@@ -0,0 +1,96 @@
+"""
+Noun-dependency extraction (schema–valence pairs).
+
+Extracts structured relationships between nouns and their modifiers
+or governing verbs.  In Spellcaster's terminology:
+
+* **Schema keyword** — the noun lemma (the concept being modified).
+* **Valence keyword** — the adjective or verb lemma that colours the
+  noun's meaning in context.
+* **Dependency type** — the syntactic relation (``amod``, ``nsubj``,
+  ``dobj``).
+
+These triples power the *valence entropy* and *semantic breadth*
+metrics in the Valence Model analyzer.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import spacy
+
+from spell_exploder.core.nlp import get_nlp
+
+
+# Type alias for a single dependency triple
+NounDependency = tuple[str, str, str]
+"""``(schema_keyword, valence_keyword, dep_type)``"""
+
+
+def extract_noun_dependencies(
+    text: str,
+    *,
+    nlp: spacy.Language | None = None,
+    model_name: str = "en_core_web_sm",
+) -> list[NounDependency]:
+    """
+    Extract noun–adjective and noun–verb dependency triples from *text*.
+
+    Three dependency patterns are captured:
+
+    1. **amod** — An adjective modifying a noun
+       (e.g. *"quick fox"* → ``("fox", "quick", "amod")``).
+    2. **nsubj** — A noun serving as subject of a verb
+       (e.g. *"The fox jumps"* → ``("fox", "jump", "nsubj")``).
+    3. **dobj** — A noun serving as direct object of a verb
+       (e.g. *"chase the mouse"* → ``("mouse", "chase", "dobj")``).
+
+    Stop-word valence keywords are filtered out.
+
+    Parameters
+    ----------
+    text : str
+        Raw input text.
+    nlp : spacy.Language or None
+        Pre-loaded pipeline (must include ``parser``).
+        When ``None``, loaded via :func:`~spell_exploder.core.nlp.get_nlp`.
+    model_name : str
+        spaCy model name (used only when *nlp* is ``None``).
+
+    Returns
+    -------
+    list[NounDependency]
+        List of ``(schema_keyword, valence_keyword, dep_type)`` tuples.
+    """
+    if nlp is None:
+        nlp = get_nlp(model_name, disable=["ner"])
+
+    doc = nlp(text)
+    dependencies: list[NounDependency] = []
+
+    for tok in doc:
+        # --- Pattern 1: adjective modifier of a noun ---
+        if tok.pos_ in {"NOUN", "PROPN"}:
+            schema = tok.lemma_
+            for child in tok.children:
+                if child.dep_ == "amod":
+                    valence = child.lemma_
+                    if not nlp.vocab[valence].is_stop:
+                        dependencies.append((schema, valence, "amod"))
+
+        # --- Patterns 2 & 3: noun as subject or object of a verb ---
+        if tok.pos_ == "VERB":
+            valence = tok.lemma_
+            if nlp.vocab[valence].is_stop:
+                continue
+            for child in tok.children:
+                if child.pos_ in {"NOUN", "PROPN"}:
+                    schema = child.lemma_
+                    if child.dep_ == "nsubj":
+                        dependencies.append((schema, valence, "nsubj"))
+                    elif child.dep_ == "dobj":
+                        dependencies.append((schema, valence, "dobj"))
+
+    return dependencies

spell_exploder/extractors/sentence_parser.py

@@ -0,0 +1,168 @@
+"""
+Sentence parsing with POS-tag extraction.
+
+Provides sentence segmentation (with a custom abbreviation-aware
+boundary detector) and per-sentence POS-tag sequences, which feed
+into the Adaptive Evolution (APE) and Keyword ERP (KEPM) analyzers.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import spacy
+
+from spell_exploder.core.nlp import get_nlp
+
+
+# ---------------------------------------------------------------------------
+# Default abbreviation set for sentence boundary detection
+# ---------------------------------------------------------------------------
+
+DEFAULT_ABBREVIATIONS: frozenset[str] = frozenset({
+    "mr.", "mrs.", "ms.", "dr.", "prof.", "rev.", "col.",
+    "gen.", "maj.", "capt.", "lt.", "sgt.", "pvt.",
+    "jr.", "sr.", "etc.", "e.g.", "i.e.",
+})
+
+
+# ---------------------------------------------------------------------------
+# Result dataclass
+# ---------------------------------------------------------------------------
+
+@dataclass(frozen=True)
+class ParsedSentence:
+    """A single sentence with its POS tag sequence."""
+
+    text: str
+    """The raw sentence text (whitespace-stripped)."""
+
+    pos_tags: list[str] = field(default_factory=list)
+    """POS tags for every token in the sentence (e.g. ``['DET', 'NOUN', 'VERB', ...]``)."""
+
+
+# ---------------------------------------------------------------------------
+# Core parsing functions
+# ---------------------------------------------------------------------------
+
+def _register_boundary_detector(
+    nlp: spacy.Language,
+    abbreviations: frozenset[str],
+) -> spacy.Language:
+    """
+    Register (once) a custom sentence-boundary component on *nlp* that
+    respects abbreviations and newline boundaries.
+
+    The component is inserted **before** the parser so that spaCy's
+    built-in sentenciser can still override when needed.
+    """
+    from spacy.language import Language as SpacyLanguage
+
+    component_name = "spell_exploder_sent_boundary"
+
+    if component_name in nlp.pipe_names:
+        return nlp
+
+    @SpacyLanguage.component(component_name)
+    def _boundary_detector(doc):
+        if len(doc) == 0:
+            return doc
+        doc[0].is_sent_start = True
+
+        for i, token in enumerate(doc[:-1]):
+            nxt = doc[i + 1]
+            # Period followed by capitalised word (not after abbreviation)
+            if (
+                token.text == "."
+                and nxt.text
+                and nxt.text[0].isupper()
+                and not nxt.is_space
+                and not nxt.is_punct
+                and token.text.lower() not in abbreviations
+            ):
+                nxt.is_sent_start = True
+            # Newline triggers new sentence
+            if token.text in ("\n", "\r", "\r\n") and i + 1 < len(doc):
+                nxt.is_sent_start = True
+
+        return doc
+
+    nlp.add_pipe(component_name, before="parser")
+    return nlp
+
+
+def parse_sentences(
+    text: str,
+    *,
+    nlp: spacy.Language | None = None,
+    model_name: str = "en_core_web_sm",
+    abbreviations: frozenset[str] | None = None,
+    use_custom_boundaries: bool = True,
+) -> list[ParsedSentence]:
+    """
+    Segment *text* into sentences with per-token POS tags.
+
+    Parameters
+    ----------
+    text : str
+        Raw input text.
+    nlp : spacy.Language or None
+        Pre-loaded pipeline. When ``None``, loaded via
+        :func:`~spell_exploder.core.nlp.get_nlp`.
+    model_name : str
+        spaCy model name (used only when *nlp* is ``None``).
+    abbreviations : frozenset[str] or None
+        Abbreviations that should **not** trigger sentence boundaries
+        (e.g. ``"dr."``).  Defaults to :data:`DEFAULT_ABBREVIATIONS`.
+    use_custom_boundaries : bool
+        If ``True``, register a custom boundary detector component
+        on the pipeline.
+
+    Returns
+    -------
+    list[ParsedSentence]
+        Ordered list of non-empty sentences with POS tags.
+    """
+    if nlp is None:
+        nlp = get_nlp(model_name)
+
+    if use_custom_boundaries:
+        abbrevs = abbreviations if abbreviations is not None else DEFAULT_ABBREVIATIONS
+        nlp = _register_boundary_detector(nlp, abbrevs)
+
+    doc = nlp(text)
+    results: list[ParsedSentence] = []
+
+    for sent in doc.sents:
+        sent_text = sent.text.strip()
+        if sent_text:
+            pos_tags = [token.pos_ for token in sent]
+            results.append(ParsedSentence(text=sent_text, pos_tags=pos_tags))
+
+    return results
+
+
+def split_sentences_simple(text: str) -> list[str]:
+    """
+    Lightweight sentence splitter using regex (no spaCy required).
+
+    Splits on periods, newlines, and bullet characters.  Used by the
+    :class:`~spell_exploder.analyzers.complexity_index.TextComplexityAnalyzer`
+    which does not need POS tags.
+
+    Parameters
+    ----------
+    text : str
+        Raw input text.
+
+    Returns
+    -------
+    list[str]
+        Non-empty, whitespace-stripped sentence strings.
+    """
+    clean = re.sub(r"<[^>]*>", "", text)
+    parts = re.split(r"[.\n•]+", clean)
+    return [s.strip() for s in parts if s.strip()]

spell_exploder/io/__init__.py

@@ -0,0 +1,14 @@
+"""
+I/O utilities: text loading and result export.
+"""
+
+from spell_exploder.io.readers import TextDocument, load_texts, texts_from_strings
+from spell_exploder.io.exporters import export_csv, export_json
+
+__all__ = [
+    "TextDocument",
+    "load_texts",
+    "texts_from_strings",
+    "export_csv",
+    "export_json",
+]

spell_exploder/io/exporters.py

@@ -0,0 +1,94 @@
+"""
+Export utilities for Spellcaster results.
+
+Provides generic serialisation of any result object to JSON or CSV,
+plus a specialised APE evolutionary-dynamics JSON report.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+
+import pandas as pd
+
+from spell_exploder.results.evolution import EvolutionResult
+
+logger = logging.getLogger(__name__)
+
+
+def export_csv(
+    result,
+    path: str,
+    **kwargs,
+) -> Path:
+    """
+    Export any result with a ``.to_dataframe()`` method to CSV.
+
+    Parameters
+    ----------
+    result
+        Any Spellcaster result object (``ComplexityComparisonResult``,
+        ``ValenceModelResult``, ``EvolutionResult``, ``KeywordERPResult``).
+    path : str
+        Output file path.
+    **kwargs
+        Forwarded to :meth:`pandas.DataFrame.to_csv`.
+
+    Returns
+    -------
+    Path
+        The written file path.
+    """
+    df = result.to_dataframe()
+    p = Path(path)
+    df.to_csv(p, index=kwargs.pop("index", False), **kwargs)
+    logger.info("Exported %d rows to %s", len(df), p)
+    return p
+
+
+def export_json(
+    result,
+    path: str,
+    indent: int = 2,
+) -> Path:
+    """
+    Export a result to JSON.
+
+    For :class:`~spell_exploder.results.evolution.EvolutionResult`, uses the
+    structured ``to_json()`` format.  For all other result types, converts
+    the ``.to_dataframe()`` output to JSON records.
+
+    Parameters
+    ----------
+    result
+        Any Spellcaster result object.
+    path : str
+        Output file path.
+    indent : int
+        JSON indentation.
+
+    Returns
+    -------
+    Path
+        The written file path.
+    """
+    p = Path(path)
+
+    if isinstance(result, EvolutionResult):
+        data = result.to_json()
+    elif hasattr(result, "to_dataframe"):
+        df = result.to_dataframe()
+        data = json.loads(df.to_json(orient="records", default_handler=str))
+    else:
+        raise TypeError(
+            f"Cannot export {type(result).__name__}: "
+            "expected a Spellcaster result with .to_dataframe() or .to_json()"
+        )
+
+    with open(p, "w", encoding="utf-8") as f:
+        json.dump(data, f, indent=indent, ensure_ascii=False, default=str)
+
+    logger.info("Exported JSON to %s", p)
+    return p

spell_exploder/io/readers.py

@@ -0,0 +1,117 @@
+"""
+Text loading utilities.
+
+Provides a uniform way to load documents from file paths or raw strings,
+producing :class:`TextDocument` instances that all analyzers accept.
+"""
+
+from __future__ import annotations
+
+import pathlib
+from dataclasses import dataclass
+
+
+@dataclass
+class TextDocument:
+    """
+    A loaded text document with provenance metadata.
+
+    Attributes
+    ----------
+    path : str
+        Original file path, or ``"<inline>"`` for text supplied directly.
+    label : str
+        Human-readable label (defaults to the filename stem).
+    text : str
+        Full text content.
+    """
+
+    path: str
+    label: str
+    text: str
+
+
+def load_texts(
+    file_paths: list[str],
+    labels: list[str] | None = None,
+    encoding: str = "utf-8",
+) -> list[TextDocument]:
+    """
+    Load text files from disk and wrap them as :class:`TextDocument` objects.
+
+    Parameters
+    ----------
+    file_paths : list[str]
+        Paths to ``.txt`` files.
+    labels : list[str] or None
+        Human-readable labels, one per file.  When ``None``, labels
+        default to the file-name stems (e.g. ``"essay1"`` for
+        ``"/data/essay1.txt"``).
+    encoding : str
+        Text encoding to use when reading files.
+
+    Returns
+    -------
+    list[TextDocument]
+
+    Raises
+    ------
+    FileNotFoundError
+        If any file does not exist.
+    ValueError
+        If *labels* is provided but its length differs from *file_paths*.
+    """
+    if labels is not None and len(labels) != len(file_paths):
+        raise ValueError(
+            f"labels length ({len(labels)}) must match "
+            f"file_paths length ({len(file_paths)})"
+        )
+
+    documents: list[TextDocument] = []
+    for i, fp in enumerate(file_paths):
+        p = pathlib.Path(fp)
+        if not p.exists():
+            raise FileNotFoundError(f"Text file not found: {fp}")
+
+        text = p.read_text(encoding=encoding, errors="ignore")
+        label = labels[i] if labels is not None else p.stem
+
+        documents.append(TextDocument(path=str(fp), label=label, text=text))
+
+    return documents
+
+
+def texts_from_strings(
+    texts: list[str],
+    labels: list[str] | None = None,
+) -> list[TextDocument]:
+    """
+    Wrap raw strings as :class:`TextDocument` objects (no file I/O).
+
+    Useful when text is already in memory — for example, from a
+    database or an API response.
+
+    Parameters
+    ----------
+    texts : list[str]
+        Raw text strings.
+    labels : list[str] or None
+        Human-readable labels.  When ``None``, defaults to
+        ``"text_0"``, ``"text_1"``, etc.
+
+    Returns
+    -------
+    list[TextDocument]
+    """
+    if labels is not None and len(labels) != len(texts):
+        raise ValueError(
+            f"labels length ({len(labels)}) must match "
+            f"texts length ({len(texts)})"
+        )
+
+    documents: list[TextDocument] = []
+    for i, text in enumerate(texts):
+        label = labels[i] if labels is not None else f"text_{i}"
+        documents.append(TextDocument(path="<inline>", label=label, text=text))
+
+    return documents

spell_exploder/results/__init__.py

@@ -0,0 +1,44 @@
+"""
+Structured result objects for all Spellcaster analyzers.
+
+Every result class provides a ``.to_dataframe()`` method that produces
+a flat :class:`pandas.DataFrame` for exploratory analysis.
+"""
+
+from spell_exploder.results.complexity import (
+    ComplexityComparisonResult,
+    ComplexityFlowResult,
+    SentenceMetrics,
+)
+from spell_exploder.results.evolution import (
+    EvolutionaryStatus,
+    EvolutionResult,
+    POSComposition,
+    SpeciesRecord,
+)
+from spell_exploder.results.keyword import (
+    CrossKeywordEntanglement,
+    FileKeywordResult,
+    KeywordERPResult,
+    KeywordMeasures,
+)
+from spell_exploder.results.valence import (
+    PostMetrics,
+    ValenceModelResult,
+)
+
+__all__ = [
+    "SentenceMetrics",
+    "ComplexityFlowResult",
+    "ComplexityComparisonResult",
+    "PostMetrics",
+    "ValenceModelResult",
+    "EvolutionaryStatus",
+    "SpeciesRecord",
+    "POSComposition",
+    "EvolutionResult",
+    "KeywordMeasures",
+    "CrossKeywordEntanglement",
+    "FileKeywordResult",
+    "KeywordERPResult",
+]

spell_exploder/results/complexity.py

@@ -0,0 +1,111 @@
+"""
+Result dataclasses for the Complexity Index (LCX) analyzer.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import numpy as np
+import pandas as pd
+
+
+@dataclass
+class SentenceMetrics:
+    """Metrics for a single sentence within a complexity flow."""
+
+    text: str
+    """Raw sentence text."""
+
+    index: int
+    """Zero-based position in the document."""
+
+    k_hist: int
+    """Cumulative compressed size of all text up to and including this sentence (bytes)."""
+
+    volatility: int
+    """Levenshtein edit distance from the previous sentence (0 for the first)."""
+
+    synergy: float
+    """Ratio of volatility to marginal compression cost (volatility / delta_k)."""
+
+
+@dataclass
+class ComplexityFlowResult:
+    """
+    Result of analysing a single text's complexity flow.
+
+    Returned by :meth:`~spell_exploder.analyzers.complexity_index.TextComplexityAnalyzer.analyze_flow`.
+    """
+
+    label: str
+    """User-supplied label for this text (e.g. file name or condition)."""
+
+    sentences: list[SentenceMetrics] = field(default_factory=list)
+    """Per-sentence metrics in document order."""
+
+    def to_dataframe(self) -> pd.DataFrame:
+        """
+        Flat DataFrame with one row per sentence.
+
+        Columns: ``index``, ``text``, ``k_hist``, ``volatility``, ``synergy``.
+        """
+        if not self.sentences:
+            return pd.DataFrame(
+                columns=["index", "text", "k_hist", "volatility", "synergy"]
+            )
+        return pd.DataFrame(
+            {
+                "index": [s.index for s in self.sentences],
+                "text": [s.text for s in self.sentences],
+                "k_hist": [s.k_hist for s in self.sentences],
+                "volatility": [s.volatility for s in self.sentences],
+                "synergy": [s.synergy for s in self.sentences],
+            }
+        )
+
+    @property
+    def k_hist_array(self) -> np.ndarray:
+        """Cumulative complexity as a NumPy array."""
+        return np.array([s.k_hist for s in self.sentences])
+
+    @property
+    def volatility_array(self) -> np.ndarray:
+        """Volatility sequence as a NumPy array."""
+        return np.array([s.volatility for s in self.sentences])
+
+    @property
+    def synergy_array(self) -> np.ndarray:
+        """Synergy sequence as a NumPy array."""
+        return np.array([s.synergy for s in self.sentences])
+
+
+@dataclass
+class ComplexityComparisonResult:
+    """
+    Result of comparing N texts via complexity flow analysis.
+
+    Returned by :meth:`~spell_exploder.analyzers.complexity_index.TextComplexityAnalyzer.compare`.
+    """
+
+    flows: list[ComplexityFlowResult] = field(default_factory=list)
+    """One :class:`ComplexityFlowResult` per input text."""
+
+    def to_dataframe(self) -> pd.DataFrame:
+        """
+        Combined DataFrame with a ``label`` column distinguishing texts.
+        """
+        frames = []
+        for flow in self.flows:
+            df = flow.to_dataframe()
+            df["label"] = flow.label
+            frames.append(df)
+        if not frames:
+            return pd.DataFrame(
+                columns=["index", "text", "k_hist", "volatility", "synergy", "label"]
+            )
+        return pd.concat(frames, ignore_index=True)
+
+    @property
+    def labels(self) -> list[str]:
+        return [f.label for f in self.flows]