pystylometry 1.3.0__py3-none-any.whl → 1.3.1__py3-none-any.whl

pystylometry/_types.py

@@ -370,6 +370,158 @@ class TTRResult:
     metadata: dict[str, Any]
 
 
+# ===== Repetition Detection Results =====
+# Related to GitHub Issue #28: Verbal tics detection for slop analysis
+# https://github.com/craigtrim/pystylometry/issues/28
+
+
+@dataclass
+class RepetitiveWord:
+    """A single word flagged as abnormally repetitive.
+
+    The repetition_score is the ratio of observed count to expected count
+    based on the word's frequency in the British National Corpus (BNC).
+    Higher scores indicate stronger overrepresentation.
+
+    Related GitHub Issue:
+        #28 - Verbal tics detection for slop analysis
+        https://github.com/craigtrim/pystylometry/issues/28
+
+    Attributes:
+        word: The flagged word (lowercased).
+        count: Observed count in the text.
+        expected_count: Expected count based on BNC relative frequency × text length.
+            0.0 if word not found in BNC.
+        repetition_score: count / expected_count. float('inf') if expected_count is 0.
+        bnc_bucket: BNC frequency bucket (1-100, 1=most frequent). None if not in BNC.
+        chunk_counts: Per-chunk occurrence counts (for distribution analysis).
+        distribution_entropy: Shannon entropy of the word's chunk distribution.
+            Low entropy = suspiciously even spread (model tic).
+            High entropy = clustered usage (human writing about a specific scene).
+        distribution_variance: Variance of per-chunk counts.
+    """
+
+    word: str
+    count: int
+    expected_count: float
+    repetition_score: float
+    bnc_bucket: int | None
+    chunk_counts: list[int]
+    distribution_entropy: float
+    distribution_variance: float
+
+
+@dataclass
+class RepetitiveUnigramsResult:
+    """Result from repetitive unigram detection.
+
+    Identifies content words that appear far more frequently than expected
+    based on their frequency in the British National Corpus (BNC, ~100M tokens).
+    This is a key indicator of AI-generated "slop" where models exhibit verbal
+    tics — repeating certain words with suspicious regularity.
+
+    Related GitHub Issue:
+        #28 - Verbal tics detection for slop analysis
+        https://github.com/craigtrim/pystylometry/issues/28
+
+    The slop_score provides a single aggregate metric:
+        slop_score = flagged_words_per_10k × mean_repetition_score
+
+    Where:
+        - flagged_words_per_10k = count of flagged words / (total content words / 10000)
+        - mean_repetition_score = mean repetition_score across all flagged words
+
+    Higher slop_score = more likely AI-generated verbal tics.
+
+    References:
+        British National Corpus Consortium. (2007). The British National Corpus,
+            version 3 (BNC XML Edition). http://www.natcorp.ox.ac.uk/
+
+    Example:
+        >>> result = compute_repetitive_unigrams(text)
+        >>> for w in result.repetitive_words[:5]:
+        ...     print(f"{w.word}: {w.count}x (expected {w.expected_count:.1f}, "
+        ...           f"score {w.repetition_score:.1f})")
+        shimmered: 23x (expected 0.1, score 266.2)
+        >>> result.slop_score
+        42.7
+    """
+
+    repetitive_words: list[RepetitiveWord]  # Sorted by repetition_score descending
+    total_content_words: int
+    flagged_count: int  # Number of words exceeding threshold
+    flagged_words_per_10k: float  # flagged_count / (total_content_words / 10000)
+    mean_repetition_score: float  # Mean score across flagged words
+    slop_score: float  # Aggregate: flagged_words_per_10k × mean_repetition_score
+    total_content_words_dist: Distribution
+    chunk_size: int
+    chunk_count: int
+    metadata: dict[str, Any]
+
+
+@dataclass
+class RepetitiveNgram:
+    """A single n-gram flagged as abnormally repetitive.
+
+    Content n-grams (bigrams, trigrams, etc.) should rarely repeat verbatim
+    in natural writing. N-grams that repeat beyond a length-scaled threshold
+    are flagged.
+
+    Related GitHub Issue:
+        #28 - Verbal tics detection for slop analysis
+        https://github.com/craigtrim/pystylometry/issues/28
+
+    Attributes:
+        ngram: The flagged n-gram as a tuple of words.
+        count: Observed count in the text.
+        frequency_per_10k: Occurrences per 10,000 n-grams.
+        chunk_counts: Per-chunk occurrence counts.
+        distribution_entropy: Shannon entropy of the n-gram's chunk distribution.
+        distribution_variance: Variance of per-chunk counts.
+    """
+
+    ngram: tuple[str, ...]
+    count: int
+    frequency_per_10k: float
+    chunk_counts: list[int]
+    distribution_entropy: float
+    distribution_variance: float
+
+
+@dataclass
+class RepetitiveNgramsResult:
+    """Result from repetitive n-gram detection.
+
+    Detects bigrams, trigrams, or higher-order n-grams that repeat more than
+    expected within the text. No external corpus is required — content n-grams
+    should not repeat verbatim often in natural writing.
+
+    N-grams composed entirely of function words (e.g., "of the", "in a") are
+    excluded since their repetition is expected.
+
+    Related GitHub Issue:
+        #28 - Verbal tics detection for slop analysis
+        https://github.com/craigtrim/pystylometry/issues/28
+
+    Example:
+        >>> result = compute_repetitive_ngrams(text, n=2)
+        >>> for ng in result.repetitive_ngrams[:5]:
+        ...     print(f"{' '.join(ng.ngram)}: {ng.count}x "
+        ...           f"({ng.frequency_per_10k:.1f} per 10k)")
+        uncomfortable truth: 8x (1.6 per 10k)
+    """
+
+    repetitive_ngrams: list[RepetitiveNgram]  # Sorted by count descending
+    n: int | tuple[int, ...]  # N-gram order(s) analyzed
+    total_ngrams: int
+    flagged_count: int
+    flagged_per_10k: float  # flagged_count / (total_ngrams / 10000)
+    total_ngrams_dist: Distribution
+    chunk_size: int
+    chunk_count: int
+    metadata: dict[str, Any]
+
+
 # ===== Readability Results =====
 
 

pystylometry/lexical/__init__.py

@@ -5,6 +5,7 @@ from .advanced_diversity import compute_hdd, compute_mattr, compute_msttr, compu
 from .function_words import compute_function_words
 from .hapax import compute_hapax_ratios, compute_hapax_with_lexicon_analysis
 from .mtld import compute_mtld
+from .repetition import compute_repetitive_ngrams, compute_repetitive_unigrams
 from .ttr import compute_ttr
 from .word_frequency_sophistication import compute_word_frequency_sophistication
 from .yule import compute_yule
@@ -21,4 +22,6 @@ __all__ = [
     "compute_hdd",
     "compute_msttr",
     "compute_word_frequency_sophistication",
+    "compute_repetitive_unigrams",
+    "compute_repetitive_ngrams",
 ]

pystylometry/lexical/repetition.py

@@ -0,0 +1,506 @@
+"""Repetitive word and n-gram detection for verbal tics / slop analysis.
+
+This module detects abnormally repetitive words and phrases in text — a common
+pattern in AI-generated content ("slop") where certain content words and phrases
+appear far more frequently than expected.
+
+Generative models exhibit "verbal tics": they repeatedly use certain words and
+phrases throughout generated text. Examples include "shimmered", "flickered",
+"obsidian", "a testament to", "an uncomfortable truth". These patterns differ
+from natural human writing where content words appear when contextually relevant,
+repetition clusters around specific scenes or topics, and unusual words don't
+appear with suspiciously even distribution.
+
+Two functions are provided:
+
+    compute_repetitive_unigrams:
+        Compares observed word frequencies against the British National Corpus
+        (BNC, ~100M tokens) baseline. Words that appear far more than their
+        BNC relative frequency predicts are flagged.
+
+    compute_repetitive_ngrams:
+        Detects content n-grams (bigrams, trigrams, etc.) that repeat more
+        than expected. No external corpus is required — content n-grams should
+        not repeat verbatim often in natural writing.
+
+Both functions support chunked analysis to reveal distribution patterns:
+    - Even distribution across text = suspicious (model's consistent tic)
+    - Clustered distribution = likely intentional (human describing a scene)
+
+Related GitHub Issue:
+    #28 - Verbal tics detection for slop analysis
+    https://github.com/craigtrim/pystylometry/issues/28
+
+Dependencies:
+    - bnc-lookup >= 1.3.0 (optional, in lexical group)
+      Provides expected_count() and bucket() for BNC baseline comparison.
+
+References:
+    British National Corpus Consortium. (2007). The British National Corpus,
+        version 3 (BNC XML Edition). http://www.natcorp.ox.ac.uk/
+    Kilgarriff, A. (2001). BNC database and word frequency lists.
+        https://www.kilgarriff.co.uk/bnc-readme.html
+"""
+
+from __future__ import annotations
+
+import math
+import statistics
+from collections import Counter
+
+from .._types import (
+    Distribution,
+    RepetitiveNgram,
+    RepetitiveNgramsResult,
+    RepetitiveUnigramsResult,
+    RepetitiveWord,
+    chunk_text,
+    make_distribution,
+)
+from .._utils import check_optional_dependency, tokenize
+from .function_words import (
+    AUXILIARIES,
+    CONJUNCTIONS,
+    DETERMINERS,
+    PARTICLES,
+    PREPOSITIONS,
+    PRONOUNS,
+)
+
+# Union of all function word sets — used to filter out non-content words
+_FUNCTION_WORDS = DETERMINERS | PREPOSITIONS | CONJUNCTIONS | PRONOUNS | AUXILIARIES | PARTICLES
+
+
+def _chunk_entropy(chunk_counts: list[int]) -> float:
+    """Compute Shannon entropy of a word's distribution across chunks.
+
+    Entropy measures how evenly a word is distributed across chunks.
+    Low entropy means the word appears evenly (suspicious for rare words).
+    High entropy means the word is concentrated in specific chunks (natural).
+
+    Formula:
+        H = -sum(p_i * log2(p_i)) for each chunk i where p_i > 0
+        p_i = count_in_chunk_i / total_count
+
+    Args:
+        chunk_counts: Per-chunk occurrence counts.
+
+    Returns:
+        Shannon entropy in bits. 0.0 if the word appears in only one chunk.
+        Returns 0.0 for empty or all-zero counts.
+    """
+    total = sum(chunk_counts)
+    if total == 0:
+        return 0.0
+
+    entropy = 0.0
+    for count in chunk_counts:
+        if count > 0:
+            p = count / total
+            entropy -= p * math.log2(p)
+
+    return entropy
+
+
+def _tokenize_content_words(text: str) -> list[str]:
+    """Tokenize text and return only lowercase alphabetic content words.
+
+    Filters out:
+        - Non-alphabetic tokens (punctuation, numbers)
+        - Function words (determiners, prepositions, conjunctions,
+          pronouns, auxiliaries, particles)
+
+    Args:
+        text: Input text.
+
+    Returns:
+        List of lowercase content word tokens.
+    """
+    tokens = tokenize(text.lower())
+    return [t for t in tokens if t.isalpha() and t not in _FUNCTION_WORDS]
+
+
+def compute_repetitive_unigrams(
+    text: str,
+    threshold: float = 3.0,
+    chunk_size: int = 1000,
+    min_count: int = 3,
+) -> RepetitiveUnigramsResult:
+    """Detect content words that repeat far more than expected based on BNC frequencies.
+
+    For each content word in the text, computes:
+        expected_count = BNC_relative_frequency(word) * text_length
+        repetition_score = observed_count / expected_count
+
+    Words exceeding the threshold score and minimum count are flagged.
+
+    This function uses native chunked analysis to capture distribution patterns
+    across the text. Words that are evenly distributed (low entropy) are more
+    suspicious than words clustered in specific sections.
+
+    Related GitHub Issue:
+        #28 - Verbal tics detection for slop analysis
+        https://github.com/craigtrim/pystylometry/issues/28
+
+    References:
+        British National Corpus Consortium. (2007). The British National Corpus,
+            version 3 (BNC XML Edition). http://www.natcorp.ox.ac.uk/
+
+    Args:
+        text: Input text to analyze.
+        threshold: Minimum repetition_score (observed/expected) to flag a word.
+            Default 3.0 means the word must appear at least 3x more than expected.
+        chunk_size: Number of words per chunk for distribution analysis (default: 1000).
+        min_count: Minimum observed count to flag a word. Prevents flagging words
+            that appear only once or twice, which aren't meaningfully repetitive
+            regardless of their score. Default: 3.
+    Returns:
+        RepetitiveUnigramsResult with flagged words, aggregate scores, and metadata.
+
+    Example:
+        >>> result = compute_repetitive_unigrams(novel_text)
+        >>> for w in result.repetitive_words[:5]:
+        ...     print(f"{w.word}: {w.count}x (expected {w.expected_count:.1f}, "
+        ...           f"score {w.repetition_score:.1f})")
+        shimmered: 23x (expected 0.1, score 266.2)
+        obsidian: 18x (expected 0.0, score 450.0)
+        >>> print(f"Slop score: {result.slop_score:.1f}")
+        Slop score: 42.7
+    """
+    check_optional_dependency("bnc_lookup", "lexical")
+
+    from bnc_lookup import bucket as bnc_bucket  # type: ignore[import-untyped]
+    from bnc_lookup import expected_count as bnc_expected_count  # type: ignore[import-untyped]
+
+    # Chunk the text
+    chunks = chunk_text(text, chunk_size)
+
+    # Tokenize each chunk into content words
+    chunk_tokens: list[list[str]] = [_tokenize_content_words(chunk) for chunk in chunks]
+
+    # Count content words per chunk
+    chunk_counters: list[Counter[str]] = [Counter(tokens) for tokens in chunk_tokens]
+    content_words_per_chunk = [len(tokens) for tokens in chunk_tokens]
+
+    # Build global content word counts
+    global_counter: Counter[str] = Counter()
+    for counter in chunk_counters:
+        global_counter.update(counter)
+
+    total_content_words = sum(global_counter.values())
+
+    # Handle empty text
+    if total_content_words == 0:
+        empty_dist = Distribution(
+            values=[],
+            mean=float("nan"),
+            median=float("nan"),
+            std=0.0,
+            range=0.0,
+            iqr=0.0,
+        )
+        return RepetitiveUnigramsResult(
+            repetitive_words=[],
+            total_content_words=0,
+            flagged_count=0,
+            flagged_words_per_10k=0.0,
+            mean_repetition_score=0.0,
+            slop_score=0.0,
+            total_content_words_dist=empty_dist,
+            chunk_size=chunk_size,
+            chunk_count=len(chunks),
+            metadata={"threshold": threshold, "min_count": min_count},
+        )
+
+    # Evaluate each content word against BNC baseline
+    flagged: list[RepetitiveWord] = []
+
+    for word, observed in global_counter.items():
+        if observed < min_count:
+            continue
+
+        # Get BNC expected count for this word given our text length
+        expected = bnc_expected_count(word, total_content_words)
+        word_bucket = bnc_bucket(word)
+
+        if expected is None or expected == 0.0:
+            # Word not in BNC or has zero expected frequency
+            # Any repeated occurrence is notable
+            score = float("inf")
+            expected_val = 0.0
+        else:
+            expected_val = expected
+            score = observed / expected_val
+
+        if score >= threshold:
+            # Build per-chunk counts for this word
+            per_chunk = [counter.get(word, 0) for counter in chunk_counters]
+            entropy = _chunk_entropy(per_chunk)
+            variance = statistics.variance(per_chunk) if len(per_chunk) > 1 else 0.0
+
+            flagged.append(
+                RepetitiveWord(
+                    word=word,
+                    count=observed,
+                    expected_count=expected_val,
+                    repetition_score=score,
+                    bnc_bucket=word_bucket,
+                    chunk_counts=per_chunk,
+                    distribution_entropy=entropy,
+                    distribution_variance=variance,
+                )
+            )
+
+    # Sort by repetition_score descending (inf sorts last with key trick)
+    flagged.sort(
+        key=lambda w: (
+            -w.repetition_score if w.repetition_score != float("inf") else -1e18,
+            -w.count,
+        )
+    )
+
+    # Compute aggregate metrics
+    flagged_count = len(flagged)
+    flagged_words_per_10k = (
+        flagged_count / (total_content_words / 10_000) if total_content_words > 0 else 0.0
+    )
+
+    # Mean repetition score (exclude inf for meaningful average)
+    finite_scores = [w.repetition_score for w in flagged if w.repetition_score != float("inf")]
+    mean_rep_score = statistics.mean(finite_scores) if finite_scores else 0.0
+
+    slop_score = flagged_words_per_10k * mean_rep_score
+
+    # Content words distribution
+    content_dist = (
+        make_distribution([float(c) for c in content_words_per_chunk])
+        if content_words_per_chunk
+        else Distribution(
+            values=[], mean=float("nan"), median=float("nan"), std=0.0, range=0.0, iqr=0.0
+        )
+    )
+
+    return RepetitiveUnigramsResult(
+        repetitive_words=flagged,
+        total_content_words=total_content_words,
+        flagged_count=flagged_count,
+        flagged_words_per_10k=flagged_words_per_10k,
+        mean_repetition_score=mean_rep_score,
+        slop_score=slop_score,
+        total_content_words_dist=content_dist,
+        chunk_size=chunk_size,
+        chunk_count=len(chunks),
+        metadata={
+            "threshold": threshold,
+            "min_count": min_count,
+            "total_unique_content_words": len(global_counter),
+            "inf_score_count": sum(1 for w in flagged if w.repetition_score == float("inf")),
+        },
+    )
+
+
+def _validate_n(n: int | tuple[int, ...]) -> tuple[int, ...]:
+    """Validate and normalize the n-gram order parameter.
+
+    Args:
+        n: Single integer or tuple of integers specifying n-gram orders.
+
+    Returns:
+        Sorted tuple of unique valid n-gram orders.
+
+    Raises:
+        ValueError: If any value is outside the range [2, 5] or input is empty.
+    """
+    values: tuple[int, ...]
+    if isinstance(n, int):
+        values = (n,)
+    else:
+        values = tuple(sorted(set(n)))
+
+    if not values:
+        raise ValueError("n must specify at least one n-gram order.")
+
+    for v in values:
+        if v < 2:
+            raise ValueError(
+                f"n-gram order {v} is too small. Minimum is 2 (bigrams). "
+                f"For single-word repetition, use compute_repetitive_unigrams() instead."
+            )
+        if v > 5:
+            raise ValueError(
+                f"n-gram order {v} is too large. Maximum is 5. "
+                f"N-grams of order 6+ are too sparse to produce meaningful repetition "
+                f"signals in typical texts (they rarely repeat even once)."
+            )
+
+    return values
+
+
+def _is_content_ngram(ngram: tuple[str, ...]) -> bool:
+    """Check if an n-gram contains at least one content word.
+
+    An n-gram composed entirely of function words (e.g., "of the", "in a")
+    is expected to repeat and should not be flagged.
+
+    Args:
+        ngram: Tuple of words.
+
+    Returns:
+        True if at least one word is not a function word.
+    """
+    return any(word not in _FUNCTION_WORDS for word in ngram)
+
+
+def compute_repetitive_ngrams(
+    text: str,
+    n: int | tuple[int, ...] = (2, 3),
+    chunk_size: int = 1000,
+    min_count: int = 3,
+) -> RepetitiveNgramsResult:
+    """Detect content n-grams that repeat more than expected within the text.
+
+    Content n-grams (bigrams, trigrams, etc.) should rarely repeat verbatim in
+    natural writing. This function flags n-grams that exceed a length-scaled
+    threshold, filtering out n-grams composed entirely of function words.
+
+    No external corpus is required — the threshold is computed internally based
+    on text length. Any content n-gram appearing more than
+    max(min_count, total_ngrams / 10000) times is flagged.
+
+    Related GitHub Issue:
+        #28 - Verbal tics detection for slop analysis
+        https://github.com/craigtrim/pystylometry/issues/28
+
+    Args:
+        text: Input text to analyze.
+        n: N-gram order(s) to analyze. Can be a single integer (e.g., 2 for
+            bigrams) or a tuple of integers (e.g., (2, 3) for bigrams and
+            trigrams). Valid range: 2 to 5. Default: (2, 3).
+            - Values below 2 are rejected (use compute_repetitive_unigrams
+              for single words).
+            - Values above 5 are rejected (n-grams of order 6+ are too sparse
+              to produce meaningful repetition signals).
+        chunk_size: Number of words per chunk for distribution analysis (default: 1000).
+        min_count: Minimum count to flag an n-gram. Default: 3.
+
+    Returns:
+        RepetitiveNgramsResult with flagged n-grams, counts, and metadata.
+
+    Example:
+        >>> result = compute_repetitive_ngrams(text, n=2)
+        >>> for ng in result.repetitive_ngrams[:5]:
+        ...     print(f"{' '.join(ng.ngram)}: {ng.count}x")
+        uncomfortable truth: 8x
+        >>> result = compute_repetitive_ngrams(text, n=(2, 3, 4))
+        >>> print(f"Flagged: {result.flagged_count} n-grams")
+    """
+    # Validate n parameter
+    n_values = _validate_n(n)
+
+    # Chunk the text
+    chunks = chunk_text(text, chunk_size)
+
+    # Tokenize each chunk — lowercase alpha only (but keep function words
+    # so n-grams spanning content+function words are preserved; we filter
+    # all-function-word n-grams separately)
+    chunk_tokens: list[list[str]] = []
+    for chunk in chunks:
+        tokens = tokenize(chunk.lower())
+        chunk_tokens.append([t for t in tokens if t.isalpha()])
+
+    # Build n-grams per chunk for each requested order
+    # chunk_ngram_counters[chunk_idx] aggregates across all n values
+    chunk_ngram_counters: list[Counter[tuple[str, ...]]] = [Counter() for _ in chunks]
+    total_ngram_count = 0
+
+    for chunk_idx, tokens in enumerate(chunk_tokens):
+        for nv in n_values:
+            for i in range(len(tokens) - nv + 1):
+                ngram = tuple(tokens[i : i + nv])
+                if _is_content_ngram(ngram):
+                    chunk_ngram_counters[chunk_idx][ngram] += 1
+                    total_ngram_count += 1
+
+    # Build global counts
+    global_ngram_counter: Counter[tuple[str, ...]] = Counter()
+    for counter in chunk_ngram_counters:
+        global_ngram_counter.update(counter)
+
+    # Determine threshold: any content n-gram appearing more than this is flagged
+    length_threshold = max(min_count, total_ngram_count // 10_000)
+
+    # Handle empty text
+    if total_ngram_count == 0:
+        empty_dist = Distribution(
+            values=[],
+            mean=float("nan"),
+            median=float("nan"),
+            std=0.0,
+            range=0.0,
+            iqr=0.0,
+        )
+        return RepetitiveNgramsResult(
+            repetitive_ngrams=[],
+            n=n,
+            total_ngrams=0,
+            flagged_count=0,
+            flagged_per_10k=0.0,
+            total_ngrams_dist=empty_dist,
+            chunk_size=chunk_size,
+            chunk_count=len(chunks),
+            metadata={"min_count": min_count, "effective_threshold": length_threshold},
+        )
+
+    # Flag n-grams exceeding threshold
+    flagged: list[RepetitiveNgram] = []
+
+    for ngram, count in global_ngram_counter.items():
+        if count >= length_threshold:
+            per_chunk = [counter.get(ngram, 0) for counter in chunk_ngram_counters]
+            entropy = _chunk_entropy(per_chunk)
+            variance = statistics.variance(per_chunk) if len(per_chunk) > 1 else 0.0
+            freq_per_10k = count / (total_ngram_count / 10_000) if total_ngram_count > 0 else 0.0
+
+            flagged.append(
+                RepetitiveNgram(
+                    ngram=ngram,
+                    count=count,
+                    frequency_per_10k=freq_per_10k,
+                    chunk_counts=per_chunk,
+                    distribution_entropy=entropy,
+                    distribution_variance=variance,
+                )
+            )
+
+    # Sort by count descending
+    flagged.sort(key=lambda ng: -ng.count)
+
+    flagged_count = len(flagged)
+    flagged_per_10k = flagged_count / (total_ngram_count / 10_000) if total_ngram_count > 0 else 0.0
+
+    # N-grams per chunk distribution
+    ngrams_per_chunk = [sum(counter.values()) for counter in chunk_ngram_counters]
+    ngrams_dist = (
+        make_distribution([float(c) for c in ngrams_per_chunk])
+        if ngrams_per_chunk
+        else Distribution(
+            values=[], mean=float("nan"), median=float("nan"), std=0.0, range=0.0, iqr=0.0
+        )
+    )
+
+    return RepetitiveNgramsResult(
+        repetitive_ngrams=flagged,
+        n=n,
+        total_ngrams=total_ngram_count,
+        flagged_count=flagged_count,
+        flagged_per_10k=flagged_per_10k,
+        total_ngrams_dist=ngrams_dist,
+        chunk_size=chunk_size,
+        chunk_count=len(chunks),
+        metadata={
+            "min_count": min_count,
+            "effective_threshold": length_threshold,
+            "n_values": list(n_values),
+            "total_unique_ngrams": len(global_ngram_counter),
+        },
+    )

pystylometry-1.3.1.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Craig Trim
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pystylometry-1.3.1.dist-info/METADATA

@@ -0,0 +1,79 @@
+Metadata-Version: 2.1
+Name: pystylometry
+Version: 1.3.1
+Summary: Comprehensive Python package for stylometric analysis
+License: MIT
+Keywords: stylometry,nlp,text-analysis,authorship,readability,lexical-diversity,readability-metrics
+Author: Craig Trim
+Author-email: craigtrim@gmail.com
+Requires-Python: >=3.9,<4.0
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Typing :: Typed
+Requires-Dist: stylometry-ttr (>=1.0.3,<2.0.0)
+Project-URL: Homepage, https://github.com/craigtrim/pystylometry
+Project-URL: Issues, https://github.com/craigtrim/pystylometry/issues
+Project-URL: Repository, https://github.com/craigtrim/pystylometry
+Description-Content-Type: text/markdown
+
+# pystylometry
+
+[![PyPI version](https://badge.fury.io/py/pystylometry.svg)](https://badge.fury.io/py/pystylometry)
+[![Downloads](https://static.pepy.tech/badge/pystylometry)](https://pepy.tech/project/pystylometry)
+[![Downloads/Month](https://static.pepy.tech/badge/pystylometry/month)](https://pepy.tech/project/pystylometry)
+[![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT)
+[![Tests](https://img.shields.io/badge/tests-1022%20passed-brightgreen)]()
+
+Stylometric analysis and authorship attribution for Python. 50+ metrics across 11 modules, from vocabulary diversity to AI-generation detection.
+
+## Install
+
+```bash
+pip install pystylometry              # Core (lexical metrics)
+pip install pystylometry[all]         # Everything
+```
+
+## Modules
+
+| Module | Metrics | Description |
+|--------|---------|-------------|
+| [**lexical**](https://github.com/craigtrim/pystylometry/tree/master/pystylometry/lexical) | TTR, MTLD, Yule's K/I, Hapax, MATTR, VocD-D, HD-D, MSTTR, function words, word frequency | Vocabulary diversity and richness |
+| [**readability**](https://github.com/craigtrim/pystylometry/tree/master/pystylometry/readability) | Flesch, Flesch-Kincaid, SMOG, Gunning Fog, Coleman-Liau, ARI, Dale-Chall, Fry, FORCAST, Linsear Write, Powers-Sumner-Kearl | Grade-level and difficulty scoring |
+| [**syntactic**](https://github.com/craigtrim/pystylometry/tree/master/pystylometry/syntactic) | POS ratios, sentence types, parse tree depth, clausal density, passive voice, T-units, dependency distance | Sentence and parse structure (requires spaCy) |
+| [**authorship**](https://github.com/craigtrim/pystylometry/tree/master/pystylometry/authorship) | Burrows' Delta, Cosine Delta, Zeta, Kilgarriff chi-squared, MinMax, John's Delta, NCD | Author attribution and text comparison |
+| [**stylistic**](https://github.com/craigtrim/pystylometry/tree/master/pystylometry/stylistic) | Contractions, hedges, intensifiers, modals, punctuation, vocabulary overlap (Jaccard/Dice/Cosine/KL), cohesion, genre/register | Style markers and text similarity |
+| [**character**](https://github.com/craigtrim/pystylometry/tree/master/pystylometry/character) | Letter frequencies, digit/uppercase ratios, special characters, whitespace | Character-level fingerprinting |
+| [**ngrams**](https://github.com/craigtrim/pystylometry/tree/master/pystylometry/ngrams) | Word/character/POS n-grams, Shannon entropy, skipgrams | N-gram profiles and entropy |
+| [**dialect**](https://github.com/craigtrim/pystylometry/tree/master/pystylometry/dialect) | British/American classification, spelling/grammar/vocabulary markers, markedness | Regional dialect detection |
+| [**consistency**](https://github.com/craigtrim/pystylometry/tree/master/pystylometry/consistency) | Sliding-window chi-squared drift, pattern classification | Intra-document style analysis |
+| [**prosody**](https://github.com/craigtrim/pystylometry/tree/master/pystylometry/prosody) | Syllable stress, rhythm regularity | Prose rhythm (requires spaCy) |
+| [**viz**](https://github.com/craigtrim/pystylometry/tree/master/pystylometry/viz) | Timeline, scatter, report (PNG + interactive HTML) | Drift detection visualization |
+
+## Development
+
+```bash
+git clone https://github.com/craigtrim/pystylometry && cd pystylometry
+pip install -e ".[dev,all]"
+make test       # 1022 tests
+make lint       # ruff + mypy
+make all        # lint + test + build
+```
+
+## License
+
+MIT
+
+## Author
+
+Craig Trim -- craigtrim@gmail.com
+

{pystylometry-1.3.0.dist-info → pystylometry-1.3.1.dist-info}/RECORD

@@ -1,7 +1,7 @@
 pystylometry/README.md,sha256=WFOtCAF3qtDTgGG3a_jTjNSwVgpQEXI1PKqbVBfyo1M,2366
 pystylometry/__init__.py,sha256=Z6zkHlX05SUeObDca9dL1Gkfq4UPBWbU2M4sp4fVj78,9220
 pystylometry/_normalize.py,sha256=7tdfgAKg5CI2d4eoDypmFqOVByoxpwgUUZD6vyBH86A,8679
-pystylometry/_types.py,sha256=HddTq-8kGeXyTXFkUd26HmOlOhCOdIgEVULHp168ais,76563
+pystylometry/_types.py,sha256=g6XzwCHeMAIBfexId6Pd9EQfJzvZ0KYMfD4kpS5T7BQ,82284
 pystylometry/_utils.py,sha256=CXTx4KDJ_6iiHcc2OXqOYs-izhLf_ZEmJFKdHyd7q34,5282
 pystylometry/authorship/README.md,sha256=zNXCpLj7nczPnYykJnCUw3y-kxfC9mWZmngi3nfw6us,1016
 pystylometry/authorship/__init__.py,sha256=D7m38hWi_62o1ZDSrghLCfob9YsykTht4K37wiVgHfg,1530
@@ -24,11 +24,12 @@ pystylometry/dialect/_data/dialect_markers.json,sha256=DthluOA6q0rG_8IrCrFIYWh_E
 pystylometry/dialect/_loader.py,sha256=M2ATp-5754v_yX9EWvBP0r5qgNf8xlL8XadVsVb_Hco,12989
 pystylometry/dialect/detector.py,sha256=9x0ZuIfTIjsmdNSx0Ezy5AC0SAFtC4kVw11iOSBd9gQ,20147
 pystylometry/lexical/README.md,sha256=cFQ7KRZV4ubsQwIlOH3YHTbhhNl5X91Sr3zcn-3x0HI,1185
-pystylometry/lexical/__init__.py,sha256=HTncnGVZgpktZqpf-r4_HI_9Jq42WkZZKXn8nho3y3s,751
+pystylometry/lexical/__init__.py,sha256=_VpemdfVYZYXHP4ulTItoyegJ-3lE85wlfzDCpseaNE,898
 pystylometry/lexical/advanced_diversity.py,sha256=rL1hlNqTnaEFcA2v4oBJlojHZMTqdvvm4jYXTFGVpYE,25664
 pystylometry/lexical/function_words.py,sha256=eel9bq_qWgWlvG0NtDiouilMt9kaFqz2rh3add2UC4U,17832
 pystylometry/lexical/hapax.py,sha256=djTqZyZIYXa3GRiPoy6TTGHPm0wCRNJ9U0Rwnf5NoDk,12173
 pystylometry/lexical/mtld.py,sha256=XpeCF8sOXZhWbaazHGuqm08mrOf_DYfkfGGAltWnyy4,7101
+pystylometry/lexical/repetition.py,sha256=A9L0oNwfnCepVkWy57kjHV47Pw4M6fZXEl25hBVdq2s,18318
 pystylometry/lexical/ttr.py,sha256=iEsXkoSPyZEyiiFwKatKA8KhLRukD7RDRvyRkRQOTsk,5848
 pystylometry/lexical/word_frequency_sophistication.py,sha256=OHOS0fBvd1Bz8zsJk-pJbWLTgImmBd-aewQnp_kq8BY,38828
 pystylometry/lexical/yule.py,sha256=NXggha8jmQCu4i-qKZpISwyJBqNpuPHyVR86BLDLgio,5192
@@ -70,7 +71,8 @@ pystylometry/viz/jsx/_base.py,sha256=nd7kEc13fUcRMom3A5jqjGyTy-djIeydq2k3oPHZIHY
 pystylometry/viz/jsx/report.py,sha256=DbbHnnNAEi5tmVg4PmiHb17vkBBXujyE4x1CfVBiOBw,25857
 pystylometry/viz/jsx/timeline.py,sha256=hor-xnBa6oVkSqN0AEZUCQFBOB-iTfHSFZHiEfeakPA,30716
 pystylometry/viz/jsx/viewer.py,sha256=3LO49d_2bRf_P-P-2oSKpKx4N8Ugo4oCLb3DtvyNxXI,43716
-pystylometry-1.3.0.dist-info/METADATA,sha256=wsQ5QTEH7i6hpePEnlfDgJFKVHJi1m-HpMcHuznQt3c,5706
-pystylometry-1.3.0.dist-info/WHEEL,sha256=3ny-bZhpXrU6vSQ1UPG34FoxZBp3lVcvK0LkgUz6VLk,88
-pystylometry-1.3.0.dist-info/entry_points.txt,sha256=iHOaFXlyiwcQM1LlID2gWGmN4DBLdTSpKGjttU8tgm8,113
-pystylometry-1.3.0.dist-info/RECORD,,
+pystylometry-1.3.1.dist-info/LICENSE,sha256=vou5JCLAT5nHcsUv-AkjUYAihYfN9mwPDXxV2DHyHBo,1067
+pystylometry-1.3.1.dist-info/METADATA,sha256=Nn-0-ABq9tykuxWpC79GkhHO71oWLnAseh0z9R3mycs,4813
+pystylometry-1.3.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+pystylometry-1.3.1.dist-info/entry_points.txt,sha256=iHOaFXlyiwcQM1LlID2gWGmN4DBLdTSpKGjttU8tgm8,113
+pystylometry-1.3.1.dist-info/RECORD,,

{pystylometry-1.3.0.dist-info → pystylometry-1.3.1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.3.0
+Generator: poetry-core 1.9.0
 Root-Is-Purelib: true
 Tag: py3-none-any

pystylometry-1.3.0.dist-info/METADATA

@@ -1,136 +0,0 @@
-Metadata-Version: 2.4
-Name: pystylometry
-Version: 1.3.0
-Summary: Comprehensive Python package for stylometric analysis
-License: MIT
-Keywords: stylometry,nlp,text-analysis,authorship,readability,lexical-diversity,readability-metrics
-Author: Craig Trim
-Author-email: craigtrim@gmail.com
-Requires-Python: >=3.9,<4.0
-Classifier: Development Status :: 4 - Beta
-Classifier: Intended Audience :: Developers
-Classifier: Intended Audience :: Science/Research
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
-Classifier: Programming Language :: Python :: 3.14
-Classifier: Topic :: Scientific/Engineering :: Information Analysis
-Classifier: Topic :: Text Processing :: Linguistic
-Classifier: Typing :: Typed
-Requires-Dist: stylometry-ttr (>=1.0.3,<2.0.0)
-Project-URL: Homepage, https://github.com/craigtrim/pystylometry
-Project-URL: Issues, https://github.com/craigtrim/pystylometry/issues
-Project-URL: Repository, https://github.com/craigtrim/pystylometry
-Description-Content-Type: text/markdown
-
-# pystylometry
-
-[![PyPI version](https://badge.fury.io/py/pystylometry.svg)](https://badge.fury.io/py/pystylometry)
-[![Downloads](https://pepy.tech/badge/pystylometry)](https://pepy.tech/project/pystylometry)
-[![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/downloads/)
-[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT)
-[![Tests](https://img.shields.io/badge/tests-1022%20passed-brightgreen)]()
-
-Stylometric analysis and authorship attribution for Python. 50+ metrics across 11 modules, from vocabulary diversity to AI-generation detection.
-
-## Install
-
-```bash
-pip install pystylometry              # Core (lexical metrics)
-pip install pystylometry[all]         # Everything
-```
-
-<details>
-<summary>Individual extras</summary>
-
-```bash
-pip install pystylometry[readability]   # Readability formulas (pronouncing, spaCy)
-pip install pystylometry[syntactic]     # POS/parse analysis (spaCy)
-pip install pystylometry[authorship]    # Attribution methods
-pip install pystylometry[ngrams]        # N-gram entropy
-pip install pystylometry[viz]           # Matplotlib visualizations
-```
-</details>
-
-## Usage
-
-```python
-from pystylometry.lexical import compute_mtld, compute_yule
-from pystylometry.readability import compute_flesch
-
-result = compute_mtld(text)
-print(result.mtld_average)       # 72.4
-
-result = compute_flesch(text)
-print(result.reading_ease)       # 65.2
-print(result.grade_level)        # 8.1
-```
-
-Every function returns a typed dataclass with the score, components, and metadata -- never a bare float.
-
-### Unified API
-
-```python
-from pystylometry import analyze
-
-results = analyze(text, lexical=True, readability=True, syntactic=True)
-```
-
-### Style Drift Detection
-
-Detect authorship changes, spliced content, and AI-generated text within a single document.
-
-```python
-from pystylometry.consistency import compute_kilgarriff_drift
-
-result = compute_kilgarriff_drift(document)
-print(result.pattern)             # "sudden_spike"
-print(result.pattern_confidence)  # 0.71
-print(result.max_location)        # Window 23 -- the splice point
-```
-
-### CLI
-
-```bash
-pystylometry-drift manuscript.txt --window-size=500 --stride=250
-pystylometry-viewer report.html
-```
-
-## Modules
-
-| Module | Metrics | Description |
-|--------|---------|-------------|
-| [**lexical**](pystylometry/lexical/) | TTR, MTLD, Yule's K/I, Hapax, MATTR, VocD-D, HD-D, MSTTR, function words, word frequency | Vocabulary diversity and richness |
-| [**readability**](pystylometry/readability/) | Flesch, Flesch-Kincaid, SMOG, Gunning Fog, Coleman-Liau, ARI, Dale-Chall, Fry, FORCAST, Linsear Write, Powers-Sumner-Kearl | Grade-level and difficulty scoring |
-| [**syntactic**](pystylometry/syntactic/) | POS ratios, sentence types, parse tree depth, clausal density, passive voice, T-units, dependency distance | Sentence and parse structure (requires spaCy) |
-| [**authorship**](pystylometry/authorship/) | Burrows' Delta, Cosine Delta, Zeta, Kilgarriff chi-squared, MinMax, John's Delta, NCD | Author attribution and text comparison |
-| [**stylistic**](pystylometry/stylistic/) | Contractions, hedges, intensifiers, modals, punctuation, vocabulary overlap (Jaccard/Dice/Cosine/KL), cohesion, genre/register | Style markers and text similarity |
-| [**character**](pystylometry/character/) | Letter frequencies, digit/uppercase ratios, special characters, whitespace | Character-level fingerprinting |
-| [**ngrams**](pystylometry/ngrams/) | Word/character/POS n-grams, Shannon entropy, skipgrams | N-gram profiles and entropy |
-| [**dialect**](pystylometry/dialect/) | British/American classification, spelling/grammar/vocabulary markers, markedness | Regional dialect detection |
-| [**consistency**](pystylometry/consistency/) | Sliding-window chi-squared drift, pattern classification | Intra-document style analysis |
-| [**prosody**](pystylometry/prosody/) | Syllable stress, rhythm regularity | Prose rhythm (requires spaCy) |
-| [**viz**](pystylometry/viz/) | Timeline, scatter, report (PNG + interactive HTML) | Drift detection visualization |
-
-## Development
-
-```bash
-git clone https://github.com/craigtrim/pystylometry && cd pystylometry
-pip install -e ".[dev,all]"
-make test       # 1022 tests
-make lint       # ruff + mypy
-make all        # lint + test + build
-```
-
-## License
-
-MIT
-
-## Author
-
-Craig Trim -- craigtrim@gmail.com
-