pystylometry 0.1.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

pystylometry/lexical/yule.py

@@ -15,7 +15,7 @@ def compute_yule(text: str) -> YuleResult:
 
     Formula:
         K = 10⁴ × (Σm²×Vm - N) / N²
-        I = (V² / Σm²×Vm) - (1/N)
+        I = V² / (Σm²×Vm - N)
 
     Where:
         - N = total tokens
@@ -33,16 +33,29 @@ def compute_yule(text: str) -> YuleResult:
     Returns:
         YuleResult with .yule_k, .yule_i, and metadata
 
+        Note: For empty input or when Σm²×Vm = N (perfectly uniform vocabulary),
+        metrics will be float('nan') to indicate undefined values.
+
     Example:
         >>> result = compute_yule("The quick brown fox jumps over the lazy dog.")
         >>> print(f"Yule's K: {result.yule_k:.2f}")
         >>> print(f"Yule's I: {result.yule_i:.2f}")
+
+        >>> # Empty input returns NaN
+        >>> import math
+        >>> result_empty = compute_yule("")
+        >>> math.isnan(result_empty.yule_k)
+        True
     """
     tokens = tokenize(text.lower())
     N = len(tokens)  # noqa: N806
 
     if N == 0:
-        return YuleResult(yule_k=0.0, yule_i=0.0, metadata={"token_count": 0, "vocabulary_size": 0})
+        return YuleResult(
+            yule_k=float("nan"),
+            yule_i=float("nan"),
+            metadata={"token_count": 0, "vocabulary_size": 0},
+        )
 
     # Count frequency of each token
     freq_counter = Counter(tokens)
@@ -50,11 +63,25 @@ def compute_yule(text: str) -> YuleResult:
 
     # Count how many words occur with each frequency
     # Vm[m] = number of words that occur exactly m times
-    # freq_of_freqs = Counter(freq_counter.values())  # TODO: Will be needed for Yule's K
-
-    # TODO: Implement Yule's K and I calculations
-    yule_k = 0.0  # Placeholder
-    yule_i = 0.0  # Placeholder
+    freq_of_freqs = Counter(freq_counter.values())
+
+    # Calculate Σm²×Vm (sum of m-squared times Vm for all m)
+    # This is the sum across all frequency levels of:
+    # (frequency)² × (count of words at that frequency)
+    sum_m2_vm = sum(m * m * vm for m, vm in freq_of_freqs.items())
+
+    # Yule's K: 10⁴ × (Σm²×Vm - N) / N²
+    # K measures vocabulary repetitiveness (higher K = more repetitive)
+    yule_k = 10_000 * (sum_m2_vm - N) / (N * N)
+
+    # Yule's I: V² / (Σm²×Vm - N)
+    # I is the inverse measure (higher I = more diverse)
+    # If Σm²×Vm = N (perfectly uniform vocabulary), denominator is 0, return NaN
+    denominator = sum_m2_vm - N
+    if denominator == 0:
+        yule_i = float("nan")
+    else:
+        yule_i = (V * V) / denominator
 
     return YuleResult(
         yule_k=yule_k,

pystylometry/ngrams/__init__.py

@@ -5,9 +5,11 @@ from .entropy import (
     compute_ngram_entropy,
     compute_word_bigram_entropy,
 )
+from .extended_ngrams import compute_extended_ngrams
 
 __all__ = [
     "compute_ngram_entropy",
     "compute_character_bigram_entropy",
     "compute_word_bigram_entropy",
+    "compute_extended_ngrams",
 ]

pystylometry/ngrams/extended_ngrams.py

@@ -0,0 +1,235 @@
+"""Extended n-gram features for authorship attribution.
+
+This module provides comprehensive n-gram analysis beyond basic bigram/trigram
+entropy. Features include frequency distributions for higher-order n-grams,
+skipgrams (n-grams with gaps), and POS n-grams, all valuable for stylometric
+analysis and authorship attribution.
+
+Related GitHub Issue:
+    #19 - Extended N-gram Features
+    https://github.com/craigtrim/pystylometry/issues/19
+
+Features implemented:
+    - Word trigrams and 4-grams (frequency distributions, top n-grams)
+    - Skipgrams (n-grams with gaps, e.g., "the * dog")
+    - POS n-grams (part-of-speech tag sequences)
+    - Character trigrams and 4-grams
+    - N-gram diversity metrics
+    - Entropy calculations for each n-gram order
+
+References:
+    Guthrie, D., Allison, B., Liu, W., Guthrie, L., & Wilks, Y. (2006).
+        A closer look at skip-gram modelling. LREC.
+    Stamatatos, E. (2009). A survey of modern authorship attribution methods.
+        JASIST, 60(3), 538-556.
+    Kešelj, V., et al. (2003). N-gram-based author profiles for authorship
+        attribution. PACLING.
+"""
+
+from .._types import ExtendedNgramResult
+
+
+def compute_extended_ngrams(
+    text: str,
+    top_n: int = 20,
+    include_pos_ngrams: bool = True,
+    spacy_model: str = "en_core_web_sm",
+) -> ExtendedNgramResult:
+    """
+    Compute extended n-gram features for stylometric analysis.
+
+    Analyzes text to extract comprehensive n-gram statistics including
+    word trigrams/4-grams, skipgrams, POS n-grams, and character n-grams.
+    These features are powerful for authorship attribution because they
+    capture both lexical and syntactic patterns.
+
+    Related GitHub Issue:
+        #19 - Extended N-gram Features
+        https://github.com/craigtrim/pystylometry/issues/19
+
+    Why extended n-grams matter:
+
+    Word N-grams:
+        - Capture phrasal patterns and collocations
+        - Trigrams/4-grams more distinctive than bigrams
+        - Reveal preferred multi-word expressions
+        - Author-specific phrase preferences
+
+    Skipgrams:
+        - N-grams with gaps (e.g., "I * to" matches "I want to", "I have to")
+        - Capture syntactic frames independent of specific words
+        - Less sparse than contiguous n-grams
+        - Model long-distance dependencies
+
+    POS N-grams:
+        - Abstract syntactic patterns (e.g., "DET ADJ NOUN")
+        - Independent of vocabulary
+        - Capture grammatical preferences
+        - Complement word n-grams
+
+    Character N-grams:
+        - Language-independent features
+        - Capture morphological patterns
+        - Effective for short texts
+        - Robust to OCR errors
+
+    N-gram Types:
+
+    Contiguous Word N-grams:
+        - Trigrams: sequences of 3 words ("in the world")
+        - 4-grams: sequences of 4 words ("at the end of")
+
+    Skipgrams:
+        - 2-skipgrams with gap 1: "word1 _ word3"
+        - 3-skipgrams with gap 1: "word1 _ word3 word4"
+        - Variable gap sizes possible
+
+    POS N-grams:
+        - POS trigrams: "DET ADJ NOUN" (the quick fox)
+        - POS 4-grams: "VERB DET ADJ NOUN" (saw the quick fox)
+
+    Character N-grams:
+        - Character trigrams: "the", "he ", "e w"
+        - Character 4-grams: "the ", "he w", "e wo"
+
+    Args:
+        text: Input text to analyze. Should contain at least 100+ words for
+              meaningful n-gram statistics. Shorter texts will have sparse
+              distributions.
+        top_n: Number of most frequent n-grams to return for each type.
+               Default is 20. Larger values provide more detail but increase
+               result size.
+        include_pos_ngrams: Whether to compute POS n-grams. Requires spaCy
+                           and is slower. Default is True. Set to False for
+                           faster computation without syntactic features.
+        spacy_model: spaCy model for POS tagging (if include_pos_ngrams=True).
+                    Default is "en_core_web_sm".
+
+    Returns:
+        ExtendedNgramResult containing:
+
+        Word n-grams:
+            - top_word_trigrams: Most frequent word trigrams with counts
+            - top_word_4grams: Most frequent word 4-grams with counts
+            - word_trigram_count: Total unique word trigrams
+            - word_4gram_count: Total unique word 4-grams
+            - word_trigram_entropy: Shannon entropy of trigram distribution
+            - word_4gram_entropy: Shannon entropy of 4-gram distribution
+
+        Skipgrams:
+            - top_skipgrams_2_1: Top 2-skipgrams with gap of 1
+            - top_skipgrams_3_1: Top 3-skipgrams with gap of 1
+            - skipgram_2_1_count: Unique 2-skipgrams
+            - skipgram_3_1_count: Unique 3-skipgrams
+
+        POS n-grams (if include_pos_ngrams=True):
+            - top_pos_trigrams: Most frequent POS trigrams with counts
+            - top_pos_4grams: Most frequent POS 4-grams with counts
+            - pos_trigram_count: Unique POS trigrams
+            - pos_4gram_count: Unique POS 4-grams
+            - pos_trigram_entropy: Shannon entropy of POS trigram distribution
+
+        Character n-grams:
+            - top_char_trigrams: Most frequent character trigrams with counts
+            - top_char_4grams: Most frequent character 4-grams with counts
+            - char_trigram_entropy: Shannon entropy of char trigram distribution
+            - char_4gram_entropy: Shannon entropy of char 4-gram distribution
+
+        Metadata:
+            - Full frequency distributions
+            - Parameters used
+            - Token counts
+            - etc.
+
+    Example:
+        >>> result = compute_extended_ngrams("Sample text for analysis...")
+        >>> print(f"Top word trigrams: {result.top_word_trigrams[:3]}")
+        Top word trigrams: [('in the world', 5), ('of the world', 4), ('at the time', 3)]
+        >>> print(f"Word trigram entropy: {result.word_trigram_entropy:.2f}")
+        Word trigram entropy: 4.32
+        >>> print(f"Top POS trigrams: {result.top_pos_trigrams[:3]}")
+        Top POS trigrams: [('DET ADJ NOUN', 12), ('VERB DET NOUN', 8), ('DET NOUN VERB', 6)]
+
+        >>> # Compare authors using n-grams
+        >>> author1 = compute_extended_ngrams("Text by author 1...")
+        >>> author2 = compute_extended_ngrams("Text by author 2...")
+        >>> # Compare top_word_trigrams for distinctive phrases
+
+    Note:
+        - Memory usage scales with text length and n-gram order
+        - Longer texts have more unique n-grams (higher counts)
+        - POS n-grams require spaCy (slower but valuable)
+        - Character n-grams include whitespace
+        - Skipgrams can be very sparse (many unique patterns)
+        - Entropy values higher for more diverse n-gram distributions
+    """
+    # TODO: Implement extended n-gram analysis
+    # GitHub Issue #19: https://github.com/craigtrim/pystylometry/issues/19
+    #
+    # Implementation steps:
+    #
+    # Word N-grams:
+    # 1. Tokenize text into words (lowercase, basic cleaning)
+    # 2. Generate word trigrams:
+    #    - Slide window of size 3 across word list
+    #    - Create tuples of 3 consecutive words
+    #    - Count frequency of each trigram
+    # 3. Generate word 4-grams (similar, window size 4)
+    # 4. Sort by frequency, extract top_n for each
+    # 5. Calculate Shannon entropy for each distribution:
+    #    H = -sum(p * log2(p)) where p = freq / total
+    #
+    # Skipgrams:
+    # 6. Generate 2-skipgrams with gap 1:
+    #    - For each position i: (word[i], word[i+2])
+    #    - Skips middle word
+    #    - Count frequencies
+    # 7. Generate 3-skipgrams with gap 1:
+    #    - For each position i: (word[i], word[i+2], word[i+3])
+    #    - Pattern: word, skip, word, word
+    #    - Count frequencies
+    # 8. Sort and extract top_n skipgrams
+    #
+    # POS N-grams (if include_pos_ngrams):
+    # 9. Load spaCy model for POS tagging
+    # 10. Parse text to get POS tags for each word
+    # 11. Generate POS trigrams (same as word trigrams, but use POS tags)
+    # 12. Generate POS 4-grams
+    # 13. Count frequencies, extract top_n
+    # 14. Calculate Shannon entropy
+    #
+    # Character N-grams:
+    # 15. Generate character trigrams:
+    #     - Slide window of size 3 across character sequence
+    #     - Include spaces and punctuation
+    #     - Count frequencies
+    # 16. Generate character 4-grams (window size 4)
+    # 17. Sort and extract top_n for each
+    # 18. Calculate Shannon entropy
+    #
+    # Diversity Metrics:
+    # 19. Count total unique n-grams for each type
+    # 20. Calculate type-token ratios (unique / total)
+    #
+    # Metadata:
+    # 21. Store full frequency distributions (optional, can be large)
+    # 22. Store parameters: top_n, include_pos_ngrams, model
+    # 23. Store token/character counts
+    #
+    # Helper Functions Needed:
+    #   - generate_ngrams(sequence, n) -> list[tuple]
+    #   - generate_skipgrams(sequence, n, gap) -> list[tuple]
+    #   - calculate_shannon_entropy(freq_dist) -> float
+    #   - get_top_n(freq_dist, n) -> list[tuple]
+    #
+    # Return ExtendedNgramResult
+    #
+    # Optimization notes:
+    #   - Use Counter from collections for frequency counting
+    #   - Consider sampling for very long texts
+    #   - Limit maximum n-gram types to prevent memory issues
+    #   - POS tagging is slowest step - make it optional
+    raise NotImplementedError(
+        "Extended n-gram features not yet implemented. "
+        "See GitHub Issue #19: https://github.com/craigtrim/pystylometry/issues/19"
+    )

pystylometry/prosody/__init__.py

@@ -0,0 +1,12 @@
+"""Rhythm and prosody metrics for written text.
+
+Related GitHub Issue:
+    #25 - Rhythm and Prosody Metrics
+    https://github.com/craigtrim/pystylometry/issues/25
+"""
+
+from .rhythm_prosody import compute_rhythm_prosody
+
+__all__ = [
+    "compute_rhythm_prosody",
+]

pystylometry/prosody/rhythm_prosody.py

@@ -0,0 +1,53 @@
+"""Rhythm and prosody metrics for written text.
+
+This module captures the musical qualities of written language, including
+stress patterns, syllable rhythms, and phonological features. While traditionally
+studied in spoken language, written text preserves many rhythmic patterns.
+
+Related GitHub Issue:
+    #25 - Rhythm and Prosody Metrics
+    https://github.com/craigtrim/pystylometry/issues/25
+
+Features analyzed:
+    - Syllable patterns and stress patterns
+    - Rhythmic regularity (coefficient of variation)
+    - Phonological features (alliteration, assonance)
+    - Syllable complexity (consonant clusters)
+    - Sentence rhythm
+    - Polysyllabic word usage
+
+References:
+    Lea, R. B., Mulligan, E. J., & Walton, J. H. (2005). Sentence rhythm and
+        text comprehension. Memory & Cognition, 33(3), 388-396.
+"""
+
+from .._types import RhythmProsodyResult
+
+
+def compute_rhythm_prosody(text: str) -> RhythmProsodyResult:
+    """
+    Compute rhythm and prosody metrics for written text.
+
+    Related GitHub Issue:
+        #25 - Rhythm and Prosody Metrics
+        https://github.com/craigtrim/pystylometry/issues/25
+
+    Args:
+        text: Input text to analyze
+
+    Returns:
+        RhythmProsodyResult with syllable patterns, rhythmic regularity,
+        phonological features, stress patterns, and complexity metrics.
+
+    Example:
+        >>> result = compute_rhythm_prosody("Sample text with rhythm...")
+        >>> print(f"Syllables/word: {result.mean_syllables_per_word:.2f}")
+        >>> print(f"Rhythmic regularity: {result.rhythmic_regularity:.3f}")
+        >>> print(f"Alliteration density: {result.alliteration_density:.2f}")
+    """
+    # TODO: Implement rhythm and prosody analysis
+    # GitHub Issue #25: https://github.com/craigtrim/pystylometry/issues/25
+    raise NotImplementedError(
+        "Rhythm and prosody metrics not yet implemented. "
+        "See GitHub Issue #25: https://github.com/craigtrim/pystylometry/issues/25"
+    )

pystylometry/readability/__init__.py

@@ -1,5 +1,12 @@
 """Readability metrics."""
 
+from .additional_formulas import (
+    compute_dale_chall,
+    compute_forcast,
+    compute_fry,
+    compute_linsear_write,
+    compute_powers_sumner_kearl,
+)
 from .ari import compute_ari
 from .coleman_liau import compute_coleman_liau
 from .flesch import compute_flesch
@@ -12,4 +19,9 @@ __all__ = [
     "compute_gunning_fog",
     "compute_coleman_liau",
     "compute_ari",
+    "compute_dale_chall",
+    "compute_linsear_write",
+    "compute_fry",
+    "compute_forcast",
+    "compute_powers_sumner_kearl",
 ]