pystylometry 0.1.0__py3-none-any.whl → 1.1.0__py3-none-any.whl

pystylometry/readability/smog.py

@@ -1,14 +1,62 @@
-"""SMOG (Simple Measure of Gobbledygook) Index."""
+"""SMOG (Simple Measure of Gobbledygook) Index.
 
-from .._types import SMOGResult
+This module implements the SMOG readability formula with native chunked
+analysis for stylometric fingerprinting.
+
+Related GitHub Issue:
+    #27 - Native chunked analysis with Distribution dataclass
+    https://github.com/craigtrim/pystylometry/issues/27
+"""
+
+import math
+
+from .._normalize import normalize_for_readability
+from .._types import Distribution, SMOGResult, chunk_text, make_distribution
 from .._utils import split_sentences, tokenize
 from .syllables import count_syllables
 
 
-def compute_smog(text: str) -> SMOGResult:
+def _compute_smog_single(text: str) -> tuple[float, float, dict]:
+    """Compute SMOG metrics for a single chunk of text.
+
+    Returns:
+        Tuple of (smog_index, grade_level, metadata_dict).
+        Returns (nan, nan, metadata) for empty/invalid input.
+    """
+    sentences = split_sentences(text)
+    tokens = tokenize(text)
+    word_tokens = normalize_for_readability(tokens)
+
+    if len(sentences) == 0 or len(word_tokens) == 0:
+        return (
+            float("nan"),
+            float("nan"),
+            {"sentence_count": 0, "word_count": 0, "polysyllable_count": 0},
+        )
+
+    # Count polysyllables (words with 3+ syllables)
+    polysyllable_count = sum(1 for word in word_tokens if count_syllables(word) >= 3)
+
+    # SMOG formula
+    smog_index = 1.043 * math.sqrt(polysyllable_count * 30 / len(sentences)) + 3.1291
+    grade_level = max(0, min(20, math.floor(smog_index + 0.5)))
+
+    metadata = {
+        "sentence_count": len(sentences),
+        "word_count": len(word_tokens),
+        "polysyllable_count": polysyllable_count,
+    }
+
+    return (smog_index, float(grade_level), metadata)
+
+
+def compute_smog(text: str, chunk_size: int = 1000) -> SMOGResult:
     """
     Compute SMOG (Simple Measure of Gobbledygook) Index.
 
+    This function uses native chunked analysis to capture variance and patterns
+    across the text, which is essential for stylometric fingerprinting.
+
     Formula:
         SMOG = 1.043 × √(polysyllables × 30/sentences) + 3.1291
 
@@ -17,55 +65,105 @@ def compute_smog(text: str) -> SMOGResult:
     The SMOG index estimates the years of education needed to understand the text.
     It's particularly useful for healthcare materials.
 
+    Related GitHub Issue:
+        #27 - Native chunked analysis with Distribution dataclass
+        https://github.com/craigtrim/pystylometry/issues/27
+
     References:
         McLaughlin, G. H. (1969). SMOG grading: A new readability formula.
         Journal of Reading, 12(8), 639-646.
 
     Args:
         text: Input text to analyze
+        chunk_size: Number of words per chunk (default: 1000).
+            The text is divided into chunks of this size, and metrics are
+            computed per-chunk.
 
     Returns:
-        SMOGResult with SMOG index and grade level
+        SMOGResult with:
+            - smog_index: Mean SMOG index across chunks
+            - grade_level: Mean grade level across chunks
+            - smog_index_dist: Distribution with per-chunk values and stats
+            - grade_level_dist: Distribution with per-chunk values and stats
+            - chunk_size: The chunk size used
+            - chunk_count: Number of chunks analyzed
 
     Example:
-        >>> result = compute_smog("The quick brown fox jumps over the lazy dog.")
-        >>> print(f"SMOG Index: {result.smog_index:.1f}")
-        >>> print(f"Grade Level: {result.grade_level}")
+        >>> result = compute_smog("Long text here...", chunk_size=1000)
+        >>> result.smog_index  # Mean across chunks
+        12.5
+        >>> result.smog_index_dist.std  # Variance reveals fingerprint
+        1.8
     """
-    sentences = split_sentences(text)
-    tokens = tokenize(text)
+    # Chunk the text
+    chunks = chunk_text(text, chunk_size)
+
+    # Compute metrics per chunk
+    smog_values = []
+    grade_values = []
+    total_sentences = 0
+    total_words = 0
+    total_polysyllables = 0
 
-    if len(sentences) < 30:
-        # SMOG requires at least 30 sentences for accuracy
-        # We'll compute anyway but note in metadata
-        pass
+    for chunk in chunks:
+        si, gl, meta = _compute_smog_single(chunk)
+        if not math.isnan(si):
+            smog_values.append(si)
+            grade_values.append(gl)
+        total_sentences += meta.get("sentence_count", 0)
+        total_words += meta.get("word_count", 0)
+        total_polysyllables += meta.get("polysyllable_count", 0)
 
-    if len(sentences) == 0 or len(tokens) == 0:
+    # Handle empty or all-invalid chunks
+    if not smog_values:
+        empty_dist = Distribution(
+            values=[],
+            mean=float("nan"),
+            median=float("nan"),
+            std=0.0,
+            range=0.0,
+            iqr=0.0,
+        )
         return SMOGResult(
-            smog_index=0.0,
-            grade_level=0,
+            smog_index=float("nan"),
+            grade_level=float("nan"),
+            smog_index_dist=empty_dist,
+            grade_level_dist=empty_dist,
+            chunk_size=chunk_size,
+            chunk_count=len(chunks),
             metadata={
+                # Backward-compatible keys
                 "sentence_count": 0,
                 "word_count": 0,
                 "polysyllable_count": 0,
+                # New prefixed keys for consistency
+                "total_sentence_count": 0,
+                "total_word_count": 0,
+                "total_polysyllable_count": 0,
                 "warning": "Insufficient text",
             },
         )
 
-    # Count polysyllables (words with 3+ syllables)
-    polysyllable_count = sum(1 for word in tokens if count_syllables(word) >= 3)
-
-    # TODO: Implement SMOG formula
-    smog_index = 0.0  # Placeholder
-    grade_level = 0  # Placeholder
+    # Build distributions
+    smog_dist = make_distribution(smog_values)
+    grade_dist = make_distribution(grade_values)
 
     return SMOGResult(
-        smog_index=smog_index,
-        grade_level=grade_level,
+        smog_index=smog_dist.mean,
+        grade_level=grade_dist.mean,
+        smog_index_dist=smog_dist,
+        grade_level_dist=grade_dist,
+        chunk_size=chunk_size,
+        chunk_count=len(chunks),
         metadata={
-            "sentence_count": len(sentences),
-            "word_count": len(tokens),
-            "polysyllable_count": polysyllable_count,
-            "warning": "Less than 30 sentences" if len(sentences) < 30 else None,
+            # Backward-compatible keys
+            "sentence_count": total_sentences,
+            "word_count": total_words,
+            "polysyllable_count": total_polysyllables,
+            # New prefixed keys for consistency
+            "total_sentence_count": total_sentences,
+            "total_word_count": total_words,
+            "total_polysyllable_count": total_polysyllables,
+            "warning": "Less than 30 sentences" if total_sentences < 30 else None,
         },
     )

pystylometry/readability/syllables.py

@@ -1,54 +1,161 @@
-"""Syllable counting utilities using CMU Pronouncing Dictionary."""
+"""
+Syllable counting using CMU Pronouncing Dictionary.
 
+Uses the pronouncing library which provides access to the CMU Pronouncing
+Dictionary for high-accuracy syllable counting based on phonetic transcriptions.
+"""
 
+import re
+from functools import lru_cache
+
+try:
+    import pronouncing  # type: ignore[import-untyped]
+except ImportError:
+    raise ImportError(
+        "The 'pronouncing' library is required for syllable counting. "
+        "Install it with: pip install pystylometry[readability]"
+    )
+
+
+@lru_cache(maxsize=4096)
 def count_syllables(word: str) -> int:
     """
-    Count syllables in a word using CMU Pronouncing Dictionary with heuristic fallback.
+    Count syllables using CMU Pronouncing Dictionary.
+
+    Uses phonetic transcriptions from CMU dictionary. For words with multiple
+    pronunciations, uses the first pronunciation (typically the most common).
+    Falls back to simple vowel counting for words not in the dictionary.
 
     Args:
-        word: The word to count syllables for
+        word: Input word (handles mixed case, strips whitespace)
 
     Returns:
-        Number of syllables in the word
+        Syllable count (minimum 1 for non-empty input)
+
+    Example:
+        >>> count_syllables("beautiful")
+        3
+        >>> count_syllables("fire")
+        2
+        >>> count_syllables("cruel")
+        1
     """
-    # TODO: Implement with pronouncing library
-    # For now, use simple heuristic fallback
-    return _heuristic_syllable_count(word)
+    word = word.lower().strip()
+    if not word:
+        return 0
 
+    # Strip common punctuation
+    word = word.strip(".,;:!?\"'()-")
+    if not word:
+        return 0
 
-def _heuristic_syllable_count(word: str) -> int:
-    """
-    Simple heuristic syllable counter for fallback.
+    # Handle contractions by removing apostrophes
+    if "'" in word:
+        word = word.replace("'", "")
 
-    This is a basic implementation that counts vowel groups.
-    Should be replaced with CMU dict lookup when pronouncing is available.
+    # Handle hyphenated compounds
+    if "-" in word:
+        return sum(count_syllables(part) for part in word.split("-") if part)
 
-    Args:
-        word: The word to count syllables for
+    # Get pronunciations from CMU dictionary
+    phones_list = pronouncing.phones_for_word(word)
 
-    Returns:
-        Estimated number of syllables
+    if phones_list:
+        # Use first pronunciation (most common)
+        # Count stress markers (0, 1, 2) in phoneme representation
+        phones = phones_list[0]
+        return pronouncing.syllable_count(phones)  # type: ignore[no-any-return]
+
+    # Fallback for words not in dictionary: simple vowel counting
+    return _fallback_count(word)
+
+
+def _fallback_count(word: str) -> int:
     """
-    word = word.lower().strip()
-    if len(word) == 0:
-        return 0
+    Simple fallback syllable counter for words not in CMU dictionary.
 
+    Uses basic vowel counting with silent-e adjustment.
+    Less accurate than CMU but handles rare/technical words.
+    """
     vowels = "aeiouy"
-    syllable_count = 0
-    previous_was_vowel = False
+    count = 0
+    prev_was_vowel = False
 
     for char in word:
         is_vowel = char in vowels
-        if is_vowel and not previous_was_vowel:
-            syllable_count += 1
-        previous_was_vowel = is_vowel
+        if is_vowel and not prev_was_vowel:
+            count += 1
+        prev_was_vowel = is_vowel
 
     # Adjust for silent 'e'
-    if word.endswith("e") and syllable_count > 1:
-        syllable_count -= 1
+    if word.endswith("e") and count > 1:
+        count -= 1
+
+    # Ensure minimum of 1
+    return max(1, count)
+
+
+def count_syllables_text(text: str) -> list[tuple[str, int]]:
+    """
+    Count syllables for all words in a text.
+
+    Args:
+        text: Input text
+
+    Returns:
+        List of (word, syllable_count) tuples
+
+    Example:
+        >>> count_syllables_text("The quick brown fox")
+        [('The', 1), ('quick', 1), ('brown', 1), ('fox', 1)]
+    """
+
+    words = re.findall(r"[a-zA-Z']+", text)
+    return [(w, count_syllables(w)) for w in words]
+
+
+def total_syllables(text: str) -> int:
+    """
+    Return total syllable count for text.
+
+    Args:
+        text: Input text
+
+    Returns:
+        Total number of syllables
+
+    Example:
+        >>> total_syllables("The quick brown fox")
+        4
+    """
+    return sum(count for _, count in count_syllables_text(text))
+
+
+def validate_accuracy(
+    test_pairs: list[tuple[str, int]],
+) -> tuple[float, list[tuple[str, int, int]]]:
+    """
+    Test accuracy against known word-syllable pairs.
+
+    Args:
+        test_pairs: List of (word, expected_syllables) tuples
+
+    Returns:
+        (accuracy_percentage, list of (word, expected, got) for failures)
+
+    Example:
+        >>> test_pairs = [("hello", 2), ("world", 1), ("beautiful", 3)]
+        >>> accuracy, failures = validate_accuracy(test_pairs)
+        >>> print(f"Accuracy: {accuracy:.1f}%")
+    """
+    failures = []
+    for word, expected in test_pairs:
+        got = count_syllables(word)
+        if got != expected:
+            failures.append((word, expected, got))
 
-    # Ensure at least one syllable
-    if syllable_count == 0:
-        syllable_count = 1
+    if not test_pairs:
+        return 0.0, []
 
-    return syllable_count
+    accuracy = (len(test_pairs) - len(failures)) / len(test_pairs) * 100
+    return accuracy, failures

pystylometry/stylistic/__init__.py

@@ -0,0 +1,20 @@
+"""Stylistic analysis metrics.
+
+Related GitHub Issues:
+    #20 - Stylistic Markers
+    #21 - Vocabulary Overlap and Similarity Metrics
+    #22 - Cohesion and Coherence Metrics
+    #23 - Genre and Register Features
+"""
+
+from .cohesion_coherence import compute_cohesion_coherence
+from .genre_register import compute_genre_register
+from .markers import compute_stylistic_markers
+from .vocabulary_overlap import compute_vocabulary_overlap
+
+__all__ = [
+    "compute_stylistic_markers",
+    "compute_vocabulary_overlap",
+    "compute_cohesion_coherence",
+    "compute_genre_register",
+]

pystylometry/stylistic/cohesion_coherence.py

@@ -0,0 +1,45 @@
+"""Cohesion and coherence metrics.
+
+This module measures how well a text holds together structurally (cohesion)
+and semantically (coherence). Important for analyzing writing quality and
+authorial sophistication.
+
+Related GitHub Issue:
+    #22 - Cohesion and Coherence Metrics
+    https://github.com/craigtrim/pystylometry/issues/22
+
+References:
+    Halliday, M. A. K., & Hasan, R. (1976). Cohesion in English. Longman.
+    Graesser, A. C., McNamara, D. S., & Kulikowich, J. M. (2011). Coh-Metrix.
+"""
+
+from .._types import CohesionCoherenceResult
+
+
+def compute_cohesion_coherence(text: str, model: str = "en_core_web_sm") -> CohesionCoherenceResult:
+    """
+    Compute cohesion and coherence metrics.
+
+    Related GitHub Issue:
+        #22 - Cohesion and Coherence Metrics
+        https://github.com/craigtrim/pystylometry/issues/22
+
+    Args:
+        text: Input text to analyze
+        model: spaCy model for linguistic analysis
+
+    Returns:
+        CohesionCoherenceResult with referential cohesion, lexical cohesion,
+        connective density, and coherence scores.
+
+    Example:
+        >>> result = compute_cohesion_coherence("Multi-paragraph text...")
+        >>> print(f"Pronoun density: {result.pronoun_density:.2f}")
+        >>> print(f"Connective density: {result.connective_density:.2f}")
+    """
+    # TODO: Implement cohesion/coherence analysis
+    # GitHub Issue #22: https://github.com/craigtrim/pystylometry/issues/22
+    raise NotImplementedError(
+        "Cohesion/coherence metrics not yet implemented. "
+        "See GitHub Issue #22: https://github.com/craigtrim/pystylometry/issues/22"
+    )

pystylometry/stylistic/genre_register.py

@@ -0,0 +1,45 @@
+"""Genre and register classification features.
+
+This module extracts features that distinguish between different text types
+(academic, journalistic, fiction, legal, etc.) and formality levels.
+
+Related GitHub Issue:
+    #23 - Genre and Register Features
+    https://github.com/craigtrim/pystylometry/issues/23
+
+References:
+    Biber, D. (1988). Variation across speech and writing. Cambridge University Press.
+    Biber, D., & Conrad, S. (2009). Register, genre, and style.
+"""
+
+from .._types import GenreRegisterResult
+
+
+def compute_genre_register(text: str, model: str = "en_core_web_sm") -> GenreRegisterResult:
+    """
+    Analyze genre and register features for text classification.
+
+    Related GitHub Issue:
+        #23 - Genre and Register Features
+        https://github.com/craigtrim/pystylometry/issues/23
+
+    Args:
+        text: Input text to analyze
+        model: spaCy model for linguistic analysis
+
+    Returns:
+        GenreRegisterResult with formality scores, register classification,
+        genre predictions, and feature scores for major genres.
+
+    Example:
+        >>> result = compute_genre_register("Academic paper text...")
+        >>> print(f"Formality score: {result.formality_score:.2f}")
+        >>> print(f"Predicted genre: {result.predicted_genre}")
+        >>> print(f"Academic score: {result.academic_score:.3f}")
+    """
+    # TODO: Implement genre/register analysis
+    # GitHub Issue #23: https://github.com/craigtrim/pystylometry/issues/23
+    raise NotImplementedError(
+        "Genre/register classification not yet implemented. "
+        "See GitHub Issue #23: https://github.com/craigtrim/pystylometry/issues/23"
+    )

pystylometry/stylistic/markers.py

@@ -0,0 +1,131 @@
+"""Stylistic markers for authorship attribution.
+
+This module identifies and analyzes specific linguistic features that authors
+use consistently and often subconsciously. These markers include contraction
+preferences, intensifier usage, hedging patterns, modal auxiliaries, negation
+patterns, and punctuation style habits.
+
+Related GitHub Issue:
+    #20 - Stylistic Markers
+    https://github.com/craigtrim/pystylometry/issues/20
+
+Categories of stylistic markers:
+    - Contraction patterns (can't vs. cannot, I'm vs. I am)
+    - Intensifiers (very, really, extremely, quite)
+    - Hedges (maybe, perhaps, probably, somewhat)
+    - Modal auxiliaries (can, could, may, might, must, should, will, would)
+    - Negation patterns (not, no, never, none, neither)
+    - Punctuation style (exclamations, questions, quotes, parentheticals)
+
+References:
+    Argamon, S., & Levitan, S. (2005). Measuring the usefulness of function
+        words for authorship attribution. ACH/ALLC.
+    Pennebaker, J. W. (2011). The secret life of pronouns. Bloomsbury Press.
+    Biber, D. (1988). Variation across speech and writing. Cambridge University Press.
+"""
+
+from .._types import StylisticMarkersResult
+
+
+def compute_stylistic_markers(text: str) -> StylisticMarkersResult:
+    """
+    Analyze stylistic markers for authorship attribution.
+
+    Identifies and quantifies specific linguistic features that reveal authorial
+    style. These features are often used subconsciously and remain consistent
+    across an author's works, making them valuable for attribution.
+
+    Related GitHub Issue:
+        #20 - Stylistic Markers
+        https://github.com/craigtrim/pystylometry/issues/20
+
+    Why stylistic markers matter:
+
+    Subconscious usage:
+        - Authors don't deliberately vary these features
+        - Remain consistent even when author tries to disguise style
+        - Difficult to consciously control
+
+    Genre-independent:
+        - Used similarly across different topics
+        - More stable than content words
+        - Complement content-based features
+
+    Psychologically meaningful:
+        - Reveal personality traits (Pennebaker's research)
+        - Indicate emotional state
+        - Show cognitive patterns
+
+    Marker Categories Analyzed:
+
+    1. Contractions:
+       - Preference for contracted vs. expanded forms
+       - Examples: can't/cannot, I'm/I am, won't/will not
+       - Formality indicator (more contractions = informal)
+
+    2. Intensifiers:
+       - Words that amplify meaning
+       - Examples: very, really, extremely, quite, rather
+       - Indicate emphatic style
+
+    3. Hedges:
+       - Words that weaken or qualify statements
+       - Examples: maybe, perhaps, probably, somewhat, kind of
+       - Indicate tentative or cautious style
+
+    4. Modal Auxiliaries:
+       - Express necessity, possibility, permission
+       - Epistemic modals: may, might, could (possibility)
+       - Deontic modals: must, should, ought (obligation)
+
+    5. Negation:
+       - Patterns of negative expression
+       - not, no, never, none, neither, nowhere
+       - Frequency and type vary by author
+
+    6. Punctuation Style:
+       - Exclamation marks: Emphatic, emotional
+       - Question marks: Interactive, rhetorical
+       - Quotation marks: Dialogue, scare quotes
+       - Parentheticals: Asides, additional info
+       - Ellipses: Trailing off, suspense
+       - Dashes: Interruptions, emphasis
+       - Semicolons/colons: Sophisticated syntax
+
+    Args:
+        text: Input text to analyze. Should contain at least 200+ words for
+              reliable statistics. Shorter texts may have unstable marker ratios.
+
+    Returns:
+        StylisticMarkersResult containing extensive marker statistics.
+        See _types.py for complete field list.
+
+    Example:
+        >>> result = compute_stylistic_markers("Sample text with markers...")
+        >>> print(f"Contraction ratio: {result.contraction_ratio * 100:.1f}%")
+        Contraction ratio: 42.3%
+        >>> print(f"Intensifiers/100 words: {result.intensifier_density:.2f}")
+        Intensifiers/100 words: 3.45
+        >>> print(f"Top intensifiers: {result.top_intensifiers[:3]}")
+        Top intensifiers: [('very', 12), ('really', 8), ('quite', 5)]
+        >>> print(f"Exclamation density: {result.exclamation_density:.2f}")
+        Exclamation density: 2.10
+
+    Note:
+        - Densities are per 100 words for interpretability
+        - Contraction detection requires pattern matching
+        - Modal auxiliaries classified as epistemic or deontic
+        - Punctuation counts include all occurrences
+        - Empty text returns NaN for ratios, 0 for counts
+    """
+    # TODO: Implement stylistic marker analysis
+    # GitHub Issue #20: https://github.com/craigtrim/pystylometry/issues/20
+    #
+    # This is a comprehensive implementation with many components.
+    # Break it down into logical sections.
+    #
+    # See GitHub issue for full implementation plan and word lists.
+    raise NotImplementedError(
+        "Stylistic markers not yet implemented. "
+        "See GitHub Issue #20: https://github.com/craigtrim/pystylometry/issues/20"
+    )

pystylometry/stylistic/vocabulary_overlap.py

@@ -0,0 +1,47 @@
+"""Vocabulary overlap and similarity metrics.
+
+This module computes similarity measures between two texts based on their
+shared vocabulary. Useful for authorship verification, plagiarism detection,
+and measuring stylistic consistency.
+
+Related GitHub Issue:
+    #21 - Vocabulary Overlap and Similarity Metrics
+    https://github.com/craigtrim/pystylometry/issues/21
+
+References:
+    Jaccard, P. (1912). The distribution of the flora in the alpine zone.
+    Salton, G., & McGill, M. J. (1983). Introduction to Modern Information Retrieval.
+"""
+
+from .._types import VocabularyOverlapResult
+
+
+def compute_vocabulary_overlap(text1: str, text2: str) -> VocabularyOverlapResult:
+    """
+    Compute vocabulary overlap and similarity between two texts.
+
+    Related GitHub Issue:
+        #21 - Vocabulary Overlap and Similarity Metrics
+        https://github.com/craigtrim/pystylometry/issues/21
+
+    Args:
+        text1: First text to compare
+        text2: Second text to compare
+
+    Returns:
+        VocabularyOverlapResult with Jaccard, Dice, cosine similarities,
+        shared vocabulary statistics, and distinctive words for each text.
+
+    Example:
+        >>> result = compute_vocabulary_overlap(text1, text2)
+        >>> print(f"Jaccard similarity: {result.jaccard_similarity:.3f}")
+        Jaccard similarity: 0.456
+        >>> print(f"Shared words: {result.shared_vocab_size}")
+        Shared words: 234
+    """
+    # TODO: Implement vocabulary overlap analysis
+    # GitHub Issue #21: https://github.com/craigtrim/pystylometry/issues/21
+    raise NotImplementedError(
+        "Vocabulary overlap not yet implemented. "
+        "See GitHub Issue #21: https://github.com/craigtrim/pystylometry/issues/21"
+    )