pystylometry 0.1.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

pystylometry/readability/gunning_fog.py

@@ -1,63 +1,232 @@
-"""Gunning Fog Index."""
+"""Gunning Fog Index with NLP-enhanced complex word detection.
 
+This module computes the Gunning Fog Index, a readability metric that
+estimates the years of formal education needed to understand text on first reading.
+
+Historical Background:
+----------------------
+The Gunning Fog Index was developed by Robert Gunning in 1952 as part of his
+work helping businesses improve the clarity of their writing. The formula produces
+a U.S. grade-level score (e.g., 12 = high school senior reading level).
+
+Reference:
+    Gunning, R. (1952). The Technique of Clear Writing.
+    McGraw-Hill, New York.
+
+Implementation Notes (PR #4):
+------------------------------
+This implementation addresses issues raised in GitHub PR #4:
+https://github.com/craigtrim/pystylometry/pull/4
+
+The original TODO implementation used simple syllable counting without proper
+exclusions for proper nouns, compounds, or inflections. This NLP-enhanced
+version uses the complex_words module for accurate detection via:
+
+1. spaCy POS tagging for proper noun detection (enhanced mode)
+2. spaCy lemmatization for morphological analysis (enhanced mode)
+3. Component-based analysis for hyphenated words (both modes)
+4. Graceful fallback to heuristics when spaCy unavailable (basic mode)
+
+See complex_words.py for detailed rationale and implementation.
+"""
+
+from .._normalize import normalize_for_readability
 from .._types import GunningFogResult
 from .._utils import split_sentences, tokenize
-from .syllables import count_syllables
 
+# Import NLP-enhanced complex word detection module
+# This module addresses PR #4 issues with proper noun and inflection detection
+from .complex_words import process_text_for_complex_words
 
-def compute_gunning_fog(text: str) -> GunningFogResult:
+# Formula coefficient from Gunning (1952)
+# Reference: Gunning, R. (1952). The Technique of Clear Writing. McGraw-Hill.
+# The 0.4 coefficient scales the combined complexity measure to approximate grade level
+_FOG_COEFFICIENT = 0.4
+
+
+def compute_gunning_fog(text: str, spacy_model: str = "en_core_web_sm") -> GunningFogResult:
     """
-    Compute Gunning Fog Index.
+    Compute Gunning Fog Index with NLP-enhanced complex word detection.
 
-    Formula:
+    The Gunning Fog Index estimates the years of formal education required
+    to understand text on first reading. It combines sentence length and
+    lexical complexity (polysyllabic words) into a single grade-level score.
+
+    Formula (Gunning, 1952):
+    ------------------------
         Fog Index = 0.4 × [(words/sentences) + 100 × (complex words/words)]
 
-    Where complex words are defined as words with 3+ syllables,
-    excluding proper nouns, compound words, and common suffixes.
+    Where:
+        - words/sentences = Average Sentence Length (ASL)
+        - complex words/words = Percentage of Hard Words (PHW)
+        - 0.4 = Scaling coefficient to approximate U.S. grade levels
+
+    The resulting score represents a U.S. education grade level:
+        - 6 = Sixth grade (age 11-12)
+        - 12 = High school senior (age 17-18)
+        - 17+ = College graduate level
+
+    Complex Words Definition (Gunning, 1952):
+    ------------------------------------------
+    Words with 3+ syllables, EXCLUDING:
+        1. Proper nouns (names, places, organizations)
+        2. Compound words (hyphenated)
+        3. Common verb forms (-es, -ed, -ing endings)
+
+    Reference:
+        Gunning, R. (1952). The Technique of Clear Writing. McGraw-Hill.
+        Pages 38-39: Complex word criteria
+
+    NLP Enhancement (PR #4):
+    ------------------------
+    This implementation addresses issues in GitHub PR #4:
+    https://github.com/craigtrim/pystylometry/pull/4
 
-    The index estimates years of formal education needed to understand the text
-    on first reading.
+    **Enhanced Mode** (when spaCy available):
+        - Uses POS tagging (PROPN) for proper noun detection
+        - Uses lemmatization for morphological analysis
+        - Analyzes hyphenated word components individually
+        - More accurate, handles edge cases (acronyms, irregular verbs)
 
-    References:
-        Gunning, R. (1952). The Technique of Clear Writing.
-        McGraw-Hill.
+    **Basic Mode** (when spaCy unavailable):
+        - Uses capitalization heuristic for proper nouns
+        - Uses simple suffix stripping for inflections
+        - Analyzes hyphenated word components individually
+        - Less accurate but requires no external dependencies
+
+    The mode used is reported in metadata for transparency.
 
     Args:
         text: Input text to analyze
+        spacy_model: spaCy model name for enhanced mode (default: "en_core_web_sm")
+                    Requires model download: python -m spacy download en_core_web_sm
+                    Other options: "en_core_web_md", "en_core_web_lg"
 
     Returns:
-        GunningFogResult with fog index and grade level
+        GunningFogResult with:
+            - fog_index: Float, the calculated Gunning Fog Index
+            - grade_level: Float, rounded U.S. grade level (0-20), or NaN if empty
+            - metadata: Dict with:
+                - sentence_count: Number of sentences
+                - word_count: Number of words (tokens)
+                - complex_word_count: Number of complex words
+                - complex_word_percentage: Percentage of complex words
+                - average_words_per_sentence: Mean sentence length
+                - reliable: Boolean, True if word_count >= 100 and sentence_count >= 3
+                - mode: "enhanced" (spaCy) or "basic" (heuristics)
+                - proper_noun_detection: Detection method used
+                - inflection_handling: Inflection analysis method used
+                - spacy_model: Model name if enhanced mode (else absent)
 
     Example:
-        >>> result = compute_gunning_fog("The quick brown fox jumps over the lazy dog.")
+        >>> # Simple text (low complexity)
+        >>> result = compute_gunning_fog("The cat sat on the mat. The dog ran.")
+        >>> print(f"Fog Index: {result.fog_index:.1f}")
+        Fog Index: 2.7
+        >>> print(f"Grade Level: {result.grade_level}")
+        Grade Level: 3
+        >>> print(f"Mode: {result.metadata['mode']}")
+        Mode: enhanced
+
+        >>> # Complex academic text (high complexity)
+        >>> text = "Understanding phenomenological hermeneutics necessitates comprehensive study."
+        >>> result = compute_gunning_fog(text)
         >>> print(f"Fog Index: {result.fog_index:.1f}")
+        Fog Index: 23.6
         >>> print(f"Grade Level: {result.grade_level}")
+        Grade Level: 20
+
+        >>> # Check which detection mode was used
+        >>> if result.metadata['mode'] == 'enhanced':
+        ...     print("Using spaCy NLP features")
+        Using spaCy NLP features
+
+    Notes:
+        - Empty text returns fog_index=NaN and grade_level=NaN (no data)
+        - Grade levels are clamped to [0, 20] range for valid input
+        - For short texts (< 100 words), results may be unreliable
+        - Gunning (1952) recommends analyzing samples of 100+ words
     """
+    # Step 1: Sentence and word tokenization
+    # Using the project's standard utilities for consistency
     sentences = split_sentences(text)
-    tokens = tokenize(text)
+    all_tokens = tokenize(text)
+
+    # Filter to only valid words (exclude punctuation, numbers, URLs, emails)
+    # Allows hyphenated words and contractions per Gunning (1952)
+    # Prevents errors in syllable counting from non-word tokens
+    tokens = normalize_for_readability(all_tokens)
 
+    # Edge case: Empty or whitespace-only input
+    # Return NaN to distinguish "no data" from actual zero scores
+    # This matches SMOG behavior and prevents conflating empty input with simple text
     if len(sentences) == 0 or len(tokens) == 0:
         return GunningFogResult(
-            fog_index=0.0,
-            grade_level=0,
-            metadata={"sentence_count": 0, "word_count": 0, "complex_word_count": 0},
+            fog_index=float("nan"),
+            grade_level=float("nan"),
+            metadata={
+                "sentence_count": 0,
+                "word_count": 0,
+                "complex_word_count": 0,
+                "complex_word_percentage": 0.0,
+                "average_words_per_sentence": 0.0,
+                "reliable": False,
+                "mode": "none",
+                "proper_noun_detection": "N/A",
+                "inflection_handling": "N/A",
+            },
         )
 
-    # Count complex words (3+ syllables)
-    # TODO: Exclude proper nouns, compound words, and -es/-ed/-ing endings
-    complex_word_count = sum(1 for word in tokens if count_syllables(word) >= 3)
+    # Step 2: Count complex words using NLP-enhanced detection
+    # This addresses PR #4 issues with proper noun and inflection detection
+    # See complex_words.py for detailed implementation
+    complex_word_count, detection_metadata = process_text_for_complex_words(
+        text, tokens, model=spacy_model
+    )
+
+    # Step 3: Calculate formula components
+    # Reference: Gunning (1952), p. 40: "The Fog Index formula"
+
+    # Average Sentence Length (ASL)
+    # Number of words divided by number of sentences
+    average_words_per_sentence = len(tokens) / len(sentences)
+
+    # Percentage of Hard Words (PHW)
+    # Number of complex words divided by total words, multiplied by 100
+    complex_word_percentage = (complex_word_count / len(tokens)) * 100
+
+    # Step 4: Apply Gunning Fog formula
+    # Fog = 0.4 × (ASL + PHW)
+    # The 0.4 coefficient scales the result to approximate U.S. grade levels
+    fog_index = _FOG_COEFFICIENT * (average_words_per_sentence + complex_word_percentage)
+
+    # Step 5: Convert to grade level
+    # Round to nearest integer using standard rounding (round half to even)
+    # Clamp to reasonable range [0, 20] to prevent extreme values
+    # Note: Texts with fog_index > 20 are considered "post-graduate" level
+    grade_level = max(0, min(20, round(fog_index)))
 
-    # TODO: Implement Gunning Fog formula
-    fog_index = 0.0  # Placeholder
-    grade_level = 0  # Placeholder
+    # Reliability heuristic: Gunning (1952) recommends 100+ word samples
+    # Also require 3+ sentences to ensure meaningful average sentence length
+    # Very long texts with few sentences can produce unstable FOG estimates
+    reliable = len(tokens) >= 100 and len(sentences) >= 3
 
+    # Step 6: Assemble result with comprehensive metadata
     return GunningFogResult(
         fog_index=fog_index,
         grade_level=grade_level,
         metadata={
+            # Core counts
             "sentence_count": len(sentences),
             "word_count": len(tokens),
             "complex_word_count": complex_word_count,
-            "complex_word_percentage": (complex_word_count / len(tokens) * 100) if tokens else 0,
+            # Derived metrics
+            "complex_word_percentage": complex_word_percentage,
+            "average_words_per_sentence": average_words_per_sentence,
+            # Reliability indicator
+            "reliable": reliable,
+            # Detection method transparency (from complex_words module)
+            # This allows users to verify which mode was used
+            **detection_metadata,
         },
     )

pystylometry/readability/smog.py

@@ -1,5 +1,8 @@
 """SMOG (Simple Measure of Gobbledygook) Index."""
 
+import math
+
+from .._normalize import normalize_for_readability
 from .._types import SMOGResult
 from .._utils import split_sentences, tokenize
 from .syllables import count_syllables
@@ -27,23 +30,29 @@ def compute_smog(text: str) -> SMOGResult:
     Returns:
         SMOGResult with SMOG index and grade level
 
+        Note: For empty input (no sentences or words), smog_index and grade_level
+        will be float('nan'). This prevents conflating "no data" with actual scores.
+
+        SMOG is designed for texts with 30+ sentences. For shorter texts, the formula
+        still computes but a warning is included in metadata. Results may be less reliable.
+
     Example:
-        >>> result = compute_smog("The quick brown fox jumps over the lazy dog.")
+        >>> text = "Caffeinated programmers debugged incomprehensible code."
+        >>> result = compute_smog(text)
         >>> print(f"SMOG Index: {result.smog_index:.1f}")
         >>> print(f"Grade Level: {result.grade_level}")
     """
     sentences = split_sentences(text)
     tokens = tokenize(text)
 
-    if len(sentences) < 30:
-        # SMOG requires at least 30 sentences for accuracy
-        # We'll compute anyway but note in metadata
-        pass
+    # Filter tokens to only valid words for syllable counting
+    # Removes numbers, URLs, emails, etc. that would cause errors
+    word_tokens = normalize_for_readability(tokens)
 
-    if len(sentences) == 0 or len(tokens) == 0:
+    if len(sentences) == 0 or len(word_tokens) == 0:
         return SMOGResult(
-            smog_index=0.0,
-            grade_level=0,
+            smog_index=float("nan"),
+            grade_level=float("nan"),
             metadata={
                 "sentence_count": 0,
                 "word_count": 0,
@@ -52,19 +61,27 @@ def compute_smog(text: str) -> SMOGResult:
             },
         )
 
-    # Count polysyllables (words with 3+ syllables)
-    polysyllable_count = sum(1 for word in tokens if count_syllables(word) >= 3)
+    # Count polysyllables (words with 3+ syllables) - safe now, only valid words
+    polysyllable_count = sum(1 for word in word_tokens if count_syllables(word) >= 3)
+
+    # SMOG formula: 1.043 × √(polysyllables × 30/sentences) + 3.1291
+    smog_index = 1.043 * math.sqrt(polysyllable_count * 30 / len(sentences)) + 3.1291
 
-    # TODO: Implement SMOG formula
-    smog_index = 0.0  # Placeholder
-    grade_level = 0  # Placeholder
+    # Use round-half-up rounding (not banker's rounding)
+    # Clamp to valid grade range [0, 20]
+    # Round half up: 4.5 → 5 (not Python's default round-half-to-even)
+    # math.floor(x + 0.5) implements round-half-up for both positive and negative values
+    # Lower bound: Prevent negative grades
+    # (though mathematically unlikely with SMOG's +3.1291 constant)
+    # Upper bound: Cap at grade 20 (post-graduate) for extreme complexity
+    grade_level = max(0, min(20, math.floor(smog_index + 0.5)))
 
     return SMOGResult(
         smog_index=smog_index,
         grade_level=grade_level,
         metadata={
             "sentence_count": len(sentences),
-            "word_count": len(tokens),
+            "word_count": len(word_tokens),
             "polysyllable_count": polysyllable_count,
             "warning": "Less than 30 sentences" if len(sentences) < 30 else None,
         },

pystylometry/readability/syllables.py

@@ -1,54 +1,161 @@
-"""Syllable counting utilities using CMU Pronouncing Dictionary."""
+"""
+Syllable counting using CMU Pronouncing Dictionary.
 
+Uses the pronouncing library which provides access to the CMU Pronouncing
+Dictionary for high-accuracy syllable counting based on phonetic transcriptions.
+"""
 
+import re
+from functools import lru_cache
+
+try:
+    import pronouncing  # type: ignore[import-untyped]
+except ImportError:
+    raise ImportError(
+        "The 'pronouncing' library is required for syllable counting. "
+        "Install it with: pip install pystylometry[readability]"
+    )
+
+
+@lru_cache(maxsize=4096)
 def count_syllables(word: str) -> int:
     """
-    Count syllables in a word using CMU Pronouncing Dictionary with heuristic fallback.
+    Count syllables using CMU Pronouncing Dictionary.
+
+    Uses phonetic transcriptions from CMU dictionary. For words with multiple
+    pronunciations, uses the first pronunciation (typically the most common).
+    Falls back to simple vowel counting for words not in the dictionary.
 
     Args:
-        word: The word to count syllables for
+        word: Input word (handles mixed case, strips whitespace)
 
     Returns:
-        Number of syllables in the word
+        Syllable count (minimum 1 for non-empty input)
+
+    Example:
+        >>> count_syllables("beautiful")
+        3
+        >>> count_syllables("fire")
+        2
+        >>> count_syllables("cruel")
+        1
     """
-    # TODO: Implement with pronouncing library
-    # For now, use simple heuristic fallback
-    return _heuristic_syllable_count(word)
+    word = word.lower().strip()
+    if not word:
+        return 0
 
+    # Strip common punctuation
+    word = word.strip(".,;:!?\"'()-")
+    if not word:
+        return 0
 
-def _heuristic_syllable_count(word: str) -> int:
-    """
-    Simple heuristic syllable counter for fallback.
+    # Handle contractions by removing apostrophes
+    if "'" in word:
+        word = word.replace("'", "")
 
-    This is a basic implementation that counts vowel groups.
-    Should be replaced with CMU dict lookup when pronouncing is available.
+    # Handle hyphenated compounds
+    if "-" in word:
+        return sum(count_syllables(part) for part in word.split("-") if part)
 
-    Args:
-        word: The word to count syllables for
+    # Get pronunciations from CMU dictionary
+    phones_list = pronouncing.phones_for_word(word)
 
-    Returns:
-        Estimated number of syllables
+    if phones_list:
+        # Use first pronunciation (most common)
+        # Count stress markers (0, 1, 2) in phoneme representation
+        phones = phones_list[0]
+        return pronouncing.syllable_count(phones)  # type: ignore[no-any-return]
+
+    # Fallback for words not in dictionary: simple vowel counting
+    return _fallback_count(word)
+
+
+def _fallback_count(word: str) -> int:
     """
-    word = word.lower().strip()
-    if len(word) == 0:
-        return 0
+    Simple fallback syllable counter for words not in CMU dictionary.
 
+    Uses basic vowel counting with silent-e adjustment.
+    Less accurate than CMU but handles rare/technical words.
+    """
     vowels = "aeiouy"
-    syllable_count = 0
-    previous_was_vowel = False
+    count = 0
+    prev_was_vowel = False
 
     for char in word:
         is_vowel = char in vowels
-        if is_vowel and not previous_was_vowel:
-            syllable_count += 1
-        previous_was_vowel = is_vowel
+        if is_vowel and not prev_was_vowel:
+            count += 1
+        prev_was_vowel = is_vowel
 
     # Adjust for silent 'e'
-    if word.endswith("e") and syllable_count > 1:
-        syllable_count -= 1
+    if word.endswith("e") and count > 1:
+        count -= 1
+
+    # Ensure minimum of 1
+    return max(1, count)
+
+
+def count_syllables_text(text: str) -> list[tuple[str, int]]:
+    """
+    Count syllables for all words in a text.
+
+    Args:
+        text: Input text
+
+    Returns:
+        List of (word, syllable_count) tuples
+
+    Example:
+        >>> count_syllables_text("The quick brown fox")
+        [('The', 1), ('quick', 1), ('brown', 1), ('fox', 1)]
+    """
+
+    words = re.findall(r"[a-zA-Z']+", text)
+    return [(w, count_syllables(w)) for w in words]
+
+
+def total_syllables(text: str) -> int:
+    """
+    Return total syllable count for text.
+
+    Args:
+        text: Input text
+
+    Returns:
+        Total number of syllables
+
+    Example:
+        >>> total_syllables("The quick brown fox")
+        4
+    """
+    return sum(count for _, count in count_syllables_text(text))
+
+
+def validate_accuracy(
+    test_pairs: list[tuple[str, int]],
+) -> tuple[float, list[tuple[str, int, int]]]:
+    """
+    Test accuracy against known word-syllable pairs.
+
+    Args:
+        test_pairs: List of (word, expected_syllables) tuples
+
+    Returns:
+        (accuracy_percentage, list of (word, expected, got) for failures)
+
+    Example:
+        >>> test_pairs = [("hello", 2), ("world", 1), ("beautiful", 3)]
+        >>> accuracy, failures = validate_accuracy(test_pairs)
+        >>> print(f"Accuracy: {accuracy:.1f}%")
+    """
+    failures = []
+    for word, expected in test_pairs:
+        got = count_syllables(word)
+        if got != expected:
+            failures.append((word, expected, got))
 
-    # Ensure at least one syllable
-    if syllable_count == 0:
-        syllable_count = 1
+    if not test_pairs:
+        return 0.0, []
 
-    return syllable_count
+    accuracy = (len(test_pairs) - len(failures)) / len(test_pairs) * 100
+    return accuracy, failures

pystylometry/stylistic/__init__.py

@@ -0,0 +1,20 @@
+"""Stylistic analysis metrics.
+
+Related GitHub Issues:
+    #20 - Stylistic Markers
+    #21 - Vocabulary Overlap and Similarity Metrics
+    #22 - Cohesion and Coherence Metrics
+    #23 - Genre and Register Features
+"""
+
+from .cohesion_coherence import compute_cohesion_coherence
+from .genre_register import compute_genre_register
+from .markers import compute_stylistic_markers
+from .vocabulary_overlap import compute_vocabulary_overlap
+
+__all__ = [
+    "compute_stylistic_markers",
+    "compute_vocabulary_overlap",
+    "compute_cohesion_coherence",
+    "compute_genre_register",
+]

pystylometry/stylistic/cohesion_coherence.py

@@ -0,0 +1,45 @@
+"""Cohesion and coherence metrics.
+
+This module measures how well a text holds together structurally (cohesion)
+and semantically (coherence). Important for analyzing writing quality and
+authorial sophistication.
+
+Related GitHub Issue:
+    #22 - Cohesion and Coherence Metrics
+    https://github.com/craigtrim/pystylometry/issues/22
+
+References:
+    Halliday, M. A. K., & Hasan, R. (1976). Cohesion in English. Longman.
+    Graesser, A. C., McNamara, D. S., & Kulikowich, J. M. (2011). Coh-Metrix.
+"""
+
+from .._types import CohesionCoherenceResult
+
+
+def compute_cohesion_coherence(text: str, model: str = "en_core_web_sm") -> CohesionCoherenceResult:
+    """
+    Compute cohesion and coherence metrics.
+
+    Related GitHub Issue:
+        #22 - Cohesion and Coherence Metrics
+        https://github.com/craigtrim/pystylometry/issues/22
+
+    Args:
+        text: Input text to analyze
+        model: spaCy model for linguistic analysis
+
+    Returns:
+        CohesionCoherenceResult with referential cohesion, lexical cohesion,
+        connective density, and coherence scores.
+
+    Example:
+        >>> result = compute_cohesion_coherence("Multi-paragraph text...")
+        >>> print(f"Pronoun density: {result.pronoun_density:.2f}")
+        >>> print(f"Connective density: {result.connective_density:.2f}")
+    """
+    # TODO: Implement cohesion/coherence analysis
+    # GitHub Issue #22: https://github.com/craigtrim/pystylometry/issues/22
+    raise NotImplementedError(
+        "Cohesion/coherence metrics not yet implemented. "
+        "See GitHub Issue #22: https://github.com/craigtrim/pystylometry/issues/22"
+    )

pystylometry/stylistic/genre_register.py

@@ -0,0 +1,45 @@
+"""Genre and register classification features.
+
+This module extracts features that distinguish between different text types
+(academic, journalistic, fiction, legal, etc.) and formality levels.
+
+Related GitHub Issue:
+    #23 - Genre and Register Features
+    https://github.com/craigtrim/pystylometry/issues/23
+
+References:
+    Biber, D. (1988). Variation across speech and writing. Cambridge University Press.
+    Biber, D., & Conrad, S. (2009). Register, genre, and style.
+"""
+
+from .._types import GenreRegisterResult
+
+
+def compute_genre_register(text: str, model: str = "en_core_web_sm") -> GenreRegisterResult:
+    """
+    Analyze genre and register features for text classification.
+
+    Related GitHub Issue:
+        #23 - Genre and Register Features
+        https://github.com/craigtrim/pystylometry/issues/23
+
+    Args:
+        text: Input text to analyze
+        model: spaCy model for linguistic analysis
+
+    Returns:
+        GenreRegisterResult with formality scores, register classification,
+        genre predictions, and feature scores for major genres.
+
+    Example:
+        >>> result = compute_genre_register("Academic paper text...")
+        >>> print(f"Formality score: {result.formality_score:.2f}")
+        >>> print(f"Predicted genre: {result.predicted_genre}")
+        >>> print(f"Academic score: {result.academic_score:.3f}")
+    """
+    # TODO: Implement genre/register analysis
+    # GitHub Issue #23: https://github.com/craigtrim/pystylometry/issues/23
+    raise NotImplementedError(
+        "Genre/register classification not yet implemented. "
+        "See GitHub Issue #23: https://github.com/craigtrim/pystylometry/issues/23"
+    )