pystylometry 0.1.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

pystylometry/readability/ari.py

@@ -1,8 +1,47 @@
 """Automated Readability Index (ARI)."""
 
+import math
+
 from .._types import ARIResult
 from .._utils import split_sentences, tokenize
 
+# Formula coefficients from Senter & Smith (1967)
+# Reference: Senter, R. J., & Smith, E. A. (1967). Automated readability index.
+#            AMRL-TR-6620. Aerospace Medical Research Laboratories.
+
+# Coefficient for characters per word
+_CHARACTER_COEFFICIENT = 4.71
+
+# Coefficient for words per sentence
+_WORD_COEFFICIENT = 0.5
+
+# Intercept to calibrate scale to U.S. grade levels
+_INTERCEPT = -21.43
+
+
+def _get_age_range(grade_level: int) -> str:
+    """
+    Map grade level to age range.
+
+    Args:
+        grade_level: U.S. grade level (0-20+)
+
+    Returns:
+        Age range string
+    """
+    if grade_level <= 0:
+        return "5-6 years (Kindergarten)"
+    elif grade_level <= 5:
+        return "6-11 years (Elementary)"
+    elif grade_level <= 8:
+        return "11-14 years (Middle School)"
+    elif grade_level <= 12:
+        return "14-18 years (High School)"
+    elif grade_level <= 14:
+        return "18-22 years (College)"
+    else:
+        return "22+ years (Graduate)"
+
 
 def compute_ari(text: str) -> ARIResult:
     """
@@ -11,15 +50,22 @@ def compute_ari(text: str) -> ARIResult:
     Formula:
         ARI = 4.71 × (characters/words) + 0.5 × (words/sentences) - 21.43
 
-    The ARI is designed to gauge the understandability of a text and produces
-    an approximate representation of the US grade level needed to comprehend the text.
+    The ARI uses character counts and word counts (similar to Coleman-Liau)
+    but adds sentence length as a factor. It produces an approximate
+    representation of the US grade level needed to comprehend the text.
+
+    **Implementation Notes:**
+    - Grade levels are clamped to [0, 20] range
+    - Uses round-half-up rounding for grade level calculation
+    - Character count includes alphanumeric characters only (letters and digits)
+    - Reliability heuristic: 100+ words recommended
 
     Grade Level to Age mapping:
-        1-5:   5-11 years
-        6-8:   11-14 years
-        9-12:  14-18 years
-        13-14: 18-22 years
-        14+:   22+ years (college level)
+        1-5:   6-11 years (Elementary)
+        6-8:   11-14 years (Middle School)
+        9-12:  14-18 years (High School)
+        13-14: 18-22 years (College)
+        15+:   22+ years (Graduate)
 
     References:
         Senter, R. J., & Smith, E. A. (1967). Automated readability index.
@@ -34,27 +80,56 @@ def compute_ari(text: str) -> ARIResult:
     Example:
         >>> result = compute_ari("The quick brown fox jumps over the lazy dog.")
         >>> print(f"ARI Score: {result.ari_score:.1f}")
+        ARI Score: 0.1
         >>> print(f"Grade Level: {result.grade_level}")
+        Grade Level: 0
         >>> print(f"Age Range: {result.age_range}")
+        Age Range: 5-6 years (Kindergarten)
+        >>> result.metadata["reliable"]
+        False
     """
     sentences = split_sentences(text)
     tokens = tokenize(text)
 
+    # Count characters (alphanumeric: letters and digits, excluding spaces/punctuation)
+    # Computed before early return to ensure metadata consistency
+    character_count = sum(1 for char in text if char.isalnum())
+
     if len(sentences) == 0 or len(tokens) == 0:
         return ARIResult(
             ari_score=0.0,
             grade_level=0,
-            age_range="Unknown",
-            metadata={"sentence_count": 0, "word_count": 0, "character_count": 0},
+            age_range="5-6 years (Kindergarten)",
+            metadata={
+                "sentence_count": len(sentences),
+                "word_count": len(tokens),
+                "character_count": character_count,
+                "characters_per_word": 0.0,
+                "words_per_sentence": 0.0,
+                "reliable": False,
+            },
         )
 
-    # Count characters (letters, numbers, excluding spaces and punctuation)
-    character_count = sum(1 for char in text if char.isalnum())
+    # Calculate ratios
+    chars_per_word = character_count / len(tokens)
+    words_per_sentence = len(tokens) / len(sentences)
+
+    # Apply ARI formula
+    ari_score = (
+        _CHARACTER_COEFFICIENT * chars_per_word
+        + _WORD_COEFFICIENT * words_per_sentence
+        + _INTERCEPT
+    )
+
+    # Use round-half-up rounding and clamp to valid grade range [0, 20]
+    # math.floor(x + 0.5) implements round-half-up for both positive and negative values
+    grade_level = max(0, min(20, math.floor(ari_score + 0.5)))
+
+    # Get age range from grade level
+    age_range = _get_age_range(grade_level)
 
-    # TODO: Implement ARI formula
-    ari_score = 0.0  # Placeholder
-    grade_level = 0  # Placeholder
-    age_range = "Unknown"  # Placeholder
+    # Reliability heuristic: like other readability metrics, 100+ words recommended
+    reliable = len(tokens) >= 100
 
     return ARIResult(
         ari_score=ari_score,
@@ -64,7 +139,8 @@ def compute_ari(text: str) -> ARIResult:
             "sentence_count": len(sentences),
             "word_count": len(tokens),
             "character_count": character_count,
-            "characters_per_word": character_count / len(tokens) if tokens else 0,
-            "words_per_sentence": len(tokens) / len(sentences) if sentences else 0,
+            "characters_per_word": chars_per_word,
+            "words_per_sentence": words_per_sentence,
+            "reliable": reliable,
         },
     )

pystylometry/readability/coleman_liau.py

@@ -1,8 +1,26 @@
 """Coleman-Liau Index."""
 
+import math
+
 from .._types import ColemanLiauResult
 from .._utils import split_sentences, tokenize
 
+# Regression coefficients from Coleman & Liau (1975)
+# Derived from empirical analysis of Cloze test results on graded texts
+# Reference: Coleman, M., & Liau, T. L. (1975). A computer readability formula
+#            designed for machine scoring. Journal of Applied Psychology, 60(2), 283.
+
+# Coefficient for letters per 100 words
+# Represents impact of word length on reading difficulty
+_LETTER_COEFFICIENT = 0.0588
+
+# Coefficient for sentences per 100 words (negative: more sentences = easier)
+# Represents impact of sentence length on reading difficulty
+_SENTENCE_COEFFICIENT = -0.296
+
+# Intercept to calibrate scale to U.S. grade levels (1-16)
+_INTERCEPT = -15.8
+
 
 def compute_coleman_liau(text: str) -> ColemanLiauResult:
     """
@@ -16,7 +34,21 @@ def compute_coleman_liau(text: str) -> ColemanLiauResult:
         S = average number of sentences per 100 words
 
     The Coleman-Liau index relies on characters rather than syllables,
-    making it easier to compute and potentially more language-agnostic.
+    making it easier to compute and not requiring syllable-counting algorithms.
+
+    **Implementation Notes:**
+    - Grade levels are NOT clamped (removed upper bound of 20 per PR #2 review).
+      The original Coleman & Liau (1975) paper calibrated to grades 1-16 but did not
+      specify an upper bound. Post-graduate texts may exceed grade 20.
+    - Uses round-half-up rounding (not banker's rounding) for grade level calculation
+    - Letter counts (Unicode alphabetic characters only) computed from tokenized words
+      to ensure measurement consistency. Both letter count and word count use identical
+      tokenization logic, preventing divergence in edge cases (emails, URLs, hyphens).
+      See PR #2 review discussion: https://github.com/craigtrim/pystylometry/pull/2
+    - Reliability heuristic based on validation study passage lengths (~100 words);
+      shorter texts flagged in metadata
+    - English-centric sentence splitting and Unicode assumptions limit true
+      cross-language applicability
 
     References:
         Coleman, M., & Liau, T. L. (1975). A computer readability formula
@@ -31,28 +63,88 @@ def compute_coleman_liau(text: str) -> ColemanLiauResult:
     Example:
         >>> result = compute_coleman_liau("The quick brown fox jumps over the lazy dog.")
         >>> print(f"CLI Index: {result.cli_index:.1f}")
+        CLI Index: 3.8
         >>> print(f"Grade Level: {result.grade_level}")
+        Grade Level: 4
+        >>> result.metadata["reliable"]
+        False
     """
     sentences = split_sentences(text)
-    tokens = tokenize(text)
+    all_tokens = tokenize(text)
+
+    # Filter to only tokens that contain at least one alphabetic character
+    # This excludes pure punctuation (. ! ?) but keeps words with mixed content
+    # (Hello123, Test@example.com) to count their letters per Coleman-Liau spec.
+    # This is different from Gunning Fog which uses stricter normalization.
+    tokens = [token for token in all_tokens if any(char.isalpha() for char in token)]
+
+    # CRITICAL: Count letters from tokenized words, NOT from raw text
+    # ===============================================================
+    # Coleman & Liau (1975) define L as "average number of letters per 100 words"
+    # where both letters and words must be measured consistently from the same text units.
+    #
+    # Original implementation (buggy):
+    #   letter_count = sum(1 for char in text if char.isalpha())
+    #   This counted letters from RAW text but words from TOKENIZED text
+    #
+    # Problem cases (PR #2 review https://github.com/craigtrim/pystylometry/pull/2):
+    #   - "test@example.com" → tokenizer may split into ['test', '@', 'example', '.', 'com']
+    #     Raw letter count: 15 letters, Token count: 5 tokens → wrong ratio
+    #   - "co-operate" → tokenizer may split into ['co', '-', 'operate']
+    #     Raw letter count: 9 letters, Token count: 3 tokens → wrong ratio
+    #   - URLs, special tokens, etc. → similar inconsistencies
+    #
+    # Fixed implementation:
+    #   Count only alphabetic characters that appear in valid word tokens (after normalization).
+    #   This ensures both letter count and word count use identical tokenization logic,
+    #   maintaining the mathematical integrity of the L term in the Coleman-Liau formula.
+    letter_count = sum(1 for token in tokens for char in token if char.isalpha())
 
     if len(sentences) == 0 or len(tokens) == 0:
         return ColemanLiauResult(
             cli_index=0.0,
             grade_level=0,
-            metadata={"sentence_count": 0, "word_count": 0, "letter_count": 0},
+            metadata={
+                "sentence_count": len(sentences),
+                "word_count": len(tokens),
+                "letter_count": letter_count,
+                "letters_per_100_words": 0.0,
+                "sentences_per_100_words": 0.0,
+                "reliable": False,
+            },
         )
 
-    # Count letters (excluding spaces and punctuation)
-    letter_count = sum(1 for char in text if char.isalpha())
-
     # Calculate per 100 words
     L = (letter_count / len(tokens)) * 100  # noqa: N806
     S = (len(sentences) / len(tokens)) * 100  # noqa: N806
 
-    # TODO: Implement Coleman-Liau formula
-    cli_index = 0.0  # Placeholder
-    grade_level = 0  # Placeholder
+    # Compute Coleman-Liau Index using empirically-derived coefficients
+    cli_index = _LETTER_COEFFICIENT * L + _SENTENCE_COEFFICIENT * S + _INTERCEPT
+
+    # Grade Level Calculation and Bounds
+    # ===================================
+    # Round-half-up rounding (not Python's default banker's rounding):
+    #   4.5 → 5 (always rounds up), not round-half-to-even
+    #   math.floor(x + 0.5) implements this for both positive and negative values
+    #
+    # Lower bound (0): Prevent negative grades for very simple texts
+    #   Coleman & Liau (1975) calibrated to U.S. grades 1-16, but simpler texts
+    #   (e.g., "Go. Run. Stop.") can produce negative CLI values. We clamp to 0
+    #   as there is no "negative grade level" in the educational system.
+    #
+    # Upper bound (REMOVED per PR #2 review):
+    #   Original implementation clamped at grade 20, but this was arbitrary.
+    #   Coleman & Liau (1975) did not specify an upper bound in their paper.
+    #   Clamping discards information: PhD dissertations (grade 25) and complex
+    #   legal documents (grade 30+) would both report as grade 20, making them
+    #   indistinguishable. The empirical formula should determine the full range.
+    #
+    # See PR #2 discussion: https://github.com/craigtrim/pystylometry/pull/2
+    grade_level = max(0, math.floor(cli_index + 0.5))
+
+    # Reliability heuristic: validation study used ~100-word passages
+    # Not a hard minimum, but shorter texts may deviate from expected behavior
+    reliable = len(tokens) >= 100
 
     return ColemanLiauResult(
         cli_index=cli_index,
@@ -63,5 +155,6 @@ def compute_coleman_liau(text: str) -> ColemanLiauResult:
             "letter_count": letter_count,
             "letters_per_100_words": L,
             "sentences_per_100_words": S,
+            "reliable": reliable,
         },
     )