pystylometry 0.1.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

pystylometry/lexical/mtld.py

@@ -4,6 +4,64 @@ from .._types import MTLDResult
 from .._utils import tokenize
 
 
+def _calculate_mtld_direction(tokens: list[str], threshold: float, forward: bool) -> float:
+    """
+    Calculate MTLD in one direction (forward or backward).
+
+    Args:
+        tokens: List of tokens to analyze
+        threshold: TTR threshold to maintain (must be in range (0, 1))
+        forward: If True, process forward; if False, process backward
+
+    Returns:
+        MTLD score for this direction
+    """
+    if len(tokens) == 0:
+        return 0.0
+
+    # Process tokens in the specified direction
+    token_list = tokens if forward else tokens[::-1]
+
+    factors = 0.0
+    current_count = 0
+    current_types = set()
+
+    for token in token_list:
+        current_count += 1
+        current_types.add(token)
+
+        # Calculate current TTR
+        ttr = len(current_types) / current_count
+
+        # If TTR drops below threshold, we've completed a factor
+        if ttr < threshold:
+            factors += 1.0
+            current_count = 0
+            current_types = set()
+
+    # Handle remaining partial factor
+    # Add proportion of a complete factor based on how close we are to threshold
+    if current_count > 0:
+        ttr = len(current_types) / current_count
+        # If we're still above threshold, add partial factor credit
+        # Formula: (1 - current_ttr) / (1 - threshold)
+        # This represents how far we've progressed toward completing a factor
+        # In theory, ttr should always be >= threshold here because drops below
+        # threshold are handled in the loop above (which resets current_count).
+        # Adding defensive check to prevent mathematical errors.
+        if ttr >= threshold:
+            factors += (1.0 - ttr) / (1.0 - threshold)
+
+    # MTLD is the mean length of factors
+    # Total tokens / number of factors
+    if factors > 0:
+        return len(tokens) / factors
+    else:
+        # If no factors were completed, return the text length
+        # This happens when TTR stays above threshold for the entire text
+        return float(len(tokens))
+
+
 def compute_mtld(
     text: str,
     threshold: float = 0.72,
@@ -16,8 +74,10 @@ def compute_mtld(
     varying lengths.
 
     Formula:
-        MTLD = mean(forward_factors, backward_factors)
-        where factors are word string lengths that maintain TTR >= threshold
+        MTLD = total_tokens / factor_count
+        where factor_count includes:
+        - Completed factors (segments where TTR dropped below threshold)
+        - Partial factor for any remaining incomplete segment (weighted by proximity to threshold)
 
     References:
         McCarthy, P. M., & Jarvis, S. (2010). MTLD, vocd-D, and HD-D:
@@ -26,16 +86,28 @@ def compute_mtld(
 
     Args:
         text: Input text to analyze
-        threshold: TTR threshold to maintain (default: 0.72)
+        threshold: TTR threshold to maintain (default: 0.72, must be in range (0, 1))
 
     Returns:
         MTLDResult with forward, backward, and average MTLD scores
 
+    Raises:
+        ValueError: If threshold is not in range (0, 1)
+
     Example:
         >>> result = compute_mtld("The quick brown fox jumps over the lazy dog...")
         >>> print(f"MTLD: {result.mtld_average:.2f}")
     """
-    tokens = tokenize(text)
+    # Validate threshold parameter
+    if not (0 < threshold < 1):
+        raise ValueError(
+            f"Threshold must be in range (0, 1), got {threshold}. "
+            "Common values: 0.72 (default), 0.5-0.8"
+        )
+
+    # Case-insensitive tokenization for consistency with other lexical metrics
+    # (compute_yule, compute_hapax_ratios both use text.lower())
+    tokens = tokenize(text.lower())
 
     if len(tokens) == 0:
         return MTLDResult(
@@ -45,9 +117,13 @@ def compute_mtld(
             metadata={"token_count": 0, "threshold": threshold},
         )
 
-    # TODO: Implement forward and backward MTLD calculation
-    mtld_forward = 0.0  # Placeholder
-    mtld_backward = 0.0  # Placeholder
+    # Calculate MTLD in forward direction
+    mtld_forward = _calculate_mtld_direction(tokens, threshold, forward=True)
+
+    # Calculate MTLD in backward direction
+    mtld_backward = _calculate_mtld_direction(tokens, threshold, forward=False)
+
+    # Average of forward and backward
     mtld_average = (mtld_forward + mtld_backward) / 2
 
     return MTLDResult(

pystylometry/lexical/ttr.py

@@ -0,0 +1,83 @@
+"""Type-Token Ratio (TTR) analysis using stylometry-ttr package.
+
+This module provides a facade wrapper around the stylometry-ttr package,
+maintaining consistent API patterns with other pystylometry metrics.
+"""
+
+from .._types import TTRResult
+
+
+def compute_ttr(text: str, text_id: str | None = None) -> TTRResult:
+    """
+    Compute Type-Token Ratio (TTR) metrics for vocabulary richness.
+
+    This is a facade wrapper around the stylometry-ttr package that provides
+    multiple TTR variants for measuring lexical diversity. TTR measures the
+    ratio of unique words (types) to total words (tokens).
+
+    Metrics computed:
+    - Raw TTR: unique_words / total_words
+    - Root TTR (Guiraud's index): unique_words / sqrt(total_words)
+    - Log TTR (Herdan's C): log(unique_words) / log(total_words)
+    - STTR: Standardized TTR across fixed-size chunks (reduces length bias)
+    - Delta Std: Standard deviation of TTR across chunks (vocabulary consistency)
+
+    References:
+        Guiraud, P. (1960). Problèmes et méthodes de la statistique linguistique.
+        Herdan, G. (1960). Type-token Mathematics: A Textbook of Mathematical
+            Linguistics. Mouton.
+        Johnson, W. (1944). Studies in language behavior: I. A program of research.
+            Psychological Monographs, 56(2), 1-15.
+
+    Args:
+        text: Input text to analyze
+        text_id: Optional identifier for the text (for tracking purposes)
+
+    Returns:
+        TTRResult with all TTR variants and metadata
+
+    Example:
+        >>> result = compute_ttr("The quick brown fox jumps over the lazy dog.")
+        >>> print(f"Raw TTR: {result.ttr:.3f}")
+        Raw TTR: 0.900
+        >>> print(f"Root TTR: {result.root_ttr:.3f}")
+        Root TTR: 2.846
+        >>> print(f"STTR: {result.sttr:.3f}")
+        STTR: 1.000
+
+        >>> # With text identifier
+        >>> result = compute_ttr("Sample text here.", text_id="sample-001")
+        >>> print(result.metadata["text_id"])
+        sample-001
+    """
+    try:
+        from stylometry_ttr import compute_ttr as _compute_ttr
+    except ImportError as e:
+        raise ImportError(
+            "TTR metrics require the stylometry-ttr package. "
+            "This should have been installed as a core dependency. "
+            "Install with: pip install stylometry-ttr"
+        ) from e
+
+    # Call the stylometry-ttr compute_ttr function
+    # Note: stylometry-ttr requires text_id to be a string, not None
+    ttr_result = _compute_ttr(text, text_id=text_id or "")
+
+    # Convert to our TTRResult dataclass
+    # The stylometry-ttr result has attributes we can access
+    # Some fields (sttr, delta_std) may be None for short texts
+    return TTRResult(
+        total_words=ttr_result.total_words,
+        unique_words=ttr_result.unique_words,
+        ttr=ttr_result.ttr,
+        root_ttr=ttr_result.root_ttr,
+        log_ttr=ttr_result.log_ttr,
+        sttr=ttr_result.sttr if ttr_result.sttr is not None else 0.0,
+        delta_std=ttr_result.delta_std if ttr_result.delta_std is not None else 0.0,
+        metadata={
+            "text_id": text_id or "",
+            "source": "stylometry-ttr",
+            "sttr_available": ttr_result.sttr is not None,
+            "delta_std_available": ttr_result.delta_std is not None,
+        },
+    )