pystylometry 0.1.0__py3-none-any.whl → 1.1.0__py3-none-any.whl

pystylometry/lexical/hapax.py

@@ -1,15 +1,87 @@
-"""Hapax legomena and related vocabulary richness metrics."""
+"""Hapax legomena and related vocabulary richness metrics.
 
+This module implements hapax metrics with native chunked analysis for
+stylometric fingerprinting.
+
+Related GitHub Issue:
+    #27 - Native chunked analysis with Distribution dataclass
+    https://github.com/craigtrim/pystylometry/issues/27
+"""
+
+import math
 from collections import Counter
 
-from .._types import HapaxResult
-from .._utils import tokenize
+from .._types import (
+    Distribution,
+    HapaxLexiconResult,
+    HapaxResult,
+    LexiconCategories,
+    chunk_text,
+    make_distribution,
+)
+from .._utils import check_optional_dependency, tokenize
+
 
+def _compute_hapax_single(text: str) -> tuple[int, float, int, float, float, float, dict]:
+    """Compute hapax metrics for a single chunk of text.
 
-def compute_hapax_ratios(text: str) -> HapaxResult:
+    Returns:
+        Tuple of (hapax_count, hapax_ratio, dis_hapax_count, dis_hapax_ratio,
+                  sichel_s, honore_r, metadata_dict).
+        Returns nans for ratios on empty input.
+    """
+    tokens = tokenize(text.lower())
+    N = len(tokens)  # noqa: N806
+
+    if N == 0:
+        return (
+            0,
+            float("nan"),
+            0,
+            float("nan"),
+            float("nan"),
+            float("nan"),
+            {"token_count": 0, "vocabulary_size": 0},
+        )
+
+    # Count frequency of each token
+    freq_counter = Counter(tokens)
+    V = len(freq_counter)  # noqa: N806
+
+    # Count hapax legomena (V₁) and dislegomena (V₂)
+    V1 = sum(1 for count in freq_counter.values() if count == 1)  # noqa: N806
+    V2 = sum(1 for count in freq_counter.values() if count == 2)  # noqa: N806
+
+    # Sichel's S: ratio of dislegomena to vocabulary size
+    sichel_s = V2 / V if V > 0 else 0.0
+
+    # Honoré's R: 100 × log(N) / (1 - V₁/V)
+    if V1 == V:
+        honore_r = float("inf")
+    else:
+        honore_r = 100 * math.log(N) / (1 - V1 / V)
+
+    hapax_ratio = V1 / N if N > 0 else 0.0
+    dis_hapax_ratio = V2 / N if N > 0 else 0.0
+
+    return (
+        V1,
+        hapax_ratio,
+        V2,
+        dis_hapax_ratio,
+        sichel_s,
+        honore_r,
+        {"token_count": N, "vocabulary_size": V},
+    )
+
+
+def compute_hapax_ratios(text: str, chunk_size: int = 1000) -> HapaxResult:
     """
     Compute hapax legomena, hapax dislegomena, and related richness metrics.
 
+    This function uses native chunked analysis to capture variance and patterns
+    across the text, which is essential for stylometric fingerprinting.
+
     Hapax legomena = words appearing exactly once
     Hapax dislegomena = words appearing exactly twice
 
@@ -17,6 +89,10 @@ def compute_hapax_ratios(text: str) -> HapaxResult:
     - Sichel's S: V₂ / V (ratio of dislegomena to total vocabulary)
     - Honoré's R: 100 × log(N) / (1 - V₁/V)
 
+    Related GitHub Issue:
+        #27 - Native chunked analysis with Distribution dataclass
+        https://github.com/craigtrim/pystylometry/issues/27
+
     References:
         Sichel, H. S. (1975). On a distribution law for word frequencies.
         Journal of the American Statistical Association, 70(351a), 542-547.
@@ -26,50 +102,251 @@ def compute_hapax_ratios(text: str) -> HapaxResult:
 
     Args:
         text: Input text to analyze
+        chunk_size: Number of words per chunk (default: 1000)
 
     Returns:
-        HapaxResult with counts, ratios, Sichel's S, Honoré's R, and metadata
+        HapaxResult with counts, ratios, distributions, and metadata
 
     Example:
-        >>> result = compute_hapax_ratios("The quick brown fox jumps over the lazy dog.")
-        >>> print(f"Hapax ratio: {result.hapax_ratio:.3f}")
-        >>> print(f"Sichel's S: {result.sichel_s:.3f}")
+        >>> result = compute_hapax_ratios("Long text here...", chunk_size=1000)
+        >>> result.hapax_ratio  # Mean across chunks
+        0.45
+        >>> result.hapax_ratio_dist.std  # Variance reveals fingerprint
+        0.08
     """
-    tokens = tokenize(text.lower())
-    N = len(tokens)  # noqa: N806
+    # Chunk the text
+    chunks = chunk_text(text, chunk_size)
 
-    if N == 0:
+    # Compute metrics per chunk
+    hapax_ratio_values = []
+    dis_hapax_ratio_values = []
+    sichel_s_values = []
+    honore_r_values = []
+    honore_r_inf_count = 0  # Track chunks where all words are unique (V₁ = V)
+    total_hapax_count = 0
+    total_dis_hapax_count = 0
+    total_tokens = 0
+    total_vocab = 0
+    valid_chunk_count = 0
+
+    for chunk in chunks:
+        h_cnt, h_rat, dh_cnt, dh_rat, sichel, honore, meta = _compute_hapax_single(chunk)
+        total_hapax_count += h_cnt
+        total_dis_hapax_count += dh_cnt
+        total_tokens += meta.get("token_count", 0)
+        total_vocab += meta.get("vocabulary_size", 0)
+
+        if not math.isnan(h_rat):
+            hapax_ratio_values.append(h_rat)
+            valid_chunk_count += 1
+        if not math.isnan(dh_rat):
+            dis_hapax_ratio_values.append(dh_rat)
+        if not math.isnan(sichel):
+            sichel_s_values.append(sichel)
+        if math.isinf(honore):
+            # Track infinite values (when V₁ = V, maximal vocabulary richness)
+            honore_r_inf_count += 1
+        elif not math.isnan(honore):
+            honore_r_values.append(honore)
+
+    # Handle empty or all-invalid chunks
+    if not hapax_ratio_values:
+        empty_dist = Distribution(
+            values=[],
+            mean=float("nan"),
+            median=float("nan"),
+            std=0.0,
+            range=0.0,
+            iqr=0.0,
+        )
         return HapaxResult(
             hapax_count=0,
-            hapax_ratio=0.0,
+            hapax_ratio=float("nan"),
             dis_hapax_count=0,
-            dis_hapax_ratio=0.0,
-            sichel_s=0.0,
-            honore_r=0.0,
-            metadata={"token_count": 0, "vocabulary_size": 0},
+            dis_hapax_ratio=float("nan"),
+            sichel_s=float("nan"),
+            honore_r=float("nan"),
+            hapax_ratio_dist=empty_dist,
+            dis_hapax_ratio_dist=empty_dist,
+            sichel_s_dist=empty_dist,
+            honore_r_dist=empty_dist,
+            chunk_size=chunk_size,
+            chunk_count=len(chunks),
+            metadata={"total_token_count": 0, "total_vocabulary_size": 0},
         )
 
-    # Count frequency of each token
+    # Build distributions
+    hapax_ratio_dist = make_distribution(hapax_ratio_values)
+    dis_hapax_ratio_dist = make_distribution(dis_hapax_ratio_values)
+    sichel_s_dist = (
+        make_distribution(sichel_s_values)
+        if sichel_s_values
+        else Distribution(
+            values=[], mean=float("nan"), median=float("nan"), std=0.0, range=0.0, iqr=0.0
+        )
+    )
+
+    # Handle honore_r specially: if all valid chunks had V₁ = V (all unique words),
+    # return infinity to indicate maximal vocabulary richness
+    if honore_r_values:
+        honore_r_dist = make_distribution(honore_r_values)
+        honore_r_final = honore_r_dist.mean
+    elif honore_r_inf_count > 0 and honore_r_inf_count == valid_chunk_count:
+        # All valid chunks had infinite honore_r (all words unique)
+        honore_r_dist = Distribution(
+            values=[], mean=float("inf"), median=float("inf"), std=0.0, range=0.0, iqr=0.0
+        )
+        honore_r_final = float("inf")
+    else:
+        honore_r_dist = Distribution(
+            values=[], mean=float("nan"), median=float("nan"), std=0.0, range=0.0, iqr=0.0
+        )
+        honore_r_final = float("nan")
+
+    return HapaxResult(
+        hapax_count=total_hapax_count,
+        hapax_ratio=hapax_ratio_dist.mean,
+        dis_hapax_count=total_dis_hapax_count,
+        dis_hapax_ratio=dis_hapax_ratio_dist.mean,
+        sichel_s=sichel_s_dist.mean,
+        honore_r=honore_r_final,
+        hapax_ratio_dist=hapax_ratio_dist,
+        dis_hapax_ratio_dist=dis_hapax_ratio_dist,
+        sichel_s_dist=sichel_s_dist,
+        honore_r_dist=honore_r_dist,
+        chunk_size=chunk_size,
+        chunk_count=len(chunks),
+        metadata={
+            "total_token_count": total_tokens,
+            "total_vocabulary_size": total_vocab,
+        },
+    )
+
+
+def compute_hapax_with_lexicon_analysis(text: str) -> HapaxLexiconResult:
+    """
+    Compute hapax legomena with lexicon-based categorization.
+
+    Extends standard hapax analysis by categorizing hapax legomena based on
+    presence in WordNet and British National Corpus (BNC). This distinguishes
+    between:
+
+    1. **Neologisms**: Words not in WordNet AND not in BNC
+       - True novel words or proper nouns
+       - High neologism ratio indicates vocabulary innovation
+
+    2. **Rare Words**: Words in BNC but not WordNet, or vice versa
+       - Technical jargon, specialized terminology
+       - Words at the edges of common vocabulary
+
+    3. **Common Words**: Words in both WordNet AND BNC
+       - Standard vocabulary that happens to appear once
+       - Low incidental usage of common words
+
+    This categorization is valuable for stylometric analysis:
+    - Authors with high neologism ratios are more innovative/creative
+    - Technical writing typically has higher rare word ratios
+    - Comparison of neologism vs common hapax distinguishes vocabulary
+      innovation from incidental word usage
+
+    Args:
+        text: Input text to analyze
+
+    Returns:
+        HapaxLexiconResult with standard hapax metrics and lexicon categorization
+
+    Raises:
+        ImportError: If bnc-lookup or wordnet-lookup packages are not installed
+
+    Example:
+        >>> text = "The xyzbot platform facilitates interdepartmental synergy."
+        >>> result = compute_hapax_with_lexicon_analysis(text)
+        >>> result.lexicon_analysis.neologisms
+        ['xyzbot', 'platform']
+        >>> result.lexicon_analysis.rare_words
+        ['facilitates', 'interdepartmental']
+        >>> result.lexicon_analysis.common_words
+        ['synergy']
+        >>> print(f"Neologism ratio: {result.lexicon_analysis.neologism_ratio:.2%}")
+        Neologism ratio: 40.00%
+
+    References:
+        British National Corpus: http://www.natcorp.ox.ac.uk/
+        WordNet: https://wordnet.princeton.edu/
+    """
+    # Check dependencies
+    check_optional_dependency("bnc_lookup", "lexical")
+    check_optional_dependency("wordnet_lookup", "lexical")
+
+    from bnc_lookup import exists as is_bnc_term  # type: ignore[import-untyped]
+    from wordnet_lookup import is_wordnet_term  # type: ignore[import-untyped]
+
+    # First compute standard hapax metrics
+    hapax_result = compute_hapax_ratios(text)
+
+    # If no hapax legomena, return empty categorization
+    if hapax_result.hapax_count == 0:
+        return HapaxLexiconResult(
+            hapax_result=hapax_result,
+            lexicon_analysis=LexiconCategories(
+                neologisms=[],
+                rare_words=[],
+                common_words=[],
+                neologism_ratio=0.0,
+                rare_word_ratio=0.0,
+                metadata={"total_hapax": 0},
+            ),
+            metadata={"note": "No hapax legomena found"},
+        )
+
+    # Get tokens and identify hapax words
+    tokens = tokenize(text.lower())
     freq_counter = Counter(tokens)
-    V = len(freq_counter)  # noqa: N806
+    hapax_words = [word for word, count in freq_counter.items() if count == 1]
 
-    # Count hapax legomena (V₁) and dislegomena (V₂)
-    V1 = sum(1 for count in freq_counter.values() if count == 1)  # noqa: N806
-    V2 = sum(1 for count in freq_counter.values() if count == 2)  # noqa: N806
+    # Categorize each hapax word by lexicon presence
+    neologisms = []
+    rare_words = []
+    common_words = []
 
-    # TODO: Implement Sichel's S and Honoré's R
-    sichel_s = 0.0  # Placeholder
-    honore_r = 0.0  # Placeholder
+    for word in hapax_words:
+        in_bnc = is_bnc_term(word)
+        in_wordnet = is_wordnet_term(word)
 
-    return HapaxResult(
-        hapax_count=V1,
-        hapax_ratio=V1 / N if N > 0 else 0.0,
-        dis_hapax_count=V2,
-        dis_hapax_ratio=V2 / N if N > 0 else 0.0,
-        sichel_s=sichel_s,
-        honore_r=honore_r,
+        if not in_bnc and not in_wordnet:
+            # Not in either lexicon → true neologism
+            neologisms.append(word)
+        elif in_bnc and in_wordnet:
+            # In both lexicons → common word
+            common_words.append(word)
+        else:
+            # In one but not the other → rare word
+            rare_words.append(word)
+
+    # Calculate ratios
+    total_hapax = len(hapax_words)
+    neologism_ratio = len(neologisms) / total_hapax if total_hapax > 0 else 0.0
+    rare_word_ratio = len(rare_words) / total_hapax if total_hapax > 0 else 0.0
+    common_word_ratio = len(common_words) / total_hapax if total_hapax > 0 else 0.0
+
+    return HapaxLexiconResult(
+        hapax_result=hapax_result,
+        lexicon_analysis=LexiconCategories(
+            neologisms=sorted(neologisms),
+            rare_words=sorted(rare_words),
+            common_words=sorted(common_words),
+            neologism_ratio=neologism_ratio,
+            rare_word_ratio=rare_word_ratio,
+            metadata={
+                "total_hapax": total_hapax,
+                "neologism_count": len(neologisms),
+                "rare_word_count": len(rare_words),
+                "common_word_count": len(common_words),
+                "common_word_ratio": common_word_ratio,
+            },
+        ),
         metadata={
-            "token_count": N,
-            "vocabulary_size": V,
+            "lexicons_used": ["bnc", "wordnet"],
+            "note": "Lexicon categorization based on BNC and WordNet presence",
         },
     )

pystylometry/lexical/mtld.py

@@ -1,23 +1,130 @@
-"""MTLD (Measure of Textual Lexical Diversity) implementation."""
+"""MTLD (Measure of Textual Lexical Diversity) implementation.
 
-from .._types import MTLDResult
+This module implements MTLD with native chunked analysis for stylometric
+fingerprinting.
+
+Related GitHub Issue:
+    #27 - Native chunked analysis with Distribution dataclass
+    https://github.com/craigtrim/pystylometry/issues/27
+"""
+
+import math
+
+from .._types import Distribution, MTLDResult, chunk_text, make_distribution
 from .._utils import tokenize
 
 
+def _calculate_mtld_direction(tokens: list[str], threshold: float, forward: bool) -> float:
+    """
+    Calculate MTLD in one direction (forward or backward).
+
+    Args:
+        tokens: List of tokens to analyze
+        threshold: TTR threshold to maintain (must be in range (0, 1))
+        forward: If True, process forward; if False, process backward
+
+    Returns:
+        MTLD score for this direction
+    """
+    if len(tokens) == 0:
+        return 0.0
+
+    # Process tokens in the specified direction
+    token_list = tokens if forward else tokens[::-1]
+
+    factors = 0.0
+    current_count = 0
+    current_types = set()
+
+    for token in token_list:
+        current_count += 1
+        current_types.add(token)
+
+        # Calculate current TTR
+        ttr = len(current_types) / current_count
+
+        # If TTR drops below threshold, we've completed a factor
+        if ttr < threshold:
+            factors += 1.0
+            current_count = 0
+            current_types = set()
+
+    # Handle remaining partial factor
+    # Add proportion of a complete factor based on how close we are to threshold
+    if current_count > 0:
+        ttr = len(current_types) / current_count
+        # If we're still above threshold, add partial factor credit
+        # Formula: (1 - current_ttr) / (1 - threshold)
+        # This represents how far we've progressed toward completing a factor
+        # In theory, ttr should always be >= threshold here because drops below
+        # threshold are handled in the loop above (which resets current_count).
+        # Adding defensive check to prevent mathematical errors.
+        if ttr >= threshold:
+            factors += (1.0 - ttr) / (1.0 - threshold)
+
+    # MTLD is the mean length of factors
+    # Total tokens / number of factors
+    if factors > 0:
+        return len(tokens) / factors
+    else:
+        # If no factors were completed, return the text length
+        # This happens when TTR stays above threshold for the entire text
+        return float(len(tokens))
+
+
+def _compute_mtld_single(text: str, threshold: float) -> tuple[float, float, float, dict]:
+    """Compute MTLD for a single chunk of text.
+
+    Returns:
+        Tuple of (mtld_forward, mtld_backward, mtld_average, metadata_dict).
+        Returns (nan, nan, nan, metadata) for empty input.
+    """
+    tokens = tokenize(text.lower())
+
+    if len(tokens) == 0:
+        return (
+            float("nan"),
+            float("nan"),
+            float("nan"),
+            {"token_count": 0},
+        )
+
+    mtld_forward = _calculate_mtld_direction(tokens, threshold, forward=True)
+    mtld_backward = _calculate_mtld_direction(tokens, threshold, forward=False)
+    mtld_average = (mtld_forward + mtld_backward) / 2
+
+    return (
+        mtld_forward,
+        mtld_backward,
+        mtld_average,
+        {"token_count": len(tokens)},
+    )
+
+
 def compute_mtld(
     text: str,
     threshold: float = 0.72,
+    chunk_size: int = 1000,
 ) -> MTLDResult:
     """
     Compute MTLD (Measure of Textual Lexical Diversity).
 
+    This function uses native chunked analysis to capture variance and patterns
+    across the text, which is essential for stylometric fingerprinting.
+
     MTLD measures the mean length of sequential word strings that maintain
     a minimum threshold TTR. It's more robust than simple TTR for texts of
     varying lengths.
 
     Formula:
-        MTLD = mean(forward_factors, backward_factors)
-        where factors are word string lengths that maintain TTR >= threshold
+        MTLD = total_tokens / factor_count
+        where factor_count includes:
+        - Completed factors (segments where TTR dropped below threshold)
+        - Partial factor for any remaining incomplete segment (weighted by proximity to threshold)
+
+    Related GitHub Issue:
+        #27 - Native chunked analysis with Distribution dataclass
+        https://github.com/craigtrim/pystylometry/issues/27
 
     References:
         McCarthy, P. M., & Jarvis, S. (2010). MTLD, vocd-D, and HD-D:
@@ -26,36 +133,87 @@ def compute_mtld(
 
     Args:
         text: Input text to analyze
-        threshold: TTR threshold to maintain (default: 0.72)
+        threshold: TTR threshold to maintain (default: 0.72, must be in range (0, 1))
+        chunk_size: Number of words per chunk (default: 1000)
 
     Returns:
-        MTLDResult with forward, backward, and average MTLD scores
+        MTLDResult with forward, backward, average MTLD scores and distributions
+
+    Raises:
+        ValueError: If threshold is not in range (0, 1)
 
     Example:
-        >>> result = compute_mtld("The quick brown fox jumps over the lazy dog...")
-        >>> print(f"MTLD: {result.mtld_average:.2f}")
+        >>> result = compute_mtld("Long text here...", chunk_size=1000)
+        >>> result.mtld_average  # Mean across chunks
+        72.5
+        >>> result.mtld_average_dist.std  # Variance reveals fingerprint
+        8.3
     """
-    tokens = tokenize(text)
+    # Validate threshold parameter
+    if not (0 < threshold < 1):
+        raise ValueError(
+            f"Threshold must be in range (0, 1), got {threshold}. "
+            "Common values: 0.72 (default), 0.5-0.8"
+        )
 
-    if len(tokens) == 0:
+    # Chunk the text
+    chunks = chunk_text(text, chunk_size)
+
+    # Compute metrics per chunk
+    forward_values = []
+    backward_values = []
+    average_values = []
+    total_tokens = 0
+
+    for chunk in chunks:
+        fwd, bwd, avg, meta = _compute_mtld_single(chunk, threshold)
+        if not math.isnan(fwd):
+            forward_values.append(fwd)
+            backward_values.append(bwd)
+            average_values.append(avg)
+        total_tokens += meta.get("token_count", 0)
+
+    # Handle empty or all-invalid chunks
+    if not forward_values:
+        empty_dist = Distribution(
+            values=[],
+            mean=float("nan"),
+            median=float("nan"),
+            std=0.0,
+            range=0.0,
+            iqr=0.0,
+        )
         return MTLDResult(
-            mtld_forward=0.0,
-            mtld_backward=0.0,
-            mtld_average=0.0,
-            metadata={"token_count": 0, "threshold": threshold},
+            mtld_forward=float("nan"),
+            mtld_backward=float("nan"),
+            mtld_average=float("nan"),
+            mtld_forward_dist=empty_dist,
+            mtld_backward_dist=empty_dist,
+            mtld_average_dist=empty_dist,
+            chunk_size=chunk_size,
+            chunk_count=len(chunks),
+            metadata={
+                "total_token_count": 0,
+                "threshold": threshold,
+            },
         )
 
-    # TODO: Implement forward and backward MTLD calculation
-    mtld_forward = 0.0  # Placeholder
-    mtld_backward = 0.0  # Placeholder
-    mtld_average = (mtld_forward + mtld_backward) / 2
+    # Build distributions
+    forward_dist = make_distribution(forward_values)
+    backward_dist = make_distribution(backward_values)
+    average_dist = make_distribution(average_values)
 
     return MTLDResult(
-        mtld_forward=mtld_forward,
-        mtld_backward=mtld_backward,
-        mtld_average=mtld_average,
+        mtld_forward=forward_dist.mean,
+        mtld_backward=backward_dist.mean,
+        mtld_average=average_dist.mean,
+        mtld_forward_dist=forward_dist,
+        mtld_backward_dist=backward_dist,
+        mtld_average_dist=average_dist,
+        chunk_size=chunk_size,
+        chunk_count=len(chunks),
         metadata={
-            "token_count": len(tokens),
+            "total_token_count": total_tokens,
             "threshold": threshold,
         },
     )