pystylometry 0.1.0__py3-none-any.whl

pystylometry/authorship/burrows_delta.py

@@ -0,0 +1,152 @@
+"""Burrows' Delta and Cosine Delta for authorship attribution."""
+
+import math
+import statistics
+from collections import Counter
+
+from .._types import BurrowsDeltaResult
+from .._utils import tokenize
+
+
+def compute_burrows_delta(
+    text1: str, text2: str, mfw: int = 500, distance_type: str = "burrows"
+) -> BurrowsDeltaResult:
+    """
+    Compute Burrows' Delta or Cosine Delta between two texts.
+
+    Burrows' Delta:
+        Delta = mean(|z₁(f) - z₂(f)|) for all features f
+        where z(f) = (frequency(f) - mean(f)) / std(f)
+
+    Cosine Delta:
+        Delta = 1 - cos(z₁, z₂)
+        Measures angular distance between z-score vectors
+
+    Both methods:
+    1. Extract most frequent words (MFW) across both texts
+    2. Calculate word frequencies in each text
+    3. Z-score normalize frequencies
+    4. Compute distance measure
+
+    Lower scores indicate more similar texts (likely same author).
+
+    References:
+        Burrows, J. (2002). 'Delta': A measure of stylistic difference and
+        a guide to likely authorship. Literary and Linguistic Computing, 17(3), 267-287.
+
+        Argamon, S. (2008). Interpreting Burrows's Delta: Geometric and
+        probabilistic foundations. Literary and Linguistic Computing, 23(2), 131-147.
+
+    Args:
+        text1: First text to compare
+        text2: Second text to compare
+        mfw: Number of most frequent words to use (default: 500)
+        distance_type: "burrows", "cosine", or "eder" (default: "burrows")
+
+    Returns:
+        BurrowsDeltaResult with delta score and metadata
+
+    Example:
+        >>> result = compute_burrows_delta(text1, text2, mfw=300)
+        >>> print(f"Delta score: {result.delta_score:.3f}")
+        >>> print(f"Lower is more similar")
+    """
+    # Tokenize and count words
+    tokens1 = [t.lower() for t in tokenize(text1)]
+    tokens2 = [t.lower() for t in tokenize(text2)]
+
+    if len(tokens1) == 0 or len(tokens2) == 0:
+        return BurrowsDeltaResult(
+            delta_score=0.0,
+            distance_type=distance_type,
+            mfw_count=0,
+            metadata={
+                "text1_token_count": len(tokens1),
+                "text2_token_count": len(tokens2),
+                "warning": "One or both texts are empty",
+            },
+        )
+
+    # Get word frequencies
+    freq1 = Counter(tokens1)
+    freq2 = Counter(tokens2)
+
+    # Get most frequent words across both texts
+    all_words: Counter[str] = Counter()
+    all_words.update(freq1)
+    all_words.update(freq2)
+    most_common_words = [word for word, _ in all_words.most_common(mfw)]
+
+    # Calculate relative frequencies for MFW
+    def get_relative_freqs(freq_counter: Counter, words: list[str], total: int) -> list[float]:
+        return [freq_counter.get(word, 0) / total for word in words]
+
+    rel_freqs1 = get_relative_freqs(freq1, most_common_words, len(tokens1))
+    rel_freqs2 = get_relative_freqs(freq2, most_common_words, len(tokens2))
+
+    # Combine for z-score calculation (treat as corpus)
+    combined_freqs = [(f1 + f2) / 2 for f1, f2 in zip(rel_freqs1, rel_freqs2)]
+
+    # Calculate standard deviation for each word position
+    combined_std = []
+    for i in range(len(most_common_words)):
+        values = [rel_freqs1[i], rel_freqs2[i]]
+        std = statistics.stdev(values) if len(set(values)) > 1 else 1e-10
+        combined_std.append(std if std > 0 else 1e-10)
+
+    # Calculate z-scores
+    z1 = [(f - mean) / std for f, mean, std in zip(rel_freqs1, combined_freqs, combined_std)]
+    z2 = [(f - mean) / std for f, mean, std in zip(rel_freqs2, combined_freqs, combined_std)]
+
+    # Calculate distance based on type
+    if distance_type == "burrows":
+        # Burrows' Delta: mean absolute difference of z-scores
+        abs_diffs = [abs(z1_val - z2_val) for z1_val, z2_val in zip(z1, z2)]
+        delta_score = statistics.mean(abs_diffs) if abs_diffs else 0.0
+    elif distance_type == "cosine":
+        # Cosine Delta: 1 - cosine similarity
+        dot_product = sum(z1_val * z2_val for z1_val, z2_val in zip(z1, z2))
+        norm1 = math.sqrt(sum(z**2 for z in z1))
+        norm2 = math.sqrt(sum(z**2 for z in z2))
+        cosine_sim = dot_product / (norm1 * norm2) if norm1 > 0 and norm2 > 0 else 0
+        delta_score = 1 - cosine_sim
+    elif distance_type == "eder":
+        # Eder's Delta: similar to Burrows but with different normalization
+        abs_diffs = [abs(z1_val - z2_val) for z1_val, z2_val in zip(z1, z2)]
+        delta_score = statistics.mean(abs_diffs) if abs_diffs else 0.0
+    else:
+        abs_diffs = [abs(z1_val - z2_val) for z1_val, z2_val in zip(z1, z2)]
+        delta_score = statistics.mean(abs_diffs) if abs_diffs else 0.0
+
+    return BurrowsDeltaResult(
+        delta_score=delta_score,
+        distance_type=distance_type,
+        mfw_count=len(most_common_words),
+        metadata={
+            "text1_token_count": len(tokens1),
+            "text2_token_count": len(tokens2),
+            "text1_vocab": len(freq1),
+            "text2_vocab": len(freq2),
+        },
+    )
+
+
+def compute_cosine_delta(text1: str, text2: str, mfw: int = 500) -> BurrowsDeltaResult:
+    """
+    Compute Cosine Delta between two texts.
+
+    Convenience function that calls compute_burrows_delta with distance_type="cosine".
+
+    Args:
+        text1: First text to compare
+        text2: Second text to compare
+        mfw: Number of most frequent words to use (default: 500)
+
+    Returns:
+        BurrowsDeltaResult with cosine delta score
+
+    Example:
+        >>> result = compute_cosine_delta(text1, text2)
+        >>> print(f"Cosine Delta: {result.delta_score:.3f}")
+    """
+    return compute_burrows_delta(text1, text2, mfw=mfw, distance_type="cosine")

pystylometry/authorship/zeta.py

@@ -0,0 +1,109 @@
+"""Zeta score for distinctive word usage in authorship attribution."""
+
+from .._types import ZetaResult
+from .._utils import tokenize
+
+
+def compute_zeta(text1: str, text2: str, segments: int = 10, top_n: int = 50) -> ZetaResult:
+    """
+    Compute Zeta score for distinctive word usage between two texts or text groups.
+
+    Zeta identifies words that are consistently used in one text/author but not another.
+
+    Algorithm:
+    1. Divide each text into segments
+    2. Calculate document proportion (DP) for each word:
+       - DP₁ = proportion of segments in text1 containing the word
+       - DP₂ = proportion of segments in text2 containing the word
+    3. Zeta score = DP₁ - DP₂
+    4. Positive Zeta = marker words (distinctive of text1)
+    5. Negative Zeta = anti-marker words (distinctive of text2)
+
+    References:
+        Burrows, J. (2007). All the way through: Testing for authorship in
+        different frequency strata. Literary and Linguistic Computing, 22(1), 27-47.
+
+        Craig, H., & Kinney, A. F. (2009). Shakespeare, Computers, and the
+        Mystery of Authorship. Cambridge University Press.
+
+    Args:
+        text1: First text (candidate author)
+        text2: Second text (comparison author/corpus)
+        segments: Number of segments to divide each text into (default: 10)
+        top_n: Number of top marker/anti-marker words to return (default: 50)
+
+    Returns:
+        ZetaResult with zeta score, marker words, and anti-marker words
+
+    Example:
+        >>> result = compute_zeta(author1_text, author2_text)
+        >>> print(f"Zeta score: {result.zeta_score:.3f}")
+        >>> print(f"Marker words: {result.marker_words[:10]}")
+        >>> print(f"Anti-markers: {result.anti_marker_words[:10]}")
+    """
+    # Tokenize texts
+    tokens1 = [t.lower() for t in tokenize(text1)]
+    tokens2 = [t.lower() for t in tokenize(text2)]
+
+    if len(tokens1) < segments or len(tokens2) < segments:
+        return ZetaResult(
+            zeta_score=0.0,
+            marker_words=[],
+            anti_marker_words=[],
+            metadata={
+                "text1_token_count": len(tokens1),
+                "text2_token_count": len(tokens2),
+                "segments": segments,
+                "top_n": top_n,
+                "warning": "Text too short for requested number of segments",
+            },
+        )
+
+    # Divide texts into segments
+    def create_segments(tokens: list[str], n_segments: int) -> list[set[str]]:
+        segment_size = len(tokens) // n_segments
+        return [set(tokens[i * segment_size : (i + 1) * segment_size]) for i in range(n_segments)]
+
+    segments1 = create_segments(tokens1, segments)
+    segments2 = create_segments(tokens2, segments)
+
+    # Get all unique words
+    all_words = set(tokens1) | set(tokens2)
+
+    # Calculate document proportion (DP) for each word
+    word_scores = {}
+    for word in all_words:
+        # DP1: proportion of segments in text1 containing the word
+        dp1 = sum(1 for seg in segments1 if word in seg) / len(segments1)
+        # DP2: proportion of segments in text2 containing the word
+        dp2 = sum(1 for seg in segments2 if word in seg) / len(segments2)
+        # Zeta score for this word
+        word_scores[word] = dp1 - dp2
+
+    # Sort words by zeta score
+    sorted_words = sorted(word_scores.items(), key=lambda x: x[1], reverse=True)
+
+    # Extract top marker words (positive zeta) and anti-marker words (negative zeta)
+    marker_words = [word for word, score in sorted_words[:top_n] if score > 0]
+    anti_marker_words = [word for word, score in sorted_words[-top_n:] if score < 0]
+    anti_marker_words.reverse()  # Most negative first
+
+    # Overall zeta score (mean of absolute zeta scores)
+    zeta_score = (
+        sum(abs(score) for score in word_scores.values()) / len(word_scores) if word_scores else 0.0
+    )
+
+    return ZetaResult(
+        zeta_score=zeta_score,
+        marker_words=marker_words,
+        anti_marker_words=anti_marker_words,
+        metadata={
+            "text1_token_count": len(tokens1),
+            "text2_token_count": len(tokens2),
+            "segments": segments,
+            "top_n": top_n,
+            "total_unique_words": len(all_words),
+            "marker_word_count": len(marker_words),
+            "anti_marker_word_count": len(anti_marker_words),
+        },
+    )

pystylometry/lexical/__init__.py

@@ -0,0 +1,17 @@
+"""Lexical diversity metrics."""
+
+# Re-export from stylometry-ttr
+# from stylometry_ttr import compute_ttr, TTRResult
+
+# Local implementations
+from .hapax import compute_hapax_ratios
+from .mtld import compute_mtld
+from .yule import compute_yule
+
+__all__ = [
+    # "compute_ttr",  # From stylometry-ttr
+    # "TTRResult",    # From stylometry-ttr
+    "compute_mtld",
+    "compute_yule",
+    "compute_hapax_ratios",
+]

pystylometry/lexical/hapax.py

@@ -0,0 +1,75 @@
+"""Hapax legomena and related vocabulary richness metrics."""
+
+from collections import Counter
+
+from .._types import HapaxResult
+from .._utils import tokenize
+
+
+def compute_hapax_ratios(text: str) -> HapaxResult:
+    """
+    Compute hapax legomena, hapax dislegomena, and related richness metrics.
+
+    Hapax legomena = words appearing exactly once
+    Hapax dislegomena = words appearing exactly twice
+
+    Also computes:
+    - Sichel's S: V₂ / V (ratio of dislegomena to total vocabulary)
+    - Honoré's R: 100 × log(N) / (1 - V₁/V)
+
+    References:
+        Sichel, H. S. (1975). On a distribution law for word frequencies.
+        Journal of the American Statistical Association, 70(351a), 542-547.
+
+        Honoré, A. (1979). Some simple measures of richness of vocabulary.
+        Association for Literary and Linguistic Computing Bulletin, 7, 172-177.
+
+    Args:
+        text: Input text to analyze
+
+    Returns:
+        HapaxResult with counts, ratios, Sichel's S, Honoré's R, and metadata
+
+    Example:
+        >>> result = compute_hapax_ratios("The quick brown fox jumps over the lazy dog.")
+        >>> print(f"Hapax ratio: {result.hapax_ratio:.3f}")
+        >>> print(f"Sichel's S: {result.sichel_s:.3f}")
+    """
+    tokens = tokenize(text.lower())
+    N = len(tokens)  # noqa: N806
+
+    if N == 0:
+        return HapaxResult(
+            hapax_count=0,
+            hapax_ratio=0.0,
+            dis_hapax_count=0,
+            dis_hapax_ratio=0.0,
+            sichel_s=0.0,
+            honore_r=0.0,
+            metadata={"token_count": 0, "vocabulary_size": 0},
+        )
+
+    # Count frequency of each token
+    freq_counter = Counter(tokens)
+    V = len(freq_counter)  # noqa: N806
+
+    # Count hapax legomena (V₁) and dislegomena (V₂)
+    V1 = sum(1 for count in freq_counter.values() if count == 1)  # noqa: N806
+    V2 = sum(1 for count in freq_counter.values() if count == 2)  # noqa: N806
+
+    # TODO: Implement Sichel's S and Honoré's R
+    sichel_s = 0.0  # Placeholder
+    honore_r = 0.0  # Placeholder
+
+    return HapaxResult(
+        hapax_count=V1,
+        hapax_ratio=V1 / N if N > 0 else 0.0,
+        dis_hapax_count=V2,
+        dis_hapax_ratio=V2 / N if N > 0 else 0.0,
+        sichel_s=sichel_s,
+        honore_r=honore_r,
+        metadata={
+            "token_count": N,
+            "vocabulary_size": V,
+        },
+    )

pystylometry/lexical/mtld.py

@@ -0,0 +1,61 @@
+"""MTLD (Measure of Textual Lexical Diversity) implementation."""
+
+from .._types import MTLDResult
+from .._utils import tokenize
+
+
+def compute_mtld(
+    text: str,
+    threshold: float = 0.72,
+) -> MTLDResult:
+    """
+    Compute MTLD (Measure of Textual Lexical Diversity).
+
+    MTLD measures the mean length of sequential word strings that maintain
+    a minimum threshold TTR. It's more robust than simple TTR for texts of
+    varying lengths.
+
+    Formula:
+        MTLD = mean(forward_factors, backward_factors)
+        where factors are word string lengths that maintain TTR >= threshold
+
+    References:
+        McCarthy, P. M., & Jarvis, S. (2010). MTLD, vocd-D, and HD-D:
+        A validation study of sophisticated approaches to lexical diversity assessment.
+        Behavior Research Methods, 42(2), 381-392.
+
+    Args:
+        text: Input text to analyze
+        threshold: TTR threshold to maintain (default: 0.72)
+
+    Returns:
+        MTLDResult with forward, backward, and average MTLD scores
+
+    Example:
+        >>> result = compute_mtld("The quick brown fox jumps over the lazy dog...")
+        >>> print(f"MTLD: {result.mtld_average:.2f}")
+    """
+    tokens = tokenize(text)
+
+    if len(tokens) == 0:
+        return MTLDResult(
+            mtld_forward=0.0,
+            mtld_backward=0.0,
+            mtld_average=0.0,
+            metadata={"token_count": 0, "threshold": threshold},
+        )
+
+    # TODO: Implement forward and backward MTLD calculation
+    mtld_forward = 0.0  # Placeholder
+    mtld_backward = 0.0  # Placeholder
+    mtld_average = (mtld_forward + mtld_backward) / 2
+
+    return MTLDResult(
+        mtld_forward=mtld_forward,
+        mtld_backward=mtld_backward,
+        mtld_average=mtld_average,
+        metadata={
+            "token_count": len(tokens),
+            "threshold": threshold,
+        },
+    )

pystylometry/lexical/yule.py

@@ -0,0 +1,66 @@
+"""Yule's K and I statistics for vocabulary richness."""
+
+from collections import Counter
+
+from .._types import YuleResult
+from .._utils import tokenize
+
+
+def compute_yule(text: str) -> YuleResult:
+    """
+    Compute Yule's K and I metrics for vocabulary richness.
+
+    Yule's K measures vocabulary repetitiveness (higher = more repetitive).
+    Yule's I is the inverse measure (higher = more diverse).
+
+    Formula:
+        K = 10⁴ × (Σm²×Vm - N) / N²
+        I = (V² / Σm²×Vm) - (1/N)
+
+    Where:
+        - N = total tokens
+        - V = vocabulary size (unique types)
+        - Vm = number of types occurring m times
+        - m = frequency count
+
+    References:
+        Yule, G. U. (1944). The Statistical Study of Literary Vocabulary.
+        Cambridge University Press.
+
+    Args:
+        text: Input text to analyze
+
+    Returns:
+        YuleResult with .yule_k, .yule_i, and metadata
+
+    Example:
+        >>> result = compute_yule("The quick brown fox jumps over the lazy dog.")
+        >>> print(f"Yule's K: {result.yule_k:.2f}")
+        >>> print(f"Yule's I: {result.yule_i:.2f}")
+    """
+    tokens = tokenize(text.lower())
+    N = len(tokens)  # noqa: N806
+
+    if N == 0:
+        return YuleResult(yule_k=0.0, yule_i=0.0, metadata={"token_count": 0, "vocabulary_size": 0})
+
+    # Count frequency of each token
+    freq_counter = Counter(tokens)
+    V = len(freq_counter)  # noqa: N806
+
+    # Count how many words occur with each frequency
+    # Vm[m] = number of words that occur exactly m times
+    # freq_of_freqs = Counter(freq_counter.values())  # TODO: Will be needed for Yule's K
+
+    # TODO: Implement Yule's K and I calculations
+    yule_k = 0.0  # Placeholder
+    yule_i = 0.0  # Placeholder
+
+    return YuleResult(
+        yule_k=yule_k,
+        yule_i=yule_i,
+        metadata={
+            "token_count": N,
+            "vocabulary_size": V,
+        },
+    )

pystylometry/ngrams/__init__.py

@@ -0,0 +1,13 @@
+"""N-gram entropy and sequence analysis metrics."""
+
+from .entropy import (
+    compute_character_bigram_entropy,
+    compute_ngram_entropy,
+    compute_word_bigram_entropy,
+)
+
+__all__ = [
+    "compute_ngram_entropy",
+    "compute_character_bigram_entropy",
+    "compute_word_bigram_entropy",
+]

pystylometry/ngrams/entropy.py

@@ -0,0 +1,130 @@
+"""N-gram entropy and perplexity calculations."""
+
+import math
+from collections import Counter
+
+from .._types import EntropyResult
+from .._utils import tokenize
+
+
+def compute_ngram_entropy(text: str, n: int = 2, ngram_type: str = "word") -> EntropyResult:
+    """
+    Compute n-gram entropy and perplexity for text.
+
+    Entropy measures the unpredictability of the next item in a sequence.
+    Higher entropy = more unpredictable = more diverse/complex text.
+
+    Formula:
+        H(X) = -Σ p(x) × log₂(p(x))
+        Perplexity = 2^H(X)
+
+    Where p(x) is the probability of n-gram x occurring.
+
+    References:
+        Shannon, C. E. (1948). A mathematical theory of communication.
+        Bell System Technical Journal, 27(3), 379-423.
+
+        Manning, C. D., & Schütze, H. (1999). Foundations of Statistical
+        Natural Language Processing. MIT Press.
+
+    Args:
+        text: Input text to analyze
+        n: N-gram size (2 for bigrams, 3 for trigrams, etc.)
+        ngram_type: "word" or "character" (default: "word")
+
+    Returns:
+        EntropyResult with entropy, perplexity, and metadata
+
+    Example:
+        >>> result = compute_ngram_entropy("The quick brown fox jumps", n=2, ngram_type="word")
+        >>> print(f"Bigram entropy: {result.entropy:.3f}")
+        >>> print(f"Perplexity: {result.perplexity:.3f}")
+    """
+    # Generate n-grams
+    if ngram_type == "character":
+        items = list(text)
+    else:  # word
+        items = tokenize(text)
+
+    if len(items) < n:
+        return EntropyResult(
+            entropy=0.0,
+            perplexity=1.0,
+            ngram_type=f"{ngram_type}_{n}gram",
+            metadata={
+                "n": n,
+                "ngram_type": ngram_type,
+                "item_count": len(items),
+                "warning": "Text too short for n-gram analysis",
+            },
+        )
+
+    # Create n-grams using sliding window
+    ngram_list = []
+    for i in range(len(items) - n + 1):
+        ngram = tuple(items[i : i + n])
+        ngram_list.append(ngram)
+
+    # Count n-gram frequencies
+    ngram_counts = Counter(ngram_list)
+    total_ngrams = len(ngram_list)
+
+    # Calculate entropy: H(X) = -Σ p(x) × log₂(p(x))
+    entropy = 0.0
+    for count in ngram_counts.values():
+        probability = count / total_ngrams
+        entropy -= probability * math.log2(probability)
+
+    # Calculate perplexity: 2^H(X)
+    perplexity = 2**entropy
+
+    return EntropyResult(
+        entropy=entropy,
+        perplexity=perplexity,
+        ngram_type=f"{ngram_type}_{n}gram",
+        metadata={
+            "n": n,
+            "ngram_type": ngram_type,
+            "item_count": len(items),
+            "unique_ngrams": len(ngram_counts),
+            "total_ngrams": total_ngrams,
+        },
+    )
+
+
+def compute_character_bigram_entropy(text: str) -> EntropyResult:
+    """
+    Compute character bigram entropy.
+
+    Convenience function that calls compute_ngram_entropy with n=2, ngram_type="character".
+
+    Args:
+        text: Input text to analyze
+
+    Returns:
+        EntropyResult with character bigram entropy and perplexity
+
+    Example:
+        >>> result = compute_character_bigram_entropy("The quick brown fox")
+        >>> print(f"Character bigram entropy: {result.entropy:.3f}")
+    """
+    return compute_ngram_entropy(text, n=2, ngram_type="character")
+
+
+def compute_word_bigram_entropy(text: str) -> EntropyResult:
+    """
+    Compute word bigram entropy.
+
+    Convenience function that calls compute_ngram_entropy with n=2, ngram_type="word".
+
+    Args:
+        text: Input text to analyze
+
+    Returns:
+        EntropyResult with word bigram entropy and perplexity
+
+    Example:
+        >>> result = compute_word_bigram_entropy("The quick brown fox jumps")
+        >>> print(f"Word bigram entropy: {result.entropy:.3f}")
+    """
+    return compute_ngram_entropy(text, n=2, ngram_type="word")

pystylometry/readability/__init__.py

@@ -0,0 +1,15 @@
+"""Readability metrics."""
+
+from .ari import compute_ari
+from .coleman_liau import compute_coleman_liau
+from .flesch import compute_flesch
+from .gunning_fog import compute_gunning_fog
+from .smog import compute_smog
+
+__all__ = [
+    "compute_flesch",
+    "compute_smog",
+    "compute_gunning_fog",
+    "compute_coleman_liau",
+    "compute_ari",
+]