pystylometry 1.0.0__py3-none-any.whl → 1.3.0__py3-none-any.whl

pystylometry/authorship/README.md

@@ -0,0 +1,21 @@
+# authorship
+
+![7 public functions](https://img.shields.io/badge/functions-7-blue)
+![No external deps](https://img.shields.io/badge/deps-none-brightgreen)
+
+Authorship attribution methods for comparing texts and determining likely authorship.
+
+## Catalogue
+
+| File | Functions | Method |
+|------|-----------|--------|
+| `burrows_delta.py` | `compute_burrows_delta`, `compute_cosine_delta` | Classic Delta and angular distance variant |
+| `zeta.py` | `compute_zeta` | Zeta method for marker word detection |
+| `kilgarriff.py` | `compute_kilgarriff` | Chi-squared corpus comparison |
+| `additional_methods.py` | `compute_minmax`, `compute_johns_delta` | MinMax distance, Quadratic/Weighted Delta |
+| `compression.py` | `compute_compression_distance` | Normalized Compression Distance (NCD) |
+
+## See Also
+
+- [`consistency/`](../consistency/) applies `compute_kilgarriff` in sliding windows for intra-document drift detection
+- [`lexical/`](../lexical/) provides the vocabulary features many attribution methods rely on

pystylometry/authorship/__init__.py

@@ -1,14 +1,38 @@
-"""Authorship attribution metrics."""
+"""Authorship attribution metrics.
 
-from .additional_methods import compute_johns_delta, compute_kilgarriff, compute_minmax
+This module provides methods for authorship attribution - comparing texts to
+determine whether they were written by the same author. Available methods
+include classic approaches (Burrows' Delta, Zeta), statistical methods
+(Kilgarriff's chi-squared), and information-theoretic methods (NCD).
+
+Related GitHub Issues:
+    #24 - Additional Authorship Attribution Methods
+    https://github.com/craigtrim/pystylometry/issues/24
+    #31 - Classical Stylometric Methods from Programming Historian
+    https://github.com/craigtrim/pystylometry/issues/31
+
+Available Functions:
+    compute_burrows_delta: Classic Delta method for authorship distance
+    compute_cosine_delta: Angular distance variant of Delta
+    compute_zeta: Zeta method for marker word detection
+    compute_kilgarriff: Chi-squared method for corpus comparison
+    compute_minmax: Burrows' original min-max distance method
+    compute_johns_delta: Delta variations (quadratic, weighted)
+    compute_compression_distance: Normalized Compression Distance (NCD)
+"""
+
+from .additional_methods import compute_johns_delta, compute_minmax
 from .burrows_delta import compute_burrows_delta, compute_cosine_delta
+from .compression import compute_compression_distance
+from .kilgarriff import compute_kilgarriff
 from .zeta import compute_zeta
 
 __all__ = [
     "compute_burrows_delta",
+    "compute_compression_distance",
     "compute_cosine_delta",
-    "compute_zeta",
+    "compute_johns_delta",
     "compute_kilgarriff",
     "compute_minmax",
-    "compute_johns_delta",
+    "compute_zeta",
 ]

pystylometry/authorship/additional_methods.py

@@ -8,65 +8,150 @@ Related GitHub Issue:
     https://github.com/craigtrim/pystylometry/issues/24
 
 Methods implemented:
-    - Kilgarriff's Chi-squared
-    - Min-Max (Burrows' original method)
-    - John Burrows' Delta variations
+    - Kilgarriff's Chi-squared -> See kilgarriff.py (Issue #31)
+    - Min-Max distance (Burrows' original method)
+    - John Burrows' Delta variations (Quadratic, Weighted)
 
 References:
     Kilgarriff, A. (2001). Comparing corpora. International Journal of Corpus Linguistics.
     Burrows, J. F. (1992). Not unless you ask nicely. Literary and Linguistic Computing.
     Burrows, J. (2005). Who wrote Shamela? Literary and Linguistic Computing.
+    Argamon, S. (2008). Interpreting Burrows's Delta. Literary and Linguistic Computing.
 """
 
-from .._types import JohnsBurrowsResult, KilgarriffResult, MinMaxResult
+from __future__ import annotations
 
+import math
+from collections import Counter
 
-def compute_kilgarriff(text1: str, text2: str, mfw: int = 100) -> KilgarriffResult:
+from .._types import JohnsBurrowsResult, MinMaxResult
+from .._utils import tokenize
+
+
+def compute_minmax(text1: str, text2: str, mfw: int = 100) -> MinMaxResult:
     """
-    Compute Kilgarriff's Chi-squared distance between two texts.
+    Compute Min-Max distance between two texts.
+
+    This is Burrows' original method from his 1992 paper, before the development
+    of Delta. It normalizes word frequencies using min-max scaling and computes
+    the mean absolute distance between normalized frequency vectors.
 
     Related GitHub Issue:
         #24 - Additional Authorship Attribution Methods
         https://github.com/craigtrim/pystylometry/issues/24
 
+    Algorithm:
+        1. Tokenize both texts and build frequency distributions
+        2. Identify the top N most frequent words in the joint corpus
+        3. Compute relative frequencies for each word in each text
+        4. Normalize each word's frequencies using min-max scaling:
+           normalized(f) = (f - min) / (max - min)
+        5. Compute mean absolute difference of normalized frequencies
+
+    Interpretation:
+        - Lower values indicate more similar texts (likely same author)
+        - Higher values indicate more different texts
+        - Scale: 0.0 (identical) to 1.0 (maximally different)
+
+    References:
+        Burrows, J. F. (1992). Not unless you ask nicely: The interpretative
+            nexus between analysis and information. Literary and Linguistic
+            Computing, 7(2), 91-109.
+
     Args:
         text1: First text for comparison
         text2: Second text for comparison
-        mfw: Number of most frequent words to analyze
+        mfw: Number of most frequent words to analyze (default: 100)
 
     Returns:
-        KilgarriffResult with chi-squared statistic, p-value, and
-        most distinctive features.
+        MinMaxResult with min-max distance and distinctive features.
+
+    Example:
+        >>> result = compute_minmax(text_by_author_a, text_by_author_b)
+        >>> print(f"MinMax distance: {result.minmax_distance:.3f}")
+        >>> print(f"Most distinctive: {result.most_distinctive_features[0]}")
     """
-    # TODO: Implement Kilgarriff's chi-squared
-    # GitHub Issue #24: https://github.com/craigtrim/pystylometry/issues/24
-    raise NotImplementedError(
-        "Kilgarriff's chi-squared not yet implemented. "
-        "See GitHub Issue #24: https://github.com/craigtrim/pystylometry/issues/24"
-    )
+    # Tokenize and lowercase
+    tokens1 = [t.lower() for t in tokenize(text1) if t.isalpha()]
+    tokens2 = [t.lower() for t in tokenize(text2) if t.isalpha()]
 
+    if not tokens1 or not tokens2:
+        return MinMaxResult(
+            minmax_distance=0.0,
+            feature_count=0,
+            most_distinctive_features=[],
+            metadata={
+                "text1_size": len(tokens1),
+                "text2_size": len(tokens2),
+                "warning": "One or both texts are empty",
+            },
+        )
 
-def compute_minmax(text1: str, text2: str, mfw: int = 100) -> MinMaxResult:
-    """
-    Compute Min-Max distance (Burrows' original method).
+    # Build frequency distributions
+    freq1 = Counter(tokens1)
+    freq2 = Counter(tokens2)
+    size1 = len(tokens1)
+    size2 = len(tokens2)
 
-    Related GitHub Issue:
-        #24 - Additional Authorship Attribution Methods
-        https://github.com/craigtrim/pystylometry/issues/24
+    # Joint corpus: top N most frequent words
+    joint: Counter[str] = Counter()
+    joint.update(freq1)
+    joint.update(freq2)
+    top_words = [word for word, _ in joint.most_common(mfw)]
 
-    Args:
-        text1: First text for comparison
-        text2: Second text for comparison
-        mfw: Number of most frequent words to analyze
+    if not top_words:
+        return MinMaxResult(
+            minmax_distance=0.0,
+            feature_count=0,
+            most_distinctive_features=[],
+            metadata={
+                "text1_size": size1,
+                "text2_size": size2,
+                "warning": "No common words found",
+            },
+        )
 
-    Returns:
-        MinMaxResult with min-max distance and distinctive features.
-    """
-    # TODO: Implement Min-Max distance
-    # GitHub Issue #24: https://github.com/craigtrim/pystylometry/issues/24
-    raise NotImplementedError(
-        "Min-Max distance not yet implemented. "
-        "See GitHub Issue #24: https://github.com/craigtrim/pystylometry/issues/24"
+    # Relative frequencies
+    rel1 = [freq1.get(w, 0) / size1 for w in top_words]
+    rel2 = [freq2.get(w, 0) / size2 for w in top_words]
+
+    # Min-Max normalization per feature across both texts
+    # Then compute absolute distance
+    contributions: list[tuple[str, float]] = []
+    total_distance = 0.0
+
+    for i, word in enumerate(top_words):
+        f1, f2 = rel1[i], rel2[i]
+        max_val = max(f1, f2)
+
+        if max_val > 0:
+            # Min-Max normalized distance for this feature
+            dist = abs(f1 - f2) / max_val
+        else:
+            dist = 0.0
+
+        total_distance += dist
+        contributions.append((word, dist))
+
+    # Sort contributions by magnitude
+    contributions.sort(key=lambda x: x[1], reverse=True)
+
+    # Mean distance across all features
+    minmax_distance = total_distance / len(top_words) if top_words else 0.0
+
+    return MinMaxResult(
+        minmax_distance=minmax_distance,
+        feature_count=len(top_words),
+        most_distinctive_features=contributions[:20],
+        metadata={
+            "text1_size": size1,
+            "text2_size": size2,
+            "text1_vocab": len(freq1),
+            "text2_vocab": len(freq2),
+            "mfw_requested": mfw,
+            "method": "minmax_1992",
+            "all_contributions": contributions,
+        },
     )
 
 
@@ -79,22 +164,157 @@ def compute_johns_delta(
     """
     Compute John Burrows' Delta variations.
 
+    This implements alternative formulations of Burrows' Delta metric beyond
+    the standard mean absolute z-score difference. The quadratic variant uses
+    squared z-score differences (Euclidean distance), while the weighted variant
+    applies inverse-rank weighting so higher-frequency words contribute more.
+
     Related GitHub Issue:
         #24 - Additional Authorship Attribution Methods
         https://github.com/craigtrim/pystylometry/issues/24
 
+    Methods:
+        - "quadratic": Euclidean distance of z-scores
+          Delta_Q = sqrt(sum((z1_i - z2_i)^2) / n)
+
+        - "weighted": Inverse-rank weighted Delta
+          Delta_W = sum(w_i * |z1_i - z2_i|) / sum(w_i)
+          where w_i = 1 / rank_i
+
+    Interpretation:
+        - Lower values indicate more similar texts (likely same author)
+        - Quadratic Delta penalizes large deviations more than standard Delta
+        - Weighted Delta emphasizes the most frequent words
+
+    References:
+        Burrows, J. (2005). Who wrote Shamela? Verifying the authorship of a
+            parodic text. Literary and Linguistic Computing, 20(4), 437-450.
+        Argamon, S. (2008). Interpreting Burrows's Delta: Geometric and
+            probabilistic foundations. Literary and Linguistic Computing,
+            23(2), 131-147.
+
     Args:
         text1: First text for comparison
         text2: Second text for comparison
-        mfw: Number of most frequent words to analyze
-        method: Delta variation ("quadratic", "weighted", "rotated")
+        mfw: Number of most frequent words to analyze (default: 100)
+        method: Delta variation ("quadratic" or "weighted")
 
     Returns:
         JohnsBurrowsResult with delta score and method details.
+
+    Example:
+        >>> result = compute_johns_delta(text1, text2, method="quadratic")
+        >>> print(f"Quadratic Delta: {result.delta_score:.3f}")
     """
-    # TODO: Implement John's Delta variations
-    # GitHub Issue #24: https://github.com/craigtrim/pystylometry/issues/24
-    raise NotImplementedError(
-        "John's Delta variations not yet implemented. "
-        "See GitHub Issue #24: https://github.com/craigtrim/pystylometry/issues/24"
+    if method not in ("quadratic", "weighted"):
+        raise ValueError(f"method must be 'quadratic' or 'weighted', got '{method}'")
+
+    # Tokenize and lowercase
+    tokens1 = [t.lower() for t in tokenize(text1) if t.isalpha()]
+    tokens2 = [t.lower() for t in tokenize(text2) if t.isalpha()]
+
+    if not tokens1 or not tokens2:
+        return JohnsBurrowsResult(
+            delta_score=0.0,
+            method=method,
+            feature_count=0,
+            most_distinctive_features=[],
+            metadata={
+                "text1_size": len(tokens1),
+                "text2_size": len(tokens2),
+                "warning": "One or both texts are empty",
+            },
+        )
+
+    # Build frequency distributions
+    freq1 = Counter(tokens1)
+    freq2 = Counter(tokens2)
+    size1 = len(tokens1)
+    size2 = len(tokens2)
+
+    # Joint corpus: top N most frequent words
+    joint: Counter[str] = Counter()
+    joint.update(freq1)
+    joint.update(freq2)
+    top_words = [word for word, _ in joint.most_common(mfw)]
+
+    if not top_words:
+        return JohnsBurrowsResult(
+            delta_score=0.0,
+            method=method,
+            feature_count=0,
+            most_distinctive_features=[],
+            metadata={
+                "text1_size": size1,
+                "text2_size": size2,
+                "warning": "No common words found",
+            },
+        )
+
+    # Relative frequencies
+    rel1 = [freq1.get(w, 0) / size1 for w in top_words]
+    rel2 = [freq2.get(w, 0) / size2 for w in top_words]
+
+    # Mean-normalized differences
+    # With only 2 texts, classical z-scores are degenerate: stdev([a,b]) is
+    # always |a-b|/sqrt(2), producing identical z-scores (±0.707) for all
+    # features with any difference. Instead, we normalize by the mean frequency
+    # of each feature across both texts, which preserves discriminative power:
+    #   normalized_i = (f1_i - f2_i) / mean(f1_i, f2_i)
+    # This weights words proportionally to how much they differ relative to
+    # their expected frequency, preventing high-frequency words from dominating
+    # through absolute differences alone.
+    z1: list[float] = []
+    z2: list[float] = []
+    for i in range(len(top_words)):
+        mean_val = (rel1[i] + rel2[i]) / 2
+        # Normalize by mean frequency; use epsilon for words absent in both
+        norm = mean_val if mean_val > 0 else 1e-10
+        z1.append((rel1[i] - mean_val) / norm)
+        z2.append((rel2[i] - mean_val) / norm)
+
+    # Compute distance based on method
+    contributions: list[tuple[str, float]] = []
+
+    if method == "quadratic":
+        # Quadratic Delta: root mean squared z-score difference
+        squared_diffs: list[float] = []
+        for i, word in enumerate(top_words):
+            diff_sq = (z1[i] - z2[i]) ** 2
+            squared_diffs.append(diff_sq)
+            contributions.append((word, diff_sq))
+
+        delta_score = math.sqrt(sum(squared_diffs) / len(squared_diffs)) if squared_diffs else 0.0
+
+    else:  # weighted
+        # Weighted Delta: inverse-rank weighted mean absolute z-score difference
+        weighted_diffs: list[float] = []
+        weights: list[float] = []
+        for i, word in enumerate(top_words):
+            weight = 1.0 / (i + 1)  # Inverse rank weighting
+            abs_diff = abs(z1[i] - z2[i])
+            weighted_diffs.append(weight * abs_diff)
+            weights.append(weight)
+            contributions.append((word, abs_diff))
+
+        delta_score = sum(weighted_diffs) / sum(weights) if weights else 0.0
+
+    # Sort contributions by magnitude
+    contributions.sort(key=lambda x: x[1], reverse=True)
+
+    return JohnsBurrowsResult(
+        delta_score=delta_score,
+        method=method,
+        feature_count=len(top_words),
+        most_distinctive_features=contributions[:20],
+        metadata={
+            "text1_size": size1,
+            "text2_size": size2,
+            "text1_vocab": len(freq1),
+            "text2_vocab": len(freq2),
+            "mfw_requested": mfw,
+            "z_scores_text1": dict(zip(top_words[:20], z1[:20])),
+            "z_scores_text2": dict(zip(top_words[:20], z2[:20])),
+            "all_contributions": contributions,
+        },
     )

pystylometry/authorship/compression.py

@@ -0,0 +1,175 @@
+"""Compression-based authorship attribution using Normalized Compression Distance.
+
+This module implements the Normalized Compression Distance (NCD) method for
+authorship attribution. NCD is a language-independent similarity metric based
+on Kolmogorov complexity, approximated through real-world compressors.
+
+Related GitHub Issue:
+    #24 - Additional Authorship Attribution Methods
+    https://github.com/craigtrim/pystylometry/issues/24
+
+The core insight is that if two texts share statistical regularities (as texts
+by the same author tend to), compressing them together yields better compression
+than compressing separately. This captures deep patterns including vocabulary,
+syntax, and stylistic preferences without requiring explicit feature engineering.
+
+References:
+    Cilibrasi, R., & Vitányi, P. M. (2005). Clustering by compression.
+        IEEE Transactions on Information Theory, 51(4), 1523-1545.
+    Benedetto, D., Caglioti, E., & Loreto, V. (2002). Language trees and
+        zipping. Physical Review Letters, 88(4), 048702.
+    Li, M., et al. (2004). The similarity metric. IEEE Transactions on
+        Information Theory, 50(12), 3250-3264.
+"""
+
+from __future__ import annotations
+
+import bz2
+import gzip
+import zlib
+
+from .._types import CompressionResult
+
+# Supported compressors mapped to their compress functions
+_COMPRESSORS: dict[str, type] = {
+    "gzip": type(None),  # placeholder, handled below
+    "zlib": type(None),
+    "bz2": type(None),
+}
+
+_VALID_COMPRESSORS = frozenset({"gzip", "zlib", "bz2"})
+
+
+def _compress(data: bytes, compressor: str) -> bytes:
+    """Compress data using the specified algorithm.
+
+    Args:
+        data: Raw bytes to compress.
+        compressor: One of "gzip", "zlib", or "bz2".
+
+    Returns:
+        Compressed bytes.
+    """
+    if compressor == "gzip":
+        return gzip.compress(data)
+    if compressor == "zlib":
+        return zlib.compress(data)
+    if compressor == "bz2":
+        return bz2.compress(data)
+    raise ValueError(f"Unknown compressor: {compressor}")  # pragma: no cover
+
+
+def compute_compression_distance(
+    text1: str,
+    text2: str,
+    compressor: str = "gzip",
+) -> CompressionResult:
+    """
+    Compute Normalized Compression Distance (NCD) between two texts.
+
+    NCD approximates the normalized information distance, a universal similarity
+    metric based on Kolmogorov complexity. Since Kolmogorov complexity is
+    uncomputable, NCD uses real-world compressors as practical approximations.
+
+    Related GitHub Issue:
+        #24 - Additional Authorship Attribution Methods
+        https://github.com/craigtrim/pystylometry/issues/24
+
+    Algorithm:
+        1. Encode both texts as UTF-8 bytes
+        2. Compress text1, text2, and their concatenation separately
+        3. Compute NCD using the formula:
+           NCD(x,y) = (C(xy) - min(C(x), C(y))) / max(C(x), C(y))
+
+    Interpretation:
+        - NCD ~ 0: Texts are maximally similar (identical information content)
+        - NCD ~ 1: Texts are maximally different (no shared information)
+        - Values slightly above 1.0 are possible due to compressor overhead
+        - Typical same-author pairs: 0.3-0.6
+        - Typical different-author pairs: 0.6-0.9
+
+    Compressor choice:
+        - "gzip" (default): Good balance of speed and accuracy; most widely used
+          in NCD literature. Uses Lempel-Ziv (LZ77) algorithm.
+        - "zlib": Same underlying algorithm as gzip but lower overhead. Slightly
+          faster, very similar results.
+        - "bz2": Uses Burrows-Wheeler transform. Better compression but slower.
+          May capture different patterns than LZ-based methods.
+
+    References:
+        Cilibrasi, R., & Vitanyi, P. M. (2005). Clustering by compression.
+            IEEE Transactions on Information Theory, 51(4), 1523-1545.
+        Benedetto, D., Caglioti, E., & Loreto, V. (2002). Language trees and
+            zipping. Physical Review Letters, 88(4), 048702.
+        Li, M., et al. (2004). The similarity metric. IEEE Transactions on
+            Information Theory, 50(12), 3250-3264.
+
+    Args:
+        text1: First text for comparison.
+        text2: Second text for comparison.
+        compressor: Compression algorithm to use ("gzip", "zlib", or "bz2").
+
+    Returns:
+        CompressionResult with NCD score and compression details.
+
+    Raises:
+        ValueError: If compressor is not one of "gzip", "zlib", "bz2".
+
+    Example:
+        >>> result = compute_compression_distance(text_by_author_a, text_by_author_b)
+        >>> print(f"NCD: {result.ncd:.3f}")
+        >>> print(f"Compressor: {result.compressor}")
+        >>> if result.ncd < 0.5:
+        ...     print("Texts likely by same author")
+    """
+    if compressor not in _VALID_COMPRESSORS:
+        raise ValueError(
+            f"compressor must be one of {sorted(_VALID_COMPRESSORS)}, got '{compressor}'"
+        )
+
+    # Encode texts as bytes
+    bytes1 = text1.encode("utf-8")
+    bytes2 = text2.encode("utf-8")
+    bytes_combined = bytes1 + bytes2
+
+    # Compress each component
+    compressed1 = _compress(bytes1, compressor)
+    compressed2 = _compress(bytes2, compressor)
+    compressed_combined = _compress(bytes_combined, compressor)
+
+    c1 = len(compressed1)
+    c2 = len(compressed2)
+    c12 = len(compressed_combined)
+
+    # NCD formula: (C(xy) - min(C(x), C(y))) / max(C(x), C(y))
+    min_c = min(c1, c2)
+    max_c = max(c1, c2)
+
+    if max_c == 0:
+        # Both texts are empty
+        ncd = 0.0
+    else:
+        ncd = (c12 - min_c) / max_c
+
+    # Compute compression ratios for metadata
+    raw1 = len(bytes1)
+    raw2 = len(bytes2)
+    raw_combined = len(bytes_combined)
+
+    return CompressionResult(
+        ncd=ncd,
+        compressor=compressor,
+        text1_compressed_size=c1,
+        text2_compressed_size=c2,
+        combined_compressed_size=c12,
+        metadata={
+            "text1_raw_size": raw1,
+            "text2_raw_size": raw2,
+            "combined_raw_size": raw_combined,
+            "text1_compression_ratio": c1 / raw1 if raw1 > 0 else 0.0,
+            "text2_compression_ratio": c2 / raw2 if raw2 > 0 else 0.0,
+            "combined_compression_ratio": c12 / raw_combined if raw_combined > 0 else 0.0,
+            "min_compressed": min_c,
+            "max_compressed": max_c,
+        },
+    )