pystylometry 1.1.0__py3-none-any.whl → 1.3.0__py3-none-any.whl

pystylometry/authorship/compression.py

@@ -0,0 +1,175 @@
+"""Compression-based authorship attribution using Normalized Compression Distance.
+
+This module implements the Normalized Compression Distance (NCD) method for
+authorship attribution. NCD is a language-independent similarity metric based
+on Kolmogorov complexity, approximated through real-world compressors.
+
+Related GitHub Issue:
+    #24 - Additional Authorship Attribution Methods
+    https://github.com/craigtrim/pystylometry/issues/24
+
+The core insight is that if two texts share statistical regularities (as texts
+by the same author tend to), compressing them together yields better compression
+than compressing separately. This captures deep patterns including vocabulary,
+syntax, and stylistic preferences without requiring explicit feature engineering.
+
+References:
+    Cilibrasi, R., & Vitányi, P. M. (2005). Clustering by compression.
+        IEEE Transactions on Information Theory, 51(4), 1523-1545.
+    Benedetto, D., Caglioti, E., & Loreto, V. (2002). Language trees and
+        zipping. Physical Review Letters, 88(4), 048702.
+    Li, M., et al. (2004). The similarity metric. IEEE Transactions on
+        Information Theory, 50(12), 3250-3264.
+"""
+
+from __future__ import annotations
+
+import bz2
+import gzip
+import zlib
+
+from .._types import CompressionResult
+
+# Supported compressors mapped to their compress functions
+_COMPRESSORS: dict[str, type] = {
+    "gzip": type(None),  # placeholder, handled below
+    "zlib": type(None),
+    "bz2": type(None),
+}
+
+_VALID_COMPRESSORS = frozenset({"gzip", "zlib", "bz2"})
+
+
+def _compress(data: bytes, compressor: str) -> bytes:
+    """Compress data using the specified algorithm.
+
+    Args:
+        data: Raw bytes to compress.
+        compressor: One of "gzip", "zlib", or "bz2".
+
+    Returns:
+        Compressed bytes.
+    """
+    if compressor == "gzip":
+        return gzip.compress(data)
+    if compressor == "zlib":
+        return zlib.compress(data)
+    if compressor == "bz2":
+        return bz2.compress(data)
+    raise ValueError(f"Unknown compressor: {compressor}")  # pragma: no cover
+
+
+def compute_compression_distance(
+    text1: str,
+    text2: str,
+    compressor: str = "gzip",
+) -> CompressionResult:
+    """
+    Compute Normalized Compression Distance (NCD) between two texts.
+
+    NCD approximates the normalized information distance, a universal similarity
+    metric based on Kolmogorov complexity. Since Kolmogorov complexity is
+    uncomputable, NCD uses real-world compressors as practical approximations.
+
+    Related GitHub Issue:
+        #24 - Additional Authorship Attribution Methods
+        https://github.com/craigtrim/pystylometry/issues/24
+
+    Algorithm:
+        1. Encode both texts as UTF-8 bytes
+        2. Compress text1, text2, and their concatenation separately
+        3. Compute NCD using the formula:
+           NCD(x,y) = (C(xy) - min(C(x), C(y))) / max(C(x), C(y))
+
+    Interpretation:
+        - NCD ~ 0: Texts are maximally similar (identical information content)
+        - NCD ~ 1: Texts are maximally different (no shared information)
+        - Values slightly above 1.0 are possible due to compressor overhead
+        - Typical same-author pairs: 0.3-0.6
+        - Typical different-author pairs: 0.6-0.9
+
+    Compressor choice:
+        - "gzip" (default): Good balance of speed and accuracy; most widely used
+          in NCD literature. Uses Lempel-Ziv (LZ77) algorithm.
+        - "zlib": Same underlying algorithm as gzip but lower overhead. Slightly
+          faster, very similar results.
+        - "bz2": Uses Burrows-Wheeler transform. Better compression but slower.
+          May capture different patterns than LZ-based methods.
+
+    References:
+        Cilibrasi, R., & Vitanyi, P. M. (2005). Clustering by compression.
+            IEEE Transactions on Information Theory, 51(4), 1523-1545.
+        Benedetto, D., Caglioti, E., & Loreto, V. (2002). Language trees and
+            zipping. Physical Review Letters, 88(4), 048702.
+        Li, M., et al. (2004). The similarity metric. IEEE Transactions on
+            Information Theory, 50(12), 3250-3264.
+
+    Args:
+        text1: First text for comparison.
+        text2: Second text for comparison.
+        compressor: Compression algorithm to use ("gzip", "zlib", or "bz2").
+
+    Returns:
+        CompressionResult with NCD score and compression details.
+
+    Raises:
+        ValueError: If compressor is not one of "gzip", "zlib", "bz2".
+
+    Example:
+        >>> result = compute_compression_distance(text_by_author_a, text_by_author_b)
+        >>> print(f"NCD: {result.ncd:.3f}")
+        >>> print(f"Compressor: {result.compressor}")
+        >>> if result.ncd < 0.5:
+        ...     print("Texts likely by same author")
+    """
+    if compressor not in _VALID_COMPRESSORS:
+        raise ValueError(
+            f"compressor must be one of {sorted(_VALID_COMPRESSORS)}, got '{compressor}'"
+        )
+
+    # Encode texts as bytes
+    bytes1 = text1.encode("utf-8")
+    bytes2 = text2.encode("utf-8")
+    bytes_combined = bytes1 + bytes2
+
+    # Compress each component
+    compressed1 = _compress(bytes1, compressor)
+    compressed2 = _compress(bytes2, compressor)
+    compressed_combined = _compress(bytes_combined, compressor)
+
+    c1 = len(compressed1)
+    c2 = len(compressed2)
+    c12 = len(compressed_combined)
+
+    # NCD formula: (C(xy) - min(C(x), C(y))) / max(C(x), C(y))
+    min_c = min(c1, c2)
+    max_c = max(c1, c2)
+
+    if max_c == 0:
+        # Both texts are empty
+        ncd = 0.0
+    else:
+        ncd = (c12 - min_c) / max_c
+
+    # Compute compression ratios for metadata
+    raw1 = len(bytes1)
+    raw2 = len(bytes2)
+    raw_combined = len(bytes_combined)
+
+    return CompressionResult(
+        ncd=ncd,
+        compressor=compressor,
+        text1_compressed_size=c1,
+        text2_compressed_size=c2,
+        combined_compressed_size=c12,
+        metadata={
+            "text1_raw_size": raw1,
+            "text2_raw_size": raw2,
+            "combined_raw_size": raw_combined,
+            "text1_compression_ratio": c1 / raw1 if raw1 > 0 else 0.0,
+            "text2_compression_ratio": c2 / raw2 if raw2 > 0 else 0.0,
+            "combined_compression_ratio": c12 / raw_combined if raw_combined > 0 else 0.0,
+            "min_compressed": min_c,
+            "max_compressed": max_c,
+        },
+    )

pystylometry/authorship/kilgarriff.py

@@ -249,6 +249,7 @@ def compute_kilgarriff(
     text1: str,
     text2: str,
     n_words: int = 500,
+    top_features: int = 20,
 ) -> KilgarriffResult:
     """
     Compute Kilgarriff's chi-squared distance between two texts.
@@ -292,6 +293,8 @@ def compute_kilgarriff(
         text2: Second text for comparison
         n_words: Number of most frequent words to analyze (default: 500).
             Higher values provide finer discrimination but require longer texts.
+        top_features: Number of most distinctive features to return (default: 20).
+            Controls the length of most_distinctive_features in the result.
 
     Returns:
         KilgarriffResult containing:
@@ -318,6 +321,10 @@ def compute_kilgarriff(
         violated in text analysis. The raw chi_squared value is more reliable
         for relative comparisons between text pairs.
     """
+    # Validate top_features
+    if top_features < 1:
+        raise ValueError("top_features must be >= 1")
+
     # Tokenize and lowercase
     # Using lowercase ensures "The" and "the" are counted together
     tokens1 = [t.lower() for t in tokenize(text1) if t.isalpha()]
@@ -337,7 +344,7 @@ def compute_kilgarriff(
         p_value=p_value,
         degrees_of_freedom=df,
         feature_count=len(contributions),
-        most_distinctive_features=contributions[:20],  # Top 20 contributors
+        most_distinctive_features=contributions[:top_features],
         metadata={
             **details,
             "all_contributions": contributions,  # Full list for detailed analysis

pystylometry/character/README.md

@@ -0,0 +1,17 @@
+# character
+
+![1 public function](https://img.shields.io/badge/functions-1-blue)
+![No external deps](https://img.shields.io/badge/deps-none-brightgreen)
+
+Character-level features for stylometric fingerprinting.
+
+## Catalogue
+
+| File | Function | What It Measures |
+|------|----------|-----------------|
+| `character_metrics.py` | `compute_character_metrics` | Letter frequencies, digit ratios, uppercase ratios, special character usage, whitespace patterns |
+
+## See Also
+
+- [`ngrams/`](../ngrams/) for character-level bigram entropy
+- [`stylistic/markers.py`](../stylistic/) for punctuation style analysis

pystylometry/consistency/README.md

@@ -0,0 +1,27 @@
+# consistency
+
+![1 public function](https://img.shields.io/badge/functions-1-blue)
+![No external deps](https://img.shields.io/badge/deps-none-brightgreen)
+
+Intra-document style drift detection using sliding-window chi-squared analysis.
+
+## Catalogue
+
+| File | Function | What It Does |
+|------|----------|-------------|
+| `drift.py` | `compute_kilgarriff_drift` | Detects stylistic drift, splice points, and AI-generation signatures |
+| `_thresholds.py` | _(internal)_ | Classification thresholds for pattern detection |
+
+## Detected Patterns
+
+| Pattern | Meaning |
+|---------|---------|
+| `consistent` | Natural human variation throughout |
+| `gradual_drift` | Style shifts progressively over the document |
+| `sudden_spike` | Abrupt discontinuity (possible splice or paste) |
+| `suspiciously_uniform` | Unnaturally low variation (possible AI generation) |
+
+## See Also
+
+- [`authorship/kilgarriff.py`](../authorship/) -- the underlying chi-squared method (between-text comparison)
+- [`viz/`](../viz/) for timeline and report visualizations of drift results

pystylometry/dialect/README.md

@@ -0,0 +1,26 @@
+# dialect
+
+![1 public function](https://img.shields.io/badge/functions-1-blue)
+![No external deps](https://img.shields.io/badge/deps-none-brightgreen)
+
+Regional dialect detection (British vs. American English) with markedness scoring.
+
+## Catalogue
+
+| File | Function | What It Does |
+|------|----------|-------------|
+| `detector.py` | `compute_dialect` | Classifies text dialect, computes British/American scores, markedness |
+| `_loader.py` | `get_markers`, `DialectMarkers` | Loads and caches extensible JSON marker database |
+| `_data/dialect_markers.json` | _(data)_ | Vocabulary, spelling, grammar, and eye-dialect markers |
+
+## Detection Categories
+
+- **Vocabulary** -- flat/apartment, lorry/truck, boot/trunk
+- **Spelling** -- colour/color, organise/organize, centre/center
+- **Grammar** -- collective noun agreement, "have got" patterns
+- **Eye dialect** -- gonna, wanna (register markers, not true dialect)
+
+## See Also
+
+- [`stylistic/`](../stylistic/) for broader style marker analysis
+- [`stylistic/genre_register.py`](../stylistic/) for formality and register classification

pystylometry/lexical/README.md

@@ -0,0 +1,23 @@
+# lexical
+
+![11 public functions](https://img.shields.io/badge/functions-11-blue)
+![No external deps](https://img.shields.io/badge/deps-none-brightgreen)
+
+Vocabulary diversity, richness, and frequency analysis.
+
+## Catalogue
+
+| File | Functions | What It Measures |
+|------|-----------|-----------------|
+| `ttr.py` | `compute_ttr` | Type-Token Ratio (basic vocabulary diversity) |
+| `mtld.py` | `compute_mtld` | Measure of Textual Lexical Diversity |
+| `yule.py` | `compute_yule` | Yule's K and I (frequency spectrum measures) |
+| `hapax.py` | `compute_hapax_ratios`, `compute_hapax_with_lexicon_analysis` | Hapax legomena, Honore's R, Sichel's S |
+| `advanced_diversity.py` | `compute_vocd_d`, `compute_mattr`, `compute_hdd`, `compute_msttr` | VocD-D, MATTR, HD-D, MSTTR |
+| `function_words.py` | `compute_function_words` | Function word frequencies by category |
+| `word_frequency_sophistication.py` | `compute_word_frequency_sophistication` | Vocabulary sophistication via frequency bands |
+
+## See Also
+
+- [`authorship/`](../authorship/) uses lexical features for attribution
+- [`stylistic/vocabulary_overlap.py`](../stylistic/) for Jaccard, Dice, and KL divergence between texts

pystylometry/ngrams/README.md

@@ -0,0 +1,18 @@
+# ngrams
+
+![4 public functions](https://img.shields.io/badge/functions-4-blue)
+![No external deps](https://img.shields.io/badge/deps-none-brightgreen)
+
+N-gram generation, entropy computation, and sequence analysis.
+
+## Catalogue
+
+| File | Functions | What It Measures |
+|------|-----------|-----------------|
+| `entropy.py` | `compute_ngram_entropy`, `compute_character_bigram_entropy`, `compute_word_bigram_entropy` | Shannon entropy at character and word n-gram levels |
+| `extended_ngrams.py` | `compute_extended_ngrams` | Word, character, and POS n-gram profiles with frequency distributions |
+
+## See Also
+
+- [`syntactic/`](../syntactic/) provides POS tags consumed by `compute_extended_ngrams(text, pos=True)`
+- [`character/`](../character/) for character-level features without n-gram structure