pystylometry 1.0.0__py3-none-any.whl → 1.3.0__py3-none-any.whl

pystylometry/cli.py

@@ -0,0 +1,427 @@
+"""Command-line interface for pystylometry.
+
+Usage:
+    pystylometry-drift <file> [--window-size=N] [--stride=N] [--mode=MODE] [--json]
+    pystylometry-drift <file> --plot [output.png]
+
+Example:
+    pystylometry-drift manuscript.txt
+    pystylometry-drift manuscript.txt --window-size=500 --stride=250
+    pystylometry-drift manuscript.txt --json
+    pystylometry-drift manuscript.txt --plot
+    pystylometry-drift manuscript.txt --plot drift_report.png
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+
+def drift_cli() -> None:
+    """CLI entry point for Kilgarriff drift detection."""
+    parser = argparse.ArgumentParser(
+        prog="pystylometry-drift",
+        description="Detect stylistic drift within a document using Kilgarriff chi-squared.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  pystylometry-drift manuscript.txt
+  pystylometry-drift manuscript.txt --window-size=500 --stride=250
+  pystylometry-drift manuscript.txt --mode=all_pairs --json
+  pystylometry-drift manuscript.txt --plot
+  pystylometry-drift manuscript.txt --plot report.png
+  pystylometry-drift manuscript.txt --plot timeline.png --plot-type=timeline
+  pystylometry-drift manuscript.txt --jsx report.html --plot-type=report
+  pystylometry-drift manuscript.txt --viz-all ./output  # All PNG + HTML
+
+Pattern Signatures:
+  consistent          Low, stable χ² across pairs (natural human writing)
+  gradual_drift       Slowly increasing trend (author fatigue, topic shift)
+  sudden_spike        One pair has high χ² (pasted content, different author)
+  suspiciously_uniform Near-zero variance (possible AI generation)
+""",
+    )
+
+    parser.add_argument(
+        "file",
+        type=Path,
+        help="Path to text file to analyze",
+    )
+    parser.add_argument(
+        "--window-size",
+        type=int,
+        default=1000,
+        help="Number of tokens per window (default: 1000)",
+    )
+    parser.add_argument(
+        "--stride",
+        type=int,
+        default=500,
+        help="Tokens to advance between windows (default: 500)",
+    )
+    parser.add_argument(
+        "--mode",
+        choices=["sequential", "all_pairs", "fixed_lag"],
+        default="sequential",
+        help="Comparison mode (default: sequential)",
+    )
+    parser.add_argument(
+        "--n-words",
+        type=int,
+        default=500,
+        help="Most frequent words to analyze (default: 500)",
+    )
+    parser.add_argument(
+        "--json",
+        action="store_true",
+        help="Output results as JSON",
+    )
+    parser.add_argument(
+        "--plot",
+        nargs="?",
+        const="",
+        default=None,
+        metavar="OUTPUT",
+        help="Generate visualization (optional: path to save, otherwise displays interactively)",
+    )
+    parser.add_argument(
+        "--plot-type",
+        choices=["report", "timeline"],
+        default="report",
+        help="Visualization type: report (multi-panel) or timeline (line chart)",
+    )
+    parser.add_argument(
+        "--jsx",
+        metavar="OUTPUT_FILE",
+        help="Export interactive visualization as standalone HTML (uses --plot-type)",
+    )
+    parser.add_argument(
+        "--viz-all",
+        metavar="OUTPUT_DIR",
+        type=Path,
+        help="Generate ALL visualizations (PNG + HTML) to directory for testing",
+    )
+
+    args = parser.parse_args()
+
+    # Validate file exists
+    if not args.file.exists():
+        print(f"Error: File not found: {args.file}", file=sys.stderr)
+        sys.exit(1)
+
+    # Read file
+    try:
+        text = args.file.read_text(encoding="utf-8")
+    except Exception as e:
+        print(f"Error reading file: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    # Determine output mode
+    if args.viz_all:
+        output_mode = "All Visualizations (PNG + HTML)"
+        output_dest = str(args.viz_all)
+    elif args.jsx:
+        output_mode = f"Interactive HTML ({args.plot_type})"
+        output_dest = args.jsx
+    elif args.plot is not None:
+        output_mode = f"Plot ({args.plot_type})"
+        output_dest = args.plot if args.plot else "interactive display"
+    elif args.json:
+        output_mode = "JSON"
+        output_dest = "stdout"
+    else:
+        output_mode = "Text Report"
+        output_dest = "stdout"
+
+    # Calculate file stats
+    token_count = len(text.split())
+    char_count = len(text)
+
+    # Print professional intro banner
+    print()
+    print("  PYSTYLOMETRY — Kilgarriff Chi-Squared Drift Detection")
+    print("  ═══════════════════════════════════════════════════════════════════════")
+    print()
+    print("  INPUT")
+    print("  ───────────────────────────────────────────────────────────────────────")
+    print(f"    File:              {args.file}")
+    print(f"    Size:              {char_count:,} characters / {token_count:,} tokens")
+    print()
+    print("  PARAMETERS")
+    print("  ───────────────────────────────────────────────────────────────────────")
+    print(f"    Window size:       {args.window_size} tokens")
+    print(f"    Stride:            {args.stride} tokens")
+    print(
+        f"    Overlap:           {((args.window_size - args.stride) / args.window_size) * 100:.0f}%"
+    )
+    print(f"    Comparison mode:   {args.mode}")
+    print(f"    Top N words:       {args.n_words}")
+    print()
+    print("  OUTPUT")
+    print("  ───────────────────────────────────────────────────────────────────────")
+    print(f"    Format:            {output_mode}")
+    print(f"    Destination:       {output_dest}")
+    print()
+    print("  Running analysis...")
+    print()
+
+    # Import here to avoid slow startup
+    from pystylometry.consistency import compute_kilgarriff_drift
+
+    # Run analysis
+    result = compute_kilgarriff_drift(
+        text,
+        window_size=args.window_size,
+        stride=args.stride,
+        comparison_mode=args.mode,
+        n_words=args.n_words,
+    )
+
+    # Handle --viz-all: generate all visualizations for testing
+    if args.viz_all:
+        output_dir = args.viz_all
+        output_dir.mkdir(parents=True, exist_ok=True)
+        label = args.file.stem
+
+        from pystylometry.viz.jsx import export_drift_timeline_jsx
+
+        generated = []
+
+        # Write chunks to subdirectory
+        chunks_dir = output_dir / "chunks"
+        chunks_dir.mkdir(parents=True, exist_ok=True)
+
+        # Re-create windows to get chunk text (simple word-based chunking)
+        words = text.split()
+        chunk_texts = []
+        start = 0
+        chunk_idx = 0
+        while start + args.window_size <= len(words):
+            chunk_words = words[start : start + args.window_size]
+            chunk_text = " ".join(chunk_words)
+            chunk_texts.append(chunk_text)
+
+            # Write chunk file
+            chunk_path = chunks_dir / f"chunk_{chunk_idx:03d}.txt"
+            chunk_path.write_text(chunk_text, encoding="utf-8")
+            chunk_idx += 1
+            start += args.stride
+
+        print(f"  Created: {chunks_dir}/ ({len(chunk_texts)} chunks)")
+
+        # Generate timeline HTML with chunk content
+        out_path = output_dir / "drift-detection.html"
+        export_drift_timeline_jsx(
+            result,
+            output_file=out_path,
+            title=f"Drift Timeline: {label}",
+            chunks=chunk_texts,
+        )
+        generated.append(out_path)
+        print(f"  Created: {out_path}")
+
+        print()
+        n_viz, n_chunks = len(generated), len(chunk_texts)
+        print(f"Generated {n_viz} visualizations + {n_chunks} chunks to: {output_dir.resolve()}")
+        sys.exit(0)
+
+    # Handle JSX export (generates standalone HTML)
+    if args.jsx:
+        from pystylometry.viz.jsx import (
+            export_drift_report_jsx,
+            export_drift_timeline_jsx,
+        )
+
+        label = args.file.stem
+
+        if args.plot_type == "timeline":
+            output_path = export_drift_timeline_jsx(
+                result,
+                output_file=args.jsx,
+                title=f"Drift Timeline: {label}",
+            )
+        else:  # report (default)
+            output_path = export_drift_report_jsx(
+                result,
+                output_file=args.jsx,
+                label=label,
+            )
+
+        abs_path = output_path.resolve()
+        file_url = f"file://{abs_path}"
+        print(f"Interactive visualization saved to: {output_path}")
+        print(f"Open in browser: {file_url}")
+        sys.exit(0)
+
+    # Handle plot output
+    if args.plot is not None:
+        try:
+            from pystylometry.viz import plot_drift_report, plot_drift_timeline
+        except ImportError:
+            print(
+                "Error: Visualization requires optional dependencies.",
+                file=sys.stderr,
+            )
+            print(
+                "Install with: pip install pystylometry[viz] or poetry install --with viz",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+        plot_output: str | None = args.plot if args.plot else None
+        label = args.file.stem
+
+        if args.plot_type == "timeline":
+            plot_drift_timeline(result, output=plot_output, title=f"Drift Timeline: {label}")
+        else:  # report (default)
+            plot_drift_report(result, label=label, output=plot_output)
+
+        if plot_output:
+            print(f"Visualization saved to: {plot_output}")
+        sys.exit(0)
+
+    if args.json:
+        # JSON output
+        output = {
+            "status": result.status,
+            "status_message": result.status_message,
+            "pattern": result.pattern,
+            "pattern_confidence": result.pattern_confidence,
+            "mean_chi_squared": result.mean_chi_squared,
+            "std_chi_squared": result.std_chi_squared,
+            "max_chi_squared": result.max_chi_squared,
+            "min_chi_squared": result.min_chi_squared,
+            "max_location": result.max_location,
+            "trend": result.trend,
+            "window_size": result.window_size,
+            "stride": result.stride,
+            "overlap_ratio": result.overlap_ratio,
+            "window_count": result.window_count,
+            "comparison_mode": result.comparison_mode,
+        }
+        print(json.dumps(output, indent=2))
+    else:
+        # Human-readable output
+        print("=" * 60)
+        print("STYLISTIC DRIFT ANALYSIS")
+        print("=" * 60)
+        print(f"File: {args.file}")
+        print(f"Status: {result.status}")
+        print()
+
+        if result.status == "insufficient_data":
+            print(f"⚠️  {result.status_message}")
+            print()
+            print(f"Windows created: {result.window_count}")
+            print("Minimum required: 3")
+            print()
+            print("Try reducing --window-size or --stride to create more windows.")
+            sys.exit(0)
+
+        print("PATTERN DETECTED")
+        print("-" * 40)
+        print(f"  Pattern: {result.pattern}")
+        print(f"  Confidence: {result.pattern_confidence:.1%}")
+        print()
+
+        if result.pattern == "consistent":
+            print("  ✓ Text shows consistent writing style throughout.")
+        elif result.pattern == "gradual_drift":
+            print("  ↗ Text shows gradual stylistic drift over its length.")
+            print("    Possible causes: author fatigue, topic evolution, revision.")
+        elif result.pattern == "sudden_spike":
+            print("  ⚡ Text contains a sudden stylistic discontinuity.")
+            loc = result.max_location
+            print(f"    Location: Between windows {loc} and {loc + 1}")
+            print("    Possible causes: pasted content, different author, major edit.")
+        elif result.pattern == "suspiciously_uniform":
+            print("  ⚠️  Text shows unusually uniform style (near-zero variance).")
+            print("    Possible causes: AI-generated content, heavy editing, templated text.")
+
+        print()
+        print("CHI-SQUARED STATISTICS")
+        print("-" * 40)
+        print(f"  Mean χ²:  {result.mean_chi_squared:.2f}")
+        print(f"  Std χ²:   {result.std_chi_squared:.2f}")
+        print(f"  Min χ²:   {result.min_chi_squared:.2f}")
+        print(f"  Max χ²:   {result.max_chi_squared:.2f}")
+        print(f"  Trend:    {result.trend:+.4f}")
+        print()
+
+        print("WINDOW CONFIGURATION")
+        print("-" * 40)
+        print(f"  Window size:    {result.window_size} tokens")
+        print(f"  Stride:         {result.stride} tokens")
+        print(f"  Overlap:        {result.overlap_ratio:.1%}")
+        print(f"  Windows:        {result.window_count}")
+        print(f"  Comparisons:    {len(result.pairwise_scores)}")
+        print()
+
+        if result.status == "marginal_data":
+            print(f"⚠️  {result.status_message}")
+            print()
+
+
+def viewer_cli() -> None:
+    """CLI entry point for generating a standalone drift viewer."""
+    parser = argparse.ArgumentParser(
+        prog="pystylometry-viewer",
+        description="Generate a standalone HTML drift analysis viewer.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+This generates a self-contained HTML file that users can open in any browser
+to analyze their own text files. No Python or server required - just share
+the HTML file and anyone can use it.
+
+Examples:
+  pystylometry-viewer drift_analyzer.html
+  pystylometry-viewer ~/Desktop/analyzer.html --title "My Drift Analyzer"
+
+The generated viewer includes:
+  - Drag-and-drop file upload
+  - Configurable analysis parameters
+  - Interactive timeline visualization
+  - Client-side Kilgarriff chi-squared implementation
+""",
+    )
+
+    parser.add_argument(
+        "output",
+        type=Path,
+        help="Path to write the HTML viewer file",
+    )
+    parser.add_argument(
+        "--title",
+        default="Stylistic Drift Analyzer",
+        help="Page title (default: 'Stylistic Drift Analyzer')",
+    )
+
+    args = parser.parse_args()
+
+    from pystylometry.viz.jsx import export_drift_viewer
+
+    output_path = export_drift_viewer(args.output, title=args.title)
+
+    abs_path = output_path.resolve()
+    file_url = f"file://{abs_path}"
+
+    print()
+    print("  PYSTYLOMETRY — Standalone Drift Viewer")
+    print("  ═══════════════════════════════════════════════════════════════════════")
+    print()
+    print(f"  Generated: {output_path}")
+    print(f"  Open in browser: {file_url}")
+    print()
+    print("  This viewer can be shared with anyone. Users can:")
+    print("    • Drag-and-drop or upload .txt files")
+    print("    • Configure analysis parameters")
+    print("    • View interactive drift timeline")
+    print("    • Click points to see chunk comparisons")
+    print()
+
+
+if __name__ == "__main__":
+    drift_cli()

pystylometry/consistency/README.md

@@ -0,0 +1,27 @@
+# consistency
+
+![1 public function](https://img.shields.io/badge/functions-1-blue)
+![No external deps](https://img.shields.io/badge/deps-none-brightgreen)
+
+Intra-document style drift detection using sliding-window chi-squared analysis.
+
+## Catalogue
+
+| File | Function | What It Does |
+|------|----------|-------------|
+| `drift.py` | `compute_kilgarriff_drift` | Detects stylistic drift, splice points, and AI-generation signatures |
+| `_thresholds.py` | _(internal)_ | Classification thresholds for pattern detection |
+
+## Detected Patterns
+
+| Pattern | Meaning |
+|---------|---------|
+| `consistent` | Natural human variation throughout |
+| `gradual_drift` | Style shifts progressively over the document |
+| `sudden_spike` | Abrupt discontinuity (possible splice or paste) |
+| `suspiciously_uniform` | Unnaturally low variation (possible AI generation) |
+
+## See Also
+
+- [`authorship/kilgarriff.py`](../authorship/) -- the underlying chi-squared method (between-text comparison)
+- [`viz/`](../viz/) for timeline and report visualizations of drift results

pystylometry/consistency/__init__.py

@@ -0,0 +1,57 @@
+"""Consistency analysis module for pystylometry.
+
+This module provides tools for analyzing internal stylistic consistency within
+a single document. Unlike the `authorship` module (which compares different texts),
+the `consistency` module focuses on detecting patterns within one text:
+
+- Stylistic drift over the course of a document
+- Sudden discontinuities suggesting pasted content or different authors
+- Suspiciously uniform style (potential AI generation signature)
+- Natural variation patterns in human writing
+
+Related GitHub Issues:
+    #36 - Kilgarriff Chi-Squared drift detection for intra-document analysis
+    https://github.com/craigtrim/pystylometry/issues/36
+    #27 - Native chunked analysis with Distribution dataclass
+    https://github.com/craigtrim/pystylometry/issues/27
+
+Marketing Names:
+    - "Style Drift Detector"
+    - "Consistency Fingerprint"
+    - "Authorship Continuity Score"
+
+Available Functions:
+    compute_kilgarriff_drift: Detect stylistic drift using chi-squared method
+
+Example Usage:
+    >>> from pystylometry.consistency import compute_kilgarriff_drift
+    >>>
+    >>> # Analyze a document for stylistic consistency
+    >>> result = compute_kilgarriff_drift(document_text)
+    >>>
+    >>> # Check the detected pattern
+    >>> print(f"Pattern: {result.pattern}")  # e.g., "consistent", "sudden_spike"
+    >>> print(f"Confidence: {result.pattern_confidence:.2f}")
+    >>>
+    >>> # Investigate potential AI generation
+    >>> if result.pattern == "suspiciously_uniform":
+    ...     print("Warning: Text shows unusually uniform style")
+    >>>
+    >>> # Find where the biggest style shift occurs
+    >>> if result.pattern == "sudden_spike":
+    ...     print(f"Major discontinuity at window boundary {result.max_location}")
+
+References:
+    Kilgarriff, Adam. "Comparing Corpora." International Journal of Corpus
+        Linguistics, vol. 6, no. 1, 2001, pp. 97-133.
+
+    Eder, Maciej. "Does Size Matter? Authorship Attribution, Small Samples,
+        Big Problem." Digital Scholarship in the Humanities, vol. 30, no. 2,
+        2015, pp. 167-182.
+"""
+
+from .drift import compute_kilgarriff_drift
+
+__all__ = [
+    "compute_kilgarriff_drift",
+]

pystylometry/consistency/_thresholds.py

@@ -0,0 +1,162 @@
+"""Threshold constants for consistency pattern classification.
+
+This module contains calibration constants used for classifying stylistic
+patterns in the consistency module. These thresholds determine how the
+`compute_kilgarriff_drift()` function classifies detected patterns.
+
+Related GitHub Issues:
+    #36 - Kilgarriff Chi-Squared drift detection for intra-document analysis
+    https://github.com/craigtrim/pystylometry/issues/36
+
+Calibration Notes:
+    These thresholds are initial estimates based on theoretical considerations
+    and limited empirical testing. They should be refined through systematic
+    evaluation on diverse corpora including:
+    - Human-written texts of various lengths and genres
+    - AI-generated texts from different models
+    - Mixed human/AI texts
+    - Multi-author documents
+
+    The thresholds are exposed as module-level constants to allow:
+    1. Transparency: Users can inspect what values are used
+    2. Customization: Advanced users can override via metadata or subclassing
+    3. Research: Easy adjustment for empirical calibration studies
+
+Pattern Classification Logic:
+    The pattern classification uses a decision tree approach:
+
+    1. First check for insufficient variance (suspiciously uniform)
+       - AI-generated text often shows near-identical statistics across chunks
+       - This is detected by very low std_chi_squared
+
+    2. Then check for sudden discontinuities (sudden spike)
+       - Pasted content or different authors cause outlier chi-squared values
+       - Detected by max_chi_squared significantly exceeding mean
+
+    3. Then check for trends (gradual drift)
+       - Author fatigue or topic evolution shows increasing chi-squared
+       - Detected by significant slope in chi-squared over time
+
+    4. Otherwise classify as consistent (natural variation)
+       - Human writing typically shows moderate, stable variance
+
+References:
+    The thresholds are informed by stylometric literature but require
+    empirical validation:
+
+    Eder, Maciej, et al. "Stylometry with R: A Package for Computational Text
+        Analysis." The R Journal, vol. 8, no. 1, 2016, pp. 107-121.
+
+    Juola, Patrick. "Authorship Attribution." Foundations and Trends in
+        Information Retrieval, vol. 1, no. 3, 2006, pp. 233-334.
+"""
+
+from __future__ import annotations
+
+# =============================================================================
+# Window Count Thresholds
+# =============================================================================
+# These determine the minimum data requirements for meaningful analysis.
+
+# Absolute minimum: Need at least 2 comparisons for any variance calculation
+# With 3 windows, we get pairs: (1,2), (2,3) = 2 comparisons
+MIN_WINDOWS = 3
+
+# Recommended minimum: Need enough data points for reliable pattern classification
+# With 5 windows, we get pairs: (1,2), (2,3), (3,4), (4,5) = 4 comparisons
+# This allows computation of mean, std, and basic trend detection
+RECOMMENDED_WINDOWS = 5
+
+
+# =============================================================================
+# Pattern Classification Thresholds
+# =============================================================================
+# These control how chi-squared patterns are classified into named patterns.
+
+# --- Suspiciously Uniform Detection ---
+# AI-generated text often shows near-zero variance in stylistic metrics.
+# Human writing naturally fluctuates; AI maintains eerie consistency.
+
+# Coefficient of variation threshold (std / mean)
+# Below this, variance is suspiciously low
+UNIFORM_CV_THRESHOLD = 0.15
+
+# Also require low absolute mean for "uniform" classification
+# (High mean with low variance is just "consistent" with high baseline)
+UNIFORM_MEAN_THRESHOLD = 50.0
+
+
+# --- Sudden Spike Detection ---
+# Pasted content, different authors, or major edits cause discontinuities.
+
+# Ratio of max to mean that indicates a spike
+# If max > SPIKE_RATIO × mean, we have an outlier
+SPIKE_RATIO = 2.5
+
+# Absolute minimum spike size (to avoid false positives on low-baseline texts)
+SPIKE_MIN_ABSOLUTE = 100.0
+
+
+# --- Gradual Drift Detection ---
+# Increasing chi-squared over time suggests evolving style or author fatigue.
+
+# Minimum slope (chi-squared units per chunk pair) to detect trend
+# Positive = increasing difference over time
+# Negative = converging style (less common)
+TREND_SLOPE_THRESHOLD = 5.0
+
+# R-squared threshold: Trend must explain this much variance to be meaningful
+TREND_R_SQUARED_THRESHOLD = 0.3
+
+
+# --- Consistent Pattern ---
+# If none of the above patterns are detected, text is classified as "consistent"
+# This represents natural human writing with normal variation.
+
+
+# =============================================================================
+# Confidence Calculation Weights
+# =============================================================================
+# These control how pattern confidence scores are computed.
+
+# Minimum window count for full confidence
+# Below this, confidence is scaled down proportionally
+CONFIDENCE_MIN_WINDOWS = 5
+
+# Maximum confidence achievable with marginal data
+MARGINAL_DATA_MAX_CONFIDENCE = 0.6
+
+
+# =============================================================================
+# Export all thresholds as a dict for easy inspection
+# =============================================================================
+
+
+def get_all_thresholds() -> dict[str, float]:
+    """
+    Return all threshold values as a dictionary.
+
+    This is useful for:
+    - Logging/debugging: Record what thresholds were used
+    - Transparency: Include in result metadata
+    - Research: Compare different threshold settings
+
+    Returns:
+        Dict mapping threshold names to their values
+
+    Example:
+        >>> thresholds = get_all_thresholds()
+        >>> print(f"Spike ratio: {thresholds['spike_ratio']}")
+    """
+    return {
+        "min_windows": MIN_WINDOWS,
+        "recommended_windows": RECOMMENDED_WINDOWS,
+        "uniform_cv_threshold": UNIFORM_CV_THRESHOLD,
+        "uniform_mean_threshold": UNIFORM_MEAN_THRESHOLD,
+        "spike_ratio": SPIKE_RATIO,
+        "spike_min_absolute": SPIKE_MIN_ABSOLUTE,
+        "trend_slope_threshold": TREND_SLOPE_THRESHOLD,
+        "trend_r_squared_threshold": TREND_R_SQUARED_THRESHOLD,
+        "confidence_min_windows": CONFIDENCE_MIN_WINDOWS,
+        "marginal_data_max_confidence": MARGINAL_DATA_MAX_CONFIDENCE,
+    }