pystylometry 1.3.0__py3-none-any.whl → 1.3.5__py3-none-any.whl

pystylometry/__init__.py

@@ -40,14 +40,13 @@ Usage:
     print(result.pattern_confidence)
 """
 
+from . import lexical  # noqa: E402
 from ._types import AnalysisResult
+from .tokenizer import TokenizationStats, Tokenizer, TokenMetadata
 
 # Version
 __version__ = "0.1.0"
 
-# Core exports - always available
-from . import lexical
-
 # Optional exports - may raise ImportError if dependencies not installed
 try:
     from . import readability  # noqa: F401
@@ -87,6 +86,41 @@ _CONSISTENCY_AVAILABLE = True
 _STYLISTIC_AVAILABLE = True
 
 
+def tokenize(text: str, **kwargs: object) -> list[str]:
+    """Tokenize text using the stylometric tokenizer.
+
+    Convenience wrapper around Tokenizer.tokenize(). All keyword arguments
+    are forwarded to the Tokenizer constructor.
+
+    Args:
+        text: Input text to tokenize.
+        **kwargs: Options forwarded to Tokenizer (lowercase, strip_numbers,
+            expand_contractions, etc.).
+
+    Returns:
+        List of token strings.
+
+    Example:
+        >>> from pystylometry import tokenize
+        >>> tokenize("Hello, world! It's a test.")
+        ['hello', 'world', "it's", 'a', 'test']
+    """
+    return Tokenizer(**kwargs).tokenize(text)  # type: ignore[arg-type]
+
+
+def tokenize_with_metadata(text: str, **kwargs: object) -> list[TokenMetadata]:
+    """Tokenize text and return tokens with positional and type metadata.
+
+    Args:
+        text: Input text to tokenize.
+        **kwargs: Options forwarded to Tokenizer.
+
+    Returns:
+        List of TokenMetadata objects.
+    """
+    return Tokenizer(**kwargs).tokenize_with_metadata(text)  # type: ignore[arg-type]
+
+
 def analyze(
     text: str,
     lexical_metrics: bool = True,
@@ -225,6 +259,11 @@ __all__ = [
     "__version__",
     "analyze",
     "get_available_modules",
+    "tokenize",
+    "tokenize_with_metadata",
+    "Tokenizer",
+    "TokenMetadata",
+    "TokenizationStats",
     "lexical",
 ]
 

pystylometry/_types.py

@@ -23,7 +23,7 @@ from __future__ import annotations
 
 import statistics
 from dataclasses import dataclass
-from typing import Any
+from typing import Any, Optional
 
 # ===== Distribution and Chunking =====
 # Related to GitHub Issue #27: Native chunked analysis with Distribution dataclass
@@ -316,8 +316,8 @@ class HapaxLexiconResult:
 class TTRResult:
     """Result from Type-Token Ratio (TTR) analysis.
 
-    Wraps stylometry-ttr package functionality to measure vocabulary richness
-    through the ratio of unique words (types) to total words (tokens).
+    Measures vocabulary richness through the ratio of unique words (types)
+    to total words (tokens).
 
     All numeric metrics include both a mean value (convenient access) and
     a full distribution with per-chunk values and statistics.
@@ -370,6 +370,208 @@ class TTRResult:
     metadata: dict[str, Any]
 
 
+@dataclass
+class TTRAggregateResult:
+    """Aggregated TTR statistics for a collection of texts.
+
+    Computes group-level summary statistics (mean, std, min, max, median)
+    across multiple ``TTRResult`` objects.  Useful for comparative analysis
+    across authors, genres, or time periods.
+
+    Related GitHub Issue:
+        #43 - Inline stylometry-ttr into pystylometry (remove external dependency)
+        https://github.com/craigtrim/pystylometry/issues/43
+
+    Example:
+        >>> from pystylometry.lexical import compute_ttr, TTRAggregator
+        >>> results = [compute_ttr(t) for t in texts]
+        >>> agg = TTRAggregator()
+        >>> stats = agg.aggregate(results, group_id="Austen")
+        >>> stats.ttr_mean
+        0.412
+    """
+
+    group_id: str
+    text_count: int
+    total_words: int
+
+    # Raw TTR statistics
+    ttr_mean: float
+    ttr_std: float
+    ttr_min: float
+    ttr_max: float
+    ttr_median: float
+
+    # Root TTR (Guiraud's index) statistics
+    root_ttr_mean: float
+    root_ttr_std: float
+
+    # Log TTR (Herdan's C) statistics
+    log_ttr_mean: float
+    log_ttr_std: float
+
+    # STTR statistics (None if no texts had enough words for STTR)
+    sttr_mean: Optional[float]
+    sttr_std: Optional[float]
+
+    # Delta std mean (None if no texts had delta metrics)
+    delta_std_mean: Optional[float]
+
+    metadata: dict[str, Any]
+
+
+# ===== Repetition Detection Results =====
+# Related to GitHub Issue #28: Verbal tics detection for slop analysis
+# https://github.com/craigtrim/pystylometry/issues/28
+
+
+@dataclass
+class RepetitiveWord:
+    """A single word flagged as abnormally repetitive.
+
+    The repetition_score is the ratio of observed count to expected count
+    based on the word's frequency in the British National Corpus (BNC).
+    Higher scores indicate stronger overrepresentation.
+
+    Related GitHub Issue:
+        #28 - Verbal tics detection for slop analysis
+        https://github.com/craigtrim/pystylometry/issues/28
+
+    Attributes:
+        word: The flagged word (lowercased).
+        count: Observed count in the text.
+        expected_count: Expected count based on BNC relative frequency × text length.
+            0.0 if word not found in BNC.
+        repetition_score: count / expected_count. float('inf') if expected_count is 0.
+        bnc_bucket: BNC frequency bucket (1-100, 1=most frequent). None if not in BNC.
+        chunk_counts: Per-chunk occurrence counts (for distribution analysis).
+        distribution_entropy: Shannon entropy of the word's chunk distribution.
+            Low entropy = suspiciously even spread (model tic).
+            High entropy = clustered usage (human writing about a specific scene).
+        distribution_variance: Variance of per-chunk counts.
+    """
+
+    word: str
+    count: int
+    expected_count: float
+    repetition_score: float
+    bnc_bucket: int | None
+    chunk_counts: list[int]
+    distribution_entropy: float
+    distribution_variance: float
+
+
+@dataclass
+class RepetitiveUnigramsResult:
+    """Result from repetitive unigram detection.
+
+    Identifies content words that appear far more frequently than expected
+    based on their frequency in the British National Corpus (BNC, ~100M tokens).
+    This is a key indicator of AI-generated "slop" where models exhibit verbal
+    tics — repeating certain words with suspicious regularity.
+
+    Related GitHub Issue:
+        #28 - Verbal tics detection for slop analysis
+        https://github.com/craigtrim/pystylometry/issues/28
+
+    The slop_score provides a single aggregate metric:
+        slop_score = flagged_words_per_10k × mean_repetition_score
+
+    Where:
+        - flagged_words_per_10k = count of flagged words / (total content words / 10000)
+        - mean_repetition_score = mean repetition_score across all flagged words
+
+    Higher slop_score = more likely AI-generated verbal tics.
+
+    References:
+        British National Corpus Consortium. (2007). The British National Corpus,
+            version 3 (BNC XML Edition). http://www.natcorp.ox.ac.uk/
+
+    Example:
+        >>> result = compute_repetitive_unigrams(text)
+        >>> for w in result.repetitive_words[:5]:
+        ...     print(f"{w.word}: {w.count}x (expected {w.expected_count:.1f}, "
+        ...           f"score {w.repetition_score:.1f})")
+        shimmered: 23x (expected 0.1, score 266.2)
+        >>> result.slop_score
+        42.7
+    """
+
+    repetitive_words: list[RepetitiveWord]  # Sorted by repetition_score descending
+    total_content_words: int
+    flagged_count: int  # Number of words exceeding threshold
+    flagged_words_per_10k: float  # flagged_count / (total_content_words / 10000)
+    mean_repetition_score: float  # Mean score across flagged words
+    slop_score: float  # Aggregate: flagged_words_per_10k × mean_repetition_score
+    total_content_words_dist: Distribution
+    chunk_size: int
+    chunk_count: int
+    metadata: dict[str, Any]
+
+
+@dataclass
+class RepetitiveNgram:
+    """A single n-gram flagged as abnormally repetitive.
+
+    Content n-grams (bigrams, trigrams, etc.) should rarely repeat verbatim
+    in natural writing. N-grams that repeat beyond a length-scaled threshold
+    are flagged.
+
+    Related GitHub Issue:
+        #28 - Verbal tics detection for slop analysis
+        https://github.com/craigtrim/pystylometry/issues/28
+
+    Attributes:
+        ngram: The flagged n-gram as a tuple of words.
+        count: Observed count in the text.
+        frequency_per_10k: Occurrences per 10,000 n-grams.
+        chunk_counts: Per-chunk occurrence counts.
+        distribution_entropy: Shannon entropy of the n-gram's chunk distribution.
+        distribution_variance: Variance of per-chunk counts.
+    """
+
+    ngram: tuple[str, ...]
+    count: int
+    frequency_per_10k: float
+    chunk_counts: list[int]
+    distribution_entropy: float
+    distribution_variance: float
+
+
+@dataclass
+class RepetitiveNgramsResult:
+    """Result from repetitive n-gram detection.
+
+    Detects bigrams, trigrams, or higher-order n-grams that repeat more than
+    expected within the text. No external corpus is required — content n-grams
+    should not repeat verbatim often in natural writing.
+
+    N-grams composed entirely of function words (e.g., "of the", "in a") are
+    excluded since their repetition is expected.
+
+    Related GitHub Issue:
+        #28 - Verbal tics detection for slop analysis
+        https://github.com/craigtrim/pystylometry/issues/28
+
+    Example:
+        >>> result = compute_repetitive_ngrams(text, n=2)
+        >>> for ng in result.repetitive_ngrams[:5]:
+        ...     print(f"{' '.join(ng.ngram)}: {ng.count}x "
+        ...           f"({ng.frequency_per_10k:.1f} per 10k)")
+        uncomfortable truth: 8x (1.6 per 10k)
+    """
+
+    repetitive_ngrams: list[RepetitiveNgram]  # Sorted by count descending
+    n: int | tuple[int, ...]  # N-gram order(s) analyzed
+    total_ngrams: int
+    flagged_count: int
+    flagged_per_10k: float  # flagged_count / (total_ngrams / 10000)
+    total_ngrams_dist: Distribution
+    chunk_size: int
+    chunk_count: int
+    metadata: dict[str, Any]
+
+
 # ===== Readability Results =====
 
 

pystylometry/cli.py

@@ -3,6 +3,7 @@
 Usage:
     pystylometry-drift <file> [--window-size=N] [--stride=N] [--mode=MODE] [--json]
     pystylometry-drift <file> --plot [output.png]
+    pystylometry-tokenize <file> [--json] [--metadata] [--stats]
 
 Example:
     pystylometry-drift manuscript.txt
@@ -10,6 +11,9 @@ Example:
     pystylometry-drift manuscript.txt --json
     pystylometry-drift manuscript.txt --plot
     pystylometry-drift manuscript.txt --plot drift_report.png
+    pystylometry-tokenize manuscript.txt
+    pystylometry-tokenize manuscript.txt --json --metadata
+    pystylometry-tokenize manuscript.txt --stats
 """
 
 from __future__ import annotations
@@ -423,5 +427,322 @@ The generated viewer includes:
     print()
 
 
+def tokenize_cli() -> None:
+    """CLI entry point for stylometric tokenization."""
+    parser = argparse.ArgumentParser(
+        prog="pystylometry-tokenize",
+        description="Tokenize text for stylometric analysis.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  pystylometry-tokenize manuscript.txt
+  pystylometry-tokenize manuscript.txt --json
+  pystylometry-tokenize manuscript.txt --json --metadata
+  pystylometry-tokenize manuscript.txt --stats
+  pystylometry-tokenize manuscript.txt -U --expand-contractions
+  pystylometry-tokenize manuscript.txt --min-length 3 --strip-numbers
+""",
+    )
+
+    parser.add_argument(
+        "file",
+        type=Path,
+        help="Path to text file to tokenize",
+    )
+
+    # Output mode
+    output_group = parser.add_argument_group("output")
+    output_group.add_argument(
+        "-j",
+        "--json",
+        action="store_true",
+        help="Output as JSON (list of strings, or list of objects with --metadata)",
+    )
+    output_group.add_argument(
+        "-m",
+        "--metadata",
+        action="store_true",
+        help="Include token type and position metadata (implies --json)",
+    )
+    output_group.add_argument(
+        "-s",
+        "--stats",
+        action="store_true",
+        help="Show tokenization statistics instead of tokens",
+    )
+
+    # Core behavior
+    behavior_group = parser.add_argument_group("behavior")
+    behavior_group.add_argument(
+        "-U",
+        "--no-lowercase",
+        action="store_true",
+        help="Preserve original case (default: lowercase)",
+    )
+    behavior_group.add_argument(
+        "-e",
+        "--expand-contractions",
+        action="store_true",
+        help="Expand contractions (it's -> it is)",
+    )
+    behavior_group.add_argument(
+        "-n",
+        "--strip-numbers",
+        action="store_true",
+        help="Remove numeric tokens",
+    )
+    behavior_group.add_argument(
+        "--keep-punctuation",
+        action="store_true",
+        help="Keep punctuation tokens (default: stripped)",
+    )
+
+    # Filtering
+    filter_group = parser.add_argument_group("filtering")
+    filter_group.add_argument(
+        "--min-length",
+        type=int,
+        default=1,
+        metavar="N",
+        help="Minimum token length (default: 1)",
+    )
+    filter_group.add_argument(
+        "--max-length",
+        type=int,
+        default=None,
+        metavar="N",
+        help="Maximum token length (default: unlimited)",
+    )
+    filter_group.add_argument(
+        "--preserve-urls",
+        action="store_true",
+        help="Keep URL tokens",
+    )
+    filter_group.add_argument(
+        "--preserve-emails",
+        action="store_true",
+        help="Keep email tokens",
+    )
+    filter_group.add_argument(
+        "--preserve-hashtags",
+        action="store_true",
+        help="Keep hashtag tokens",
+    )
+    filter_group.add_argument(
+        "--preserve-mentions",
+        action="store_true",
+        help="Keep @mention tokens",
+    )
+
+    # Advanced
+    advanced_group = parser.add_argument_group("advanced")
+    advanced_group.add_argument(
+        "--expand-abbreviations",
+        action="store_true",
+        help="Expand abbreviations (Dr. -> Doctor)",
+    )
+    advanced_group.add_argument(
+        "--strip-accents",
+        action="store_true",
+        help="Remove accents from characters",
+    )
+    advanced_group.add_argument(
+        "--no-clean",
+        action="store_true",
+        help="Skip text cleaning (italics, brackets, page markers)",
+    )
+    advanced_group.add_argument(
+        "--no-unicode-normalize",
+        action="store_true",
+        help="Skip unicode normalization",
+    )
+
+    args = parser.parse_args()
+
+    # --- ANSI colors ---
+    use_color = sys.stderr.isatty()
+
+    def _c(code: str, text: str) -> str:
+        return f"\033[{code}m{text}\033[0m" if use_color else text
+
+    bold = lambda t: _c("1", t)  # noqa: E731
+    dim = lambda t: _c("2", t)  # noqa: E731
+    cyan = lambda t: _c("36", t)  # noqa: E731
+    green = lambda t: _c("32", t)  # noqa: E731
+    yellow = lambda t: _c("33", t)  # noqa: E731
+
+    # --- Validate file ---
+    if not args.file.exists():
+        print(f"Error: File not found: {args.file}", file=sys.stderr)
+        sys.exit(1)
+
+    try:
+        text = args.file.read_text(encoding="utf-8")
+    except Exception as e:
+        print(f"Error reading file: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    # --- Build Tokenizer kwargs ---
+    tokenizer_kwargs = {
+        "lowercase": not args.no_lowercase,
+        "min_length": args.min_length,
+        "max_length": args.max_length,
+        "strip_numbers": args.strip_numbers,
+        "strip_punctuation": not args.keep_punctuation,
+        "preserve_urls": args.preserve_urls,
+        "preserve_emails": args.preserve_emails,
+        "preserve_hashtags": args.preserve_hashtags,
+        "preserve_mentions": args.preserve_mentions,
+        "expand_contractions": args.expand_contractions,
+        "expand_abbreviations": args.expand_abbreviations,
+        "strip_accents": args.strip_accents,
+        "normalize_unicode": not args.no_unicode_normalize,
+        "clean_text": not args.no_clean,
+    }
+
+    # Collect active options for banner
+    active_opts = []
+    if args.no_lowercase:
+        active_opts.append("preserve case")
+    if args.expand_contractions:
+        active_opts.append("expand contractions")
+    if args.expand_abbreviations:
+        active_opts.append("expand abbreviations")
+    if args.strip_numbers:
+        active_opts.append("strip numbers")
+    if args.keep_punctuation:
+        active_opts.append("keep punctuation")
+    if args.strip_accents:
+        active_opts.append("strip accents")
+    if args.no_clean:
+        active_opts.append("skip cleaning")
+    if args.no_unicode_normalize:
+        active_opts.append("skip unicode normalization")
+    if args.preserve_urls:
+        active_opts.append("preserve URLs")
+    if args.preserve_emails:
+        active_opts.append("preserve emails")
+    if args.preserve_hashtags:
+        active_opts.append("preserve hashtags")
+    if args.preserve_mentions:
+        active_opts.append("preserve mentions")
+    if args.min_length > 1:
+        active_opts.append(f"min length {args.min_length}")
+    if args.max_length is not None:
+        active_opts.append(f"max length {args.max_length}")
+
+    # Determine output format
+    if args.stats:
+        output_format = "Statistics"
+    elif args.metadata:
+        output_format = "JSON (with metadata)"
+    elif args.json:
+        output_format = "JSON"
+    else:
+        output_format = "One token per line"
+
+    # --- Banner (to stderr so stdout stays pipeable) ---
+    char_count = len(text)
+    line_count = text.count("\n") + 1
+
+    banner = sys.stderr
+    print(file=banner)
+    print(f"  {bold('PYSTYLOMETRY')} {dim('—')} {cyan('Stylometric Tokenizer')}", file=banner)
+    print(f"  {dim('═' * 71)}", file=banner)
+    print(file=banner)
+    print(f"  {bold('INPUT')}", file=banner)
+    print(f"  {dim('─' * 71)}", file=banner)
+    print(f"    File:              {args.file}", file=banner)
+    print(f"    Size:              {char_count:,} characters / {line_count:,} lines", file=banner)
+    print(file=banner)
+    print(f"  {bold('CONFIGURATION')}", file=banner)
+    print(f"  {dim('─' * 71)}", file=banner)
+    print(f"    Case:              {'preserve' if args.no_lowercase else 'lowercase'}", file=banner)
+    print(
+        f"    Punctuation:       {'keep' if args.keep_punctuation else 'strip'}",
+        file=banner,
+    )
+    print(
+        f"    Contractions:      {'expand' if args.expand_contractions else 'preserve'}",
+        file=banner,
+    )
+    print(f"    Numbers:           {'strip' if args.strip_numbers else 'keep'}", file=banner)
+    if active_opts:
+        print(f"    Active options:    {', '.join(active_opts)}", file=banner)
+    print(file=banner)
+    print(f"  {bold('OUTPUT')}", file=banner)
+    print(f"  {dim('─' * 71)}", file=banner)
+    print(f"    Format:            {output_format}", file=banner)
+    print(file=banner)
+
+    # --- Tokenize ---
+    from pystylometry.tokenizer import Tokenizer
+
+    tokenizer = Tokenizer(**tokenizer_kwargs)
+
+    if args.stats:
+        stats = tokenizer.get_statistics(text)
+        print(f"  {bold('RESULTS')}", file=banner)
+        print(f"  {dim('─' * 71)}", file=banner)
+        print(f"    Total tokens:      {green(f'{stats.total_tokens:,}')}", file=banner)
+        print(f"    Unique tokens:     {green(f'{stats.unique_tokens:,}')}", file=banner)
+        print(f"    Word tokens:       {stats.word_tokens:,}", file=banner)
+        print(f"    Number tokens:     {stats.number_tokens:,}", file=banner)
+        print(f"    Punctuation:       {stats.punctuation_tokens:,}", file=banner)
+        print(f"    URLs:              {stats.url_tokens:,}", file=banner)
+        print(f"    Emails:            {stats.email_tokens:,}", file=banner)
+        print(f"    Hashtags:          {stats.hashtag_tokens:,}", file=banner)
+        print(f"    Mentions:          {stats.mention_tokens:,}", file=banner)
+        print(f"    Avg length:        {stats.average_token_length:.1f}", file=banner)
+        print(f"    Min length:        {stats.min_token_length}", file=banner)
+        print(f"    Max length:        {stats.max_token_length}", file=banner)
+        print(file=banner)
+
+        if args.json:
+            import dataclasses
+
+            print(json.dumps(dataclasses.asdict(stats), indent=2))
+
+    elif args.metadata or (args.json and args.metadata):
+        metadata_list = tokenizer.tokenize_with_metadata(text)
+        count = len(metadata_list)
+        print(
+            f"  {yellow('Tokenizing...')} {green(f'{count:,}')} tokens extracted",
+            file=banner,
+        )
+        print(file=banner)
+        output = [
+            {
+                "token": m.token,
+                "start": m.start,
+                "end": m.end,
+                "type": m.token_type,
+            }
+            for m in metadata_list
+        ]
+        print(json.dumps(output, indent=2))
+
+    elif args.json:
+        tokens = tokenizer.tokenize(text)
+        count = len(tokens)
+        print(
+            f"  {yellow('Tokenizing...')} {green(f'{count:,}')} tokens extracted",
+            file=banner,
+        )
+        print(file=banner)
+        print(json.dumps(tokens, indent=2))
+
+    else:
+        tokens = tokenizer.tokenize(text)
+        count = len(tokens)
+        print(
+            f"  {yellow('Tokenizing...')} {green(f'{count:,}')} tokens extracted",
+            file=banner,
+        )
+        print(file=banner)
+        for token in tokens:
+            print(token)
+
+
 if __name__ == "__main__":
     drift_cli()

pystylometry/lexical/__init__.py

@@ -5,12 +5,14 @@ from .advanced_diversity import compute_hdd, compute_mattr, compute_msttr, compu
 from .function_words import compute_function_words
 from .hapax import compute_hapax_ratios, compute_hapax_with_lexicon_analysis
 from .mtld import compute_mtld
-from .ttr import compute_ttr
+from .repetition import compute_repetitive_ngrams, compute_repetitive_unigrams
+from .ttr import TTRAggregator, compute_ttr
 from .word_frequency_sophistication import compute_word_frequency_sophistication
 from .yule import compute_yule
 
 __all__ = [
     "compute_ttr",
+    "TTRAggregator",
     "compute_mtld",
     "compute_yule",
     "compute_hapax_ratios",
@@ -21,4 +23,6 @@ __all__ = [
     "compute_hdd",
     "compute_msttr",
     "compute_word_frequency_sophistication",
+    "compute_repetitive_unigrams",
+    "compute_repetitive_ngrams",
 ]