pystylometry 1.3.1__py3-none-any.whl → 1.3.6__py3-none-any.whl

pystylometry/__init__.py

@@ -40,14 +40,13 @@ Usage:
     print(result.pattern_confidence)
 """
 
+from . import lexical  # noqa: E402
 from ._types import AnalysisResult
+from .tokenizer import TokenizationStats, Tokenizer, TokenMetadata
 
 # Version
 __version__ = "0.1.0"
 
-# Core exports - always available
-from . import lexical
-
 # Optional exports - may raise ImportError if dependencies not installed
 try:
     from . import readability  # noqa: F401
@@ -87,6 +86,41 @@ _CONSISTENCY_AVAILABLE = True
 _STYLISTIC_AVAILABLE = True
 
 
+def tokenize(text: str, **kwargs: object) -> list[str]:
+    """Tokenize text using the stylometric tokenizer.
+
+    Convenience wrapper around Tokenizer.tokenize(). All keyword arguments
+    are forwarded to the Tokenizer constructor.
+
+    Args:
+        text: Input text to tokenize.
+        **kwargs: Options forwarded to Tokenizer (lowercase, strip_numbers,
+            expand_contractions, etc.).
+
+    Returns:
+        List of token strings.
+
+    Example:
+        >>> from pystylometry import tokenize
+        >>> tokenize("Hello, world! It's a test.")
+        ['hello', 'world', "it's", 'a', 'test']
+    """
+    return Tokenizer(**kwargs).tokenize(text)  # type: ignore[arg-type]
+
+
+def tokenize_with_metadata(text: str, **kwargs: object) -> list[TokenMetadata]:
+    """Tokenize text and return tokens with positional and type metadata.
+
+    Args:
+        text: Input text to tokenize.
+        **kwargs: Options forwarded to Tokenizer.
+
+    Returns:
+        List of TokenMetadata objects.
+    """
+    return Tokenizer(**kwargs).tokenize_with_metadata(text)  # type: ignore[arg-type]
+
+
 def analyze(
     text: str,
     lexical_metrics: bool = True,
@@ -225,6 +259,11 @@ __all__ = [
     "__version__",
     "analyze",
     "get_available_modules",
+    "tokenize",
+    "tokenize_with_metadata",
+    "Tokenizer",
+    "TokenMetadata",
+    "TokenizationStats",
     "lexical",
 ]
 

pystylometry/_types.py

@@ -23,7 +23,7 @@ from __future__ import annotations
 
 import statistics
 from dataclasses import dataclass
-from typing import Any
+from typing import Any, Optional
 
 # ===== Distribution and Chunking =====
 # Related to GitHub Issue #27: Native chunked analysis with Distribution dataclass
@@ -316,8 +316,8 @@ class HapaxLexiconResult:
 class TTRResult:
     """Result from Type-Token Ratio (TTR) analysis.
 
-    Wraps stylometry-ttr package functionality to measure vocabulary richness
-    through the ratio of unique words (types) to total words (tokens).
+    Measures vocabulary richness through the ratio of unique words (types)
+    to total words (tokens).
 
     All numeric metrics include both a mean value (convenient access) and
     a full distribution with per-chunk values and statistics.
@@ -370,6 +370,56 @@ class TTRResult:
     metadata: dict[str, Any]
 
 
+@dataclass
+class TTRAggregateResult:
+    """Aggregated TTR statistics for a collection of texts.
+
+    Computes group-level summary statistics (mean, std, min, max, median)
+    across multiple ``TTRResult`` objects.  Useful for comparative analysis
+    across authors, genres, or time periods.
+
+    Related GitHub Issue:
+        #43 - Inline stylometry-ttr into pystylometry (remove external dependency)
+        https://github.com/craigtrim/pystylometry/issues/43
+
+    Example:
+        >>> from pystylometry.lexical import compute_ttr, TTRAggregator
+        >>> results = [compute_ttr(t) for t in texts]
+        >>> agg = TTRAggregator()
+        >>> stats = agg.aggregate(results, group_id="Austen")
+        >>> stats.ttr_mean
+        0.412
+    """
+
+    group_id: str
+    text_count: int
+    total_words: int
+
+    # Raw TTR statistics
+    ttr_mean: float
+    ttr_std: float
+    ttr_min: float
+    ttr_max: float
+    ttr_median: float
+
+    # Root TTR (Guiraud's index) statistics
+    root_ttr_mean: float
+    root_ttr_std: float
+
+    # Log TTR (Herdan's C) statistics
+    log_ttr_mean: float
+    log_ttr_std: float
+
+    # STTR statistics (None if no texts had enough words for STTR)
+    sttr_mean: Optional[float]
+    sttr_std: Optional[float]
+
+    # Delta std mean (None if no texts had delta metrics)
+    delta_std_mean: Optional[float]
+
+    metadata: dict[str, Any]
+
+
 # ===== Repetition Detection Results =====
 # Related to GitHub Issue #28: Verbal tics detection for slop analysis
 # https://github.com/craigtrim/pystylometry/issues/28