pystylometry 0.1.0__py3-none-any.whl → 1.1.0__py3-none-any.whl

pystylometry/syntactic/pos_ratios.py

@@ -1,10 +1,17 @@
-"""Part-of-Speech ratio analysis using spaCy."""
+"""Part-of-Speech ratio analysis using spaCy.
 
-from .._types import POSResult
+Related GitHub Issue:
+    #27 - Native chunked analysis with Distribution dataclass
+    https://github.com/craigtrim/pystylometry/issues/27
+"""
+
+from .._types import Distribution, POSResult, make_distribution
 from .._utils import check_optional_dependency
 
 
-def compute_pos_ratios(text: str, model: str = "en_core_web_sm") -> POSResult:
+def compute_pos_ratios(
+    text: str, model: str = "en_core_web_sm", chunk_size: int = 1000
+) -> POSResult:
     """
     Compute Part-of-Speech ratios and lexical density using spaCy.
 
@@ -18,6 +25,10 @@ def compute_pos_ratios(text: str, model: str = "en_core_web_sm") -> POSResult:
     - Lexical density: (nouns + verbs + adjectives + adverbs) / total words
     - Function word ratio: (determiners + prepositions + conjunctions) / total words
 
+    Related GitHub Issue:
+        #27 - Native chunked analysis with Distribution dataclass
+        https://github.com/craigtrim/pystylometry/issues/27
+
     References:
         Biber, D. (1988). Variation across speech and writing.
         Cambridge University Press.
@@ -25,9 +36,13 @@ def compute_pos_ratios(text: str, model: str = "en_core_web_sm") -> POSResult:
     Args:
         text: Input text to analyze
         model: spaCy model name (default: "en_core_web_sm")
+        chunk_size: Number of words per chunk (default: 1000).
+            Note: POS analysis is performed on the full text for accuracy,
+            so this parameter is included for API consistency but actual
+            results are from a single pass.
 
     Returns:
-        POSResult with all POS ratios and metadata
+        POSResult with all POS ratios, distributions, and metadata
 
     Raises:
         ImportError: If spaCy is not installed
@@ -40,22 +55,162 @@ def compute_pos_ratios(text: str, model: str = "en_core_web_sm") -> POSResult:
     """
     check_optional_dependency("spacy", "syntactic")
 
-    # TODO: Implement spaCy-based POS analysis
-    # import spacy
-    # nlp = spacy.load(model)
-    # doc = nlp(text)
+    import spacy
+
+    # Load spaCy model
+    try:
+        nlp = spacy.load(model)
+    except OSError:
+        raise OSError(
+            f"spaCy model '{model}' not found. Download it with: python -m spacy download {model}"
+        )
+
+    # Process text with spaCy
+    doc = nlp(text)
+
+    # Count POS tags
+    noun_count = 0
+    verb_count = 0
+    adj_count = 0
+    adv_count = 0
+    det_count = 0
+    adp_count = 0  # Adpositions (prepositions)
+    conj_count = 0  # Conjunctions (coordinating and subordinating)
+    total_tokens = 0
+
+    for token in doc:
+        # Only count alphabetic tokens (skip punctuation, numbers, etc.)
+        if not token.is_alpha:
+            continue
+
+        total_tokens += 1
+        pos = token.pos_
+
+        if pos == "NOUN" or pos == "PROPN":
+            noun_count += 1
+        elif pos == "VERB":
+            verb_count += 1
+        elif pos == "ADJ":
+            adj_count += 1
+        elif pos == "ADV":
+            adv_count += 1
+        elif pos == "DET":
+            det_count += 1
+        elif pos == "ADP":
+            adp_count += 1
+        elif pos in ("CCONJ", "SCONJ"):
+            conj_count += 1
+
+    # Handle empty text
+    if total_tokens == 0:
+        empty_dist = Distribution(
+            values=[],
+            mean=float("nan"),
+            median=float("nan"),
+            std=0.0,
+            range=0.0,
+            iqr=0.0,
+        )
+        return POSResult(
+            noun_ratio=float("nan"),
+            verb_ratio=float("nan"),
+            adjective_ratio=float("nan"),
+            adverb_ratio=float("nan"),
+            noun_verb_ratio=float("nan"),
+            adjective_noun_ratio=float("nan"),
+            lexical_density=float("nan"),
+            function_word_ratio=float("nan"),
+            noun_ratio_dist=empty_dist,
+            verb_ratio_dist=empty_dist,
+            adjective_ratio_dist=empty_dist,
+            adverb_ratio_dist=empty_dist,
+            noun_verb_ratio_dist=empty_dist,
+            adjective_noun_ratio_dist=empty_dist,
+            lexical_density_dist=empty_dist,
+            function_word_ratio_dist=empty_dist,
+            chunk_size=chunk_size,
+            chunk_count=0,
+            metadata={
+                "model": model,
+                "token_count": 0,
+                "noun_count": 0,
+                "verb_count": 0,
+                "adjective_count": 0,
+                "adverb_count": 0,
+            },
+        )
+
+    # Calculate ratios
+    noun_ratio = noun_count / total_tokens
+    verb_ratio = verb_count / total_tokens
+    adj_ratio = adj_count / total_tokens
+    adv_ratio = adv_count / total_tokens
+
+    # Noun-verb ratio (handle division by zero)
+    noun_verb_ratio = noun_count / verb_count if verb_count > 0 else float("nan")
+
+    # Adjective-noun ratio (handle division by zero)
+    adj_noun_ratio = adj_count / noun_count if noun_count > 0 else float("nan")
+
+    # Lexical density: (content words) / total words
+    # Content words = nouns + verbs + adjectives + adverbs
+    lexical_words = noun_count + verb_count + adj_count + adv_count
+    lexical_density = lexical_words / total_tokens
+
+    # Function word ratio: (determiners + prepositions + conjunctions) / total words
+    function_words = det_count + adp_count + conj_count
+    function_word_ratio = function_words / total_tokens
+
+    # Create single-value distributions (POS analysis is done on full text)
+    noun_ratio_dist = make_distribution([noun_ratio])
+    verb_ratio_dist = make_distribution([verb_ratio])
+    adj_ratio_dist = make_distribution([adj_ratio])
+    adv_ratio_dist = make_distribution([adv_ratio])
+    noun_verb_dist = (
+        make_distribution([noun_verb_ratio])
+        if not (noun_verb_ratio != noun_verb_ratio)
+        else Distribution(
+            values=[], mean=float("nan"), median=float("nan"), std=0.0, range=0.0, iqr=0.0
+        )
+    )
+    adj_noun_dist = (
+        make_distribution([adj_noun_ratio])
+        if not (adj_noun_ratio != adj_noun_ratio)
+        else Distribution(
+            values=[], mean=float("nan"), median=float("nan"), std=0.0, range=0.0, iqr=0.0
+        )
+    )
+    lexical_density_dist = make_distribution([lexical_density])
+    function_word_dist = make_distribution([function_word_ratio])
 
     return POSResult(
-        noun_ratio=0.0,
-        verb_ratio=0.0,
-        adjective_ratio=0.0,
-        adverb_ratio=0.0,
-        noun_verb_ratio=0.0,
-        adjective_noun_ratio=0.0,
-        lexical_density=0.0,
-        function_word_ratio=0.0,
+        noun_ratio=noun_ratio,
+        verb_ratio=verb_ratio,
+        adjective_ratio=adj_ratio,
+        adverb_ratio=adv_ratio,
+        noun_verb_ratio=noun_verb_ratio,
+        adjective_noun_ratio=adj_noun_ratio,
+        lexical_density=lexical_density,
+        function_word_ratio=function_word_ratio,
+        noun_ratio_dist=noun_ratio_dist,
+        verb_ratio_dist=verb_ratio_dist,
+        adjective_ratio_dist=adj_ratio_dist,
+        adverb_ratio_dist=adv_ratio_dist,
+        noun_verb_ratio_dist=noun_verb_dist,
+        adjective_noun_ratio_dist=adj_noun_dist,
+        lexical_density_dist=lexical_density_dist,
+        function_word_ratio_dist=function_word_dist,
+        chunk_size=chunk_size,
+        chunk_count=1,  # Single pass analysis
         metadata={
             "model": model,
-            "token_count": 0,
+            "token_count": total_tokens,
+            "noun_count": noun_count,
+            "verb_count": verb_count,
+            "adjective_count": adj_count,
+            "adverb_count": adv_count,
+            "determiner_count": det_count,
+            "adposition_count": adp_count,
+            "conjunction_count": conj_count,
         },
     )

pystylometry/syntactic/sentence_stats.py

@@ -1,10 +1,17 @@
-"""Sentence-level statistics using spaCy."""
+"""Sentence-level statistics using spaCy.
 
-from .._types import SentenceStatsResult
-from .._utils import check_optional_dependency, split_sentences
+Related GitHub Issue:
+    #27 - Native chunked analysis with Distribution dataclass
+    https://github.com/craigtrim/pystylometry/issues/27
+"""
 
+from .._types import Distribution, SentenceStatsResult, make_distribution
+from .._utils import check_optional_dependency
 
-def compute_sentence_stats(text: str, model: str = "en_core_web_sm") -> SentenceStatsResult:
+
+def compute_sentence_stats(
+    text: str, model: str = "en_core_web_sm", chunk_size: int = 1000
+) -> SentenceStatsResult:
     """
     Compute sentence-level statistics using spaCy.
 
@@ -16,6 +23,10 @@ def compute_sentence_stats(text: str, model: str = "en_core_web_sm") -> Sentence
     - Maximum sentence length
     - Total sentence count
 
+    Related GitHub Issue:
+        #27 - Native chunked analysis with Distribution dataclass
+        https://github.com/craigtrim/pystylometry/issues/27
+
     References:
         Hunt, K. W. (1965). Grammatical structures written at three grade levels.
         NCTE Research Report No. 3.
@@ -23,9 +34,13 @@ def compute_sentence_stats(text: str, model: str = "en_core_web_sm") -> Sentence
     Args:
         text: Input text to analyze
         model: spaCy model name (default: "en_core_web_sm")
+        chunk_size: Number of words per chunk (default: 1000).
+            Note: Sentence analysis is performed on the full text for accuracy,
+            so this parameter is included for API consistency but actual
+            results are from a single pass.
 
     Returns:
-        SentenceStatsResult with sentence statistics and metadata
+        SentenceStatsResult with sentence statistics, distributions, and metadata
 
     Raises:
         ImportError: If spaCy is not installed
@@ -38,23 +53,95 @@ def compute_sentence_stats(text: str, model: str = "en_core_web_sm") -> Sentence
     """
     check_optional_dependency("spacy", "syntactic")
 
-    # TODO: Implement spaCy-based sentence analysis
-    # import spacy
-    # nlp = spacy.load(model)
-    # doc = nlp(text)
-    # sentences = list(doc.sents)
+    import spacy
+
+    # Load spaCy model
+    try:
+        nlp = spacy.load(model)
+    except OSError:
+        raise OSError(
+            f"spaCy model '{model}' not found. Download it with: python -m spacy download {model}"
+        )
+
+    # Process text with spaCy
+    doc = nlp(text)
+
+    # Extract sentences and count words in each
+    sentence_lengths = []
+    for sent in doc.sents:
+        # Count only alphabetic tokens (exclude punctuation)
+        word_count = sum(1 for token in sent if token.is_alpha)
+        if word_count > 0:  # Only include non-empty sentences
+            sentence_lengths.append(word_count)
+
+    # Handle empty text
+    if len(sentence_lengths) == 0:
+        empty_dist = Distribution(
+            values=[],
+            mean=float("nan"),
+            median=float("nan"),
+            std=0.0,
+            range=0.0,
+            iqr=0.0,
+        )
+        return SentenceStatsResult(
+            mean_sentence_length=float("nan"),
+            sentence_length_std=float("nan"),
+            sentence_length_range=0.0,
+            min_sentence_length=0.0,
+            max_sentence_length=0.0,
+            sentence_count=0,
+            mean_sentence_length_dist=empty_dist,
+            sentence_length_std_dist=empty_dist,
+            sentence_length_range_dist=empty_dist,
+            min_sentence_length_dist=empty_dist,
+            max_sentence_length_dist=empty_dist,
+            chunk_size=chunk_size,
+            chunk_count=0,
+            metadata={
+                "model": model,
+            },
+        )
+
+    # Calculate statistics
+    mean_length = sum(sentence_lengths) / len(sentence_lengths)
+
+    # Standard deviation
+    if len(sentence_lengths) > 1:
+        variance = sum((x - mean_length) ** 2 for x in sentence_lengths) / (
+            len(sentence_lengths) - 1
+        )
+        std_dev = variance**0.5
+    else:
+        std_dev = 0.0
+
+    min_length = float(min(sentence_lengths))
+    max_length = float(max(sentence_lengths))
+    length_range = max_length - min_length
 
-    # For now, use simple fallback
-    sentences = split_sentences(text)
+    # Create single-value distributions (sentence analysis is done on full text)
+    mean_dist = make_distribution([mean_length])
+    std_dist = make_distribution([std_dev])
+    range_dist = make_distribution([length_range])
+    min_dist = make_distribution([min_length])
+    max_dist = make_distribution([max_length])
 
     return SentenceStatsResult(
-        mean_sentence_length=0.0,
-        sentence_length_std=0.0,
-        sentence_length_range=0,
-        min_sentence_length=0,
-        max_sentence_length=0,
-        sentence_count=len(sentences),
+        mean_sentence_length=mean_length,
+        sentence_length_std=std_dev,
+        sentence_length_range=length_range,
+        min_sentence_length=min_length,
+        max_sentence_length=max_length,
+        sentence_count=len(sentence_lengths),
+        mean_sentence_length_dist=mean_dist,
+        sentence_length_std_dist=std_dist,
+        sentence_length_range_dist=range_dist,
+        min_sentence_length_dist=min_dist,
+        max_sentence_length_dist=max_dist,
+        chunk_size=chunk_size,
+        chunk_count=1,  # Single pass analysis
         metadata={
             "model": model,
+            "sentence_lengths": sentence_lengths,
         },
     )