pystylometry 0.1.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

pystylometry/syntactic/pos_ratios.py

@@ -40,22 +40,113 @@ def compute_pos_ratios(text: str, model: str = "en_core_web_sm") -> POSResult:
     """
     check_optional_dependency("spacy", "syntactic")
 
-    # TODO: Implement spaCy-based POS analysis
-    # import spacy
-    # nlp = spacy.load(model)
-    # doc = nlp(text)
+    import spacy
+
+    # Load spaCy model
+    try:
+        nlp = spacy.load(model)
+    except OSError:
+        raise OSError(
+            f"spaCy model '{model}' not found. "
+            f"Download it with: python -m spacy download {model}"
+        )
+
+    # Process text with spaCy
+    doc = nlp(text)
+
+    # Count POS tags
+    noun_count = 0
+    verb_count = 0
+    adj_count = 0
+    adv_count = 0
+    det_count = 0
+    adp_count = 0  # Adpositions (prepositions)
+    conj_count = 0  # Conjunctions (coordinating and subordinating)
+    total_tokens = 0
+
+    for token in doc:
+        # Only count alphabetic tokens (skip punctuation, numbers, etc.)
+        if not token.is_alpha:
+            continue
+
+        total_tokens += 1
+        pos = token.pos_
+
+        if pos == "NOUN" or pos == "PROPN":
+            noun_count += 1
+        elif pos == "VERB":
+            verb_count += 1
+        elif pos == "ADJ":
+            adj_count += 1
+        elif pos == "ADV":
+            adv_count += 1
+        elif pos == "DET":
+            det_count += 1
+        elif pos == "ADP":
+            adp_count += 1
+        elif pos in ("CCONJ", "SCONJ"):
+            conj_count += 1
+
+    # Handle empty text
+    if total_tokens == 0:
+        return POSResult(
+            noun_ratio=float("nan"),
+            verb_ratio=float("nan"),
+            adjective_ratio=float("nan"),
+            adverb_ratio=float("nan"),
+            noun_verb_ratio=float("nan"),
+            adjective_noun_ratio=float("nan"),
+            lexical_density=float("nan"),
+            function_word_ratio=float("nan"),
+            metadata={
+                "model": model,
+                "token_count": 0,
+                "noun_count": 0,
+                "verb_count": 0,
+                "adjective_count": 0,
+                "adverb_count": 0,
+            },
+        )
+
+    # Calculate ratios
+    noun_ratio = noun_count / total_tokens
+    verb_ratio = verb_count / total_tokens
+    adj_ratio = adj_count / total_tokens
+    adv_ratio = adv_count / total_tokens
+
+    # Noun-verb ratio (handle division by zero)
+    noun_verb_ratio = noun_count / verb_count if verb_count > 0 else float("nan")
+
+    # Adjective-noun ratio (handle division by zero)
+    adj_noun_ratio = adj_count / noun_count if noun_count > 0 else float("nan")
+
+    # Lexical density: (content words) / total words
+    # Content words = nouns + verbs + adjectives + adverbs
+    lexical_words = noun_count + verb_count + adj_count + adv_count
+    lexical_density = lexical_words / total_tokens
+
+    # Function word ratio: (determiners + prepositions + conjunctions) / total words
+    function_words = det_count + adp_count + conj_count
+    function_word_ratio = function_words / total_tokens
 
     return POSResult(
-        noun_ratio=0.0,
-        verb_ratio=0.0,
-        adjective_ratio=0.0,
-        adverb_ratio=0.0,
-        noun_verb_ratio=0.0,
-        adjective_noun_ratio=0.0,
-        lexical_density=0.0,
-        function_word_ratio=0.0,
+        noun_ratio=noun_ratio,
+        verb_ratio=verb_ratio,
+        adjective_ratio=adj_ratio,
+        adverb_ratio=adv_ratio,
+        noun_verb_ratio=noun_verb_ratio,
+        adjective_noun_ratio=adj_noun_ratio,
+        lexical_density=lexical_density,
+        function_word_ratio=function_word_ratio,
         metadata={
             "model": model,
-            "token_count": 0,
+            "token_count": total_tokens,
+            "noun_count": noun_count,
+            "verb_count": verb_count,
+            "adjective_count": adj_count,
+            "adverb_count": adv_count,
+            "determiner_count": det_count,
+            "adposition_count": adp_count,
+            "conjunction_count": conj_count,
         },
     )

pystylometry/syntactic/sentence_stats.py

@@ -38,23 +38,67 @@ def compute_sentence_stats(text: str, model: str = "en_core_web_sm") -> Sentence
     """
     check_optional_dependency("spacy", "syntactic")
 
-    # TODO: Implement spaCy-based sentence analysis
-    # import spacy
-    # nlp = spacy.load(model)
-    # doc = nlp(text)
-    # sentences = list(doc.sents)
+    import spacy
 
-    # For now, use simple fallback
-    sentences = split_sentences(text)
+    # Load spaCy model
+    try:
+        nlp = spacy.load(model)
+    except OSError:
+        raise OSError(
+            f"spaCy model '{model}' not found. "
+            f"Download it with: python -m spacy download {model}"
+        )
+
+    # Process text with spaCy
+    doc = nlp(text)
+
+    # Extract sentences and count words in each
+    sentence_lengths = []
+    for sent in doc.sents:
+        # Count only alphabetic tokens (exclude punctuation)
+        word_count = sum(1 for token in sent if token.is_alpha)
+        if word_count > 0:  # Only include non-empty sentences
+            sentence_lengths.append(word_count)
+
+    # Handle empty text
+    if len(sentence_lengths) == 0:
+        return SentenceStatsResult(
+            mean_sentence_length=float("nan"),
+            sentence_length_std=float("nan"),
+            sentence_length_range=0,
+            min_sentence_length=0,
+            max_sentence_length=0,
+            sentence_count=0,
+            metadata={
+                "model": model,
+            },
+        )
+
+    # Calculate statistics
+    mean_length = sum(sentence_lengths) / len(sentence_lengths)
+
+    # Standard deviation
+    if len(sentence_lengths) > 1:
+        variance = sum((x - mean_length) ** 2 for x in sentence_lengths) / (
+            len(sentence_lengths) - 1
+        )
+        std_dev = variance**0.5
+    else:
+        std_dev = 0.0
+
+    min_length = min(sentence_lengths)
+    max_length = max(sentence_lengths)
+    length_range = max_length - min_length
 
     return SentenceStatsResult(
-        mean_sentence_length=0.0,
-        sentence_length_std=0.0,
-        sentence_length_range=0,
-        min_sentence_length=0,
-        max_sentence_length=0,
-        sentence_count=len(sentences),
+        mean_sentence_length=mean_length,
+        sentence_length_std=std_dev,
+        sentence_length_range=length_range,
+        min_sentence_length=min_length,
+        max_sentence_length=max_length,
+        sentence_count=len(sentence_lengths),
         metadata={
             "model": model,
+            "sentence_lengths": sentence_lengths,
         },
     )

pystylometry/syntactic/sentence_types.py

@@ -0,0 +1,470 @@
+"""Sentence type classification for syntactic analysis.
+
+This module classifies sentences by their grammatical structure (simple, compound,
+complex, compound-complex) and communicative function (declarative, interrogative,
+imperative, exclamatory). These classifications reveal authorial preferences and
+genre-specific patterns.
+
+Related GitHub Issue:
+    #18 - Sentence Type Classification
+    https://github.com/craigtrim/pystylometry/issues/18
+
+Structural classifications:
+    - Simple: One independent clause
+    - Compound: Multiple independent clauses joined by coordination
+    - Complex: One independent clause + one or more dependent clauses
+    - Compound-Complex: Multiple independent + dependent clauses
+
+Functional classifications:
+    - Declarative: Makes a statement (ends with period)
+    - Interrogative: Asks a question (ends with question mark)
+    - Imperative: Gives a command (subject often implicit "you")
+    - Exclamatory: Expresses strong emotion (ends with exclamation mark)
+
+References:
+    Biber, D. (1988). Variation across speech and writing. Cambridge University Press.
+    Huddleston, R., & Pullum, G. K. (2002). The Cambridge Grammar of the English Language.
+    Quirk, R., et al. (1985). A Comprehensive Grammar of the English Language. Longman.
+"""
+
+from .._types import SentenceTypeResult
+from .._utils import check_optional_dependency
+
+
+def compute_sentence_types(
+    text: str,
+    model: str = "en_core_web_sm",
+) -> SentenceTypeResult:
+    """
+    Classify sentences by structure and function.
+
+    Analyzes text to determine the distribution of sentence types, both
+    structural (based on clause organization) and functional (based on
+    communicative purpose). Different authors and genres show characteristic
+    patterns in sentence type usage.
+
+    Related GitHub Issue:
+        #18 - Sentence Type Classification
+        https://github.com/craigtrim/pystylometry/issues/18
+
+    Why sentence types matter:
+
+    Structural complexity:
+        - Simple sentences: Direct, clear, easy to process
+        - Compound sentences: Coordinate ideas of equal importance
+        - Complex sentences: Subordinate ideas, show relationships
+        - Compound-complex: Sophisticated, academic style
+
+    Functional diversity:
+        - Declarative dominance: Expository/academic writing
+        - Interrogative use: Interactive, rhetorical questions
+        - Imperative use: Instructional texts, commands
+        - Exclamatory use: Emotional, emphatic style
+
+    Genre patterns:
+        - Academic: High proportion of complex sentences
+        - Fiction: Mix of simple and complex for variety
+        - Journalism: Mostly simple and compound for clarity
+        - Technical: Predominantly declarative complex sentences
+
+    Structural Classification Algorithm:
+
+    Simple Sentence:
+        - Contains exactly one independent clause
+        - No dependent clauses
+        - Example: "The cat sat on the mat."
+
+    Compound Sentence:
+        - Contains two or more independent clauses
+        - Joined by coordinating conjunction or semicolon
+        - No dependent clauses
+        - Example: "I came, and I saw."
+
+    Complex Sentence:
+        - Contains one independent clause
+        - Plus one or more dependent clauses
+        - Example: "When I arrived, I saw her."
+
+    Compound-Complex Sentence:
+        - Contains two or more independent clauses
+        - Plus one or more dependent clauses
+        - Example: "I came when called, and I stayed because I wanted to."
+
+    Functional Classification Algorithm:
+
+    Declarative:
+        - Makes a statement
+        - Typically ends with period
+        - Subject before verb
+        - Example: "The sky is blue."
+
+    Interrogative:
+        - Asks a question
+        - Ends with question mark
+        - Often inverted word order or question words
+        - Example: "Is the sky blue?"
+
+    Imperative:
+        - Gives a command or instruction
+        - Subject typically implicit ("you")
+        - Often begins with base verb
+        - Example: "Look at the sky!"
+
+    Exclamatory:
+        - Expresses strong emotion
+        - Ends with exclamation mark
+        - May have inverted structure
+        - Example: "What a blue sky!"
+
+    Args:
+        text: Input text to analyze. Should contain multiple sentences for
+              meaningful distributions. Single-sentence texts will have ratios
+              of 1.0 for one type and 0.0 for others.
+        model: spaCy model with dependency parser. Default is "en_core_web_sm".
+               Larger models provide better clause detection accuracy.
+
+    Returns:
+        SentenceTypeResult containing:
+
+        Structural ratios (sum to 1.0):
+            - simple_ratio: Simple sentences / total
+            - compound_ratio: Compound sentences / total
+            - complex_ratio: Complex sentences / total
+            - compound_complex_ratio: Compound-complex / total
+
+        Functional ratios (sum to 1.0):
+            - declarative_ratio: Declarative sentences / total
+            - interrogative_ratio: Questions / total
+            - imperative_ratio: Commands / total
+            - exclamatory_ratio: Exclamations / total
+
+        Counts:
+            - simple_count, compound_count, complex_count, compound_complex_count
+            - declarative_count, interrogative_count, imperative_count, exclamatory_count
+            - total_sentences
+
+        Diversity metrics:
+            - structural_diversity: Shannon entropy of structural distribution
+            - functional_diversity: Shannon entropy of functional distribution
+
+        Metadata:
+            - sentence_by_sentence_classifications
+            - clause_counts_per_sentence
+            - etc.
+
+    Example:
+        >>> result = compute_sentence_types("Mix of sentence types here...")
+        >>> print(f"Simple: {result.simple_ratio * 100:.1f}%")
+        Simple: 35.2%
+        >>> print(f"Complex: {result.complex_ratio * 100:.1f}%")
+        Complex: 41.3%
+        >>> print(f"Questions: {result.interrogative_ratio * 100:.1f}%")
+        Questions: 8.5%
+        >>> print(f"Structural diversity: {result.structural_diversity:.3f}")
+        Structural diversity: 0.847
+
+        >>> # Compare genres
+        >>> academic = compute_sentence_types("Academic paper text...")
+        >>> fiction = compute_sentence_types("Fiction narrative...")
+        >>> print(f"Academic complex: {academic.complex_ratio:.2f}")
+        >>> print(f"Fiction simple: {fiction.simple_ratio:.2f}")
+
+    Note:
+        - Requires spaCy with dependency parser
+        - Clause detection based on dependency relations
+        - Coordinating conjunctions: and, but, or, nor, for, yet, so
+        - Dependent clause markers: ccomp, advcl, acl, relcl
+        - Punctuation used for functional classification
+        - Imperative detection uses missing subject + base verb pattern
+        - Empty text returns NaN for ratios, 0 for counts
+    """
+    check_optional_dependency("spacy", "syntactic")
+
+    try:
+        import spacy  # type: ignore
+    except ImportError as e:
+        raise ImportError(
+            "spaCy is required for sentence type classification. "
+            "Install with: pip install spacy && python -m spacy download en_core_web_sm"
+        ) from e
+
+    # Load spaCy model
+    try:
+        nlp = spacy.load(model)
+    except OSError as e:
+        raise OSError(
+            f"spaCy model '{model}' not found. "
+            f"Download with: python -m spacy download {model}"
+        ) from e
+
+    # Parse text
+    doc = nlp(text)
+    sentences = list(doc.sents)
+
+    # Handle empty text
+    if len(sentences) == 0:
+        return SentenceTypeResult(
+            simple_ratio=float("nan"),
+            compound_ratio=float("nan"),
+            complex_ratio=float("nan"),
+            compound_complex_ratio=float("nan"),
+            declarative_ratio=float("nan"),
+            interrogative_ratio=float("nan"),
+            imperative_ratio=float("nan"),
+            exclamatory_ratio=float("nan"),
+            simple_count=0,
+            compound_count=0,
+            complex_count=0,
+            compound_complex_count=0,
+            declarative_count=0,
+            interrogative_count=0,
+            imperative_count=0,
+            exclamatory_count=0,
+            total_sentences=0,
+            structural_diversity=float("nan"),
+            functional_diversity=float("nan"),
+            metadata={
+                "warning": "Empty text or no sentences found",
+            },
+        )
+
+    # Classify each sentence
+    structural_counts = {"simple": 0, "compound": 0, "complex": 0, "compound_complex": 0}
+    functional_counts = {"declarative": 0, "interrogative": 0, "imperative": 0, "exclamatory": 0}
+    sentence_classifications = []
+    clause_counts_per_sentence = []
+
+    for sent in sentences:
+        # Count clauses
+        independent_count = _count_independent_clauses(sent)
+        dependent_count = _count_dependent_clauses(sent)
+        clause_counts_per_sentence.append((independent_count, dependent_count))
+
+        # Structural classification
+        structural_type = _classify_structural(independent_count, dependent_count)
+        structural_counts[structural_type] += 1
+
+        # Functional classification
+        functional_type = _classify_functional(sent)
+        functional_counts[functional_type] += 1
+
+        # Store classification
+        sentence_classifications.append({
+            "text": sent.text,
+            "structural_type": structural_type,
+            "functional_type": functional_type,
+            "independent_clauses": independent_count,
+            "dependent_clauses": dependent_count,
+        })
+
+    # Calculate ratios
+    total_sentences = len(sentences)
+    simple_ratio = structural_counts["simple"] / total_sentences
+    compound_ratio = structural_counts["compound"] / total_sentences
+    complex_ratio = structural_counts["complex"] / total_sentences
+    compound_complex_ratio = structural_counts["compound_complex"] / total_sentences
+
+    declarative_ratio = functional_counts["declarative"] / total_sentences
+    interrogative_ratio = functional_counts["interrogative"] / total_sentences
+    imperative_ratio = functional_counts["imperative"] / total_sentences
+    exclamatory_ratio = functional_counts["exclamatory"] / total_sentences
+
+    # Calculate diversity metrics
+    structural_ratios = [simple_ratio, compound_ratio, complex_ratio, compound_complex_ratio]
+    functional_ratios = [declarative_ratio, interrogative_ratio, imperative_ratio, exclamatory_ratio]
+
+    structural_diversity = _calculate_shannon_entropy(structural_ratios)
+    functional_diversity = _calculate_shannon_entropy(functional_ratios)
+
+    # Collect metadata
+    metadata = {
+        "sentence_count": total_sentences,
+        "sentence_classifications": sentence_classifications,
+        "clause_counts_per_sentence": clause_counts_per_sentence,
+        "structural_counts": structural_counts,
+        "functional_counts": functional_counts,
+        "model_used": model,
+    }
+
+    return SentenceTypeResult(
+        simple_ratio=simple_ratio,
+        compound_ratio=compound_ratio,
+        complex_ratio=complex_ratio,
+        compound_complex_ratio=compound_complex_ratio,
+        declarative_ratio=declarative_ratio,
+        interrogative_ratio=interrogative_ratio,
+        imperative_ratio=imperative_ratio,
+        exclamatory_ratio=exclamatory_ratio,
+        simple_count=structural_counts["simple"],
+        compound_count=structural_counts["compound"],
+        complex_count=structural_counts["complex"],
+        compound_complex_count=structural_counts["compound_complex"],
+        declarative_count=functional_counts["declarative"],
+        interrogative_count=functional_counts["interrogative"],
+        imperative_count=functional_counts["imperative"],
+        exclamatory_count=functional_counts["exclamatory"],
+        total_sentences=total_sentences,
+        structural_diversity=structural_diversity,
+        functional_diversity=functional_diversity,
+        metadata=metadata,
+    )
+
+
+def _count_independent_clauses(sent) -> int:
+    """
+    Count independent clauses in a sentence.
+
+    Independent clauses are:
+    1. The root clause (always 1)
+    2. Coordinated clauses (conj with VERB POS and cc child)
+
+    Args:
+        sent: spaCy Span representing a sentence
+
+    Returns:
+        Number of independent clauses
+    """
+    count = 1  # Always start with root clause
+
+    for token in sent:
+        # Coordinated independent clause
+        if token.dep_ == "conj" and token.pos_ == "VERB":
+            # Check if coordinating conjunction present
+            if any(child.dep_ == "cc" for child in token.head.children):
+                count += 1
+
+    return count
+
+
+def _count_dependent_clauses(sent) -> int:
+    """
+    Count dependent clauses in a sentence.
+
+    Dependent clauses are identified by dependency labels:
+    - ccomp: clausal complement
+    - advcl: adverbial clause
+    - acl: adnominal clause
+    - relcl: relative clause
+    - xcomp: open clausal complement (sometimes)
+
+    Args:
+        sent: spaCy Span representing a sentence
+
+    Returns:
+        Number of dependent clauses
+    """
+    dependent_labels = {"ccomp", "advcl", "acl", "relcl", "xcomp"}
+    count = sum(1 for token in sent if token.dep_ in dependent_labels)
+    return count
+
+
+def _classify_structural(independent: int, dependent: int) -> str:
+    """
+    Classify sentence structure based on clause counts.
+
+    Args:
+        independent: Number of independent clauses
+        dependent: Number of dependent clauses
+
+    Returns:
+        One of: "simple", "compound", "complex", "compound_complex"
+    """
+    if independent == 1 and dependent == 0:
+        return "simple"
+    elif independent >= 2 and dependent == 0:
+        return "compound"
+    elif independent == 1 and dependent >= 1:
+        return "complex"
+    elif independent >= 2 and dependent >= 1:
+        return "compound_complex"
+    else:
+        # Fallback (shouldn't happen with valid counts)
+        return "simple"
+
+
+def _classify_functional(sent) -> str:
+    """
+    Classify sentence function based on punctuation and structure.
+
+    Args:
+        sent: spaCy Span representing a sentence
+
+    Returns:
+        One of: "declarative", "interrogative", "imperative", "exclamatory"
+    """
+    # Get last token for punctuation
+    last_token = sent[-1]
+
+    # Check for question mark (interrogative)
+    if last_token.text == "?":
+        return "interrogative"
+
+    # Check for exclamation mark
+    if last_token.text == "!":
+        # Could be imperative or exclamatory
+        # Check if imperative structure
+        if _is_imperative_structure(sent):
+            return "imperative"
+        return "exclamatory"
+
+    # Check for imperative structure (missing subject + base verb)
+    if _is_imperative_structure(sent):
+        return "imperative"
+
+    # Default: declarative
+    return "declarative"
+
+
+def _is_imperative_structure(sent) -> bool:
+    """
+    Check if sentence has imperative structure.
+
+    Imperatives typically:
+    - Missing nominal subject (nsubj)
+    - Root verb is base form (VB) or imperative
+
+    Args:
+        sent: spaCy Span representing a sentence
+
+    Returns:
+        True if imperative structure detected
+    """
+    # Check for nominal subject
+    has_nominal_subject = any(token.dep_ == "nsubj" for token in sent)
+
+    # Get root verb
+    root_verb = sent.root
+
+    # If no nominal subject and root is a verb
+    if not has_nominal_subject and root_verb.pos_ == "VERB":
+        # Check if root is base form (VB) or present tense without subject
+        if root_verb.tag_ in {"VB", "VBP"}:
+            return True
+
+    return False
+
+
+def _calculate_shannon_entropy(probabilities: list[float]) -> float:
+    """
+    Calculate Shannon entropy for a probability distribution.
+
+    H = -sum(p * log2(p)) for p > 0
+
+    Args:
+        probabilities: List of probabilities (should sum to 1.0)
+
+    Returns:
+        Shannon entropy in bits (0.0 to log2(n) where n is number of categories)
+    """
+    import math
+
+    # Filter out zero probabilities (log(0) undefined)
+    non_zero_probs = [p for p in probabilities if p > 0]
+
+    if not non_zero_probs:
+        return 0.0
+
+    # Calculate entropy
+    entropy = -sum(p * math.log2(p) for p in non_zero_probs)
+
+    return entropy