pystylometry 0.1.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

pystylometry/lexical/function_words.py

@@ -0,0 +1,391 @@
+"""Function word analysis for authorship attribution.
+
+Function words (determiners, prepositions, conjunctions, pronouns, auxiliary
+verbs) are highly frequent, content-independent words that authors use
+subconsciously and consistently across different topics. This makes them
+powerful markers for authorship attribution.
+
+Related GitHub Issue:
+    #13 - Function Word Analysis
+    https://github.com/craigtrim/pystylometry/issues/13
+
+Features implemented:
+    - Frequency profiles for all function word categories
+    - Ratios for specific grammatical categories
+    - Most/least frequently used function words
+    - Function word diversity metrics
+
+Function word categories:
+    - Determiners: the, a, an, this, that, these, those, my, your, etc.
+    - Prepositions: in, on, at, by, for, with, from, to, of, etc.
+    - Conjunctions: and, but, or, nor, for, yet, so, because, although, etc.
+    - Pronouns: I, you, he, she, it, we, they, me, him, her, us, them, etc.
+    - Auxiliary verbs: be, have, do, can, will, shall, may, must, etc.
+    - Particles: up, down, out, off, over, away, back, etc.
+
+References:
+    Mosteller, F., & Wallace, D. L. (1964). Inference and disputed authorship:
+        The Federalist. Addison-Wesley.
+    Burrows, J. (2002). 'Delta': A measure of stylistic difference and a guide
+        to likely authorship. Literary and Linguistic Computing, 17(3), 267-287.
+    Argamon, S., & Levitan, S. (2005). Measuring the usefulness of function
+        words for authorship attribution. ACH/ALLC.
+"""
+
+from .._types import FunctionWordResult
+
+
+# Function word lists for English
+# GitHub Issue #13: https://github.com/craigtrim/pystylometry/issues/13
+# These lists should be comprehensive and cover all major function word categories.
+# Consider loading from external resource files for easier maintenance.
+
+# Determiners (articles, demonstratives, possessives, quantifiers)
+DETERMINERS = {
+    "the", "a", "an",  # Articles
+    "this", "that", "these", "those",  # Demonstratives
+    "my", "your", "his", "her", "its", "our", "their",  # Possessive determiners
+    "some", "any", "no", "every", "each", "either", "neither",  # Quantifiers
+    "much", "many", "more", "most", "few", "fewer", "less", "least",
+    "all", "both", "half", "several", "enough",
+}
+
+# Prepositions (locative, temporal, other)
+PREPOSITIONS = {
+    "in", "on", "at", "by", "for", "with", "from", "to", "of",
+    "about", "above", "across", "after", "against", "along", "among",
+    "around", "as", "before", "behind", "below", "beneath", "beside",
+    "between", "beyond", "but", "concerning", "considering", "despite",
+    "down", "during", "except", "inside", "into", "like", "near",
+    "off", "onto", "out", "outside", "over", "past", "regarding",
+    "since", "through", "throughout", "till", "toward", "under",
+    "underneath", "until", "up", "upon", "via", "within", "without",
+}
+
+# Conjunctions (coordinating, subordinating, correlative)
+CONJUNCTIONS = {
+    # Coordinating
+    "and", "but", "or", "nor", "for", "yet", "so",
+    # Subordinating
+    "although", "because", "since", "unless", "while", "if", "when",
+    "where", "after", "before", "once", "until", "as", "though",
+    "even", "whereas", "wherever", "whenever",
+    # Correlative components
+    "either", "neither", "both", "whether",
+}
+
+# Pronouns (personal, possessive, reflexive, demonstrative, relative, indefinite)
+PRONOUNS = {
+    # Personal (subject)
+    "i", "you", "he", "she", "it", "we", "they",
+    # Personal (object)
+    "me", "him", "her", "us", "them",
+    # Possessive
+    "mine", "yours", "his", "hers", "its", "ours", "theirs",
+    # Reflexive
+    "myself", "yourself", "himself", "herself", "itself",
+    "ourselves", "yourselves", "themselves",
+    # Demonstrative
+    "this", "that", "these", "those",
+    # Relative
+    "who", "whom", "whose", "which", "that",
+    # Indefinite
+    "anybody", "anyone", "anything", "everybody", "everyone",
+    "everything", "nobody", "no one", "nothing", "somebody",
+    "someone", "something", "one",
+}
+
+# Auxiliary verbs (modal, primary)
+AUXILIARIES = {
+    # Modals
+    "can", "could", "may", "might", "must", "shall", "should",
+    "will", "would", "ought",
+    # Primary auxiliaries (be, have, do)
+    "am", "is", "are", "was", "were", "be", "being", "been",
+    "have", "has", "had", "having",
+    "do", "does", "did", "doing",
+}
+
+# Particles (often used with phrasal verbs)
+PARTICLES = {
+    "up", "down", "out", "off", "over", "in", "away",
+    "back", "on", "along", "forth", "apart", "aside",
+}
+
+
+def compute_function_words(text: str) -> FunctionWordResult:
+    """
+    Compute function word frequency profiles for authorship analysis.
+
+    Function words are closed-class words (determiners, prepositions,
+    conjunctions, pronouns, auxiliaries) that authors use largely
+    subconsciously and consistently. Their frequency patterns are
+    powerful authorship markers because they're independent of topic.
+
+    Related GitHub Issue:
+        #13 - Function Word Analysis
+        https://github.com/craigtrim/pystylometry/issues/13
+
+    Why function words matter for authorship:
+        1. Topic-independent: Used consistently across different subjects
+        2. Subconscious usage: Authors don't deliberately vary their use
+        3. High frequency: Appear often enough for reliable statistics
+        4. Stable over time: Authors' function word patterns remain consistent
+        5. Discriminative power: Different authors show distinct patterns
+
+    Classic example: Mosteller & Wallace (1964) used function word
+    frequencies to resolve the disputed authorship of the Federalist Papers,
+    distinguishing between Hamilton and Madison based on their use of
+    "while" vs. "whilst", "upon" vs. "on", etc.
+
+    Args:
+        text: Input text to analyze. Should be at least a few hundred words
+              for reliable statistics. Function word analysis works best with
+              longer texts (1000+ words) where frequency patterns stabilize.
+
+    Returns:
+        FunctionWordResult containing:
+            - Ratios for each function word category (per total words)
+            - Total function word ratio
+            - Function word diversity (unique / total function words)
+            - Most/least frequent function words with counts
+            - Full distribution of all function words used
+            - Metadata with category-specific counts
+
+    Example:
+        >>> result = compute_function_words("Sample text for analysis...")
+        >>> print(f"Determiner ratio: {result.determiner_ratio:.3f}")
+        Determiner ratio: 0.156
+        >>> print(f"Preposition ratio: {result.preposition_ratio:.3f}")
+        Preposition ratio: 0.112
+        >>> print(f"Total function words: {result.total_function_word_ratio:.3f}")
+        Total function words: 0.487
+        >>> print(f"Most frequent: {result.most_frequent_function_words[:3]}")
+        Most frequent: [('the', 45), ('of', 32), ('to', 28)]
+
+        >>> # Authorship comparison example
+        >>> text1 = "Text by author 1..."
+        >>> text2 = "Text by author 2..."
+        >>> r1 = compute_function_words(text1)
+        >>> r2 = compute_function_words(text2)
+        >>> # Compare determiner ratios, preposition preferences, etc.
+
+    Note:
+        - Case-insensitive matching (all text lowercased for matching)
+        - Tokenization by whitespace and punctuation
+        - Words must match exactly (no stemming or lemmatization)
+        - Multi-word function words like "no one" are handled as separate tokens
+        - Empty or very short texts may have unreliable ratios
+        - Some words appear in multiple categories (e.g., "that" is both
+          determiner and pronoun) - each category is counted independently
+    """
+    # Step 1: Create union set of all function words (for total ratio calculation)
+    ALL_FUNCTION_WORDS = (
+        DETERMINERS
+        | PREPOSITIONS
+        | CONJUNCTIONS
+        | PRONOUNS
+        | AUXILIARIES
+        | PARTICLES
+    )
+
+    # Step 2: Tokenize text (lowercase, split on whitespace, strip punctuation)
+    if not text or not text.strip():
+        # Handle empty text edge case
+        return FunctionWordResult(
+            determiner_ratio=0.0,
+            preposition_ratio=0.0,
+            conjunction_ratio=0.0,
+            pronoun_ratio=0.0,
+            auxiliary_ratio=0.0,
+            particle_ratio=0.0,
+            total_function_word_ratio=0.0,
+            function_word_diversity=0.0,
+            most_frequent_function_words=[],
+            least_frequent_function_words=[],
+            function_word_distribution={},
+            metadata={
+                "total_word_count": 0,
+                "total_function_word_count": 0,
+                "unique_function_word_count": 0,
+                "determiner_count": 0,
+                "preposition_count": 0,
+                "conjunction_count": 0,
+                "pronoun_count": 0,
+                "auxiliary_count": 0,
+                "particle_count": 0,
+                "determiner_list": [],
+                "preposition_list": [],
+                "conjunction_list": [],
+                "pronoun_list": [],
+                "auxiliary_list": [],
+                "particle_list": [],
+                "overlapping_words": [],
+                "overlapping_word_categories": {},
+            },
+        )
+
+    # Lowercase entire text
+    text_lower = text.lower()
+
+    # Split on whitespace
+    raw_tokens = text_lower.split()
+
+    # Comprehensive punctuation set for stripping
+    PUNCTUATION = set(
+        ".,!?;:'\"()[]{}/-—–…*&@#$%^~`\\|<>«»„""''‚'"
+    )
+
+    # Strip punctuation from each token
+    tokens = []
+    for token in raw_tokens:
+        # Strip leading and trailing punctuation
+        clean_token = token.strip("".join(PUNCTUATION))
+        if clean_token:  # Only add non-empty tokens
+            tokens.append(clean_token)
+
+    total_words = len(tokens)
+
+    # Step 3: Initialize counters for each category
+    determiner_count = 0
+    preposition_count = 0
+    conjunction_count = 0
+    pronoun_count = 0
+    auxiliary_count = 0
+    particle_count = 0
+
+    # Step 4: Count tokens in each category (overlapping allowed)
+    for token in tokens:
+        if token in DETERMINERS:
+            determiner_count += 1
+        if token in PREPOSITIONS:
+            preposition_count += 1
+        if token in CONJUNCTIONS:
+            conjunction_count += 1
+        if token in PRONOUNS:
+            pronoun_count += 1
+        if token in AUXILIARIES:
+            auxiliary_count += 1
+        if token in PARTICLES:
+            particle_count += 1
+
+    # Step 5: Build distribution (count each function word only once per token)
+    function_word_counts: dict[str, int] = {}
+    for token in tokens:
+        if token in ALL_FUNCTION_WORDS:
+            function_word_counts[token] = function_word_counts.get(token, 0) + 1
+
+    # Step 6: Calculate ratios
+    if total_words > 0:
+        determiner_ratio = determiner_count / total_words
+        preposition_ratio = preposition_count / total_words
+        conjunction_ratio = conjunction_count / total_words
+        pronoun_ratio = pronoun_count / total_words
+        auxiliary_ratio = auxiliary_count / total_words
+        particle_ratio = particle_count / total_words
+
+        total_function_word_count = sum(function_word_counts.values())
+        total_function_word_ratio = total_function_word_count / total_words
+    else:
+        determiner_ratio = 0.0
+        preposition_ratio = 0.0
+        conjunction_ratio = 0.0
+        pronoun_ratio = 0.0
+        auxiliary_ratio = 0.0
+        particle_ratio = 0.0
+        total_function_word_count = 0
+        total_function_word_ratio = 0.0
+
+    # Step 7: Calculate diversity
+    unique_function_word_count = len(function_word_counts)
+    if total_function_word_count > 0:
+        function_word_diversity = unique_function_word_count / total_function_word_count
+    else:
+        function_word_diversity = 0.0
+
+    # Step 8: Find most/least frequent function words
+    if function_word_counts:
+        # Sort by count descending
+        sorted_by_count = sorted(
+            function_word_counts.items(), key=lambda x: x[1], reverse=True
+        )
+
+        # Top 10 most frequent
+        most_frequent = sorted_by_count[:10]
+
+        # Bottom 10 least frequent (reverse to get ascending order)
+        least_frequent = sorted_by_count[-10:]
+        least_frequent.reverse()
+    else:
+        most_frequent = []
+        least_frequent = []
+
+    # Step 9: Build category word lists (sorted)
+    determiner_list = sorted([w for w in function_word_counts if w in DETERMINERS])
+    preposition_list = sorted([w for w in function_word_counts if w in PREPOSITIONS])
+    conjunction_list = sorted([w for w in function_word_counts if w in CONJUNCTIONS])
+    pronoun_list = sorted([w for w in function_word_counts if w in PRONOUNS])
+    auxiliary_list = sorted([w for w in function_word_counts if w in AUXILIARIES])
+    particle_list = sorted([w for w in function_word_counts if w in PARTICLES])
+
+    # Step 10: Find overlapping words (words in multiple categories)
+    overlapping_words = []
+    overlapping_word_categories: dict[str, list[str]] = {}
+
+    for word in function_word_counts:
+        categories = []
+        if word in DETERMINERS:
+            categories.append("determiner")
+        if word in PREPOSITIONS:
+            categories.append("preposition")
+        if word in CONJUNCTIONS:
+            categories.append("conjunction")
+        if word in PRONOUNS:
+            categories.append("pronoun")
+        if word in AUXILIARIES:
+            categories.append("auxiliary")
+        if word in PARTICLES:
+            categories.append("particle")
+
+        if len(categories) > 1:
+            overlapping_words.append(word)
+            overlapping_word_categories[word] = categories
+
+    overlapping_words.sort()
+
+    # Step 11: Build metadata
+    metadata = {
+        "total_word_count": total_words,
+        "total_function_word_count": total_function_word_count,
+        "unique_function_word_count": unique_function_word_count,
+        "determiner_count": determiner_count,
+        "preposition_count": preposition_count,
+        "conjunction_count": conjunction_count,
+        "pronoun_count": pronoun_count,
+        "auxiliary_count": auxiliary_count,
+        "particle_count": particle_count,
+        "determiner_list": determiner_list,
+        "preposition_list": preposition_list,
+        "conjunction_list": conjunction_list,
+        "pronoun_list": pronoun_list,
+        "auxiliary_list": auxiliary_list,
+        "particle_list": particle_list,
+        "overlapping_words": overlapping_words,
+        "overlapping_word_categories": overlapping_word_categories,
+    }
+
+    # Step 12: Return result
+    return FunctionWordResult(
+        determiner_ratio=determiner_ratio,
+        preposition_ratio=preposition_ratio,
+        conjunction_ratio=conjunction_ratio,
+        pronoun_ratio=pronoun_ratio,
+        auxiliary_ratio=auxiliary_ratio,
+        particle_ratio=particle_ratio,
+        total_function_word_ratio=total_function_word_ratio,
+        function_word_diversity=function_word_diversity,
+        most_frequent_function_words=most_frequent,
+        least_frequent_function_words=least_frequent,
+        function_word_distribution=function_word_counts,
+        metadata=metadata,
+    )

pystylometry/lexical/hapax.py

@@ -1,9 +1,10 @@
 """Hapax legomena and related vocabulary richness metrics."""
 
+import math
 from collections import Counter
 
-from .._types import HapaxResult
-from .._utils import tokenize
+from .._types import HapaxLexiconResult, HapaxResult, LexiconCategories
+from .._utils import check_optional_dependency, tokenize
 
 
 def compute_hapax_ratios(text: str) -> HapaxResult:
@@ -30,10 +31,18 @@ def compute_hapax_ratios(text: str) -> HapaxResult:
     Returns:
         HapaxResult with counts, ratios, Sichel's S, Honoré's R, and metadata
 
+        Note: When all words are unique (V₁ = V), Honoré's R returns float('inf')
+        to indicate maximal vocabulary richness (division by zero case).
+
     Example:
-        >>> result = compute_hapax_ratios("The quick brown fox jumps over the lazy dog.")
-        >>> print(f"Hapax ratio: {result.hapax_ratio:.3f}")
+        >>> text = "The quick brown fox jumps over the lazy dog"
+        >>> result = compute_hapax_ratios(text)
+        >>> result.hapax_count  # Words appearing once
+        7
+        >>> result.dis_hapax_count  # Words appearing twice
+        1
         >>> print(f"Sichel's S: {result.sichel_s:.3f}")
+        Sichel's S: 0.125
     """
     tokens = tokenize(text.lower())
     N = len(tokens)  # noqa: N806
@@ -57,9 +66,18 @@ def compute_hapax_ratios(text: str) -> HapaxResult:
     V1 = sum(1 for count in freq_counter.values() if count == 1)  # noqa: N806
     V2 = sum(1 for count in freq_counter.values() if count == 2)  # noqa: N806
 
-    # TODO: Implement Sichel's S and Honoré's R
-    sichel_s = 0.0  # Placeholder
-    honore_r = 0.0  # Placeholder
+    # Sichel's S: ratio of dislegomena to vocabulary size
+    # S = V₂ / V
+    sichel_s = V2 / V if V > 0 else 0.0
+
+    # Honoré's R: 100 × log(N) / (1 - V₁/V)
+    # R = 100 × log(N) / (1 - V₁/V)
+    # If V₁ = V (all words appear once), denominator is 0, return infinity
+    # This indicates maximal vocabulary richness (every word unique)
+    if V1 == V:
+        honore_r = float("inf")
+    else:
+        honore_r = 100 * math.log(N) / (1 - V1 / V)
 
     return HapaxResult(
         hapax_count=V1,
@@ -73,3 +91,132 @@ def compute_hapax_ratios(text: str) -> HapaxResult:
             "vocabulary_size": V,
         },
     )
+
+
+def compute_hapax_with_lexicon_analysis(text: str) -> HapaxLexiconResult:
+    """
+    Compute hapax legomena with lexicon-based categorization.
+
+    Extends standard hapax analysis by categorizing hapax legomena based on
+    presence in WordNet and British National Corpus (BNC). This distinguishes
+    between:
+
+    1. **Neologisms**: Words not in WordNet AND not in BNC
+       - True novel words or proper nouns
+       - High neologism ratio indicates vocabulary innovation
+
+    2. **Rare Words**: Words in BNC but not WordNet, or vice versa
+       - Technical jargon, specialized terminology
+       - Words at the edges of common vocabulary
+
+    3. **Common Words**: Words in both WordNet AND BNC
+       - Standard vocabulary that happens to appear once
+       - Low incidental usage of common words
+
+    This categorization is valuable for stylometric analysis:
+    - Authors with high neologism ratios are more innovative/creative
+    - Technical writing typically has higher rare word ratios
+    - Comparison of neologism vs common hapax distinguishes vocabulary
+      innovation from incidental word usage
+
+    Args:
+        text: Input text to analyze
+
+    Returns:
+        HapaxLexiconResult with standard hapax metrics and lexicon categorization
+
+    Raises:
+        ImportError: If bnc-lookup or wordnet-lookup packages are not installed
+
+    Example:
+        >>> text = "The xyzbot platform facilitates interdepartmental synergy."
+        >>> result = compute_hapax_with_lexicon_analysis(text)
+        >>> result.lexicon_analysis.neologisms
+        ['xyzbot', 'platform']
+        >>> result.lexicon_analysis.rare_words
+        ['facilitates', 'interdepartmental']
+        >>> result.lexicon_analysis.common_words
+        ['synergy']
+        >>> print(f"Neologism ratio: {result.lexicon_analysis.neologism_ratio:.2%}")
+        Neologism ratio: 40.00%
+
+    References:
+        British National Corpus: http://www.natcorp.ox.ac.uk/
+        WordNet: https://wordnet.princeton.edu/
+    """
+    # Check dependencies
+    check_optional_dependency("bnc_lookup", "lexical")
+    check_optional_dependency("wordnet_lookup", "lexical")
+
+    from bnc_lookup import is_bnc_term  # type: ignore[import-not-found]
+    from wordnet_lookup import is_wordnet_term  # type: ignore[import-not-found]
+
+    # First compute standard hapax metrics
+    hapax_result = compute_hapax_ratios(text)
+
+    # If no hapax legomena, return empty categorization
+    if hapax_result.hapax_count == 0:
+        return HapaxLexiconResult(
+            hapax_result=hapax_result,
+            lexicon_analysis=LexiconCategories(
+                neologisms=[],
+                rare_words=[],
+                common_words=[],
+                neologism_ratio=0.0,
+                rare_word_ratio=0.0,
+                metadata={"total_hapax": 0},
+            ),
+            metadata={"note": "No hapax legomena found"},
+        )
+
+    # Get tokens and identify hapax words
+    tokens = tokenize(text.lower())
+    freq_counter = Counter(tokens)
+    hapax_words = [word for word, count in freq_counter.items() if count == 1]
+
+    # Categorize each hapax word by lexicon presence
+    neologisms = []
+    rare_words = []
+    common_words = []
+
+    for word in hapax_words:
+        in_bnc = is_bnc_term(word)
+        in_wordnet = is_wordnet_term(word)
+
+        if not in_bnc and not in_wordnet:
+            # Not in either lexicon → true neologism
+            neologisms.append(word)
+        elif in_bnc and in_wordnet:
+            # In both lexicons → common word
+            common_words.append(word)
+        else:
+            # In one but not the other → rare word
+            rare_words.append(word)
+
+    # Calculate ratios
+    total_hapax = len(hapax_words)
+    neologism_ratio = len(neologisms) / total_hapax if total_hapax > 0 else 0.0
+    rare_word_ratio = len(rare_words) / total_hapax if total_hapax > 0 else 0.0
+    common_word_ratio = len(common_words) / total_hapax if total_hapax > 0 else 0.0
+
+    return HapaxLexiconResult(
+        hapax_result=hapax_result,
+        lexicon_analysis=LexiconCategories(
+            neologisms=sorted(neologisms),
+            rare_words=sorted(rare_words),
+            common_words=sorted(common_words),
+            neologism_ratio=neologism_ratio,
+            rare_word_ratio=rare_word_ratio,
+            metadata={
+                "total_hapax": total_hapax,
+                "neologism_count": len(neologisms),
+                "rare_word_count": len(rare_words),
+                "common_word_count": len(common_words),
+                "common_word_ratio": common_word_ratio,
+            },
+        ),
+        metadata={
+            "lexicons_used": ["bnc", "wordnet"],
+            "note": "Lexicon categorization based on BNC and WordNet presence",
+        },
+    )