bnc-lookup 1.3.0__tar.gz → 1.3.2__tar.gz

{bnc_lookup-1.3.0 → bnc_lookup-1.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bnc-lookup
-Version: 1.3.0
+Version: 1.3.2
 Summary: Static Hash-Based Lookup for BNC Terms
 Home-page: https://github.com/craigtrim/bnc-lookup
 License: MIT
@@ -33,6 +33,7 @@ Description-Content-Type: text/markdown
 
 [![PyPI version](https://badge.fury.io/py/bnc-lookup.svg)](https://badge.fury.io/py/bnc-lookup)
 [![Downloads](https://pepy.tech/badge/bnc-lookup)](https://pepy.tech/project/bnc-lookup)
+[![Downloads/Month](https://pepy.tech/badge/bnc-lookup/month)](https://pepy.tech/project/bnc-lookup)
 [![Python 3.7+](https://img.shields.io/badge/python-3.7%2B-blue.svg)](https://www.python.org/downloads/)
 [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT)
 [![Tests](https://img.shields.io/badge/tests-32%20passed-brightgreen)]()

{bnc_lookup-1.3.0 → bnc_lookup-1.3.2}/README.md

@@ -2,6 +2,7 @@
 
 [![PyPI version](https://badge.fury.io/py/bnc-lookup.svg)](https://badge.fury.io/py/bnc-lookup)
 [![Downloads](https://pepy.tech/badge/bnc-lookup)](https://pepy.tech/project/bnc-lookup)
+[![Downloads/Month](https://pepy.tech/badge/bnc-lookup/month)](https://pepy.tech/project/bnc-lookup)
 [![Python 3.7+](https://img.shields.io/badge/python-3.7%2B-blue.svg)](https://www.python.org/downloads/)
 [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT)
 [![Tests](https://img.shields.io/badge/tests-32%20passed-brightgreen)]()

{bnc_lookup-1.3.0 → bnc_lookup-1.3.2}/bnc_lookup/find_bnc.py

@@ -14,6 +14,8 @@ the singular form (with trailing 's' removed) is also checked.
 import hashlib
 import importlib
 
+from bnc_lookup.normalize import normalize
+
 _cache = {}
 
 
@@ -33,7 +35,10 @@ def _get_hash_set(prefix: str) -> frozenset:
 
 
 def _calculate_md5(input_text: str) -> str:
-    """Compute the MD5 hex digest of a normalized (lowercase, stripped) word.
+    """Compute the MD5 hex digest of a normalized word.
+
+    Normalization includes apostrophe variant conversion, lowercase,
+    and whitespace stripping.
 
     Args:
         input_text: The word to hash.
@@ -41,7 +46,7 @@ def _calculate_md5(input_text: str) -> str:
     Returns:
         32-character hexadecimal MD5 digest string.
     """
-    return hashlib.md5(input_text.lower().strip().encode()).hexdigest()
+    return hashlib.md5(normalize(input_text).encode()).hexdigest()
 
 
 def _hash_exists(input_text: str) -> bool:
@@ -90,7 +95,7 @@ class FindBnc:
         Returns:
             True if the word (or its singular form) exists in the BNC.
         """
-        input_text = input_text.lower().strip()
+        input_text = normalize(input_text)
 
         if _hash_exists(input_text):
             return True

{bnc_lookup-1.3.0 → bnc_lookup-1.3.2}/bnc_lookup/find_freq.py

@@ -14,6 +14,8 @@ bucket files (f_00.py through f_ff.py) maps hash suffixes to bucket numbers.
 import hashlib
 import importlib
 
+from bnc_lookup.normalize import normalize
+
 _cache = {}
 
 
@@ -33,7 +35,10 @@ def _get_bucket_dict(prefix: str) -> dict:
 
 
 def _calculate_md5(input_text: str) -> str:
-    """Compute the MD5 hex digest of a normalized (lowercase, stripped) word.
+    """Compute the MD5 hex digest of a normalized word.
+
+    Normalization includes apostrophe variant conversion, lowercase,
+    and whitespace stripping.
 
     Args:
         input_text: The word to hash.
@@ -41,7 +46,7 @@ def _calculate_md5(input_text: str) -> str:
     Returns:
         32-character hexadecimal MD5 digest string.
     """
-    return hashlib.md5(input_text.lower().strip().encode()).hexdigest()
+    return hashlib.md5(normalize(input_text).encode()).hexdigest()
 
 
 def _lookup_bucket(input_text: str) -> int | None:
@@ -89,7 +94,7 @@ class FindFreq:
             1-100: Bucket number (1=most frequent, 100=least frequent)
             None: Word not found in BNC
         """
-        input_text = input_text.lower().strip()
+        input_text = normalize(input_text)
 
         result = _lookup_bucket(input_text)
         if result is not None:

{bnc_lookup-1.3.0 → bnc_lookup-1.3.2}/bnc_lookup/find_rf.py

@@ -17,6 +17,8 @@ through rf_ff.py) maps hash suffixes to relative frequency values.
 import hashlib
 import importlib
 
+from bnc_lookup.normalize import normalize
+
 _cache = {}
 
 
@@ -36,7 +38,10 @@ def _get_rf_dict(prefix: str) -> dict:
 
 
 def _calculate_md5(input_text: str) -> str:
-    """Compute the MD5 hex digest of a normalized (lowercase, stripped) word.
+    """Compute the MD5 hex digest of a normalized word.
+
+    Normalization includes apostrophe variant conversion, lowercase,
+    and whitespace stripping.
 
     Args:
         input_text: The word to hash.
@@ -44,7 +49,7 @@ def _calculate_md5(input_text: str) -> str:
     Returns:
         32-character hexadecimal MD5 digest string.
     """
-    return hashlib.md5(input_text.lower().strip().encode()).hexdigest()
+    return hashlib.md5(normalize(input_text).encode()).hexdigest()
 
 
 def _lookup_rf(input_text: str) -> float | None:
@@ -91,7 +96,7 @@ class FindRF:
         Returns:
             Float in range (0, 1), or None if word not in BNC.
         """
-        input_text = input_text.lower().strip()
+        input_text = normalize(input_text)
 
         result = _lookup_rf(input_text)
         if result is not None:

bnc_lookup-1.3.2/bnc_lookup/normalize.py

@@ -0,0 +1,68 @@
+# !/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""Text normalization utilities for BNC lookup.
+
+Handles normalization of Unicode apostrophe variants and other text
+transformations to ensure consistent matching against the BNC corpus.
+"""
+
+# Unicode characters that should normalize to ASCII apostrophe (U+0027)
+# Ordered by likelihood of occurrence in English text
+APOSTROPHE_VARIANTS = (
+    '\u2019'  # RIGHT SINGLE QUOTATION MARK (most common smart quote)
+    '\u2018'  # LEFT SINGLE QUOTATION MARK
+    '\u0060'  # GRAVE ACCENT
+    '\u00B4'  # ACUTE ACCENT
+    '\u201B'  # SINGLE HIGH-REVERSED-9 QUOTATION MARK
+    '\u2032'  # PRIME
+    '\u2035'  # REVERSED PRIME
+    '\u02B9'  # MODIFIER LETTER PRIME
+    '\u02BC'  # MODIFIER LETTER APOSTROPHE
+    '\u02C8'  # MODIFIER LETTER VERTICAL LINE
+    '\u0313'  # COMBINING COMMA ABOVE
+    '\u0315'  # COMBINING COMMA ABOVE RIGHT
+    '\u055A'  # ARMENIAN APOSTROPHE
+    '\u05F3'  # HEBREW PUNCTUATION GERESH
+    '\u07F4'  # NKO HIGH TONE APOSTROPHE
+    '\u07F5'  # NKO LOW TONE APOSTROPHE
+    '\uFF07'  # FULLWIDTH APOSTROPHE
+    '\u1FBF'  # GREEK PSILI
+    '\u1FBD'  # GREEK KORONIS
+    '\uA78C'  # LATIN SMALL LETTER SALTILLO
+)
+
+# Pre-compiled translation table for fast apostrophe normalization
+_APOSTROPHE_TABLE = str.maketrans({char: "'" for char in APOSTROPHE_VARIANTS})
+
+
+def normalize_apostrophes(text: str) -> str:
+    """Normalize Unicode apostrophe variants to ASCII apostrophe.
+
+    Converts curly quotes, prime marks, and other apostrophe-like
+    characters to the standard ASCII apostrophe (U+0027) used in
+    the BNC corpus data.
+
+    Args:
+        text: Input text potentially containing Unicode apostrophes.
+
+    Returns:
+        Text with all apostrophe variants converted to ASCII apostrophe.
+    """
+    return text.translate(_APOSTROPHE_TABLE)
+
+
+def normalize(text: str) -> str:
+    """Normalize text for BNC lookup.
+
+    Applies all normalization steps:
+    - Convert apostrophe variants to ASCII
+    - Convert to lowercase
+    - Strip leading/trailing whitespace
+
+    Args:
+        text: Input text to normalize.
+
+    Returns:
+        Normalized text ready for BNC lookup.
+    """
+    return normalize_apostrophes(text).lower().strip()

{bnc_lookup-1.3.0 → bnc_lookup-1.3.2}/pyproject.toml

@@ -11,7 +11,7 @@ maintainers = [
 description = "Static Hash-Based Lookup for BNC Terms"
 name = "bnc-lookup"
 readme = "README.md"
-version = "1.3.0"
+version = "1.3.2"
 license = "MIT"
 
 keywords = [

bnc_lookup-1.3.2/setup.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+from setuptools import setup
+
+packages = \
+['bnc_lookup',
+ 'bnc_lookup.bw',
+ 'bnc_lookup.freq',
+ 'bnc_lookup.hs',
+ 'bnc_lookup.rf']
+
+package_data = \
+{'': ['*']}
+
+entry_points = \
+{'console_scripts': ['bnc-bucket = bnc_lookup.cli:bucket',
+                     'bnc-exists = bnc_lookup.cli:exists',
+                     'bnc-expected = bnc_lookup.cli:expected',
+                     'bnc-freq = bnc_lookup.cli:freq']}
+
+setup_kwargs = {
+    'name': 'bnc-lookup',
+    'version': '1.3.2',
+    'description': 'Static Hash-Based Lookup for BNC Terms',
+    'long_description': '# BNC Lookup\n\n[![PyPI version](https://badge.fury.io/py/bnc-lookup.svg)](https://badge.fury.io/py/bnc-lookup)\n[![Downloads](https://pepy.tech/badge/bnc-lookup)](https://pepy.tech/project/bnc-lookup)\n[![Downloads/Month](https://pepy.tech/badge/bnc-lookup/month)](https://pepy.tech/project/bnc-lookup)\n[![Python 3.7+](https://img.shields.io/badge/python-3.7%2B-blue.svg)](https://www.python.org/downloads/)\n[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT)\n[![Tests](https://img.shields.io/badge/tests-32%20passed-brightgreen)]()\n\n**Is this token a word? O(1) answer. No setup. No dependencies.**\n\nA simple question deserves a simple answer. This library gives you instant yes/no validation against 669,000 word forms from the British National Corpus, plus frequency ranking.\n\n## Quick Start\n\n```bash\npip install bnc-lookup\n```\n\n```python\nimport bnc_lookup as bnc\n\n# Check if a word exists\nbnc.exists(\'the\')          # True\nbnc.exists(\'however\')      # True\nbnc.exists(\'xyzabc123\')    # False\n\n# Get frequency bucket (1=most common, 100=least common)\nbnc.bucket(\'the\')          # 1\nbnc.bucket(\'python\')       # 4\nbnc.bucket(\'qwerty\')       # 12\nbnc.bucket(\'xyzabc123\')    # None (not found)\n\n# Relative frequency (per-word precision)\nbnc.relative_frequency(\'the\')              # 0.0618\nbnc.relative_frequency(\'shimmered\')        # 9.79e-07\n\n# Expected occurrences in a text of given length\nbnc.expected_count(\'the\', 50000)           # 3090.7\nbnc.expected_count(\'the\', 50000, rounded=True)  # 3091\n\n# Handles plurals and case automatically\nbnc.exists(\'computers\')    # True\nbnc.exists(\'THE\')          # True\n```\n\n## Features\n\n- **Zero Dependencies** - Pure Python, no external packages\n- **Zero I/O** - No filesystem access, no database queries\n- **Zero Setup** - No corpus downloads or configuration\n- **Microsecond Lookups** - O(1) dictionary access\n- **Smart Plurals** - Automatically checks singular forms\n- **Frequency Ranking** - 100 buckets from most to least common\n- **Relative Frequency** - Per-word precision for quantitative analysis\n- **Expected Counts** - Predict word occurrences in any text length\n- **CLI Tools** - `bnc-exists`, `bnc-bucket`, `bnc-freq`, `bnc-expected`\n\n## The Problem This Solves\n\nIn NLP, you frequently need to answer the question: **"Is this token a real word?"**\n\nNot "what does it mean?" Not "give me synonyms." Just: is this a word?\n\n<table>\n<tr>\n<td align="center"><code>bnc.exists(\'computer\')</code></td>\n<td align="center"><code>bnc.exists(\'asdfgh\')</code></td>\n</tr>\n<tr>\n<td align="center"><img src="https://raw.githubusercontent.com/craigtrim/bnc-lookup/master/docs/images/yes-hot-dog.png" width="180"></td>\n<td align="center"><img src="https://raw.githubusercontent.com/craigtrim/bnc-lookup/master/docs/images/not-hot-dog.png" width="180"></td>\n</tr>\n<tr>\n<td align="center"><code>True</code></td>\n<td align="center"><code>False</code></td>\n</tr>\n</table>\n\nThat\'s it. O(1) response. No ambiguity.\n\n## Frequency Buckets\n\nWords are ranked into 100 buckets based on their frequency in the BNC corpus:\n\n| Bucket | Description | Examples |\n|--------|-------------|----------|\n| 1 | Most frequent (~6,700 words) | the, of, and, is, computer |\n| 2-10 | Very common | algorithm, python, beautiful |\n| 11-50 | Common | qwerty, specialized terms |\n| 51-99 | Less common | Rare but valid words |\n| 100 | Least frequent | Obscure terms |\n\n```python\nimport bnc_lookup as bnc\n\n# Filter by frequency\ndef is_common_word(word):\n    bucket = bnc.bucket(word)\n    return bucket is not None and bucket <= 10\n```\n\n## Why BNC?\n\nThe British National Corpus isn\'t an academic wordlist (too narrow). It\'s not a web scrape (too noisy). It\'s not slang (too ephemeral).\n\nIt\'s a **100-million-word corpus of real British English** collected from written and spoken sources between 1991-1994. Books, newspapers, academic papers, conversations. The BNC frequency list captures ~669,000 unique word forms actually used by native speakers.\n\nIf a token passes the BNC test, you can be confident it\'s a word that real people actually use.\n\n## Real Words vs Dictionary Words\n\nHow much of real-world English is in the dictionary? We compared BNC against WordNet:\n\n![BNC Vocabulary Zones by WordNet Coverage](https://raw.githubusercontent.com/craigtrim/bnc-lookup/master/docs/images/vocabulary_zones.png)\n\n**93%** of common words (bucket 1-10) are in WordNet. But dictionaries miss proper nouns, technical terms, compounds, and domain jargon that appear constantly in real text.\n\nThat\'s the gap BNC fills. [Full analysis](https://github.com/craigtrim/bnc-lookup/issues/1)\n\n## When to Use This\n\n- **Tokenization filtering**: Keep real words, discard garbage\n- **Input validation**: Reject nonsense in user input\n- **NLP preprocessing**: Filter candidates before expensive operations\n- **Spell-check pre-filtering**: Quick reject obvious non-words before fuzzy matching\n- **Data cleaning**: Identify malformed or corrupted text\n- **Frequency-based filtering**: Prefer common words over obscure ones\n\n## What This Doesn\'t Do\n\n- No definitions, synonyms, or semantic relationships (use spaCy for that)\n- No spell-checking or suggestions (just existence check)\n- No irregular plural handling ("mice" → "mouse")\n\n## CLI\n\n```bash\nbnc-exists the            # True (exit code 0)\nbnc-bucket python         # 4\nbnc-freq the              # 6.181373e-02\nbnc-expected the 50000    # 3090.6865\nbnc-expected the 50000 --rounded  # 3091\n```\n\n## Documentation\n\nFor detailed usage, performance benchmarks, and advanced features, see the [API Documentation](https://github.com/craigtrim/bnc-lookup/blob/master/docs/API.md).\n\n## Development\n\n```bash\ngit clone https://github.com/craigtrim/bnc-lookup.git\ncd bnc-lookup\nmake install  # Install dependencies\nmake test     # Run tests\nmake all      # Full build pipeline\n```\n\nSee [API Documentation](https://github.com/craigtrim/bnc-lookup/blob/master/docs/API.md) for detailed development information.\n\n## License\n\nThis package is dual-licensed:\n- **Software**: MIT License\n- **BNC Data**: BNC User Licence\n\nSee [LICENSE](https://github.com/craigtrim/bnc-lookup/blob/master/LICENSE) for complete terms.\n\n## Attribution\n\nThis package contains data derived from the British National Corpus frequency lists:\n\n> BNC frequency lists compiled by Adam Kilgarriff.\n> Source: https://www.kilgarriff.co.uk/BNClists/all.num.gz\n>\n> The British National Corpus, version 3 (BNC XML Edition). 2007. Distributed by Bodleian Libraries, University of Oxford, on behalf of the BNC Consortium.\n\n**Note:** This is a static snapshot of BNC frequency data. The data is not automatically updated.\n\n## See Also\n\n- **[wordnet-lookup](https://github.com/craigtrim/wordnet-lookup)** - Similar O(1) lookup using the WordNet lexicon\n- **[BNC vs WordNet Analysis](https://github.com/craigtrim/bnc-lookup/issues/1)** - Deep dive on what each captures\n\n## Links\n\n- **Repository**: [github.com/craigtrim/bnc-lookup](https://github.com/craigtrim/bnc-lookup)\n- **PyPI**: [pypi.org/project/bnc-lookup](https://pypi.org/project/bnc-lookup)\n- **BNC**: [natcorp.ox.ac.uk](http://www.natcorp.ox.ac.uk)\n- **Author**: Craig Trim ([craigtrim@gmail.com](mailto:craigtrim@gmail.com))\n',
+    'author': 'Craig Trim',
+    'author_email': 'craigtrim@gmail.com',
+    'maintainer': 'Craig Trim',
+    'maintainer_email': 'craigtrim@gmail.com',
+    'url': 'https://github.com/craigtrim/bnc-lookup',
+    'packages': packages,
+    'package_data': package_data,
+    'entry_points': entry_points,
+    'python_requires': '>=3.7,<4.0',
+}
+
+
+setup(**setup_kwargs)

bnc_lookup-1.3.0/setup.py

@@ -1,37 +0,0 @@
-# -*- coding: utf-8 -*-
-from setuptools import setup
-
-packages = \
-['bnc_lookup',
- 'bnc_lookup.bw',
- 'bnc_lookup.freq',
- 'bnc_lookup.hs',
- 'bnc_lookup.rf']
-
-package_data = \
-{'': ['*']}
-
-entry_points = \
-{'console_scripts': ['bnc-bucket = bnc_lookup.cli:bucket',
-                     'bnc-exists = bnc_lookup.cli:exists',
-                     'bnc-expected = bnc_lookup.cli:expected',
-                     'bnc-freq = bnc_lookup.cli:freq']}
-
-setup_kwargs = {
-    'name': 'bnc-lookup',
-    'version': '1.3.0',
-    'description': 'Static Hash-Based Lookup for BNC Terms',
-    'long_description': '# BNC Lookup\n\n[![PyPI version](https://badge.fury.io/py/bnc-lookup.svg)](https://badge.fury.io/py/bnc-lookup)\n[![Downloads](https://pepy.tech/badge/bnc-lookup)](https://pepy.tech/project/bnc-lookup)\n[![Python 3.7+](https://img.shields.io/badge/python-3.7%2B-blue.svg)](https://www.python.org/downloads/)\n[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT)\n[![Tests](https://img.shields.io/badge/tests-32%20passed-brightgreen)]()\n\n**Is this token a word? O(1) answer. No setup. No dependencies.**\n\nA simple question deserves a simple answer. This library gives you instant yes/no validation against 669,000 word forms from the British National Corpus, plus frequency ranking.\n\n## Quick Start\n\n```bash\npip install bnc-lookup\n```\n\n```python\nimport bnc_lookup as bnc\n\n# Check if a word exists\nbnc.exists(\'the\')          # True\nbnc.exists(\'however\')      # True\nbnc.exists(\'xyzabc123\')    # False\n\n# Get frequency bucket (1=most common, 100=least common)\nbnc.bucket(\'the\')          # 1\nbnc.bucket(\'python\')       # 4\nbnc.bucket(\'qwerty\')       # 12\nbnc.bucket(\'xyzabc123\')    # None (not found)\n\n# Relative frequency (per-word precision)\nbnc.relative_frequency(\'the\')              # 0.0618\nbnc.relative_frequency(\'shimmered\')        # 9.79e-07\n\n# Expected occurrences in a text of given length\nbnc.expected_count(\'the\', 50000)           # 3090.7\nbnc.expected_count(\'the\', 50000, rounded=True)  # 3091\n\n# Handles plurals and case automatically\nbnc.exists(\'computers\')    # True\nbnc.exists(\'THE\')          # True\n```\n\n## Features\n\n- **Zero Dependencies** - Pure Python, no external packages\n- **Zero I/O** - No filesystem access, no database queries\n- **Zero Setup** - No corpus downloads or configuration\n- **Microsecond Lookups** - O(1) dictionary access\n- **Smart Plurals** - Automatically checks singular forms\n- **Frequency Ranking** - 100 buckets from most to least common\n- **Relative Frequency** - Per-word precision for quantitative analysis\n- **Expected Counts** - Predict word occurrences in any text length\n- **CLI Tools** - `bnc-exists`, `bnc-bucket`, `bnc-freq`, `bnc-expected`\n\n## The Problem This Solves\n\nIn NLP, you frequently need to answer the question: **"Is this token a real word?"**\n\nNot "what does it mean?" Not "give me synonyms." Just: is this a word?\n\n<table>\n<tr>\n<td align="center"><code>bnc.exists(\'computer\')</code></td>\n<td align="center"><code>bnc.exists(\'asdfgh\')</code></td>\n</tr>\n<tr>\n<td align="center"><img src="https://raw.githubusercontent.com/craigtrim/bnc-lookup/master/docs/images/yes-hot-dog.png" width="180"></td>\n<td align="center"><img src="https://raw.githubusercontent.com/craigtrim/bnc-lookup/master/docs/images/not-hot-dog.png" width="180"></td>\n</tr>\n<tr>\n<td align="center"><code>True</code></td>\n<td align="center"><code>False</code></td>\n</tr>\n</table>\n\nThat\'s it. O(1) response. No ambiguity.\n\n## Frequency Buckets\n\nWords are ranked into 100 buckets based on their frequency in the BNC corpus:\n\n| Bucket | Description | Examples |\n|--------|-------------|----------|\n| 1 | Most frequent (~6,700 words) | the, of, and, is, computer |\n| 2-10 | Very common | algorithm, python, beautiful |\n| 11-50 | Common | qwerty, specialized terms |\n| 51-99 | Less common | Rare but valid words |\n| 100 | Least frequent | Obscure terms |\n\n```python\nimport bnc_lookup as bnc\n\n# Filter by frequency\ndef is_common_word(word):\n    bucket = bnc.bucket(word)\n    return bucket is not None and bucket <= 10\n```\n\n## Why BNC?\n\nThe British National Corpus isn\'t an academic wordlist (too narrow). It\'s not a web scrape (too noisy). It\'s not slang (too ephemeral).\n\nIt\'s a **100-million-word corpus of real British English** collected from written and spoken sources between 1991-1994. Books, newspapers, academic papers, conversations. The BNC frequency list captures ~669,000 unique word forms actually used by native speakers.\n\nIf a token passes the BNC test, you can be confident it\'s a word that real people actually use.\n\n## Real Words vs Dictionary Words\n\nHow much of real-world English is in the dictionary? We compared BNC against WordNet:\n\n![BNC Vocabulary Zones by WordNet Coverage](https://raw.githubusercontent.com/craigtrim/bnc-lookup/master/docs/images/vocabulary_zones.png)\n\n**93%** of common words (bucket 1-10) are in WordNet. But dictionaries miss proper nouns, technical terms, compounds, and domain jargon that appear constantly in real text.\n\nThat\'s the gap BNC fills. [Full analysis](https://github.com/craigtrim/bnc-lookup/issues/1)\n\n## When to Use This\n\n- **Tokenization filtering**: Keep real words, discard garbage\n- **Input validation**: Reject nonsense in user input\n- **NLP preprocessing**: Filter candidates before expensive operations\n- **Spell-check pre-filtering**: Quick reject obvious non-words before fuzzy matching\n- **Data cleaning**: Identify malformed or corrupted text\n- **Frequency-based filtering**: Prefer common words over obscure ones\n\n## What This Doesn\'t Do\n\n- No definitions, synonyms, or semantic relationships (use spaCy for that)\n- No spell-checking or suggestions (just existence check)\n- No irregular plural handling ("mice" → "mouse")\n\n## CLI\n\n```bash\nbnc-exists the            # True (exit code 0)\nbnc-bucket python         # 4\nbnc-freq the              # 6.181373e-02\nbnc-expected the 50000    # 3090.6865\nbnc-expected the 50000 --rounded  # 3091\n```\n\n## Documentation\n\nFor detailed usage, performance benchmarks, and advanced features, see the [API Documentation](https://github.com/craigtrim/bnc-lookup/blob/master/docs/API.md).\n\n## Development\n\n```bash\ngit clone https://github.com/craigtrim/bnc-lookup.git\ncd bnc-lookup\nmake install  # Install dependencies\nmake test     # Run tests\nmake all      # Full build pipeline\n```\n\nSee [API Documentation](https://github.com/craigtrim/bnc-lookup/blob/master/docs/API.md) for detailed development information.\n\n## License\n\nThis package is dual-licensed:\n- **Software**: MIT License\n- **BNC Data**: BNC User Licence\n\nSee [LICENSE](https://github.com/craigtrim/bnc-lookup/blob/master/LICENSE) for complete terms.\n\n## Attribution\n\nThis package contains data derived from the British National Corpus frequency lists:\n\n> BNC frequency lists compiled by Adam Kilgarriff.\n> Source: https://www.kilgarriff.co.uk/BNClists/all.num.gz\n>\n> The British National Corpus, version 3 (BNC XML Edition). 2007. Distributed by Bodleian Libraries, University of Oxford, on behalf of the BNC Consortium.\n\n**Note:** This is a static snapshot of BNC frequency data. The data is not automatically updated.\n\n## See Also\n\n- **[wordnet-lookup](https://github.com/craigtrim/wordnet-lookup)** - Similar O(1) lookup using the WordNet lexicon\n- **[BNC vs WordNet Analysis](https://github.com/craigtrim/bnc-lookup/issues/1)** - Deep dive on what each captures\n\n## Links\n\n- **Repository**: [github.com/craigtrim/bnc-lookup](https://github.com/craigtrim/bnc-lookup)\n- **PyPI**: [pypi.org/project/bnc-lookup](https://pypi.org/project/bnc-lookup)\n- **BNC**: [natcorp.ox.ac.uk](http://www.natcorp.ox.ac.uk)\n- **Author**: Craig Trim ([craigtrim@gmail.com](mailto:craigtrim@gmail.com))\n',
-    'author': 'Craig Trim',
-    'author_email': 'craigtrim@gmail.com',
-    'maintainer': 'Craig Trim',
-    'maintainer_email': 'craigtrim@gmail.com',
-    'url': 'https://github.com/craigtrim/bnc-lookup',
-    'packages': packages,
-    'package_data': package_data,
-    'entry_points': entry_points,
-    'python_requires': '>=3.7,<4.0',
-}
-
-
-setup(**setup_kwargs)