bnc-lookup 1.0.5__tar.gz

bnc_lookup-1.0.5/LICENSE

@@ -0,0 +1,46 @@
+BNC-LOOKUP LICENSE
+
+This package contains two components with different licenses:
+
+===============================================================================
+1. SOFTWARE LICENSE (MIT)
+===============================================================================
+
+Copyright (c) 2026 Craig Trim
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+===============================================================================
+2. BNC DATA LICENSE
+===============================================================================
+
+This package contains data derived from the British National Corpus frequency
+lists compiled by Adam Kilgarriff.
+
+Source: https://www.kilgarriff.co.uk/BNClists/all.num.gz
+
+The British National Corpus, version 3 (BNC XML Edition). 2007. Distributed by
+Bodleian Libraries, University of Oxford, on behalf of the BNC Consortium.
+
+The BNC frequency data is used in accordance with standard academic citation
+practices. Users of this package should cite the BNC appropriately in any
+published work.
+
+For licensing information regarding the full British National Corpus, see:
+http://www.natcorp.ox.ac.uk/corpus/licence.html

bnc_lookup-1.0.5/PKG-INFO

@@ -0,0 +1,169 @@
+Metadata-Version: 2.1
+Name: bnc-lookup
+Version: 1.0.5
+Summary: Static Hash-Based Lookup for BNC Terms
+Home-page: https://github.com/craigtrim/bnc-lookup
+License: MIT
+Keywords: bnc,lexicon,nlp,natural-language-processing,dictionary,vocabulary,validation,linguistics
+Author: Craig Trim
+Author-email: craigtrim@gmail.com
+Maintainer: Craig Trim
+Maintainer-email: craigtrim@gmail.com
+Requires-Python: >=3.7,<4.0
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Natural Language :: English
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing :: Linguistic
+Project-URL: Repository, https://github.com/craigtrim/bnc-lookup
+Description-Content-Type: text/markdown
+
+# BNC Lookup
+
+[![PyPI version](https://img.shields.io/pypi/v/bnc-lookup.svg)](https://pypi.org/project/bnc-lookup/)
+[![PyPI downloads](https://img.shields.io/pypi/dm/bnc-lookup.svg)](https://pypi.org/project/bnc-lookup/)
+[![Python versions](https://img.shields.io/pypi/pyversions/bnc-lookup.svg)](https://pypi.org/project/bnc-lookup/)
+[![License](https://img.shields.io/badge/License-MIT%20%2B%20BNC-yellow.svg)](LICENSE)
+[![Code style: ruff](https://img.shields.io/badge/code%20style-ruff-000000.svg)](https://github.com/astral-sh/ruff)
+[![Pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit)
+[![Tests](https://img.shields.io/badge/tests-passing-brightgreen.svg)](https://github.com/craigtrim/bnc-lookup)
+
+**Is this token a word? O(1) answer. No setup. No dependencies.**
+
+A simple question deserves a simple answer. This library gives you instant yes/no validation against 669,000 word forms from the British National Corpus.
+
+## Quick Start
+
+```bash
+pip install bnc-lookup
+```
+
+```python
+from bnc_lookup import is_bnc_term
+
+# That's it. Start validating.
+is_bnc_term('the')          # True
+is_bnc_term('however')      # True
+is_bnc_term('nonetheless')  # True
+is_bnc_term('xyzabc123')    # False
+
+# Handles plurals automatically
+is_bnc_term('computers')    # True
+
+# Case insensitive
+is_bnc_term('THE')          # True
+```
+
+## Features
+
+- **Zero Dependencies** - Pure Python, no external packages
+- **Zero I/O** - No filesystem access, no database queries
+- **Zero Setup** - No corpus downloads or configuration
+- **Microsecond Lookups** - O(1) dictionary access
+- **Smart Plurals** - Automatically checks singular forms
+- **Simple API** - One function does it all
+
+## The Problem This Solves
+
+In NLP, you frequently need to answer the question: **"Is this token a real word?"**
+
+Not "what does it mean?" Not "give me synonyms." Just: is this a word?
+
+<table>
+<tr>
+<td align="center"><code>is_bnc_term('computer')</code></td>
+<td align="center"><code>is_bnc_term('asdfgh')</code></td>
+</tr>
+<tr>
+<td align="center"><img src="docs/images/yes-hot-dog.png" width="180"></td>
+<td align="center"><img src="docs/images/not-hot-dog.png" width="180"></td>
+</tr>
+<tr>
+<td align="center"><code>True</code></td>
+<td align="center"><code>False</code></td>
+</tr>
+</table>
+
+That's it. O(1) response. No ambiguity.
+
+## Why BNC?
+
+The British National Corpus isn't an academic wordlist (too narrow). It's not a web scrape (too noisy). It's not slang (too ephemeral).
+
+It's a **100-million-word corpus of real British English** collected from written and spoken sources between 1991-1994. Books, newspapers, academic papers, conversations. The BNC frequency list captures ~669,000 unique word forms actually used by native speakers.
+
+If a token passes the BNC test, you can be confident it's a word that real people actually use.
+
+## When to Use This
+
+- **Tokenization filtering**: Keep real words, discard garbage
+- **Input validation**: Reject nonsense in user input
+- **NLP preprocessing**: Filter candidates before expensive operations
+- **Spell-check pre-filtering**: Quick reject obvious non-words before fuzzy matching
+- **Data cleaning**: Identify malformed or corrupted text
+
+## What This Doesn't Do
+
+- No definitions, synonyms, or semantic relationships (use spaCy for that)
+- No frequency counts or rankings (just yes/no)
+- No spell-checking or suggestions (just existence check)
+
+## Documentation
+
+For detailed usage, performance benchmarks, and advanced features, see the [API Documentation](docs/API.md).
+
+## How It Works
+
+BNC terms are stored as MD5 hash suffixes in 256 `frozenset` buckets (by first two hex characters of the hash). Lookups hash the input, route to the correct bucket, and perform O(1) set membership. Modules are lazy-loaded on first access per bucket.
+
+For the gory details, see [Implementation Notes](docs/IMPLEMENTATION.md).
+
+## Development
+
+```bash
+git clone https://github.com/craigtrim/bnc-lookup.git
+cd bnc-lookup
+make install  # Install dependencies
+make test     # Run tests
+make all      # Full build pipeline
+```
+
+See [API Documentation](docs/API.md) for detailed development information.
+
+## License
+
+This package is dual-licensed:
+- **Software**: MIT License
+- **BNC Data**: BNC User Licence
+
+See [LICENSE](LICENSE) for complete terms.
+
+## Attribution
+
+This package contains data derived from the British National Corpus frequency lists:
+
+> BNC frequency lists compiled by Adam Kilgarriff.
+> Source: https://www.kilgarriff.co.uk/BNClists/all.num.gz
+>
+> The British National Corpus, version 3 (BNC XML Edition). 2007. Distributed by Bodleian Libraries, University of Oxford, on behalf of the BNC Consortium.
+
+**Note:** This is a static snapshot of BNC frequency data. The data is not automatically updated.
+
+## Links
+
+- **Repository**: [github.com/craigtrim/bnc-lookup](https://github.com/craigtrim/bnc-lookup)
+- **PyPI**: [pypi.org/project/bnc-lookup](https://pypi.org/project/bnc-lookup)
+- **BNC**: [natcorp.ox.ac.uk](http://www.natcorp.ox.ac.uk)
+- **Author**: Craig Trim ([craigtrim@gmail.com](mailto:craigtrim@gmail.com))
+

bnc_lookup-1.0.5/README.md

@@ -0,0 +1,137 @@
+# BNC Lookup
+
+[![PyPI version](https://img.shields.io/pypi/v/bnc-lookup.svg)](https://pypi.org/project/bnc-lookup/)
+[![PyPI downloads](https://img.shields.io/pypi/dm/bnc-lookup.svg)](https://pypi.org/project/bnc-lookup/)
+[![Python versions](https://img.shields.io/pypi/pyversions/bnc-lookup.svg)](https://pypi.org/project/bnc-lookup/)
+[![License](https://img.shields.io/badge/License-MIT%20%2B%20BNC-yellow.svg)](LICENSE)
+[![Code style: ruff](https://img.shields.io/badge/code%20style-ruff-000000.svg)](https://github.com/astral-sh/ruff)
+[![Pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit)
+[![Tests](https://img.shields.io/badge/tests-passing-brightgreen.svg)](https://github.com/craigtrim/bnc-lookup)
+
+**Is this token a word? O(1) answer. No setup. No dependencies.**
+
+A simple question deserves a simple answer. This library gives you instant yes/no validation against 669,000 word forms from the British National Corpus.
+
+## Quick Start
+
+```bash
+pip install bnc-lookup
+```
+
+```python
+from bnc_lookup import is_bnc_term
+
+# That's it. Start validating.
+is_bnc_term('the')          # True
+is_bnc_term('however')      # True
+is_bnc_term('nonetheless')  # True
+is_bnc_term('xyzabc123')    # False
+
+# Handles plurals automatically
+is_bnc_term('computers')    # True
+
+# Case insensitive
+is_bnc_term('THE')          # True
+```
+
+## Features
+
+- **Zero Dependencies** - Pure Python, no external packages
+- **Zero I/O** - No filesystem access, no database queries
+- **Zero Setup** - No corpus downloads or configuration
+- **Microsecond Lookups** - O(1) dictionary access
+- **Smart Plurals** - Automatically checks singular forms
+- **Simple API** - One function does it all
+
+## The Problem This Solves
+
+In NLP, you frequently need to answer the question: **"Is this token a real word?"**
+
+Not "what does it mean?" Not "give me synonyms." Just: is this a word?
+
+<table>
+<tr>
+<td align="center"><code>is_bnc_term('computer')</code></td>
+<td align="center"><code>is_bnc_term('asdfgh')</code></td>
+</tr>
+<tr>
+<td align="center"><img src="docs/images/yes-hot-dog.png" width="180"></td>
+<td align="center"><img src="docs/images/not-hot-dog.png" width="180"></td>
+</tr>
+<tr>
+<td align="center"><code>True</code></td>
+<td align="center"><code>False</code></td>
+</tr>
+</table>
+
+That's it. O(1) response. No ambiguity.
+
+## Why BNC?
+
+The British National Corpus isn't an academic wordlist (too narrow). It's not a web scrape (too noisy). It's not slang (too ephemeral).
+
+It's a **100-million-word corpus of real British English** collected from written and spoken sources between 1991-1994. Books, newspapers, academic papers, conversations. The BNC frequency list captures ~669,000 unique word forms actually used by native speakers.
+
+If a token passes the BNC test, you can be confident it's a word that real people actually use.
+
+## When to Use This
+
+- **Tokenization filtering**: Keep real words, discard garbage
+- **Input validation**: Reject nonsense in user input
+- **NLP preprocessing**: Filter candidates before expensive operations
+- **Spell-check pre-filtering**: Quick reject obvious non-words before fuzzy matching
+- **Data cleaning**: Identify malformed or corrupted text
+
+## What This Doesn't Do
+
+- No definitions, synonyms, or semantic relationships (use spaCy for that)
+- No frequency counts or rankings (just yes/no)
+- No spell-checking or suggestions (just existence check)
+
+## Documentation
+
+For detailed usage, performance benchmarks, and advanced features, see the [API Documentation](docs/API.md).
+
+## How It Works
+
+BNC terms are stored as MD5 hash suffixes in 256 `frozenset` buckets (by first two hex characters of the hash). Lookups hash the input, route to the correct bucket, and perform O(1) set membership. Modules are lazy-loaded on first access per bucket.
+
+For the gory details, see [Implementation Notes](docs/IMPLEMENTATION.md).
+
+## Development
+
+```bash
+git clone https://github.com/craigtrim/bnc-lookup.git
+cd bnc-lookup
+make install  # Install dependencies
+make test     # Run tests
+make all      # Full build pipeline
+```
+
+See [API Documentation](docs/API.md) for detailed development information.
+
+## License
+
+This package is dual-licensed:
+- **Software**: MIT License
+- **BNC Data**: BNC User Licence
+
+See [LICENSE](LICENSE) for complete terms.
+
+## Attribution
+
+This package contains data derived from the British National Corpus frequency lists:
+
+> BNC frequency lists compiled by Adam Kilgarriff.
+> Source: https://www.kilgarriff.co.uk/BNClists/all.num.gz
+>
+> The British National Corpus, version 3 (BNC XML Edition). 2007. Distributed by Bodleian Libraries, University of Oxford, on behalf of the BNC Consortium.
+
+**Note:** This is a static snapshot of BNC frequency data. The data is not automatically updated.
+
+## Links
+
+- **Repository**: [github.com/craigtrim/bnc-lookup](https://github.com/craigtrim/bnc-lookup)
+- **PyPI**: [pypi.org/project/bnc-lookup](https://pypi.org/project/bnc-lookup)
+- **BNC**: [natcorp.ox.ac.uk](http://www.natcorp.ox.ac.uk)
+- **Author**: Craig Trim ([craigtrim@gmail.com](mailto:craigtrim@gmail.com))

bnc_lookup-1.0.5/bnc_lookup/__init__.py

@@ -0,0 +1,5 @@
+from bnc_lookup.find_bnc import FindBnc
+
+
+def is_bnc_term(input_text: str) -> bool:
+    return FindBnc().exists(input_text)

bnc_lookup-1.0.5/bnc_lookup/find_bnc.py

@@ -0,0 +1,49 @@
+# !/usr/bin/env python
+# -*- coding: UTF-8 -*-
+""" Facade to find BNC Data on Disk """
+
+import hashlib
+import importlib
+
+_cache = {}
+
+
+def _get_hash_set(prefix: str) -> frozenset:
+    if prefix not in _cache:
+        module = importlib.import_module(f'bnc_lookup.hs.h_{prefix}')
+        _cache[prefix] = getattr(module, f'hashes_{prefix}')
+    return _cache[prefix]
+
+
+def _calculate_md5(input_text: str) -> str:
+    return hashlib.md5(input_text.lower().strip().encode()).hexdigest()
+
+
+def _hash_exists(input_text: str) -> bool:
+    if not input_text:
+        return False
+    h = _calculate_md5(input_text)
+    prefix, suffix = h[:2], h[2:]
+    try:
+        return suffix in _get_hash_set(prefix)
+    except ModuleNotFoundError:
+        return False
+
+
+class FindBnc:
+    """ Facade to find BNC Data on Disk """
+
+    def __init__(self):
+        pass
+
+    def exists(self, input_text: str) -> bool:
+        input_text = input_text.lower().strip()
+
+        if _hash_exists(input_text):
+            return True
+
+        if input_text.endswith('s') and len(input_text) > 3:
+            if _hash_exists(input_text[:-1]):
+                return True
+
+        return False