bnc-lookup 1.3.0__py3-none-any.whl → 1.3.2__py3-none-any.whl

bnc_lookup/find_bnc.py

@@ -14,6 +14,8 @@ the singular form (with trailing 's' removed) is also checked.
 import hashlib
 import importlib
 
+from bnc_lookup.normalize import normalize
+
 _cache = {}
 
 
@@ -33,7 +35,10 @@ def _get_hash_set(prefix: str) -> frozenset:
 
 
 def _calculate_md5(input_text: str) -> str:
-    """Compute the MD5 hex digest of a normalized (lowercase, stripped) word.
+    """Compute the MD5 hex digest of a normalized word.
+
+    Normalization includes apostrophe variant conversion, lowercase,
+    and whitespace stripping.
 
     Args:
         input_text: The word to hash.
@@ -41,7 +46,7 @@ def _calculate_md5(input_text: str) -> str:
     Returns:
         32-character hexadecimal MD5 digest string.
     """
-    return hashlib.md5(input_text.lower().strip().encode()).hexdigest()
+    return hashlib.md5(normalize(input_text).encode()).hexdigest()
 
 
 def _hash_exists(input_text: str) -> bool:
@@ -90,7 +95,7 @@ class FindBnc:
         Returns:
             True if the word (or its singular form) exists in the BNC.
         """
-        input_text = input_text.lower().strip()
+        input_text = normalize(input_text)
 
         if _hash_exists(input_text):
             return True

bnc_lookup/find_freq.py

@@ -14,6 +14,8 @@ bucket files (f_00.py through f_ff.py) maps hash suffixes to bucket numbers.
 import hashlib
 import importlib
 
+from bnc_lookup.normalize import normalize
+
 _cache = {}
 
 
@@ -33,7 +35,10 @@ def _get_bucket_dict(prefix: str) -> dict:
 
 
 def _calculate_md5(input_text: str) -> str:
-    """Compute the MD5 hex digest of a normalized (lowercase, stripped) word.
+    """Compute the MD5 hex digest of a normalized word.
+
+    Normalization includes apostrophe variant conversion, lowercase,
+    and whitespace stripping.
 
     Args:
         input_text: The word to hash.
@@ -41,7 +46,7 @@ def _calculate_md5(input_text: str) -> str:
     Returns:
         32-character hexadecimal MD5 digest string.
     """
-    return hashlib.md5(input_text.lower().strip().encode()).hexdigest()
+    return hashlib.md5(normalize(input_text).encode()).hexdigest()
 
 
 def _lookup_bucket(input_text: str) -> int | None:
@@ -89,7 +94,7 @@ class FindFreq:
             1-100: Bucket number (1=most frequent, 100=least frequent)
             None: Word not found in BNC
         """
-        input_text = input_text.lower().strip()
+        input_text = normalize(input_text)
 
         result = _lookup_bucket(input_text)
         if result is not None:

bnc_lookup/find_rf.py

@@ -17,6 +17,8 @@ through rf_ff.py) maps hash suffixes to relative frequency values.
 import hashlib
 import importlib
 
+from bnc_lookup.normalize import normalize
+
 _cache = {}
 
 
@@ -36,7 +38,10 @@ def _get_rf_dict(prefix: str) -> dict:
 
 
 def _calculate_md5(input_text: str) -> str:
-    """Compute the MD5 hex digest of a normalized (lowercase, stripped) word.
+    """Compute the MD5 hex digest of a normalized word.
+
+    Normalization includes apostrophe variant conversion, lowercase,
+    and whitespace stripping.
 
     Args:
         input_text: The word to hash.
@@ -44,7 +49,7 @@ def _calculate_md5(input_text: str) -> str:
     Returns:
         32-character hexadecimal MD5 digest string.
     """
-    return hashlib.md5(input_text.lower().strip().encode()).hexdigest()
+    return hashlib.md5(normalize(input_text).encode()).hexdigest()
 
 
 def _lookup_rf(input_text: str) -> float | None:
@@ -91,7 +96,7 @@ class FindRF:
         Returns:
             Float in range (0, 1), or None if word not in BNC.
         """
-        input_text = input_text.lower().strip()
+        input_text = normalize(input_text)
 
         result = _lookup_rf(input_text)
         if result is not None:

bnc_lookup/normalize.py

@@ -0,0 +1,68 @@
+# !/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""Text normalization utilities for BNC lookup.
+
+Handles normalization of Unicode apostrophe variants and other text
+transformations to ensure consistent matching against the BNC corpus.
+"""
+
+# Unicode characters that should normalize to ASCII apostrophe (U+0027)
+# Ordered by likelihood of occurrence in English text
+APOSTROPHE_VARIANTS = (
+    '\u2019'  # RIGHT SINGLE QUOTATION MARK (most common smart quote)
+    '\u2018'  # LEFT SINGLE QUOTATION MARK
+    '\u0060'  # GRAVE ACCENT
+    '\u00B4'  # ACUTE ACCENT
+    '\u201B'  # SINGLE HIGH-REVERSED-9 QUOTATION MARK
+    '\u2032'  # PRIME
+    '\u2035'  # REVERSED PRIME
+    '\u02B9'  # MODIFIER LETTER PRIME
+    '\u02BC'  # MODIFIER LETTER APOSTROPHE
+    '\u02C8'  # MODIFIER LETTER VERTICAL LINE
+    '\u0313'  # COMBINING COMMA ABOVE
+    '\u0315'  # COMBINING COMMA ABOVE RIGHT
+    '\u055A'  # ARMENIAN APOSTROPHE
+    '\u05F3'  # HEBREW PUNCTUATION GERESH
+    '\u07F4'  # NKO HIGH TONE APOSTROPHE
+    '\u07F5'  # NKO LOW TONE APOSTROPHE
+    '\uFF07'  # FULLWIDTH APOSTROPHE
+    '\u1FBF'  # GREEK PSILI
+    '\u1FBD'  # GREEK KORONIS
+    '\uA78C'  # LATIN SMALL LETTER SALTILLO
+)
+
+# Pre-compiled translation table for fast apostrophe normalization
+_APOSTROPHE_TABLE = str.maketrans({char: "'" for char in APOSTROPHE_VARIANTS})
+
+
+def normalize_apostrophes(text: str) -> str:
+    """Normalize Unicode apostrophe variants to ASCII apostrophe.
+
+    Converts curly quotes, prime marks, and other apostrophe-like
+    characters to the standard ASCII apostrophe (U+0027) used in
+    the BNC corpus data.
+
+    Args:
+        text: Input text potentially containing Unicode apostrophes.
+
+    Returns:
+        Text with all apostrophe variants converted to ASCII apostrophe.
+    """
+    return text.translate(_APOSTROPHE_TABLE)
+
+
+def normalize(text: str) -> str:
+    """Normalize text for BNC lookup.
+
+    Applies all normalization steps:
+    - Convert apostrophe variants to ASCII
+    - Convert to lowercase
+    - Strip leading/trailing whitespace
+
+    Args:
+        text: Input text to normalize.
+
+    Returns:
+        Normalized text ready for BNC lookup.
+    """
+    return normalize_apostrophes(text).lower().strip()

{bnc_lookup-1.3.0.dist-info → bnc_lookup-1.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bnc-lookup
-Version: 1.3.0
+Version: 1.3.2
 Summary: Static Hash-Based Lookup for BNC Terms
 Home-page: https://github.com/craigtrim/bnc-lookup
 License: MIT
@@ -33,6 +33,7 @@ Description-Content-Type: text/markdown
 
 [![PyPI version](https://badge.fury.io/py/bnc-lookup.svg)](https://badge.fury.io/py/bnc-lookup)
 [![Downloads](https://pepy.tech/badge/bnc-lookup)](https://pepy.tech/project/bnc-lookup)
+[![Downloads/Month](https://pepy.tech/badge/bnc-lookup/month)](https://pepy.tech/project/bnc-lookup)
 [![Python 3.7+](https://img.shields.io/badge/python-3.7%2B-blue.svg)](https://www.python.org/downloads/)
 [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT)
 [![Tests](https://img.shields.io/badge/tests-32%20passed-brightgreen)]()

{bnc_lookup-1.3.0.dist-info → bnc_lookup-1.3.2.dist-info}/RECORD

@@ -101,9 +101,9 @@ bnc_lookup/bw/bw_97.py,sha256=pYgPQTbjKGEIXiaGU0P0B7wOVuY1yyt7PwT2iU3gJTA,110752
 bnc_lookup/bw/bw_98.py,sha256=WMhy-xQJHGrMwC3MUAVyHFtcfwbKFUNxKL2wbJBkoJo,110145
 bnc_lookup/bw/bw_99.py,sha256=MKGgX_qSTvI8-Q6dVh0WHyKSaRzHDL0uKtN-x_98xbU,143227
 bnc_lookup/cli.py,sha256=36Ma2LBEgsEt2-9u0pf_u8wwM4wa0Bqx80RefBa0l7E,2211
-bnc_lookup/find_bnc.py,sha256=ylKY4lpQG0NV6MMZeAXU9y_Dv7gtYPPrRdBwBKt2ylc,3067
-bnc_lookup/find_freq.py,sha256=fe9H4N8NJYr36nwvQ2ITZ_aUxC_7VHrw50FVQNfvGZ0,3028
-bnc_lookup/find_rf.py,sha256=YOEvZu7-nyzLHoEtN2yKykVs11S2mcX4doj7r1YuEUQ,4117
+bnc_lookup/find_bnc.py,sha256=9IGZa6gig17F_WpO7RtQ05dACM152KAD0ohsXRljNmk,3179
+bnc_lookup/find_freq.py,sha256=_ko6xTpf9Kb7zT3gAt65_HZJInLSZUPNDl4LOeh90M8,3140
+bnc_lookup/find_rf.py,sha256=TMWBtpA2bWwLEuaGUp_9JYKlFnyOeiMQAOjo4OfYTuo,4229
 bnc_lookup/find_words.py,sha256=_fRBHCuR1G3yFI69CJFEJD1jLKK47UQDPMC7-e--USQ,2206
 bnc_lookup/freq/__init__.py,sha256=Nvot78FaLDAGJfcZ1vAGroICATR8xqtv9dzhvUeXQsM,12127
 bnc_lookup/freq/f_00.py,sha256=Y-uDv7i6rrRQNN05fKfaX3u3EL08IXN1XBx7gLy0X00,106727
@@ -619,6 +619,7 @@ bnc_lookup/hs/h_fc.py,sha256=36PmEBdpRmYq8PCEL7g9m08mCLCQuq50uMUllCAuUdI,101225
 bnc_lookup/hs/h_fd.py,sha256=N2Sh5oNn6mEBf1FKfSF9V9jWXsaCMOrj3cWgeVHvRUE,97843
 bnc_lookup/hs/h_fe.py,sha256=90Vr4qZcGfni2nxaGUR4MxuF6Z6aRar8awM5g-Z61T0,100807
 bnc_lookup/hs/h_ff.py,sha256=vwrosNEVBRctHXd3lC0bsY0bKMUiJnNhOLLv9Be2ouY,105595
+bnc_lookup/normalize.py,sha256=QG4CT_9oaQ2He5uJRwDaG_G9xgIj2oRynflvQ1tGnEQ,2207
 bnc_lookup/rf/__init__.py,sha256=hJzc1pdn9tJzJnq6ZQ6fqZABgyqrYQEBDrAbxTmyNhM,14431
 bnc_lookup/rf/rf_00.py,sha256=W068APGlhNzO_vjmaIILu2F7ph5QZtgtde3dubNx_eA,132337
 bnc_lookup/rf/rf_01.py,sha256=nIKNAxxpvWzWuFeOhYEe1m9ygCvlZo25Maak88jro9Y,137329
@@ -876,8 +877,8 @@ bnc_lookup/rf/rf_fc.py,sha256=8nHyAedubjFHX4_IILOxSHi3h6LONhIBqEmAX2F63Ik,138473
 bnc_lookup/rf/rf_fd.py,sha256=auLl-Ldci6mJD5cG1a4bGW5opdh8A3hfIpe-JxFcCh4,133845
 bnc_lookup/rf/rf_fe.py,sha256=Pz2T5g7IPOPsOQsJ4cF02fu9dmKj_AECR_zeE8HeZ8M,137901
 bnc_lookup/rf/rf_ff.py,sha256=jbZ8YdL3gRuBNDMs5lLYhdy6LbUIPtKFuHnVXxP96Xo,144453
-bnc_lookup-1.3.0.dist-info/LICENSE,sha256=AYOkuUBnkCcFiW6x4tIrmFPNbW7EG2T5HkGYtxD6cFo,2124
-bnc_lookup-1.3.0.dist-info/METADATA,sha256=Yy5pG4cEktF1zi4Rr579t1oz1KLbWuYkK3J0ax2vD8M,8463
-bnc_lookup-1.3.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-bnc_lookup-1.3.0.dist-info/entry_points.txt,sha256=hh9_DCGsghIy9vqaXQH9k4GWbkyDVZWWLYhi9CC_ZJE,151
-bnc_lookup-1.3.0.dist-info/RECORD,,
+bnc_lookup-1.3.2.dist-info/LICENSE,sha256=AYOkuUBnkCcFiW6x4tIrmFPNbW7EG2T5HkGYtxD6cFo,2124
+bnc_lookup-1.3.2.dist-info/METADATA,sha256=qzvj-zy-bS_nAlcf_vSNSf9hc8zltT27gXv6OUSaErA,8564
+bnc_lookup-1.3.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+bnc_lookup-1.3.2.dist-info/entry_points.txt,sha256=hh9_DCGsghIy9vqaXQH9k4GWbkyDVZWWLYhi9CC_ZJE,151
+bnc_lookup-1.3.2.dist-info/RECORD,,