rigour 2.1.2__tar.gz → 2.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. {rigour-2.1.2 → rigour-2.2.0}/PKG-INFO +1 -1
  2. {rigour-2.1.2 → rigour-2.2.0}/pyproject.toml +4 -2
  3. rigour-2.2.0/rigour/names/name.py +24 -0
  4. {rigour-2.1.2 → rigour-2.2.0}/rigour/names/org_types.py +20 -11
  5. rigour-2.2.0/rigour/names/part.py +23 -0
  6. {rigour-2.1.2 → rigour-2.2.0}/rigour/names/pick.py +1 -2
  7. {rigour-2.1.2 → rigour-2.2.0}/rigour/reset.py +2 -2
  8. {rigour-2.1.2 → rigour-2.2.0}/rigour/territories/util.py +2 -6
  9. {rigour-2.1.2 → rigour-2.2.0}/rigour/text/normalize.py +10 -5
  10. {rigour-2.1.2 → rigour-2.2.0}/rigour/text/scripts.py +9 -37
  11. {rigour-2.1.2 → rigour-2.2.0}/rigour/text/translit.py +4 -4
  12. {rigour-2.1.2 → rigour-2.2.0}/rust/Cargo.lock +21 -32
  13. {rigour-2.1.2 → rigour-2.2.0}/rust/Cargo.toml +7 -5
  14. {rigour-2.1.2 → rigour-2.2.0}/rust/build.rs +10 -18
  15. {rigour-2.1.2 → rigour-2.2.0}/rust/data/names/person_names.txt +2313 -1202
  16. {rigour-2.1.2 → rigour-2.2.0}/rust/data/territories/data.jsonl +114 -39
  17. {rigour-2.1.2 → rigour-2.2.0}/rust/data/text/stopwords.json +2 -1
  18. rigour-2.2.0/rust/src/constants.rs +6 -0
  19. {rigour-2.1.2 → rigour-2.2.0}/rust/src/lib.rs +1 -6
  20. {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/alignment.rs +41 -4
  21. {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/analyze.rs +23 -7
  22. {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/compare.rs +17 -9
  23. {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/name.rs +44 -4
  24. {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/ordering.rs +17 -23
  25. {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/org_types.rs +231 -55
  26. {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/pairing.rs +451 -70
  27. {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/part.rs +31 -10
  28. {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/person_names.rs +5 -20
  29. {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/pick.rs +29 -64
  30. {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/symbol.rs +24 -0
  31. {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/symbols.rs +0 -9
  32. {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/tagger.rs +65 -22
  33. {rigour-2.1.2 → rigour-2.2.0}/rust/src/territories.rs +6 -10
  34. {rigour-2.1.2 → rigour-2.2.0}/rust/src/text/distance.rs +19 -27
  35. {rigour-2.1.2 → rigour-2.2.0}/rust/src/text/normalize.rs +74 -23
  36. {rigour-2.1.2 → rigour-2.2.0}/rust/src/text/scripts.rs +1 -1
  37. {rigour-2.1.2 → rigour-2.2.0}/rust/src/text/tokenize.rs +8 -13
  38. {rigour-2.1.2 → rigour-2.2.0}/rust/src/text/translit.rs +42 -28
  39. rigour-2.1.2/rigour/names/name.py +0 -26
  40. rigour-2.1.2/rigour/names/part.py +0 -23
  41. rigour-2.1.2/rust/src/constants.rs +0 -3
  42. {rigour-2.1.2 → rigour-2.2.0}/LICENSE +0 -0
  43. {rigour-2.1.2 → rigour-2.2.0}/README.md +0 -0
  44. {rigour-2.1.2 → rigour-2.2.0}/rigour/__init__.py +0 -0
  45. {rigour-2.1.2 → rigour-2.2.0}/rigour/_core.pyi +0 -0
  46. {rigour-2.1.2 → rigour-2.2.0}/rigour/addresses/__init__.py +0 -0
  47. {rigour-2.1.2 → rigour-2.2.0}/rigour/addresses/cleaning.py +0 -0
  48. {rigour-2.1.2 → rigour-2.2.0}/rigour/addresses/format.py +0 -0
  49. {rigour-2.1.2 → rigour-2.2.0}/rigour/addresses/normalize.py +0 -0
  50. {rigour-2.1.2 → rigour-2.2.0}/rigour/boolean.py +0 -0
  51. {rigour-2.1.2 → rigour-2.2.0}/rigour/data/__init__.py +0 -0
  52. {rigour-2.1.2 → rigour-2.2.0}/rigour/data/addresses/__init__.py +0 -0
  53. {rigour-2.1.2 → rigour-2.2.0}/rigour/data/addresses/data.py +0 -0
  54. {rigour-2.1.2 → rigour-2.2.0}/rigour/data/addresses/formats.yml +0 -0
  55. {rigour-2.1.2 → rigour-2.2.0}/rigour/data/langs/__init__.py +0 -0
  56. {rigour-2.1.2 → rigour-2.2.0}/rigour/data/langs/iso639.py +0 -0
  57. {rigour-2.1.2 → rigour-2.2.0}/rigour/env.py +0 -0
  58. {rigour-2.1.2 → rigour-2.2.0}/rigour/ids/__init__.py +0 -0
  59. {rigour-2.1.2 → rigour-2.2.0}/rigour/ids/common.py +0 -0
  60. {rigour-2.1.2 → rigour-2.2.0}/rigour/ids/imo.py +0 -0
  61. {rigour-2.1.2 → rigour-2.2.0}/rigour/ids/npi.py +0 -0
  62. {rigour-2.1.2 → rigour-2.2.0}/rigour/ids/ogrn.py +0 -0
  63. {rigour-2.1.2 → rigour-2.2.0}/rigour/ids/stdnum_.py +0 -0
  64. {rigour-2.1.2 → rigour-2.2.0}/rigour/ids/strict.py +0 -0
  65. {rigour-2.1.2 → rigour-2.2.0}/rigour/ids/uei.py +0 -0
  66. {rigour-2.1.2 → rigour-2.2.0}/rigour/ids/wikidata.py +0 -0
  67. {rigour-2.1.2 → rigour-2.2.0}/rigour/langs/__init__.py +0 -0
  68. {rigour-2.1.2 → rigour-2.2.0}/rigour/langs/synonyms.py +0 -0
  69. {rigour-2.1.2 → rigour-2.2.0}/rigour/langs/text.py +0 -0
  70. {rigour-2.1.2 → rigour-2.2.0}/rigour/langs/util.py +0 -0
  71. {rigour-2.1.2 → rigour-2.2.0}/rigour/mime/__init__.py +0 -0
  72. {rigour-2.1.2 → rigour-2.2.0}/rigour/mime/filename.py +0 -0
  73. {rigour-2.1.2 → rigour-2.2.0}/rigour/mime/mappings.py +0 -0
  74. {rigour-2.1.2 → rigour-2.2.0}/rigour/mime/mime.py +0 -0
  75. {rigour-2.1.2 → rigour-2.2.0}/rigour/mime/parse.py +0 -0
  76. {rigour-2.1.2 → rigour-2.2.0}/rigour/mime/types.py +0 -0
  77. {rigour-2.1.2 → rigour-2.2.0}/rigour/names/__init__.py +0 -0
  78. {rigour-2.1.2 → rigour-2.2.0}/rigour/names/analyze.py +0 -0
  79. {rigour-2.1.2 → rigour-2.2.0}/rigour/names/check.py +0 -0
  80. {rigour-2.1.2 → rigour-2.2.0}/rigour/names/compare.py +0 -0
  81. {rigour-2.1.2 → rigour-2.2.0}/rigour/names/ordering.py +0 -0
  82. {rigour-2.1.2 → rigour-2.2.0}/rigour/names/prefix.py +0 -0
  83. {rigour-2.1.2 → rigour-2.2.0}/rigour/names/split_phrases.py +0 -0
  84. {rigour-2.1.2 → rigour-2.2.0}/rigour/names/symbol.py +0 -0
  85. {rigour-2.1.2 → rigour-2.2.0}/rigour/names/tag.py +0 -0
  86. {rigour-2.1.2 → rigour-2.2.0}/rigour/names/tokenize.py +0 -0
  87. {rigour-2.1.2 → rigour-2.2.0}/rigour/py.typed +0 -0
  88. {rigour-2.1.2 → rigour-2.2.0}/rigour/territories/__init__.py +0 -0
  89. {rigour-2.1.2 → rigour-2.2.0}/rigour/territories/lookup.py +0 -0
  90. {rigour-2.1.2 → rigour-2.2.0}/rigour/territories/match.py +0 -0
  91. {rigour-2.1.2 → rigour-2.2.0}/rigour/territories/territory.py +0 -0
  92. {rigour-2.1.2 → rigour-2.2.0}/rigour/text/__init__.py +0 -0
  93. {rigour-2.1.2 → rigour-2.2.0}/rigour/text/checksum.py +0 -0
  94. {rigour-2.1.2 → rigour-2.2.0}/rigour/text/cleaning.py +0 -0
  95. {rigour-2.1.2 → rigour-2.2.0}/rigour/text/distance.py +0 -0
  96. {rigour-2.1.2 → rigour-2.2.0}/rigour/text/phonetics.py +0 -0
  97. {rigour-2.1.2 → rigour-2.2.0}/rigour/text/stopwords.py +0 -0
  98. {rigour-2.1.2 → rigour-2.2.0}/rigour/time.py +0 -0
  99. {rigour-2.1.2 → rigour-2.2.0}/rigour/units.py +0 -0
  100. {rigour-2.1.2 → rigour-2.2.0}/rigour/urls/__init__.py +0 -0
  101. {rigour-2.1.2 → rigour-2.2.0}/rigour/urls/cleaning.py +0 -0
  102. {rigour-2.1.2 → rigour-2.2.0}/rigour/urls/compare.py +0 -0
  103. {rigour-2.1.2 → rigour-2.2.0}/rigour/urls/util.py +0 -0
  104. {rigour-2.1.2 → rigour-2.2.0}/rigour/util.py +0 -0
  105. {rigour-2.1.2 → rigour-2.2.0}/rust/data/names/compare.json +0 -0
  106. {rigour-2.1.2 → rigour-2.2.0}/rust/data/names/org_types.json +0 -0
  107. {rigour-2.1.2 → rigour-2.2.0}/rust/data/names/stopwords.json +0 -0
  108. {rigour-2.1.2 → rigour-2.2.0}/rust/data/names/symbols.json +0 -0
  109. {rigour-2.1.2 → rigour-2.2.0}/rust/data/text/ordinals.json +0 -0
  110. {rigour-2.1.2 → rigour-2.2.0}/rust/src/generated/.gitkeep +0 -0
  111. {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/matcher.rs +0 -0
  112. {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/mod.rs +0 -0
  113. {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/prefix.rs +0 -0
  114. {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/stopwords.rs +0 -0
  115. {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/tag.rs +0 -0
  116. {rigour-2.1.2 → rigour-2.2.0}/rust/src/text/mod.rs +0 -0
  117. {rigour-2.1.2 → rigour-2.2.0}/rust/src/text/numbers.rs +0 -0
  118. {rigour-2.1.2 → rigour-2.2.0}/rust/src/text/ordinals.rs +0 -0
  119. {rigour-2.1.2 → rigour-2.2.0}/rust/src/text/phonetics.rs +0 -0
  120. {rigour-2.1.2 → rigour-2.2.0}/rust/src/text/stopwords.rs +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: rigour
3
- Version: 2.1.2
3
+ Version: 2.2.0
4
4
  Classifier: Intended Audience :: Developers
5
5
  Classifier: Operating System :: OS Independent
6
6
  Classifier: License :: OSI Approved :: MIT License
@@ -4,7 +4,7 @@ build-backend = "maturin"
4
4
 
5
5
  [project]
6
6
  name = "rigour"
7
- version = "2.1.2"
7
+ version = "2.2.0"
8
8
  description = "Financial crime domain data validation and normalization library."
9
9
  readme = "README.md"
10
10
  license = { file = "LICENSE" }
@@ -60,7 +60,9 @@ docs = [
60
60
  ]
61
61
 
62
62
  [tool.maturin]
63
- features = ["pyo3/extension-module", "python"]
63
+ # pyo3/generate-import-lib lets the Windows wheel job link without a
64
+ # real Python install (PyO3 generates a stub import library).
65
+ features = ["pyo3/extension-module", "pyo3/generate-import-lib", "python"]
64
66
  module-name = "rigour._core"
65
67
  manifest-path = "rust/Cargo.toml"
66
68
  # `rigour/` is the Python package directory, sitting at the repo root. Maturin
@@ -0,0 +1,24 @@
1
+ """Name — the top-level object in the `rigour.names` object graph.
2
+
3
+ Rust-backed via `rigour._core`. Python-side surface:
4
+
5
+ * `Name(original, form=None, tag=NameTypeTag.UNK, phonetics=True)` —
6
+ `form` defaults to `casefold(original)` and is tokenised; each
7
+ token becomes a fresh `NamePart`.
8
+ * eager attributes: `original`, `form`, `tag` (mutable), `parts`,
9
+ `spans`, `comparable`, `norm_form`
10
+ * dynamic property: `symbols` (recomputed each access from `spans`)
11
+ * mutation: `tag_text(text, tag, max_matches=1)`,
12
+ `apply_phrase(phrase, symbol)`, `apply_part(part, symbol)`
13
+ * `contains(other)` for PER-aware subset checks
14
+ * `Name.consolidate_names(names)` classmethod — drops names that are
15
+ substrings of longer names in the same iterable
16
+ * `__hash__` / `__eq__` by `form` (stable across tag mutation)
17
+
18
+ `parts` is a tuple built once at construction — attribute reads are
19
+ zero-copy INCREFs. `spans` starts empty and grows via `apply_phrase`
20
+ / `apply_part`.
21
+ """
22
+ from rigour._core import Name
23
+
24
+ __all__ = ["Name"]
@@ -88,27 +88,36 @@ def replace_org_types_display(
88
88
  """Replace organisation types in a name with their short display form.
89
89
 
90
90
  Spelt-out legal forms are shortened into common abbreviations
91
- (e.g. ``"Siemens Aktiengesellschaft"`` → ``"Siemens AG"``), preserving
92
- the case of non-matched portions. If the whole input is uppercase
93
- (`str.isupper()`), the whole output is re-uppercased.
94
-
95
- Matches case-insensitively across Unicode by casefolding a copy of
96
- the input internally for the match step — `normalize_flags` must
97
- therefore include `Normalize.CASEFOLD` so the alias table is
98
- casefolded too. The default does this.
91
+ (e.g. ``"Siemens Aktiengesellschaft"`` → ``"Siemens AG"``), and
92
+ case-variant forms are canonicalised (``"Siemens GMBH"``
93
+ ``"Siemens GmbH"``). Everything outside a matched span is returned
94
+ verbatim — case, spacing and stray characters untouched. If the
95
+ whole input is uppercase (`str.isupper()`), the whole output is
96
+ re-uppercased, so ``"ACME COMPANY LIMITED"`` becomes
97
+ ``"ACME COMPANY LTD"`` rather than the odd-looking
98
+ ``"ACME COMPANY Ltd"``.
99
+
100
+ Matching is Unicode-case- and whitespace-insensitive: the input is
101
+ normalised internally with the same `normalize_flags` as the alias
102
+ table (which must therefore include `Normalize.CASEFOLD`; the
103
+ default does), and matches are mapped back to the original text.
104
+ Legal forms broken by copy/paste residue — soft hyphens, zero-width
105
+ characters, doubled spaces — are still recognised, and the
106
+ replacement swallows the residue.
99
107
 
100
108
  Args:
101
109
  name: The text to be processed.
102
110
  normalize_flags: `Normalize` flag
103
- set applied to the alias list at build time. Must include
104
- `Normalize.CASEFOLD` for Unicode-case-insensitive matching.
111
+ set applied to the alias list at build time and mirrored on
112
+ the input at match time. Must include `Normalize.CASEFOLD`
113
+ for Unicode-case-insensitive matching.
105
114
  Default `CASEFOLD | SQUASH_SPACES`.
106
115
  cleanup: `Cleanup` variant applied
107
116
  during alias normalisation. Default `Cleanup.Noop`.
108
117
 
109
118
  Returns:
110
119
  The text with recognised organisation types substituted for
111
- their display form. Non-matched regions keep their original case.
120
+ their display form and all non-matched regions verbatim.
112
121
  """
113
122
  return _replace_org_types_display(name, int(normalize_flags), int(cleanup))
114
123
 
@@ -0,0 +1,23 @@
1
+ """NamePart / Span — the leaf classes of the `rigour.names` object graph.
2
+
3
+ Rust-backed via `rigour._core`. Python-side surface:
4
+
5
+ * `NamePart(form, index, tag=NamePartTag.UNSET, phonetics=True)` —
6
+ `index` is the part's position within its name and is required.
7
+ * eager attributes: `form`, `index`, `tag` (mutable), `latinize`,
8
+ `numeric`, `ascii`, `integer`, `comparable`, `metaphone`
9
+ * `NamePart.tag_sort(parts)` classmethod — stable-sort parts into
10
+ human display order by tag (honorifics, given, middle, family, …)
11
+ * tag compatibility checks live on the tag: `part.tag.can_match(other.tag)`
12
+ * `__hash__` / `__eq__` by the precomputed `_hash` (tuple of
13
+ `(index, form)`)
14
+ * `Span(parts, symbol)` with precomputed `comparable`
15
+
16
+ `phonetics=False` at construction skips the metaphone computation —
17
+ `part.metaphone` is `None` in that case. `ascii` uses
18
+ `rigour._core.maybe_ascii` (narrow 6-script transliteration); parts
19
+ whose form is outside the admitted scripts resolve `ascii` to `None`.
20
+ """
21
+ from rigour._core import NamePart, Span
22
+
23
+ __all__ = ["NamePart", "Span"]
@@ -1,5 +1,6 @@
1
1
  from typing import Dict, List, Optional
2
2
 
3
+ from rigour._core import pick_case as _pick_case
3
4
  from rigour._core import pick_name as pick_name
4
5
  from rigour._core import reduce_names as reduce_names
5
6
  from rigour.langs import LangStr, PREFERRED_LANG, PREFERRED_LANGS
@@ -152,8 +153,6 @@ def pick_case(names: List[str]) -> str:
152
153
  Returns:
153
154
  str: The best name for display.
154
155
  """
155
- from rigour._core import pick_case as _pick_case
156
-
157
156
  result = _pick_case(names)
158
157
  if result is None:
159
158
  raise ValueError("Cannot pick a name from an empty list.")
@@ -5,7 +5,7 @@ from rigour.text.phonetics import soundex, metaphone
5
5
  from rigour.territories.territory import get_index
6
6
  from rigour.territories.lookup import lookup_territory
7
7
  from rigour.territories.lookup import _get_identifier_map, _get_territory_names
8
- from rigour.text.scripts import can_latinize_cp
8
+ from rigour.text.scripts import codepoint_script
9
9
  from rigour.names.tokenize import normalize_name
10
10
  from rigour.names.prefix import (
11
11
  _person_prefix_regex,
@@ -29,7 +29,7 @@ def reset_caches() -> None:
29
29
  jaro_winkler.cache_clear()
30
30
  soundex.cache_clear()
31
31
  metaphone.cache_clear()
32
- can_latinize_cp.cache_clear()
32
+ codepoint_script.cache_clear()
33
33
  _load_formats.cache_clear()
34
34
  _load_template.cache_clear()
35
35
  _address_replacer.cache_clear()
@@ -2,7 +2,7 @@ import unicodedata
2
2
  from typing import List
3
3
  from normality import latinize_text, squash_spaces
4
4
 
5
- from rigour.text.scripts import can_latinize_cp
5
+ from rigour.text.scripts import can_latinize
6
6
 
7
7
  SKIP_CHARACTERS = ".()[],;:_-/ʻ'’"
8
8
 
@@ -20,7 +20,6 @@ def clean_codes(codes: List[str]) -> List[str]:
20
20
  def normalize_territory_name(name: str) -> str:
21
21
  """Normalize a territory name for lookup."""
22
22
  name = unicodedata.normalize("NFKD", name).casefold()
23
- can_latinize = True
24
23
  filtered: List[str] = []
25
24
  for char in name:
26
25
  if char in SKIP_CHARACTERS:
@@ -28,12 +27,9 @@ def normalize_territory_name(name: str) -> str:
28
27
  if not char.isalnum():
29
28
  filtered.append(" ")
30
29
  continue
31
- cp = ord(char)
32
- if not can_latinize_cp(cp):
33
- can_latinize = False
34
30
  filtered.append(char)
35
31
  normalized = "".join(filtered)
36
- if can_latinize:
32
+ if can_latinize(normalized):
37
33
  normalized = latinize_text(normalized)
38
34
  normalized = unicodedata.normalize("NFKC", normalized)
39
35
  normalized = squash_spaces(normalized)
@@ -41,7 +41,8 @@ value:
41
41
  listed in its dispatch (NFKD)
42
42
  3. CASEFOLD — Unicode full casefold (ß → ss, not lowercase)
43
43
  4. Cleanup — category_replace, unless Cleanup.Noop
44
- 5. SQUASH_SPACES — collapse whitespace runs, trim ends
44
+ 5. SQUASH_SPACES — delete invisible format characters,
45
+ collapse whitespace runs, trim ends
45
46
  6. NAME — tokenize via
46
47
  [tokenize_name][rigour.names.tokenize.tokenize_name]
47
48
  and rejoin with a single ASCII space
@@ -132,10 +133,14 @@ class Normalize(IntFlag):
132
133
  Attributes:
133
134
  STRIP: Trim leading and trailing whitespace.
134
135
  SQUASH_SPACES: Collapse runs of whitespace (including newlines,
135
- tabs, Unicode whitespace) into single spaces and trim the
136
- edges. Runs as the final pipeline step, so cleaning up
137
- whitespace introduced by earlier steps (e.g. category
138
- replacement) comes out right.
136
+ tabs, Unicode whitespace, and the U+001C-001F information
137
+ separators) into single spaces and trim the edges. Also
138
+ deletes invisible format characters zero-width
139
+ space/joiners (U+200B-U+200D), BOM (U+FEFF), soft hyphen
140
+ (U+00AD), and word joiners (U+2060-U+2064) — so that text
141
+ pasted from PDFs or web pages (``Gm\\u00ADbH``) compares
142
+ equal to its clean form (``GmbH``). Cleans up whitespace
143
+ introduced by earlier steps (e.g. category replacement).
139
144
  CASEFOLD: Unicode full casefold (e.g. ``ß → ss``). This is
140
145
  *not* the same as `str.lower()` — casefold is the correct
141
146
  tool for case-insensitive comparison across Unicode.
@@ -3,15 +3,11 @@ from typing import Optional
3
3
 
4
4
  from rigour._core import codepoint_script as _codepoint_script
5
5
  from rigour._core import common_scripts as _common_scripts
6
+ from rigour._core import should_ascii as _should_ascii
6
7
  from rigour._core import text_scripts as _text_scripts
7
8
  from rigour.util import MEMO_MEDIUM
8
9
 
9
10
 
10
- #: Scripts whose romanisation is well-defined enough that
11
- #: rigour will automatically transliterate them to ASCII via
12
- #: :func:`rigour.text.translit.maybe_ascii`.
13
- LATINIZE_SCRIPTS = {"Hangul", "Cyrillic", "Greek", "Armenian", "Latin", "Georgian"}
14
-
15
11
  #: Letter-based writing systems with explicit vowels — these
16
12
  #: transliterate reliably to Latin without language hints.
17
13
  #: Excludes Hangul (syllabic) and the dense logographic scripts.
@@ -99,38 +95,14 @@ def common_scripts(a: str, b: str) -> set[str]:
99
95
  return _common_scripts(a, b)
100
96
 
101
97
 
102
- @lru_cache(maxsize=MEMO_MEDIUM)
103
- def can_latinize_cp(cp: int) -> Optional[bool]:
104
- """Check whether a single codepoint can be latinized.
105
-
106
- Three-valued: distinguishing-script-bearing codepoints get a
107
- True/False; others (digits, punctuation, combining marks)
108
- return `None` because the question doesn't apply to them.
109
-
110
- Args:
111
- cp: Codepoint as an integer.
112
-
113
- Returns:
114
- `True` if the codepoint's script is in
115
- :data:`LATINIZE_SCRIPTS`, `False` if it has a real script
116
- outside that set, `None` for non-alphanumeric or
117
- Common/Inherited/Unknown codepoints.
118
- """
119
- char = chr(cp)
120
- if not char.isalnum():
121
- return None
122
- script = codepoint_script(cp)
123
- if script is None or script in ("Common", "Inherited"):
124
- return None
125
- return script in LATINIZE_SCRIPTS
126
-
127
-
128
98
  def can_latinize(word: str) -> bool:
129
99
  """Check whether every script in a word is latinizable.
130
100
 
131
- Equivalent to `text_scripts(word) <= LATINIZE_SCRIPTS`. When
132
- True, :func:`rigour.text.translit.maybe_ascii` will produce
133
- an ASCII output; when False, it returns the input unchanged.
101
+ Alias for `should_ascii` in [rigour.text.translit][] — the two
102
+ call the same Rust predicate, which admits Latin, Cyrillic,
103
+ Greek, Armenian, Georgian, and Hangul. When True, `maybe_ascii`
104
+ will produce an ASCII output; when False, it returns the input
105
+ unchanged.
134
106
  Characters with no distinguishing script (digits, punctuation,
135
107
  spaces, combining marks) are ignored. Empty input and
136
108
  pure-Common input (`"123"`) return True vacuously.
@@ -139,10 +111,10 @@ def can_latinize(word: str) -> bool:
139
111
  word: A string.
140
112
 
141
113
  Returns:
142
- True iff every distinguishing script is in
143
- :data:`LATINIZE_SCRIPTS`.
114
+ True iff every distinguishing script is admitted for
115
+ transliteration.
144
116
  """
145
- return text_scripts(word) <= LATINIZE_SCRIPTS
117
+ return _should_ascii(word)
146
118
 
147
119
 
148
120
  def is_modern_alphabet(word: str) -> bool:
@@ -1,10 +1,10 @@
1
1
  """Opportunistic transliteration: `should_ascii` + `maybe_ascii`.
2
2
 
3
3
  The minimal transliteration surface rigour exposes going forward.
4
- Covers only the scripts listed in `rigour.text.scripts.LATINIZE_SCRIPTS`
5
- (Latin, Cyrillic, Greek, Armenian, Georgian, Hangul). Anything
6
- outside that set passes through unchanged (default) or becomes
7
- empty (``drop=True``).
4
+ Covers only six admitted scripts (Latin, Cyrillic, Greek, Armenian,
5
+ Georgian, Hangul); the admission policy is defined in the Rust core.
6
+ Anything outside that set passes through unchanged (default) or
7
+ becomes empty (``drop=True``).
8
8
 
9
9
  For broader-script, lossy transliteration (Han, Arabic, Devanagari,
10
10
  etc.) use `normality.ascii_text` / `normality.latinize_text` —
@@ -25,9 +25,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
25
25
 
26
26
  [[package]]
27
27
  name = "bitflags"
28
- version = "2.12.1"
28
+ version = "2.13.1"
29
29
  source = "registry+https://github.com/rust-lang/crates.io-index"
30
- checksum = "84d7ced0ae9557296835c32bf1b1e02b44c746701f898460fb000d7eaa84f00a"
30
+ checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
31
31
 
32
32
  [[package]]
33
33
  name = "calendrical_calculations"
@@ -623,9 +623,9 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"
623
623
 
624
624
  [[package]]
625
625
  name = "lru"
626
- version = "0.18.0"
626
+ version = "0.18.1"
627
627
  source = "registry+https://github.com/rust-lang/crates.io-index"
628
- checksum = "8a860605968fce16869fd239cf4237a82f3ac470723415db603b0e8b6c8d4fb9"
628
+ checksum = "0b6180140927ee907000b0aa540091f6ea512ead4447c92b8fc35bc72788a5a6"
629
629
  dependencies = [
630
630
  "hashbrown",
631
631
  ]
@@ -724,9 +724,9 @@ dependencies = [
724
724
 
725
725
  [[package]]
726
726
  name = "pyo3"
727
- version = "0.28.3"
727
+ version = "0.29.0"
728
728
  source = "registry+https://github.com/rust-lang/crates.io-index"
729
- checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12"
729
+ checksum = "cd274650b21d4bfc26a0a47587962c1edb425f69287324355cd040c3ea66071c"
730
730
  dependencies = [
731
731
  "libc",
732
732
  "once_cell",
@@ -738,19 +738,18 @@ dependencies = [
738
738
 
739
739
  [[package]]
740
740
  name = "pyo3-build-config"
741
- version = "0.28.3"
741
+ version = "0.29.0"
742
742
  source = "registry+https://github.com/rust-lang/crates.io-index"
743
- checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e"
743
+ checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078"
744
744
  dependencies = [
745
- "python3-dll-a",
746
745
  "target-lexicon",
747
746
  ]
748
747
 
749
748
  [[package]]
750
749
  name = "pyo3-ffi"
751
- version = "0.28.3"
750
+ version = "0.29.0"
752
751
  source = "registry+https://github.com/rust-lang/crates.io-index"
753
- checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e"
752
+ checksum = "ca85c467da1bbc8d866eea5deff9cf29ea5f7785054a17da36e65bda9c05845b"
754
753
  dependencies = [
755
754
  "libc",
756
755
  "pyo3-build-config",
@@ -758,9 +757,9 @@ dependencies = [
758
757
 
759
758
  [[package]]
760
759
  name = "pyo3-macros"
761
- version = "0.28.3"
760
+ version = "0.29.0"
762
761
  source = "registry+https://github.com/rust-lang/crates.io-index"
763
- checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813"
762
+ checksum = "9ac53762fd065daa3194dd09337a38bd793a188100fd1a9304c4ab312d901771"
764
763
  dependencies = [
765
764
  "proc-macro2",
766
765
  "pyo3-macros-backend",
@@ -770,26 +769,16 @@ dependencies = [
770
769
 
771
770
  [[package]]
772
771
  name = "pyo3-macros-backend"
773
- version = "0.28.3"
772
+ version = "0.29.0"
774
773
  source = "registry+https://github.com/rust-lang/crates.io-index"
775
- checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb"
774
+ checksum = "4ca3a1557399783172dc5bf39cfca835157732532cba56b71d2292161e53b362"
776
775
  dependencies = [
777
776
  "heck",
778
777
  "proc-macro2",
779
- "pyo3-build-config",
780
778
  "quote",
781
779
  "syn",
782
780
  ]
783
781
 
784
- [[package]]
785
- name = "python3-dll-a"
786
- version = "0.2.15"
787
- source = "registry+https://github.com/rust-lang/crates.io-index"
788
- checksum = "d80ba7540edb18890d444c5aa8e1f1f99b1bdf26fb26ae383135325f4a36042b"
789
- dependencies = [
790
- "cc",
791
- ]
792
-
793
782
  [[package]]
794
783
  name = "quote"
795
784
  version = "1.0.45"
@@ -813,9 +802,9 @@ checksum = "270e04e5ea61d40841942bb15e451c29ee1618637bcf97fc7ede5dd4a9b1601b"
813
802
 
814
803
  [[package]]
815
804
  name = "regex"
816
- version = "1.12.3"
805
+ version = "1.13.1"
817
806
  source = "registry+https://github.com/rust-lang/crates.io-index"
818
- checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
807
+ checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
819
808
  dependencies = [
820
809
  "aho-corasick",
821
810
  "memchr",
@@ -825,9 +814,9 @@ dependencies = [
825
814
 
826
815
  [[package]]
827
816
  name = "regex-automata"
828
- version = "0.4.14"
817
+ version = "0.4.16"
829
818
  source = "registry+https://github.com/rust-lang/crates.io-index"
830
- checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
819
+ checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad"
831
820
  dependencies = [
832
821
  "aho-corasick",
833
822
  "memchr",
@@ -836,13 +825,13 @@ dependencies = [
836
825
 
837
826
  [[package]]
838
827
  name = "regex-syntax"
839
- version = "0.8.10"
828
+ version = "0.8.11"
840
829
  source = "registry+https://github.com/rust-lang/crates.io-index"
841
- checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
830
+ checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
842
831
 
843
832
  [[package]]
844
833
  name = "rigour-core"
845
- version = "2.1.2"
834
+ version = "2.2.0"
846
835
  dependencies = [
847
836
  "aho-corasick",
848
837
  "bitflags",
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "rigour-core"
3
- version = "2.1.2"
3
+ version = "2.2.0"
4
4
  edition = "2024"
5
5
  rust-version = "1.86"
6
6
  license = "MIT"
@@ -17,9 +17,11 @@ crate-type = ["cdylib", "rlib"]
17
17
  python = ["dep:pyo3"]
18
18
 
19
19
  [dependencies]
20
- # generate-import-lib avoids needing a real Python install on the Windows CI
21
- # runner at link timePyO3 generates a stub import library instead.
22
- pyo3 = { version = "0.28", features = ["extension-module", "generate-import-lib"], optional = true }
20
+ # Wheel-only pyo3 features (extension-module, generate-import-lib) are
21
+ # deliberately NOT set herethey live in `[tool.maturin].features` in
22
+ # pyproject.toml. Keeping them out of the crate lets `cargo test
23
+ # --features python` link a standalone test binary against libpython.
24
+ pyo3 = { version = "0.29", optional = true }
23
25
 
24
26
  # Phonetic algorithms (metaphone, soundex). Port of Apache commons-codec, pure
25
27
  # Rust. The upstream jellyfish Rust crate can't be used directly because it
@@ -43,7 +45,7 @@ bitflags = "2"
43
45
  # `rigour.text.distance` and `rigour.territories.lookup` consume.
44
46
  rapidfuzz = "0.5"
45
47
 
46
- # LRU caches. Currently used for `maybe_ascii`'s per-thread result
48
+ # LRU caches. Currently used for `maybe_ascii`'s process-wide result
47
49
  # cache. Tiny crate (~500 LoC, no deps), O(1) get/put.
48
50
  lru = "0.18"
49
51
 
@@ -9,10 +9,10 @@
9
9
  // - `data/names/org_types.json` (~125 KiB → ~15 KiB)
10
10
  // - `data/territories/data.jsonl` (~783 KiB → ~214 KiB)
11
11
  //
12
- // If a source file is missing (fresh checkout before
13
- // `contrib/namesdb/Makefile::dump` has been run, for example), we emit
14
- // an empty blob and print a cargo warning. Callers of the corresponding
15
- // `raw()` loader see an empty string.
12
+ // All source files are committed, so a missing one means a broken
13
+ // checkout fail the build rather than embed an empty blob that
14
+ // would ship as a silently non-functional wheel (empty tagger, zero
15
+ // territories).
16
16
 
17
17
  use std::env;
18
18
  use std::fs;
@@ -21,36 +21,32 @@ use std::path::PathBuf;
21
21
  struct Compress {
22
22
  src: &'static str, // path under rust/
23
23
  dst: &'static str, // filename under OUT_DIR
24
- missing: &'static str, // cargo warning if source file is absent
24
+ missing: &'static str, // error detail if source file is absent
25
25
  }
26
26
 
27
27
  const FILES: &[Compress] = &[
28
28
  Compress {
29
29
  src: "data/names/person_names.txt",
30
30
  dst: "person_names.txt.zst",
31
- missing: "rust/data/names/person_names.txt not found \
32
- compiling empty person-names corpus. Run \
31
+ missing: "rust/data/names/person_names.txt not found. Run \
33
32
  `make -C contrib/namesdb dump` to regenerate.",
34
33
  },
35
34
  Compress {
36
35
  src: "data/territories/data.jsonl",
37
36
  dst: "territories.jsonl.zst",
38
- missing: "rust/data/territories/data.jsonl not found \
39
- compiling empty territories blob. Run \
37
+ missing: "rust/data/territories/data.jsonl not found. Run \
40
38
  `make build-territories` to regenerate.",
41
39
  },
42
40
  Compress {
43
41
  src: "data/names/symbols.json",
44
42
  dst: "symbols.json.zst",
45
- missing: "rust/data/names/symbols.json not found \
46
- compiling empty symbols blob. Run \
43
+ missing: "rust/data/names/symbols.json not found. Run \
47
44
  `make build-names` to regenerate.",
48
45
  },
49
46
  Compress {
50
47
  src: "data/names/org_types.json",
51
48
  dst: "org_types.json.zst",
52
- missing: "rust/data/names/org_types.json not found \
53
- compiling empty org-types blob. Run \
49
+ missing: "rust/data/names/org_types.json not found. Run \
54
50
  `make build-names` to regenerate.",
55
51
  },
56
52
  ];
@@ -64,11 +60,7 @@ fn main() {
64
60
  let dest_path = out_dir.join(entry.dst);
65
61
  println!("cargo:rerun-if-changed={}", entry.src);
66
62
 
67
- let Ok(bytes) = fs::read(&source_path) else {
68
- println!("cargo:warning={}", entry.missing);
69
- fs::write(&dest_path, [] as [u8; 0]).expect("write empty placeholder");
70
- continue;
71
- };
63
+ let bytes = fs::read(&source_path).unwrap_or_else(|_| panic!("{}", entry.missing));
72
64
 
73
65
  // Level 19 is the high-ratio tier — slow compression, but this
74
66
  // runs at build time only and gives the best wheel size.