rigour 2.1.2__tar.gz → 2.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rigour-2.1.2 → rigour-2.2.0}/PKG-INFO +1 -1
- {rigour-2.1.2 → rigour-2.2.0}/pyproject.toml +4 -2
- rigour-2.2.0/rigour/names/name.py +24 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/names/org_types.py +20 -11
- rigour-2.2.0/rigour/names/part.py +23 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/names/pick.py +1 -2
- {rigour-2.1.2 → rigour-2.2.0}/rigour/reset.py +2 -2
- {rigour-2.1.2 → rigour-2.2.0}/rigour/territories/util.py +2 -6
- {rigour-2.1.2 → rigour-2.2.0}/rigour/text/normalize.py +10 -5
- {rigour-2.1.2 → rigour-2.2.0}/rigour/text/scripts.py +9 -37
- {rigour-2.1.2 → rigour-2.2.0}/rigour/text/translit.py +4 -4
- {rigour-2.1.2 → rigour-2.2.0}/rust/Cargo.lock +21 -32
- {rigour-2.1.2 → rigour-2.2.0}/rust/Cargo.toml +7 -5
- {rigour-2.1.2 → rigour-2.2.0}/rust/build.rs +10 -18
- {rigour-2.1.2 → rigour-2.2.0}/rust/data/names/person_names.txt +2313 -1202
- {rigour-2.1.2 → rigour-2.2.0}/rust/data/territories/data.jsonl +114 -39
- {rigour-2.1.2 → rigour-2.2.0}/rust/data/text/stopwords.json +2 -1
- rigour-2.2.0/rust/src/constants.rs +6 -0
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/lib.rs +1 -6
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/alignment.rs +41 -4
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/analyze.rs +23 -7
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/compare.rs +17 -9
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/name.rs +44 -4
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/ordering.rs +17 -23
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/org_types.rs +231 -55
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/pairing.rs +451 -70
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/part.rs +31 -10
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/person_names.rs +5 -20
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/pick.rs +29 -64
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/symbol.rs +24 -0
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/symbols.rs +0 -9
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/tagger.rs +65 -22
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/territories.rs +6 -10
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/text/distance.rs +19 -27
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/text/normalize.rs +74 -23
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/text/scripts.rs +1 -1
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/text/tokenize.rs +8 -13
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/text/translit.rs +42 -28
- rigour-2.1.2/rigour/names/name.py +0 -26
- rigour-2.1.2/rigour/names/part.py +0 -23
- rigour-2.1.2/rust/src/constants.rs +0 -3
- {rigour-2.1.2 → rigour-2.2.0}/LICENSE +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/README.md +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/__init__.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/_core.pyi +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/addresses/__init__.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/addresses/cleaning.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/addresses/format.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/addresses/normalize.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/boolean.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/data/__init__.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/data/addresses/__init__.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/data/addresses/data.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/data/addresses/formats.yml +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/data/langs/__init__.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/data/langs/iso639.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/env.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/ids/__init__.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/ids/common.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/ids/imo.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/ids/npi.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/ids/ogrn.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/ids/stdnum_.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/ids/strict.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/ids/uei.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/ids/wikidata.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/langs/__init__.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/langs/synonyms.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/langs/text.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/langs/util.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/mime/__init__.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/mime/filename.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/mime/mappings.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/mime/mime.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/mime/parse.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/mime/types.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/names/__init__.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/names/analyze.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/names/check.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/names/compare.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/names/ordering.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/names/prefix.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/names/split_phrases.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/names/symbol.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/names/tag.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/names/tokenize.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/py.typed +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/territories/__init__.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/territories/lookup.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/territories/match.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/territories/territory.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/text/__init__.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/text/checksum.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/text/cleaning.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/text/distance.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/text/phonetics.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/text/stopwords.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/time.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/units.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/urls/__init__.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/urls/cleaning.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/urls/compare.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/urls/util.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rigour/util.py +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rust/data/names/compare.json +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rust/data/names/org_types.json +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rust/data/names/stopwords.json +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rust/data/names/symbols.json +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rust/data/text/ordinals.json +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/generated/.gitkeep +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/matcher.rs +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/mod.rs +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/prefix.rs +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/stopwords.rs +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/names/tag.rs +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/text/mod.rs +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/text/numbers.rs +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/text/ordinals.rs +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/text/phonetics.rs +0 -0
- {rigour-2.1.2 → rigour-2.2.0}/rust/src/text/stopwords.rs +0 -0
|
@@ -4,7 +4,7 @@ build-backend = "maturin"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "rigour"
|
|
7
|
-
version = "2.
|
|
7
|
+
version = "2.2.0"
|
|
8
8
|
description = "Financial crime domain data validation and normalization library."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = { file = "LICENSE" }
|
|
@@ -60,7 +60,9 @@ docs = [
|
|
|
60
60
|
]
|
|
61
61
|
|
|
62
62
|
[tool.maturin]
|
|
63
|
-
|
|
63
|
+
# pyo3/generate-import-lib lets the Windows wheel job link without a
|
|
64
|
+
# real Python install (PyO3 generates a stub import library).
|
|
65
|
+
features = ["pyo3/extension-module", "pyo3/generate-import-lib", "python"]
|
|
64
66
|
module-name = "rigour._core"
|
|
65
67
|
manifest-path = "rust/Cargo.toml"
|
|
66
68
|
# `rigour/` is the Python package directory, sitting at the repo root. Maturin
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""Name — the top-level object in the `rigour.names` object graph.
|
|
2
|
+
|
|
3
|
+
Rust-backed via `rigour._core`. Python-side surface:
|
|
4
|
+
|
|
5
|
+
* `Name(original, form=None, tag=NameTypeTag.UNK, phonetics=True)` —
|
|
6
|
+
`form` defaults to `casefold(original)` and is tokenised; each
|
|
7
|
+
token becomes a fresh `NamePart`.
|
|
8
|
+
* eager attributes: `original`, `form`, `tag` (mutable), `parts`,
|
|
9
|
+
`spans`, `comparable`, `norm_form`
|
|
10
|
+
* dynamic property: `symbols` (recomputed each access from `spans`)
|
|
11
|
+
* mutation: `tag_text(text, tag, max_matches=1)`,
|
|
12
|
+
`apply_phrase(phrase, symbol)`, `apply_part(part, symbol)`
|
|
13
|
+
* `contains(other)` for PER-aware subset checks
|
|
14
|
+
* `Name.consolidate_names(names)` classmethod — drops names that are
|
|
15
|
+
substrings of longer names in the same iterable
|
|
16
|
+
* `__hash__` / `__eq__` by `form` (stable across tag mutation)
|
|
17
|
+
|
|
18
|
+
`parts` is a tuple built once at construction — attribute reads are
|
|
19
|
+
zero-copy INCREFs. `spans` starts empty and grows via `apply_phrase`
|
|
20
|
+
/ `apply_part`.
|
|
21
|
+
"""
|
|
22
|
+
from rigour._core import Name
|
|
23
|
+
|
|
24
|
+
__all__ = ["Name"]
|
|
@@ -88,27 +88,36 @@ def replace_org_types_display(
|
|
|
88
88
|
"""Replace organisation types in a name with their short display form.
|
|
89
89
|
|
|
90
90
|
Spelt-out legal forms are shortened into common abbreviations
|
|
91
|
-
(e.g. ``"Siemens Aktiengesellschaft"`` → ``"Siemens AG"``),
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
91
|
+
(e.g. ``"Siemens Aktiengesellschaft"`` → ``"Siemens AG"``), and
|
|
92
|
+
case-variant forms are canonicalised (``"Siemens GMBH"`` →
|
|
93
|
+
``"Siemens GmbH"``). Everything outside a matched span is returned
|
|
94
|
+
verbatim — case, spacing and stray characters untouched. If the
|
|
95
|
+
whole input is uppercase (`str.isupper()`), the whole output is
|
|
96
|
+
re-uppercased, so ``"ACME COMPANY LIMITED"`` becomes
|
|
97
|
+
``"ACME COMPANY LTD"`` rather than the odd-looking
|
|
98
|
+
``"ACME COMPANY Ltd"``.
|
|
99
|
+
|
|
100
|
+
Matching is Unicode-case- and whitespace-insensitive: the input is
|
|
101
|
+
normalised internally with the same `normalize_flags` as the alias
|
|
102
|
+
table (which must therefore include `Normalize.CASEFOLD`; the
|
|
103
|
+
default does), and matches are mapped back to the original text.
|
|
104
|
+
Legal forms broken by copy/paste residue — soft hyphens, zero-width
|
|
105
|
+
characters, doubled spaces — are still recognised, and the
|
|
106
|
+
replacement swallows the residue.
|
|
99
107
|
|
|
100
108
|
Args:
|
|
101
109
|
name: The text to be processed.
|
|
102
110
|
normalize_flags: `Normalize` flag
|
|
103
|
-
set applied to the alias list at build time
|
|
104
|
-
`Normalize.CASEFOLD`
|
|
111
|
+
set applied to the alias list at build time and mirrored on
|
|
112
|
+
the input at match time. Must include `Normalize.CASEFOLD`
|
|
113
|
+
for Unicode-case-insensitive matching.
|
|
105
114
|
Default `CASEFOLD | SQUASH_SPACES`.
|
|
106
115
|
cleanup: `Cleanup` variant applied
|
|
107
116
|
during alias normalisation. Default `Cleanup.Noop`.
|
|
108
117
|
|
|
109
118
|
Returns:
|
|
110
119
|
The text with recognised organisation types substituted for
|
|
111
|
-
their display form
|
|
120
|
+
their display form and all non-matched regions verbatim.
|
|
112
121
|
"""
|
|
113
122
|
return _replace_org_types_display(name, int(normalize_flags), int(cleanup))
|
|
114
123
|
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""NamePart / Span — the leaf classes of the `rigour.names` object graph.
|
|
2
|
+
|
|
3
|
+
Rust-backed via `rigour._core`. Python-side surface:
|
|
4
|
+
|
|
5
|
+
* `NamePart(form, index, tag=NamePartTag.UNSET, phonetics=True)` —
|
|
6
|
+
`index` is the part's position within its name and is required.
|
|
7
|
+
* eager attributes: `form`, `index`, `tag` (mutable), `latinize`,
|
|
8
|
+
`numeric`, `ascii`, `integer`, `comparable`, `metaphone`
|
|
9
|
+
* `NamePart.tag_sort(parts)` classmethod — stable-sort parts into
|
|
10
|
+
human display order by tag (honorifics, given, middle, family, …)
|
|
11
|
+
* tag compatibility checks live on the tag: `part.tag.can_match(other.tag)`
|
|
12
|
+
* `__hash__` / `__eq__` by the precomputed `_hash` (tuple of
|
|
13
|
+
`(index, form)`)
|
|
14
|
+
* `Span(parts, symbol)` with precomputed `comparable`
|
|
15
|
+
|
|
16
|
+
`phonetics=False` at construction skips the metaphone computation —
|
|
17
|
+
`part.metaphone` is `None` in that case. `ascii` uses
|
|
18
|
+
`rigour._core.maybe_ascii` (narrow 6-script transliteration); parts
|
|
19
|
+
whose form is outside the admitted scripts resolve `ascii` to `None`.
|
|
20
|
+
"""
|
|
21
|
+
from rigour._core import NamePart, Span
|
|
22
|
+
|
|
23
|
+
__all__ = ["NamePart", "Span"]
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from typing import Dict, List, Optional
|
|
2
2
|
|
|
3
|
+
from rigour._core import pick_case as _pick_case
|
|
3
4
|
from rigour._core import pick_name as pick_name
|
|
4
5
|
from rigour._core import reduce_names as reduce_names
|
|
5
6
|
from rigour.langs import LangStr, PREFERRED_LANG, PREFERRED_LANGS
|
|
@@ -152,8 +153,6 @@ def pick_case(names: List[str]) -> str:
|
|
|
152
153
|
Returns:
|
|
153
154
|
str: The best name for display.
|
|
154
155
|
"""
|
|
155
|
-
from rigour._core import pick_case as _pick_case
|
|
156
|
-
|
|
157
156
|
result = _pick_case(names)
|
|
158
157
|
if result is None:
|
|
159
158
|
raise ValueError("Cannot pick a name from an empty list.")
|
|
@@ -5,7 +5,7 @@ from rigour.text.phonetics import soundex, metaphone
|
|
|
5
5
|
from rigour.territories.territory import get_index
|
|
6
6
|
from rigour.territories.lookup import lookup_territory
|
|
7
7
|
from rigour.territories.lookup import _get_identifier_map, _get_territory_names
|
|
8
|
-
from rigour.text.scripts import
|
|
8
|
+
from rigour.text.scripts import codepoint_script
|
|
9
9
|
from rigour.names.tokenize import normalize_name
|
|
10
10
|
from rigour.names.prefix import (
|
|
11
11
|
_person_prefix_regex,
|
|
@@ -29,7 +29,7 @@ def reset_caches() -> None:
|
|
|
29
29
|
jaro_winkler.cache_clear()
|
|
30
30
|
soundex.cache_clear()
|
|
31
31
|
metaphone.cache_clear()
|
|
32
|
-
|
|
32
|
+
codepoint_script.cache_clear()
|
|
33
33
|
_load_formats.cache_clear()
|
|
34
34
|
_load_template.cache_clear()
|
|
35
35
|
_address_replacer.cache_clear()
|
|
@@ -2,7 +2,7 @@ import unicodedata
|
|
|
2
2
|
from typing import List
|
|
3
3
|
from normality import latinize_text, squash_spaces
|
|
4
4
|
|
|
5
|
-
from rigour.text.scripts import
|
|
5
|
+
from rigour.text.scripts import can_latinize
|
|
6
6
|
|
|
7
7
|
SKIP_CHARACTERS = ".()[],;:_-/ʻ'’"
|
|
8
8
|
|
|
@@ -20,7 +20,6 @@ def clean_codes(codes: List[str]) -> List[str]:
|
|
|
20
20
|
def normalize_territory_name(name: str) -> str:
|
|
21
21
|
"""Normalize a territory name for lookup."""
|
|
22
22
|
name = unicodedata.normalize("NFKD", name).casefold()
|
|
23
|
-
can_latinize = True
|
|
24
23
|
filtered: List[str] = []
|
|
25
24
|
for char in name:
|
|
26
25
|
if char in SKIP_CHARACTERS:
|
|
@@ -28,12 +27,9 @@ def normalize_territory_name(name: str) -> str:
|
|
|
28
27
|
if not char.isalnum():
|
|
29
28
|
filtered.append(" ")
|
|
30
29
|
continue
|
|
31
|
-
cp = ord(char)
|
|
32
|
-
if not can_latinize_cp(cp):
|
|
33
|
-
can_latinize = False
|
|
34
30
|
filtered.append(char)
|
|
35
31
|
normalized = "".join(filtered)
|
|
36
|
-
if can_latinize:
|
|
32
|
+
if can_latinize(normalized):
|
|
37
33
|
normalized = latinize_text(normalized)
|
|
38
34
|
normalized = unicodedata.normalize("NFKC", normalized)
|
|
39
35
|
normalized = squash_spaces(normalized)
|
|
@@ -41,7 +41,8 @@ value:
|
|
|
41
41
|
listed in its dispatch (NFKD)
|
|
42
42
|
3. CASEFOLD — Unicode full casefold (ß → ss, not lowercase)
|
|
43
43
|
4. Cleanup — category_replace, unless Cleanup.Noop
|
|
44
|
-
5. SQUASH_SPACES —
|
|
44
|
+
5. SQUASH_SPACES — delete invisible format characters,
|
|
45
|
+
collapse whitespace runs, trim ends
|
|
45
46
|
6. NAME — tokenize via
|
|
46
47
|
[tokenize_name][rigour.names.tokenize.tokenize_name]
|
|
47
48
|
and rejoin with a single ASCII space
|
|
@@ -132,10 +133,14 @@ class Normalize(IntFlag):
|
|
|
132
133
|
Attributes:
|
|
133
134
|
STRIP: Trim leading and trailing whitespace.
|
|
134
135
|
SQUASH_SPACES: Collapse runs of whitespace (including newlines,
|
|
135
|
-
tabs, Unicode whitespace
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
136
|
+
tabs, Unicode whitespace, and the U+001C-001F information
|
|
137
|
+
separators) into single spaces and trim the edges. Also
|
|
138
|
+
deletes invisible format characters — zero-width
|
|
139
|
+
space/joiners (U+200B-U+200D), BOM (U+FEFF), soft hyphen
|
|
140
|
+
(U+00AD), and word joiners (U+2060-U+2064) — so that text
|
|
141
|
+
pasted from PDFs or web pages (``Gm\\u00ADbH``) compares
|
|
142
|
+
equal to its clean form (``GmbH``). Cleans up whitespace
|
|
143
|
+
introduced by earlier steps (e.g. category replacement).
|
|
139
144
|
CASEFOLD: Unicode full casefold (e.g. ``ß → ss``). This is
|
|
140
145
|
*not* the same as `str.lower()` — casefold is the correct
|
|
141
146
|
tool for case-insensitive comparison across Unicode.
|
|
@@ -3,15 +3,11 @@ from typing import Optional
|
|
|
3
3
|
|
|
4
4
|
from rigour._core import codepoint_script as _codepoint_script
|
|
5
5
|
from rigour._core import common_scripts as _common_scripts
|
|
6
|
+
from rigour._core import should_ascii as _should_ascii
|
|
6
7
|
from rigour._core import text_scripts as _text_scripts
|
|
7
8
|
from rigour.util import MEMO_MEDIUM
|
|
8
9
|
|
|
9
10
|
|
|
10
|
-
#: Scripts whose romanisation is well-defined enough that
|
|
11
|
-
#: rigour will automatically transliterate them to ASCII via
|
|
12
|
-
#: :func:`rigour.text.translit.maybe_ascii`.
|
|
13
|
-
LATINIZE_SCRIPTS = {"Hangul", "Cyrillic", "Greek", "Armenian", "Latin", "Georgian"}
|
|
14
|
-
|
|
15
11
|
#: Letter-based writing systems with explicit vowels — these
|
|
16
12
|
#: transliterate reliably to Latin without language hints.
|
|
17
13
|
#: Excludes Hangul (syllabic) and the dense logographic scripts.
|
|
@@ -99,38 +95,14 @@ def common_scripts(a: str, b: str) -> set[str]:
|
|
|
99
95
|
return _common_scripts(a, b)
|
|
100
96
|
|
|
101
97
|
|
|
102
|
-
@lru_cache(maxsize=MEMO_MEDIUM)
|
|
103
|
-
def can_latinize_cp(cp: int) -> Optional[bool]:
|
|
104
|
-
"""Check whether a single codepoint can be latinized.
|
|
105
|
-
|
|
106
|
-
Three-valued: distinguishing-script-bearing codepoints get a
|
|
107
|
-
True/False; others (digits, punctuation, combining marks)
|
|
108
|
-
return `None` because the question doesn't apply to them.
|
|
109
|
-
|
|
110
|
-
Args:
|
|
111
|
-
cp: Codepoint as an integer.
|
|
112
|
-
|
|
113
|
-
Returns:
|
|
114
|
-
`True` if the codepoint's script is in
|
|
115
|
-
:data:`LATINIZE_SCRIPTS`, `False` if it has a real script
|
|
116
|
-
outside that set, `None` for non-alphanumeric or
|
|
117
|
-
Common/Inherited/Unknown codepoints.
|
|
118
|
-
"""
|
|
119
|
-
char = chr(cp)
|
|
120
|
-
if not char.isalnum():
|
|
121
|
-
return None
|
|
122
|
-
script = codepoint_script(cp)
|
|
123
|
-
if script is None or script in ("Common", "Inherited"):
|
|
124
|
-
return None
|
|
125
|
-
return script in LATINIZE_SCRIPTS
|
|
126
|
-
|
|
127
|
-
|
|
128
98
|
def can_latinize(word: str) -> bool:
|
|
129
99
|
"""Check whether every script in a word is latinizable.
|
|
130
100
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
101
|
+
Alias for `should_ascii` in [rigour.text.translit][] — the two
|
|
102
|
+
call the same Rust predicate, which admits Latin, Cyrillic,
|
|
103
|
+
Greek, Armenian, Georgian, and Hangul. When True, `maybe_ascii`
|
|
104
|
+
will produce an ASCII output; when False, it returns the input
|
|
105
|
+
unchanged.
|
|
134
106
|
Characters with no distinguishing script (digits, punctuation,
|
|
135
107
|
spaces, combining marks) are ignored. Empty input and
|
|
136
108
|
pure-Common input (`"123"`) return True vacuously.
|
|
@@ -139,10 +111,10 @@ def can_latinize(word: str) -> bool:
|
|
|
139
111
|
word: A string.
|
|
140
112
|
|
|
141
113
|
Returns:
|
|
142
|
-
True iff every distinguishing script is
|
|
143
|
-
|
|
114
|
+
True iff every distinguishing script is admitted for
|
|
115
|
+
transliteration.
|
|
144
116
|
"""
|
|
145
|
-
return
|
|
117
|
+
return _should_ascii(word)
|
|
146
118
|
|
|
147
119
|
|
|
148
120
|
def is_modern_alphabet(word: str) -> bool:
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
"""Opportunistic transliteration: `should_ascii` + `maybe_ascii`.
|
|
2
2
|
|
|
3
3
|
The minimal transliteration surface rigour exposes going forward.
|
|
4
|
-
Covers only
|
|
5
|
-
|
|
6
|
-
outside that set passes through unchanged (default) or
|
|
7
|
-
empty (``drop=True``).
|
|
4
|
+
Covers only six admitted scripts (Latin, Cyrillic, Greek, Armenian,
|
|
5
|
+
Georgian, Hangul); the admission policy is defined in the Rust core.
|
|
6
|
+
Anything outside that set passes through unchanged (default) or
|
|
7
|
+
becomes empty (``drop=True``).
|
|
8
8
|
|
|
9
9
|
For broader-script, lossy transliteration (Han, Arabic, Devanagari,
|
|
10
10
|
etc.) use `normality.ascii_text` / `normality.latinize_text` —
|
|
@@ -25,9 +25,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
|
|
|
25
25
|
|
|
26
26
|
[[package]]
|
|
27
27
|
name = "bitflags"
|
|
28
|
-
version = "2.
|
|
28
|
+
version = "2.13.1"
|
|
29
29
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
30
|
-
checksum = "
|
|
30
|
+
checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
|
|
31
31
|
|
|
32
32
|
[[package]]
|
|
33
33
|
name = "calendrical_calculations"
|
|
@@ -623,9 +623,9 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"
|
|
|
623
623
|
|
|
624
624
|
[[package]]
|
|
625
625
|
name = "lru"
|
|
626
|
-
version = "0.18.
|
|
626
|
+
version = "0.18.1"
|
|
627
627
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
628
|
-
checksum = "
|
|
628
|
+
checksum = "0b6180140927ee907000b0aa540091f6ea512ead4447c92b8fc35bc72788a5a6"
|
|
629
629
|
dependencies = [
|
|
630
630
|
"hashbrown",
|
|
631
631
|
]
|
|
@@ -724,9 +724,9 @@ dependencies = [
|
|
|
724
724
|
|
|
725
725
|
[[package]]
|
|
726
726
|
name = "pyo3"
|
|
727
|
-
version = "0.
|
|
727
|
+
version = "0.29.0"
|
|
728
728
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
729
|
-
checksum = "
|
|
729
|
+
checksum = "cd274650b21d4bfc26a0a47587962c1edb425f69287324355cd040c3ea66071c"
|
|
730
730
|
dependencies = [
|
|
731
731
|
"libc",
|
|
732
732
|
"once_cell",
|
|
@@ -738,19 +738,18 @@ dependencies = [
|
|
|
738
738
|
|
|
739
739
|
[[package]]
|
|
740
740
|
name = "pyo3-build-config"
|
|
741
|
-
version = "0.
|
|
741
|
+
version = "0.29.0"
|
|
742
742
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
743
|
-
checksum = "
|
|
743
|
+
checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078"
|
|
744
744
|
dependencies = [
|
|
745
|
-
"python3-dll-a",
|
|
746
745
|
"target-lexicon",
|
|
747
746
|
]
|
|
748
747
|
|
|
749
748
|
[[package]]
|
|
750
749
|
name = "pyo3-ffi"
|
|
751
|
-
version = "0.
|
|
750
|
+
version = "0.29.0"
|
|
752
751
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
753
|
-
checksum = "
|
|
752
|
+
checksum = "ca85c467da1bbc8d866eea5deff9cf29ea5f7785054a17da36e65bda9c05845b"
|
|
754
753
|
dependencies = [
|
|
755
754
|
"libc",
|
|
756
755
|
"pyo3-build-config",
|
|
@@ -758,9 +757,9 @@ dependencies = [
|
|
|
758
757
|
|
|
759
758
|
[[package]]
|
|
760
759
|
name = "pyo3-macros"
|
|
761
|
-
version = "0.
|
|
760
|
+
version = "0.29.0"
|
|
762
761
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
763
|
-
checksum = "
|
|
762
|
+
checksum = "9ac53762fd065daa3194dd09337a38bd793a188100fd1a9304c4ab312d901771"
|
|
764
763
|
dependencies = [
|
|
765
764
|
"proc-macro2",
|
|
766
765
|
"pyo3-macros-backend",
|
|
@@ -770,26 +769,16 @@ dependencies = [
|
|
|
770
769
|
|
|
771
770
|
[[package]]
|
|
772
771
|
name = "pyo3-macros-backend"
|
|
773
|
-
version = "0.
|
|
772
|
+
version = "0.29.0"
|
|
774
773
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
775
|
-
checksum = "
|
|
774
|
+
checksum = "4ca3a1557399783172dc5bf39cfca835157732532cba56b71d2292161e53b362"
|
|
776
775
|
dependencies = [
|
|
777
776
|
"heck",
|
|
778
777
|
"proc-macro2",
|
|
779
|
-
"pyo3-build-config",
|
|
780
778
|
"quote",
|
|
781
779
|
"syn",
|
|
782
780
|
]
|
|
783
781
|
|
|
784
|
-
[[package]]
|
|
785
|
-
name = "python3-dll-a"
|
|
786
|
-
version = "0.2.15"
|
|
787
|
-
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
788
|
-
checksum = "d80ba7540edb18890d444c5aa8e1f1f99b1bdf26fb26ae383135325f4a36042b"
|
|
789
|
-
dependencies = [
|
|
790
|
-
"cc",
|
|
791
|
-
]
|
|
792
|
-
|
|
793
782
|
[[package]]
|
|
794
783
|
name = "quote"
|
|
795
784
|
version = "1.0.45"
|
|
@@ -813,9 +802,9 @@ checksum = "270e04e5ea61d40841942bb15e451c29ee1618637bcf97fc7ede5dd4a9b1601b"
|
|
|
813
802
|
|
|
814
803
|
[[package]]
|
|
815
804
|
name = "regex"
|
|
816
|
-
version = "1.
|
|
805
|
+
version = "1.13.1"
|
|
817
806
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
818
|
-
checksum = "
|
|
807
|
+
checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
|
|
819
808
|
dependencies = [
|
|
820
809
|
"aho-corasick",
|
|
821
810
|
"memchr",
|
|
@@ -825,9 +814,9 @@ dependencies = [
|
|
|
825
814
|
|
|
826
815
|
[[package]]
|
|
827
816
|
name = "regex-automata"
|
|
828
|
-
version = "0.4.
|
|
817
|
+
version = "0.4.16"
|
|
829
818
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
830
|
-
checksum = "
|
|
819
|
+
checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad"
|
|
831
820
|
dependencies = [
|
|
832
821
|
"aho-corasick",
|
|
833
822
|
"memchr",
|
|
@@ -836,13 +825,13 @@ dependencies = [
|
|
|
836
825
|
|
|
837
826
|
[[package]]
|
|
838
827
|
name = "regex-syntax"
|
|
839
|
-
version = "0.8.
|
|
828
|
+
version = "0.8.11"
|
|
840
829
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
841
|
-
checksum = "
|
|
830
|
+
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
|
|
842
831
|
|
|
843
832
|
[[package]]
|
|
844
833
|
name = "rigour-core"
|
|
845
|
-
version = "2.
|
|
834
|
+
version = "2.2.0"
|
|
846
835
|
dependencies = [
|
|
847
836
|
"aho-corasick",
|
|
848
837
|
"bitflags",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "rigour-core"
|
|
3
|
-
version = "2.
|
|
3
|
+
version = "2.2.0"
|
|
4
4
|
edition = "2024"
|
|
5
5
|
rust-version = "1.86"
|
|
6
6
|
license = "MIT"
|
|
@@ -17,9 +17,11 @@ crate-type = ["cdylib", "rlib"]
|
|
|
17
17
|
python = ["dep:pyo3"]
|
|
18
18
|
|
|
19
19
|
[dependencies]
|
|
20
|
-
# generate-import-lib
|
|
21
|
-
#
|
|
22
|
-
|
|
20
|
+
# Wheel-only pyo3 features (extension-module, generate-import-lib) are
|
|
21
|
+
# deliberately NOT set here — they live in `[tool.maturin].features` in
|
|
22
|
+
# pyproject.toml. Keeping them out of the crate lets `cargo test
|
|
23
|
+
# --features python` link a standalone test binary against libpython.
|
|
24
|
+
pyo3 = { version = "0.29", optional = true }
|
|
23
25
|
|
|
24
26
|
# Phonetic algorithms (metaphone, soundex). Port of Apache commons-codec, pure
|
|
25
27
|
# Rust. The upstream jellyfish Rust crate can't be used directly because it
|
|
@@ -43,7 +45,7 @@ bitflags = "2"
|
|
|
43
45
|
# `rigour.text.distance` and `rigour.territories.lookup` consume.
|
|
44
46
|
rapidfuzz = "0.5"
|
|
45
47
|
|
|
46
|
-
# LRU caches. Currently used for `maybe_ascii`'s
|
|
48
|
+
# LRU caches. Currently used for `maybe_ascii`'s process-wide result
|
|
47
49
|
# cache. Tiny crate (~500 LoC, no deps), O(1) get/put.
|
|
48
50
|
lru = "0.18"
|
|
49
51
|
|
|
@@ -9,10 +9,10 @@
|
|
|
9
9
|
// - `data/names/org_types.json` (~125 KiB → ~15 KiB)
|
|
10
10
|
// - `data/territories/data.jsonl` (~783 KiB → ~214 KiB)
|
|
11
11
|
//
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
//
|
|
12
|
+
// All source files are committed, so a missing one means a broken
|
|
13
|
+
// checkout — fail the build rather than embed an empty blob that
|
|
14
|
+
// would ship as a silently non-functional wheel (empty tagger, zero
|
|
15
|
+
// territories).
|
|
16
16
|
|
|
17
17
|
use std::env;
|
|
18
18
|
use std::fs;
|
|
@@ -21,36 +21,32 @@ use std::path::PathBuf;
|
|
|
21
21
|
struct Compress {
|
|
22
22
|
src: &'static str, // path under rust/
|
|
23
23
|
dst: &'static str, // filename under OUT_DIR
|
|
24
|
-
missing: &'static str, //
|
|
24
|
+
missing: &'static str, // error detail if source file is absent
|
|
25
25
|
}
|
|
26
26
|
|
|
27
27
|
const FILES: &[Compress] = &[
|
|
28
28
|
Compress {
|
|
29
29
|
src: "data/names/person_names.txt",
|
|
30
30
|
dst: "person_names.txt.zst",
|
|
31
|
-
missing: "rust/data/names/person_names.txt not found
|
|
32
|
-
compiling empty person-names corpus. Run \
|
|
31
|
+
missing: "rust/data/names/person_names.txt not found. Run \
|
|
33
32
|
`make -C contrib/namesdb dump` to regenerate.",
|
|
34
33
|
},
|
|
35
34
|
Compress {
|
|
36
35
|
src: "data/territories/data.jsonl",
|
|
37
36
|
dst: "territories.jsonl.zst",
|
|
38
|
-
missing: "rust/data/territories/data.jsonl not found
|
|
39
|
-
compiling empty territories blob. Run \
|
|
37
|
+
missing: "rust/data/territories/data.jsonl not found. Run \
|
|
40
38
|
`make build-territories` to regenerate.",
|
|
41
39
|
},
|
|
42
40
|
Compress {
|
|
43
41
|
src: "data/names/symbols.json",
|
|
44
42
|
dst: "symbols.json.zst",
|
|
45
|
-
missing: "rust/data/names/symbols.json not found
|
|
46
|
-
compiling empty symbols blob. Run \
|
|
43
|
+
missing: "rust/data/names/symbols.json not found. Run \
|
|
47
44
|
`make build-names` to regenerate.",
|
|
48
45
|
},
|
|
49
46
|
Compress {
|
|
50
47
|
src: "data/names/org_types.json",
|
|
51
48
|
dst: "org_types.json.zst",
|
|
52
|
-
missing: "rust/data/names/org_types.json not found
|
|
53
|
-
compiling empty org-types blob. Run \
|
|
49
|
+
missing: "rust/data/names/org_types.json not found. Run \
|
|
54
50
|
`make build-names` to regenerate.",
|
|
55
51
|
},
|
|
56
52
|
];
|
|
@@ -64,11 +60,7 @@ fn main() {
|
|
|
64
60
|
let dest_path = out_dir.join(entry.dst);
|
|
65
61
|
println!("cargo:rerun-if-changed={}", entry.src);
|
|
66
62
|
|
|
67
|
-
let
|
|
68
|
-
println!("cargo:warning={}", entry.missing);
|
|
69
|
-
fs::write(&dest_path, [] as [u8; 0]).expect("write empty placeholder");
|
|
70
|
-
continue;
|
|
71
|
-
};
|
|
63
|
+
let bytes = fs::read(&source_path).unwrap_or_else(|_| panic!("{}", entry.missing));
|
|
72
64
|
|
|
73
65
|
// Level 19 is the high-ratio tier — slow compression, but this
|
|
74
66
|
// runs at build time only and gives the best wheel size.
|