rigour 2.1.2__tar.gz → 2.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rigour-2.1.2 → rigour-2.2.2}/PKG-INFO +2 -1
- {rigour-2.1.2 → rigour-2.2.2}/pyproject.toml +5 -2
- rigour-2.2.2/rigour/dates.py +203 -0
- rigour-2.2.2/rigour/ids/imo.py +125 -0
- rigour-2.2.2/rigour/names/name.py +24 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/names/org_types.py +20 -11
- rigour-2.2.2/rigour/names/part.py +23 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/names/pick.py +1 -2
- {rigour-2.1.2 → rigour-2.2.2}/rigour/reset.py +2 -2
- {rigour-2.1.2 → rigour-2.2.2}/rigour/territories/util.py +2 -6
- {rigour-2.1.2 → rigour-2.2.2}/rigour/text/normalize.py +10 -5
- {rigour-2.1.2 → rigour-2.2.2}/rigour/text/scripts.py +9 -37
- {rigour-2.1.2 → rigour-2.2.2}/rigour/text/translit.py +4 -4
- {rigour-2.1.2 → rigour-2.2.2}/rust/Cargo.lock +21 -32
- {rigour-2.1.2 → rigour-2.2.2}/rust/Cargo.toml +7 -5
- {rigour-2.1.2 → rigour-2.2.2}/rust/build.rs +10 -18
- {rigour-2.1.2 → rigour-2.2.2}/rust/data/names/person_names.txt +2313 -1202
- {rigour-2.1.2 → rigour-2.2.2}/rust/data/territories/data.jsonl +114 -39
- {rigour-2.1.2 → rigour-2.2.2}/rust/data/text/stopwords.json +2 -1
- rigour-2.2.2/rust/src/constants.rs +6 -0
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/lib.rs +1 -6
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/names/alignment.rs +41 -4
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/names/analyze.rs +23 -7
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/names/compare.rs +17 -9
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/names/name.rs +44 -4
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/names/ordering.rs +17 -23
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/names/org_types.rs +231 -55
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/names/pairing.rs +451 -70
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/names/part.rs +31 -10
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/names/person_names.rs +5 -20
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/names/pick.rs +29 -64
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/names/symbol.rs +24 -0
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/names/symbols.rs +0 -9
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/names/tagger.rs +65 -22
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/territories.rs +6 -10
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/text/distance.rs +19 -27
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/text/normalize.rs +74 -23
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/text/scripts.rs +1 -1
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/text/tokenize.rs +8 -13
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/text/translit.rs +42 -28
- rigour-2.1.2/rigour/ids/imo.py +0 -60
- rigour-2.1.2/rigour/names/name.py +0 -26
- rigour-2.1.2/rigour/names/part.py +0 -23
- rigour-2.1.2/rust/src/constants.rs +0 -3
- {rigour-2.1.2 → rigour-2.2.2}/LICENSE +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/README.md +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/__init__.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/_core.pyi +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/addresses/__init__.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/addresses/cleaning.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/addresses/format.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/addresses/normalize.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/boolean.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/data/__init__.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/data/addresses/__init__.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/data/addresses/data.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/data/addresses/formats.yml +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/data/langs/__init__.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/data/langs/iso639.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/env.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/ids/__init__.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/ids/common.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/ids/npi.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/ids/ogrn.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/ids/stdnum_.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/ids/strict.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/ids/uei.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/ids/wikidata.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/langs/__init__.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/langs/synonyms.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/langs/text.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/langs/util.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/mime/__init__.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/mime/filename.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/mime/mappings.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/mime/mime.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/mime/parse.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/mime/types.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/names/__init__.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/names/analyze.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/names/check.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/names/compare.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/names/ordering.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/names/prefix.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/names/split_phrases.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/names/symbol.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/names/tag.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/names/tokenize.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/py.typed +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/territories/__init__.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/territories/lookup.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/territories/match.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/territories/territory.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/text/__init__.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/text/checksum.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/text/cleaning.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/text/distance.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/text/phonetics.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/text/stopwords.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/time.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/units.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/urls/__init__.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/urls/cleaning.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/urls/compare.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/urls/util.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rigour/util.py +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rust/data/names/compare.json +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rust/data/names/org_types.json +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rust/data/names/stopwords.json +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rust/data/names/symbols.json +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rust/data/text/ordinals.json +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/generated/.gitkeep +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/names/matcher.rs +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/names/mod.rs +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/names/prefix.rs +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/names/stopwords.rs +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/names/tag.rs +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/text/mod.rs +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/text/numbers.rs +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/text/ordinals.rs +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/text/phonetics.rs +0 -0
- {rigour-2.1.2 → rigour-2.2.2}/rust/src/text/stopwords.rs +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rigour
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.2.2
|
|
4
4
|
Classifier: Intended Audience :: Developers
|
|
5
5
|
Classifier: Operating System :: OS Independent
|
|
6
6
|
Classifier: License :: OSI Approved :: MIT License
|
|
@@ -12,6 +12,7 @@ Classifier: Programming Language :: Python :: 3.14
|
|
|
12
12
|
Classifier: Programming Language :: Rust
|
|
13
13
|
Requires-Dist: pyyaml>=5.0.0,<7.0.0
|
|
14
14
|
Requires-Dist: normality>=3.0.1,<4.0.0
|
|
15
|
+
Requires-Dist: prefixdate>=0.5.0,<1.0.0
|
|
15
16
|
Requires-Dist: orjson>=3.0.0,<4.0.0
|
|
16
17
|
Requires-Dist: python-stdnum>=2.0,<3.0.0
|
|
17
18
|
Requires-Dist: jinja2>=3.1.0,<4.0.0
|
|
@@ -4,7 +4,7 @@ build-backend = "maturin"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "rigour"
|
|
7
|
-
version = "2.
|
|
7
|
+
version = "2.2.2"
|
|
8
8
|
description = "Financial crime domain data validation and normalization library."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = { file = "LICENSE" }
|
|
@@ -24,6 +24,7 @@ requires-python = ">= 3.10"
|
|
|
24
24
|
dependencies = [
|
|
25
25
|
"pyyaml >= 5.0.0, < 7.0.0",
|
|
26
26
|
"normality >= 3.0.1, < 4.0.0",
|
|
27
|
+
"prefixdate >= 0.5.0, < 1.0.0",
|
|
27
28
|
"orjson >= 3.0.0, < 4.0.0",
|
|
28
29
|
"python-stdnum >= 2.0, < 3.0.0",
|
|
29
30
|
"jinja2 >= 3.1.0, < 4.0.0",
|
|
@@ -60,7 +61,9 @@ docs = [
|
|
|
60
61
|
]
|
|
61
62
|
|
|
62
63
|
[tool.maturin]
|
|
63
|
-
|
|
64
|
+
# pyo3/generate-import-lib lets the Windows wheel job link without a
|
|
65
|
+
# real Python install (PyO3 generates a stub import library).
|
|
66
|
+
features = ["pyo3/extension-module", "pyo3/generate-import-lib", "python"]
|
|
64
67
|
module-name = "rigour._core"
|
|
65
68
|
manifest-path = "rust/Cargo.toml"
|
|
66
69
|
# `rigour/` is the Python package directory, sitting at the repo root. Maturin
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
"""Compare prefix dates without pretending they are exact instants.
|
|
2
|
+
|
|
3
|
+
A prefix date such as ``2026`` or ``2026-06`` represents an interval, rather
|
|
4
|
+
than the first instant of that year or month. Use the string wrappers for
|
|
5
|
+
one-off comparisons, or parse a [DateInterval][rigour.dates.DateInterval] once
|
|
6
|
+
when comparing the same value repeatedly.
|
|
7
|
+
|
|
8
|
+
Age and recency checks use the same comparisons with a shifted reference:
|
|
9
|
+
|
|
10
|
+
```python
|
|
11
|
+
cutoff = now - max_age
|
|
12
|
+
is_recent = not ended_before(value, cutoff)
|
|
13
|
+
```
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
import re
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
from datetime import datetime, timedelta, timezone
|
|
19
|
+
from functools import lru_cache
|
|
20
|
+
|
|
21
|
+
from prefixdate import Precision, parse
|
|
22
|
+
|
|
23
|
+
from rigour.util import MEMO_SMALL
|
|
24
|
+
|
|
25
|
+
_TIMEZONE_SUFFIX_RE = re.compile(r"(?:Z|[+-]\d{2}(?::?\d{2})?)$")
|
|
26
|
+
_SECOND_TIMESTAMP_RE = re.compile(
|
|
27
|
+
r"^[12]\d{3}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}"
|
|
28
|
+
r"(?:Z|[+-]\d{2}(?::?\d{2})?)?$"
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True)
|
|
33
|
+
class DateInterval:
|
|
34
|
+
"""Represent every possible instant described by an imprecise date.
|
|
35
|
+
|
|
36
|
+
Use this parsed form when applying multiple comparisons to the same prefix
|
|
37
|
+
date. Both bounds are timezone-aware UTC datetimes and ``end`` is exclusive.
|
|
38
|
+
|
|
39
|
+
Attributes:
|
|
40
|
+
start: Earliest instant represented by the date.
|
|
41
|
+
end: First instant after the represented interval.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
start: datetime
|
|
45
|
+
end: datetime
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _ensure_utc(value: datetime) -> datetime:
|
|
49
|
+
"""Adapt legacy naive-UTC values to aware UTC datetimes."""
|
|
50
|
+
if value.tzinfo is None or value.utcoffset() is None:
|
|
51
|
+
return value.replace(tzinfo=timezone.utc)
|
|
52
|
+
return value.astimezone(timezone.utc)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def parse_utc(value: str) -> datetime:
|
|
56
|
+
"""Parse an exact timestamp as an aware UTC datetime.
|
|
57
|
+
|
|
58
|
+
Use this at a prefix-date boundary where an exact timestamp may carry an
|
|
59
|
+
offset. Naive timestamps use the ecosystem convention of implicit UTC.
|
|
60
|
+
|
|
61
|
+
Args:
|
|
62
|
+
value: Canonical second-precision timestamp.
|
|
63
|
+
|
|
64
|
+
Returns:
|
|
65
|
+
The represented instant as a timezone-aware UTC datetime.
|
|
66
|
+
|
|
67
|
+
Raises:
|
|
68
|
+
ValueError: The timestamp is invalid or is not precise to the second.
|
|
69
|
+
"""
|
|
70
|
+
if _SECOND_TIMESTAMP_RE.fullmatch(value) is None:
|
|
71
|
+
raise ValueError(f"Invalid exact timestamp: {value!r}")
|
|
72
|
+
try:
|
|
73
|
+
timestamp = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
74
|
+
except ValueError as exc:
|
|
75
|
+
raise ValueError(f"Invalid timestamp: {value!r}") from exc
|
|
76
|
+
return _ensure_utc(timestamp)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@lru_cache(maxsize=MEMO_SMALL)
|
|
80
|
+
def prefix_interval(value: str) -> DateInterval:
|
|
81
|
+
"""Expand a canonical prefix date into its represented UTC interval.
|
|
82
|
+
|
|
83
|
+
Use this at the boundary between prefix-date strings and interval comparison.
|
|
84
|
+
Feed it normalized property values only, not uncleaned source strings. Exact
|
|
85
|
+
timestamps with offsets are converted to UTC before their one-second interval
|
|
86
|
+
is constructed.
|
|
87
|
+
|
|
88
|
+
Args:
|
|
89
|
+
value: Canonical prefix date, from year through second precision.
|
|
90
|
+
|
|
91
|
+
Returns:
|
|
92
|
+
The timezone-aware UTC, half-open interval represented by the value.
|
|
93
|
+
|
|
94
|
+
Raises:
|
|
95
|
+
ValueError: The value is invalid or non-canonical, or an imprecise value
|
|
96
|
+
carries a timezone.
|
|
97
|
+
"""
|
|
98
|
+
if _SECOND_TIMESTAMP_RE.fullmatch(value) is not None:
|
|
99
|
+
start = parse_utc(value)
|
|
100
|
+
return DateInterval(start=start, end=start + timedelta(seconds=1))
|
|
101
|
+
has_timezone = value.endswith("Z") or (
|
|
102
|
+
"T" in value and _TIMEZONE_SUFFIX_RE.search(value) is not None
|
|
103
|
+
)
|
|
104
|
+
if has_timezone:
|
|
105
|
+
raise ValueError("timezone suffixes require second precision")
|
|
106
|
+
|
|
107
|
+
prefix = parse(value)
|
|
108
|
+
if prefix.dt is None or prefix.text is None or prefix.precision == Precision.EMPTY:
|
|
109
|
+
raise ValueError(f"Invalid prefix date: {value!r}")
|
|
110
|
+
if prefix.text != value:
|
|
111
|
+
raise ValueError(f"Prefix date is not canonical: {value!r}")
|
|
112
|
+
|
|
113
|
+
start = prefix.dt.replace(tzinfo=timezone.utc)
|
|
114
|
+
precision = prefix.precision
|
|
115
|
+
if precision == Precision.YEAR:
|
|
116
|
+
end = start.replace(year=start.year + 1)
|
|
117
|
+
elif precision == Precision.MONTH:
|
|
118
|
+
if start.month == 12:
|
|
119
|
+
end = start.replace(year=start.year + 1, month=1)
|
|
120
|
+
else:
|
|
121
|
+
end = start.replace(month=start.month + 1)
|
|
122
|
+
elif precision == Precision.DAY:
|
|
123
|
+
end = start + timedelta(days=1)
|
|
124
|
+
elif precision == Precision.HOUR:
|
|
125
|
+
end = start + timedelta(hours=1)
|
|
126
|
+
elif precision == Precision.MINUTE:
|
|
127
|
+
end = start + timedelta(minutes=1)
|
|
128
|
+
elif precision == Precision.SECOND:
|
|
129
|
+
end = start + timedelta(seconds=1)
|
|
130
|
+
else:
|
|
131
|
+
raise ValueError(f"Unsupported prefix date precision: {value!r}")
|
|
132
|
+
return DateInterval(start=start, end=end)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def interval_ended_before(value: DateInterval, reference: datetime) -> bool:
|
|
136
|
+
"""Check whether an interval has completely elapsed before a point in time.
|
|
137
|
+
|
|
138
|
+
Use this for end dates and age cutoffs when the prefix date has already been
|
|
139
|
+
parsed.
|
|
140
|
+
|
|
141
|
+
Args:
|
|
142
|
+
value: Interval to compare.
|
|
143
|
+
reference: Timezone-aware UTC point in time.
|
|
144
|
+
|
|
145
|
+
Returns:
|
|
146
|
+
``True`` if every instant represented by the interval precedes the
|
|
147
|
+
reference.
|
|
148
|
+
"""
|
|
149
|
+
return value.end <= reference
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def interval_starts_after(value: DateInterval, reference: datetime) -> bool:
|
|
153
|
+
"""Check whether an interval begins strictly after a point in time.
|
|
154
|
+
|
|
155
|
+
Use this for start dates and future-date guardrails when the prefix date has
|
|
156
|
+
already been parsed.
|
|
157
|
+
|
|
158
|
+
Args:
|
|
159
|
+
value: Interval to compare.
|
|
160
|
+
reference: Timezone-aware UTC point in time.
|
|
161
|
+
|
|
162
|
+
Returns:
|
|
163
|
+
``True`` if the earliest instant in the interval follows the reference.
|
|
164
|
+
"""
|
|
165
|
+
return value.start > reference
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def ended_before(value: str, reference: datetime) -> bool:
|
|
169
|
+
"""Check whether a prefix date has completely elapsed.
|
|
170
|
+
|
|
171
|
+
Use this string wrapper for one-off end-date and age-cutoff checks. Parse
|
|
172
|
+
once with [prefix_interval][rigour.dates.prefix_interval] when making
|
|
173
|
+
several comparisons against the same value.
|
|
174
|
+
|
|
175
|
+
Args:
|
|
176
|
+
value: Canonical prefix date, from year through second precision.
|
|
177
|
+
reference: UTC point in time. Legacy naive values are interpreted as
|
|
178
|
+
UTC; aware values are converted to UTC.
|
|
179
|
+
|
|
180
|
+
Returns:
|
|
181
|
+
``True`` if every instant represented by the date precedes the
|
|
182
|
+
reference.
|
|
183
|
+
"""
|
|
184
|
+
return interval_ended_before(prefix_interval(value), _ensure_utc(reference))
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def starts_after(value: str, reference: datetime) -> bool:
|
|
188
|
+
"""Check whether a prefix date begins after a point in time.
|
|
189
|
+
|
|
190
|
+
Use this string wrapper for one-off start-date and future-date checks. Parse
|
|
191
|
+
once with [prefix_interval][rigour.dates.prefix_interval] when reusing the
|
|
192
|
+
same value.
|
|
193
|
+
|
|
194
|
+
Args:
|
|
195
|
+
value: Canonical prefix date, from year through second precision.
|
|
196
|
+
reference: UTC point in time. Legacy naive values are interpreted as
|
|
197
|
+
UTC; aware values are converted to UTC.
|
|
198
|
+
|
|
199
|
+
Returns:
|
|
200
|
+
``True`` if the earliest instant represented by the date follows the
|
|
201
|
+
reference.
|
|
202
|
+
"""
|
|
203
|
+
return interval_starts_after(prefix_interval(value), _ensure_utc(reference))
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from typing import Callable, List, Optional, Tuple
|
|
3
|
+
|
|
4
|
+
from rigour.ids.common import IdentifierFormat
|
|
5
|
+
|
|
6
|
+
IMO_RE = re.compile(r"(IMO[\s:.#-]*)?(?<!\d)(\d{5,7})(?!\d)")
|
|
7
|
+
|
|
8
|
+
Checksum = Callable[[List[int]], bool]
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _vessel_checksum(digits: List[int]) -> bool:
|
|
12
|
+
return sum(d * (7 - i) for i, d in enumerate(digits[:6])) % 10 == digits[6]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _company_checksum(digits: List[int]) -> bool:
|
|
16
|
+
weights = (8, 6, 4, 2, 9, 7)
|
|
17
|
+
checksum = sum(d * w for d, w in zip(digits, weights))
|
|
18
|
+
return (11 - checksum % 11) % 10 == digits[6]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class IMO(IdentifierFormat):
|
|
22
|
+
"""An IMO number for a ship or a shipping company.
|
|
23
|
+
|
|
24
|
+
Ship and company IMO numbers are both seven digits but use different check
|
|
25
|
+
digit algorithms; this format admits both. Use
|
|
26
|
+
[is_valid_vessel][rigour.ids.imo.IMO.is_valid_vessel] or
|
|
27
|
+
[is_valid_company][rigour.ids.imo.IMO.is_valid_company] when the value
|
|
28
|
+
must belong to one specific scheme, e.g. when minting scheme-specific
|
|
29
|
+
entity IDs.
|
|
30
|
+
|
|
31
|
+
Extraction considers digit runs of five to seven digits, preferring longer
|
|
32
|
+
runs, then runs carrying an `IMO` prefix, then earlier ones; the first
|
|
33
|
+
candidate that passes a checksum wins. Runs shorter than seven digits are
|
|
34
|
+
left-padded with zeros, since data sources commonly strip leading zeros
|
|
35
|
+
from IMO fields. The all-zero value is rejected.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
NAME = "imo"
|
|
39
|
+
TITLE = "IMO"
|
|
40
|
+
STRONG = True
|
|
41
|
+
|
|
42
|
+
@classmethod
|
|
43
|
+
def is_valid(cls, text: str) -> bool:
|
|
44
|
+
"""Determine if the given text contains a valid IMO number.
|
|
45
|
+
|
|
46
|
+
Args:
|
|
47
|
+
text: A string that may contain an IMO number.
|
|
48
|
+
|
|
49
|
+
Returns:
|
|
50
|
+
True if an IMO number valid in either checksum scheme can be
|
|
51
|
+
extracted from the text.
|
|
52
|
+
"""
|
|
53
|
+
return cls._extract(text, (_vessel_checksum, _company_checksum)) is not None
|
|
54
|
+
|
|
55
|
+
@classmethod
|
|
56
|
+
def is_valid_vessel(cls, text: str) -> bool:
|
|
57
|
+
"""Determine if the given text contains a valid ship IMO number.
|
|
58
|
+
|
|
59
|
+
Args:
|
|
60
|
+
text: A string that may contain an IMO number.
|
|
61
|
+
|
|
62
|
+
Returns:
|
|
63
|
+
True if an IMO number valid in the vessel checksum scheme can be
|
|
64
|
+
extracted from the text.
|
|
65
|
+
"""
|
|
66
|
+
return cls._extract(text, (_vessel_checksum,)) is not None
|
|
67
|
+
|
|
68
|
+
@classmethod
|
|
69
|
+
def is_valid_company(cls, text: str) -> bool:
|
|
70
|
+
"""Determine if the given text contains a valid company IMO number.
|
|
71
|
+
|
|
72
|
+
Args:
|
|
73
|
+
text: A string that may contain an IMO number.
|
|
74
|
+
|
|
75
|
+
Returns:
|
|
76
|
+
True if an IMO number valid in the company checksum scheme can be
|
|
77
|
+
extracted from the text.
|
|
78
|
+
"""
|
|
79
|
+
return cls._extract(text, (_company_checksum,)) is not None
|
|
80
|
+
|
|
81
|
+
@classmethod
|
|
82
|
+
def normalize(cls, text: str) -> Optional[str]:
|
|
83
|
+
"""Extract and normalize an IMO number from the given text.
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
text: A string that may contain an IMO number.
|
|
87
|
+
|
|
88
|
+
Returns:
|
|
89
|
+
The number in `IMO0000000` form, or None if the text contains no
|
|
90
|
+
checksum-valid IMO number.
|
|
91
|
+
"""
|
|
92
|
+
value = cls._extract(text, (_vessel_checksum, _company_checksum))
|
|
93
|
+
if value is None:
|
|
94
|
+
return None
|
|
95
|
+
return f"IMO{value}"
|
|
96
|
+
|
|
97
|
+
@classmethod
|
|
98
|
+
def _extract(cls, text: str, checksums: Tuple[Checksum, ...]) -> Optional[str]:
|
|
99
|
+
matches = sorted(
|
|
100
|
+
IMO_RE.finditer(text),
|
|
101
|
+
key=lambda m: (-len(m.group(2)), m.group(1) is None, m.start()),
|
|
102
|
+
)
|
|
103
|
+
for match in matches:
|
|
104
|
+
value = match.group(2).zfill(7)
|
|
105
|
+
digits = [int(d) for d in value]
|
|
106
|
+
if not any(digits):
|
|
107
|
+
continue
|
|
108
|
+
if any(check(digits) for check in checksums):
|
|
109
|
+
return value
|
|
110
|
+
return None
|
|
111
|
+
|
|
112
|
+
@classmethod
|
|
113
|
+
def format(cls, value: str) -> str:
|
|
114
|
+
"""Format the given value as an IMO number.
|
|
115
|
+
|
|
116
|
+
Args:
|
|
117
|
+
value: A pre-validated IMO number.
|
|
118
|
+
|
|
119
|
+
Returns:
|
|
120
|
+
The value with an `IMO` prefix.
|
|
121
|
+
"""
|
|
122
|
+
value = value.replace(" ", "")
|
|
123
|
+
if not value.startswith("IMO"):
|
|
124
|
+
value = f"IMO{value}"
|
|
125
|
+
return value
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""Name — the top-level object in the `rigour.names` object graph.
|
|
2
|
+
|
|
3
|
+
Rust-backed via `rigour._core`. Python-side surface:
|
|
4
|
+
|
|
5
|
+
* `Name(original, form=None, tag=NameTypeTag.UNK, phonetics=True)` —
|
|
6
|
+
`form` defaults to `casefold(original)` and is tokenised; each
|
|
7
|
+
token becomes a fresh `NamePart`.
|
|
8
|
+
* eager attributes: `original`, `form`, `tag` (mutable), `parts`,
|
|
9
|
+
`spans`, `comparable`, `norm_form`
|
|
10
|
+
* dynamic property: `symbols` (recomputed each access from `spans`)
|
|
11
|
+
* mutation: `tag_text(text, tag, max_matches=1)`,
|
|
12
|
+
`apply_phrase(phrase, symbol)`, `apply_part(part, symbol)`
|
|
13
|
+
* `contains(other)` for PER-aware subset checks
|
|
14
|
+
* `Name.consolidate_names(names)` classmethod — drops names that are
|
|
15
|
+
substrings of longer names in the same iterable
|
|
16
|
+
* `__hash__` / `__eq__` by `form` (stable across tag mutation)
|
|
17
|
+
|
|
18
|
+
`parts` is a tuple built once at construction — attribute reads are
|
|
19
|
+
zero-copy INCREFs. `spans` starts empty and grows via `apply_phrase`
|
|
20
|
+
/ `apply_part`.
|
|
21
|
+
"""
|
|
22
|
+
from rigour._core import Name
|
|
23
|
+
|
|
24
|
+
__all__ = ["Name"]
|
|
@@ -88,27 +88,36 @@ def replace_org_types_display(
|
|
|
88
88
|
"""Replace organisation types in a name with their short display form.
|
|
89
89
|
|
|
90
90
|
Spelt-out legal forms are shortened into common abbreviations
|
|
91
|
-
(e.g. ``"Siemens Aktiengesellschaft"`` → ``"Siemens AG"``),
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
91
|
+
(e.g. ``"Siemens Aktiengesellschaft"`` → ``"Siemens AG"``), and
|
|
92
|
+
case-variant forms are canonicalised (``"Siemens GMBH"`` →
|
|
93
|
+
``"Siemens GmbH"``). Everything outside a matched span is returned
|
|
94
|
+
verbatim — case, spacing and stray characters untouched. If the
|
|
95
|
+
whole input is uppercase (`str.isupper()`), the whole output is
|
|
96
|
+
re-uppercased, so ``"ACME COMPANY LIMITED"`` becomes
|
|
97
|
+
``"ACME COMPANY LTD"`` rather than the odd-looking
|
|
98
|
+
``"ACME COMPANY Ltd"``.
|
|
99
|
+
|
|
100
|
+
Matching is Unicode-case- and whitespace-insensitive: the input is
|
|
101
|
+
normalised internally with the same `normalize_flags` as the alias
|
|
102
|
+
table (which must therefore include `Normalize.CASEFOLD`; the
|
|
103
|
+
default does), and matches are mapped back to the original text.
|
|
104
|
+
Legal forms broken by copy/paste residue — soft hyphens, zero-width
|
|
105
|
+
characters, doubled spaces — are still recognised, and the
|
|
106
|
+
replacement swallows the residue.
|
|
99
107
|
|
|
100
108
|
Args:
|
|
101
109
|
name: The text to be processed.
|
|
102
110
|
normalize_flags: `Normalize` flag
|
|
103
|
-
set applied to the alias list at build time
|
|
104
|
-
`Normalize.CASEFOLD`
|
|
111
|
+
set applied to the alias list at build time and mirrored on
|
|
112
|
+
the input at match time. Must include `Normalize.CASEFOLD`
|
|
113
|
+
for Unicode-case-insensitive matching.
|
|
105
114
|
Default `CASEFOLD | SQUASH_SPACES`.
|
|
106
115
|
cleanup: `Cleanup` variant applied
|
|
107
116
|
during alias normalisation. Default `Cleanup.Noop`.
|
|
108
117
|
|
|
109
118
|
Returns:
|
|
110
119
|
The text with recognised organisation types substituted for
|
|
111
|
-
their display form
|
|
120
|
+
their display form and all non-matched regions verbatim.
|
|
112
121
|
"""
|
|
113
122
|
return _replace_org_types_display(name, int(normalize_flags), int(cleanup))
|
|
114
123
|
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""NamePart / Span — the leaf classes of the `rigour.names` object graph.
|
|
2
|
+
|
|
3
|
+
Rust-backed via `rigour._core`. Python-side surface:
|
|
4
|
+
|
|
5
|
+
* `NamePart(form, index, tag=NamePartTag.UNSET, phonetics=True)` —
|
|
6
|
+
`index` is the part's position within its name and is required.
|
|
7
|
+
* eager attributes: `form`, `index`, `tag` (mutable), `latinize`,
|
|
8
|
+
`numeric`, `ascii`, `integer`, `comparable`, `metaphone`
|
|
9
|
+
* `NamePart.tag_sort(parts)` classmethod — stable-sort parts into
|
|
10
|
+
human display order by tag (honorifics, given, middle, family, …)
|
|
11
|
+
* tag compatibility checks live on the tag: `part.tag.can_match(other.tag)`
|
|
12
|
+
* `__hash__` / `__eq__` by the precomputed `_hash` (tuple of
|
|
13
|
+
`(index, form)`)
|
|
14
|
+
* `Span(parts, symbol)` with precomputed `comparable`
|
|
15
|
+
|
|
16
|
+
`phonetics=False` at construction skips the metaphone computation —
|
|
17
|
+
`part.metaphone` is `None` in that case. `ascii` uses
|
|
18
|
+
`rigour._core.maybe_ascii` (narrow 6-script transliteration); parts
|
|
19
|
+
whose form is outside the admitted scripts resolve `ascii` to `None`.
|
|
20
|
+
"""
|
|
21
|
+
from rigour._core import NamePart, Span
|
|
22
|
+
|
|
23
|
+
__all__ = ["NamePart", "Span"]
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from typing import Dict, List, Optional
|
|
2
2
|
|
|
3
|
+
from rigour._core import pick_case as _pick_case
|
|
3
4
|
from rigour._core import pick_name as pick_name
|
|
4
5
|
from rigour._core import reduce_names as reduce_names
|
|
5
6
|
from rigour.langs import LangStr, PREFERRED_LANG, PREFERRED_LANGS
|
|
@@ -152,8 +153,6 @@ def pick_case(names: List[str]) -> str:
|
|
|
152
153
|
Returns:
|
|
153
154
|
str: The best name for display.
|
|
154
155
|
"""
|
|
155
|
-
from rigour._core import pick_case as _pick_case
|
|
156
|
-
|
|
157
156
|
result = _pick_case(names)
|
|
158
157
|
if result is None:
|
|
159
158
|
raise ValueError("Cannot pick a name from an empty list.")
|
|
@@ -5,7 +5,7 @@ from rigour.text.phonetics import soundex, metaphone
|
|
|
5
5
|
from rigour.territories.territory import get_index
|
|
6
6
|
from rigour.territories.lookup import lookup_territory
|
|
7
7
|
from rigour.territories.lookup import _get_identifier_map, _get_territory_names
|
|
8
|
-
from rigour.text.scripts import
|
|
8
|
+
from rigour.text.scripts import codepoint_script
|
|
9
9
|
from rigour.names.tokenize import normalize_name
|
|
10
10
|
from rigour.names.prefix import (
|
|
11
11
|
_person_prefix_regex,
|
|
@@ -29,7 +29,7 @@ def reset_caches() -> None:
|
|
|
29
29
|
jaro_winkler.cache_clear()
|
|
30
30
|
soundex.cache_clear()
|
|
31
31
|
metaphone.cache_clear()
|
|
32
|
-
|
|
32
|
+
codepoint_script.cache_clear()
|
|
33
33
|
_load_formats.cache_clear()
|
|
34
34
|
_load_template.cache_clear()
|
|
35
35
|
_address_replacer.cache_clear()
|
|
@@ -2,7 +2,7 @@ import unicodedata
|
|
|
2
2
|
from typing import List
|
|
3
3
|
from normality import latinize_text, squash_spaces
|
|
4
4
|
|
|
5
|
-
from rigour.text.scripts import
|
|
5
|
+
from rigour.text.scripts import can_latinize
|
|
6
6
|
|
|
7
7
|
SKIP_CHARACTERS = ".()[],;:_-/ʻ'’"
|
|
8
8
|
|
|
@@ -20,7 +20,6 @@ def clean_codes(codes: List[str]) -> List[str]:
|
|
|
20
20
|
def normalize_territory_name(name: str) -> str:
|
|
21
21
|
"""Normalize a territory name for lookup."""
|
|
22
22
|
name = unicodedata.normalize("NFKD", name).casefold()
|
|
23
|
-
can_latinize = True
|
|
24
23
|
filtered: List[str] = []
|
|
25
24
|
for char in name:
|
|
26
25
|
if char in SKIP_CHARACTERS:
|
|
@@ -28,12 +27,9 @@ def normalize_territory_name(name: str) -> str:
|
|
|
28
27
|
if not char.isalnum():
|
|
29
28
|
filtered.append(" ")
|
|
30
29
|
continue
|
|
31
|
-
cp = ord(char)
|
|
32
|
-
if not can_latinize_cp(cp):
|
|
33
|
-
can_latinize = False
|
|
34
30
|
filtered.append(char)
|
|
35
31
|
normalized = "".join(filtered)
|
|
36
|
-
if can_latinize:
|
|
32
|
+
if can_latinize(normalized):
|
|
37
33
|
normalized = latinize_text(normalized)
|
|
38
34
|
normalized = unicodedata.normalize("NFKC", normalized)
|
|
39
35
|
normalized = squash_spaces(normalized)
|
|
@@ -41,7 +41,8 @@ value:
|
|
|
41
41
|
listed in its dispatch (NFKD)
|
|
42
42
|
3. CASEFOLD — Unicode full casefold (ß → ss, not lowercase)
|
|
43
43
|
4. Cleanup — category_replace, unless Cleanup.Noop
|
|
44
|
-
5. SQUASH_SPACES —
|
|
44
|
+
5. SQUASH_SPACES — delete invisible format characters,
|
|
45
|
+
collapse whitespace runs, trim ends
|
|
45
46
|
6. NAME — tokenize via
|
|
46
47
|
[tokenize_name][rigour.names.tokenize.tokenize_name]
|
|
47
48
|
and rejoin with a single ASCII space
|
|
@@ -132,10 +133,14 @@ class Normalize(IntFlag):
|
|
|
132
133
|
Attributes:
|
|
133
134
|
STRIP: Trim leading and trailing whitespace.
|
|
134
135
|
SQUASH_SPACES: Collapse runs of whitespace (including newlines,
|
|
135
|
-
tabs, Unicode whitespace
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
136
|
+
tabs, Unicode whitespace, and the U+001C-001F information
|
|
137
|
+
separators) into single spaces and trim the edges. Also
|
|
138
|
+
deletes invisible format characters — zero-width
|
|
139
|
+
space/joiners (U+200B-U+200D), BOM (U+FEFF), soft hyphen
|
|
140
|
+
(U+00AD), and word joiners (U+2060-U+2064) — so that text
|
|
141
|
+
pasted from PDFs or web pages (``Gm\\u00ADbH``) compares
|
|
142
|
+
equal to its clean form (``GmbH``). Cleans up whitespace
|
|
143
|
+
introduced by earlier steps (e.g. category replacement).
|
|
139
144
|
CASEFOLD: Unicode full casefold (e.g. ``ß → ss``). This is
|
|
140
145
|
*not* the same as `str.lower()` — casefold is the correct
|
|
141
146
|
tool for case-insensitive comparison across Unicode.
|
|
@@ -3,15 +3,11 @@ from typing import Optional
|
|
|
3
3
|
|
|
4
4
|
from rigour._core import codepoint_script as _codepoint_script
|
|
5
5
|
from rigour._core import common_scripts as _common_scripts
|
|
6
|
+
from rigour._core import should_ascii as _should_ascii
|
|
6
7
|
from rigour._core import text_scripts as _text_scripts
|
|
7
8
|
from rigour.util import MEMO_MEDIUM
|
|
8
9
|
|
|
9
10
|
|
|
10
|
-
#: Scripts whose romanisation is well-defined enough that
|
|
11
|
-
#: rigour will automatically transliterate them to ASCII via
|
|
12
|
-
#: :func:`rigour.text.translit.maybe_ascii`.
|
|
13
|
-
LATINIZE_SCRIPTS = {"Hangul", "Cyrillic", "Greek", "Armenian", "Latin", "Georgian"}
|
|
14
|
-
|
|
15
11
|
#: Letter-based writing systems with explicit vowels — these
|
|
16
12
|
#: transliterate reliably to Latin without language hints.
|
|
17
13
|
#: Excludes Hangul (syllabic) and the dense logographic scripts.
|
|
@@ -99,38 +95,14 @@ def common_scripts(a: str, b: str) -> set[str]:
|
|
|
99
95
|
return _common_scripts(a, b)
|
|
100
96
|
|
|
101
97
|
|
|
102
|
-
@lru_cache(maxsize=MEMO_MEDIUM)
|
|
103
|
-
def can_latinize_cp(cp: int) -> Optional[bool]:
|
|
104
|
-
"""Check whether a single codepoint can be latinized.
|
|
105
|
-
|
|
106
|
-
Three-valued: distinguishing-script-bearing codepoints get a
|
|
107
|
-
True/False; others (digits, punctuation, combining marks)
|
|
108
|
-
return `None` because the question doesn't apply to them.
|
|
109
|
-
|
|
110
|
-
Args:
|
|
111
|
-
cp: Codepoint as an integer.
|
|
112
|
-
|
|
113
|
-
Returns:
|
|
114
|
-
`True` if the codepoint's script is in
|
|
115
|
-
:data:`LATINIZE_SCRIPTS`, `False` if it has a real script
|
|
116
|
-
outside that set, `None` for non-alphanumeric or
|
|
117
|
-
Common/Inherited/Unknown codepoints.
|
|
118
|
-
"""
|
|
119
|
-
char = chr(cp)
|
|
120
|
-
if not char.isalnum():
|
|
121
|
-
return None
|
|
122
|
-
script = codepoint_script(cp)
|
|
123
|
-
if script is None or script in ("Common", "Inherited"):
|
|
124
|
-
return None
|
|
125
|
-
return script in LATINIZE_SCRIPTS
|
|
126
|
-
|
|
127
|
-
|
|
128
98
|
def can_latinize(word: str) -> bool:
|
|
129
99
|
"""Check whether every script in a word is latinizable.
|
|
130
100
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
101
|
+
Alias for `should_ascii` in [rigour.text.translit][] — the two
|
|
102
|
+
call the same Rust predicate, which admits Latin, Cyrillic,
|
|
103
|
+
Greek, Armenian, Georgian, and Hangul. When True, `maybe_ascii`
|
|
104
|
+
will produce an ASCII output; when False, it returns the input
|
|
105
|
+
unchanged.
|
|
134
106
|
Characters with no distinguishing script (digits, punctuation,
|
|
135
107
|
spaces, combining marks) are ignored. Empty input and
|
|
136
108
|
pure-Common input (`"123"`) return True vacuously.
|
|
@@ -139,10 +111,10 @@ def can_latinize(word: str) -> bool:
|
|
|
139
111
|
word: A string.
|
|
140
112
|
|
|
141
113
|
Returns:
|
|
142
|
-
True iff every distinguishing script is
|
|
143
|
-
|
|
114
|
+
True iff every distinguishing script is admitted for
|
|
115
|
+
transliteration.
|
|
144
116
|
"""
|
|
145
|
-
return
|
|
117
|
+
return _should_ascii(word)
|
|
146
118
|
|
|
147
119
|
|
|
148
120
|
def is_modern_alphabet(word: str) -> bool:
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
"""Opportunistic transliteration: `should_ascii` + `maybe_ascii`.
|
|
2
2
|
|
|
3
3
|
The minimal transliteration surface rigour exposes going forward.
|
|
4
|
-
Covers only
|
|
5
|
-
|
|
6
|
-
outside that set passes through unchanged (default) or
|
|
7
|
-
empty (``drop=True``).
|
|
4
|
+
Covers only six admitted scripts (Latin, Cyrillic, Greek, Armenian,
|
|
5
|
+
Georgian, Hangul); the admission policy is defined in the Rust core.
|
|
6
|
+
Anything outside that set passes through unchanged (default) or
|
|
7
|
+
becomes empty (``drop=True``).
|
|
8
8
|
|
|
9
9
|
For broader-script, lossy transliteration (Han, Arabic, Devanagari,
|
|
10
10
|
etc.) use `normality.ascii_text` / `normality.latinize_text` —
|