confusable 0.1.0a1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- confusable-0.1.0a1/LICENSE +21 -0
- confusable-0.1.0a1/LICENSES/Unicode-3.0.txt +39 -0
- confusable-0.1.0a1/PKG-INFO +203 -0
- confusable-0.1.0a1/README.md +171 -0
- confusable-0.1.0a1/pyproject.toml +137 -0
- confusable-0.1.0a1/pyproject.toml.orig +125 -0
- confusable-0.1.0a1/src/confusable/__init__.py +52 -0
- confusable-0.1.0a1/src/confusable/_bidi.py +584 -0
- confusable-0.1.0a1/src/confusable/_data/__init__.py +8 -0
- confusable-0.1.0a1/src/confusable/_data/bidi.py +2203 -0
- confusable-0.1.0a1/src/confusable/_data/confusables.py +6722 -0
- confusable-0.1.0a1/src/confusable/_data/emoji.py +14 -0
- confusable-0.1.0a1/src/confusable/_data/graphemes.py +2843 -0
- confusable-0.1.0a1/src/confusable/_data/homoglyphs.py +19106 -0
- confusable-0.1.0a1/src/confusable/_data/identifiers.py +8404 -0
- confusable-0.1.0a1/src/confusable/_data/joining.py +1098 -0
- confusable-0.1.0a1/src/confusable/_data/normalization.py +4064 -0
- confusable-0.1.0a1/src/confusable/_data/properties.py +4368 -0
- confusable-0.1.0a1/src/confusable/_data/scripts.py +2036 -0
- confusable-0.1.0a1/src/confusable/_data/variation.py +2218 -0
- confusable-0.1.0a1/src/confusable/_graphemes.py +225 -0
- confusable-0.1.0a1/src/confusable/_identifiers.py +189 -0
- confusable-0.1.0a1/src/confusable/_invisible.py +243 -0
- confusable-0.1.0a1/src/confusable/_joining.py +150 -0
- confusable-0.1.0a1/src/confusable/_lookup.py +33 -0
- confusable-0.1.0a1/src/confusable/_marks.py +146 -0
- confusable-0.1.0a1/src/confusable/_normalize.py +119 -0
- confusable-0.1.0a1/src/confusable/_numbers.py +58 -0
- confusable-0.1.0a1/src/confusable/_restriction.py +85 -0
- confusable-0.1.0a1/src/confusable/_scripts.py +78 -0
- confusable-0.1.0a1/src/confusable/_skeleton.py +153 -0
- confusable-0.1.0a1/src/confusable/compat/__init__.py +1 -0
- confusable-0.1.0a1/src/confusable/compat/homoglyphs/__init__.py +29 -0
- confusable-0.1.0a1/src/confusable/compat/homoglyphs/categories.py +75 -0
- confusable-0.1.0a1/src/confusable/compat/homoglyphs/confusables.py +138 -0
- confusable-0.1.0a1/src/confusable/py.typed +0 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 BLIPBOX
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
UNICODE LICENSE V3
|
|
2
|
+
|
|
3
|
+
COPYRIGHT AND PERMISSION NOTICE
|
|
4
|
+
|
|
5
|
+
Copyright © 1991-2026 Unicode, Inc.
|
|
6
|
+
|
|
7
|
+
NOTICE TO USER: Carefully read the following legal agreement. BY
|
|
8
|
+
DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR
|
|
9
|
+
SOFTWARE, YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE
|
|
10
|
+
TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT
|
|
11
|
+
DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE.
|
|
12
|
+
|
|
13
|
+
Permission is hereby granted, free of charge, to any person obtaining a
|
|
14
|
+
copy of data files and any associated documentation (the "Data Files") or
|
|
15
|
+
software and any associated documentation (the "Software") to deal in the
|
|
16
|
+
Data Files or Software without restriction, including without limitation
|
|
17
|
+
the rights to use, copy, modify, merge, publish, distribute, and/or sell
|
|
18
|
+
copies of the Data Files or Software, and to permit persons to whom the
|
|
19
|
+
Data Files or Software are furnished to do so, provided that either (a)
|
|
20
|
+
this copyright and permission notice appear with all copies of the Data
|
|
21
|
+
Files or Software, or (b) this copyright and permission notice appear in
|
|
22
|
+
associated Documentation.
|
|
23
|
+
|
|
24
|
+
THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
|
|
25
|
+
KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
26
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
|
|
27
|
+
THIRD PARTY RIGHTS.
|
|
28
|
+
|
|
29
|
+
IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE
|
|
30
|
+
BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES,
|
|
31
|
+
OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
|
|
32
|
+
WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION,
|
|
33
|
+
ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA
|
|
34
|
+
FILES OR SOFTWARE.
|
|
35
|
+
|
|
36
|
+
Except as contained in this notice, the name of a copyright holder shall
|
|
37
|
+
not be used in advertising or otherwise to promote the sale, use or other
|
|
38
|
+
dealings in these Data Files or Software without prior written
|
|
39
|
+
authorization of the copyright holder.
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: confusable
|
|
3
|
+
Version: 0.1.0a1
|
|
4
|
+
Summary: Unicode spoofing and confusable detection (UTS #39) in pure Python.
|
|
5
|
+
Keywords: unicode,confusables,homoglyph,homograph,spoofing,uts39,identifier,security
|
|
6
|
+
Author: Atilla Guzel
|
|
7
|
+
Author-email: Atilla Guzel <33524244+atillaguzel@users.noreply.github.com>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
License-File: LICENSES/Unicode-3.0.txt
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
22
|
+
Classifier: Topic :: Security
|
|
23
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
24
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
25
|
+
Classifier: Typing :: Typed
|
|
26
|
+
Requires-Python: >=3.10
|
|
27
|
+
Project-URL: Homepage, https://github.com/blip-box/confusable
|
|
28
|
+
Project-URL: Repository, https://github.com/blip-box/confusable
|
|
29
|
+
Project-URL: Issues, https://github.com/blip-box/confusable/issues
|
|
30
|
+
Project-URL: Changelog, https://github.com/blip-box/confusable/releases
|
|
31
|
+
Description-Content-Type: text/markdown
|
|
32
|
+
|
|
33
|
+
# confusable
|
|
34
|
+
|
|
35
|
+
Unicode spoofing and confusable detection for Python, following
|
|
36
|
+
[Unicode Technical Standard #39](https://www.unicode.org/reports/tr39/). Pure Python,
|
|
37
|
+
no runtime dependencies.
|
|
38
|
+
|
|
39
|
+
> **Status: pre-alpha.** Look-alike detection (`skeleton`, `match`), mixed-script
|
|
40
|
+
> detection, restriction levels, the identifier checks, the mixed-number and
|
|
41
|
+
> combining-mark checks, invisible-character detection, grapheme clusters and the
|
|
42
|
+
> compatibility module work and pass their tests. Nothing is on PyPI yet.
|
|
43
|
+
|
|
44
|
+
## Usage
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
import confusable
|
|
48
|
+
|
|
49
|
+
confusable.skeleton("p\u0430ypal") # 'paypal' (with CYRILLIC SMALL LETTER A)
|
|
50
|
+
confusable.match("paypal", "p\u0430ypal") # Match.MIXED_SCRIPT
|
|
51
|
+
confusable.match("rn", "m") # Match.SINGLE_SCRIPT
|
|
52
|
+
confusable.match("paypal", "paypa1x") # None: not confusable
|
|
53
|
+
|
|
54
|
+
# The skeleton follows display order, so a right-to-left override is seen
|
|
55
|
+
# the way a reader sees it:
|
|
56
|
+
confusable.skeleton("invoice\u202egpj.exe") # 'invoiceexe.jpg'
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Restriction levels (UTS #39 section 5.2) rank how safely a string mixes scripts,
|
|
60
|
+
from `ASCII_ONLY` to `UNRESTRICTED`, using the General Security Profile:
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
confusable.restriction_level("caf\u00e9") # RestrictionLevel.SINGLE_SCRIPT
|
|
64
|
+
confusable.restriction_level("abc\u3042") # HIGHLY_RESTRICTIVE: Latin with Japanese
|
|
65
|
+
confusable.restriction_level(
|
|
66
|
+
"p\u0430ypal"
|
|
67
|
+
) # MINIMALLY_RESTRICTIVE: Latin with Cyrillic
|
|
68
|
+
confusable.is_mixed_script("p\u0430ypal") # True
|
|
69
|
+
|
|
70
|
+
# Levels are ordered, most restrictive first:
|
|
71
|
+
confusable.restriction_level(name) <= confusable.RestrictionLevel.HIGHLY_RESTRICTIVE
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
The identifier checks follow UTS #39's General Security Profile, under canonical
|
|
75
|
+
equivalence, so the answer is the same however the text is encoded:
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
confusable.is_allowed("paypal") # True
|
|
79
|
+
confusable.is_allowed("pay pal") # False: a space is Restricted
|
|
80
|
+
confusable.is_allowed("\u0114") # True: U+0114 is Restricted, E + U+0306 is not
|
|
81
|
+
confusable.identifier_types("\u200d") # {IdentifierType.DEFAULT_IGNORABLE}
|
|
82
|
+
confusable.isxidstart("_"), confusable.isxidcontinue("_") # (False, True)
|
|
83
|
+
|
|
84
|
+
# Persian, Malayalam and Sinhala need ZERO WIDTH (NON-)JOINER in some words. The
|
|
85
|
+
# profile excludes both; joining_controls=True allows them only where UTS #39
|
|
86
|
+
# section 3.1.1.1 says they change how the word looks:
|
|
87
|
+
confusable.is_allowed("\u0dc1\u0dca\u200d\u0dbb\u0dd3", joining_controls=True) # True
|
|
88
|
+
confusable.is_allowed("pay\u200dpal", joining_controls=True) # False
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
Three more checks from UTS #39 sections 5.3 and 5.4 catch digits from different
|
|
92
|
+
number systems and combining marks that pile up or hide:
|
|
93
|
+
|
|
94
|
+
```python
|
|
95
|
+
confusable.has_mixed_numbers("1\u0661") # True: ASCII and Arabic-Indic digits
|
|
96
|
+
confusable.has_unlikely_marks("e\u0301\u0301") # True: the same accent twice
|
|
97
|
+
confusable.has_hidden_overlay("i\u0307") # True: a dot above, lost in i's own dot
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Invisible and control characters, for text that is not held to the identifier
|
|
101
|
+
profile, such as display names and file names:
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
confusable.find_invisible("pay\u200bpal") # [(3, Invisible.DEFAULT_IGNORABLE)]: ZWSP
|
|
105
|
+
confusable.find_invisible("invoice\u202egpj.exe") # [(7, Invisible.BIDI_CONTROL)]
|
|
106
|
+
confusable.find_invisible("hi\U000e0068\U000e0069") # two Invisible.TAG: hidden "hi"
|
|
107
|
+
confusable.has_invisible(
|
|
108
|
+
"\U0001f468\u200d\U0001f469\u200d\U0001f467"
|
|
109
|
+
) # False: an emoji
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
Grapheme clusters, the characters a reader sees, and truncation that never cuts
|
|
113
|
+
one apart. A cluster can be any size, so pair a grapheme limit with a size limit:
|
|
114
|
+
|
|
115
|
+
```python
|
|
116
|
+
[
|
|
117
|
+
str(g) for g in confusable.iter_graphemes("e\u0301\U0001f1fa\U0001f1f8")
|
|
118
|
+
] # ['e' + accent, a flag]
|
|
119
|
+
confusable.truncate(name, 32) # at most 32 graphemes
|
|
120
|
+
confusable.truncate(bio, 300, max_utf8_bytes=3000) # and at most 3000 bytes
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
`skeleton(text, direction)` takes the paragraph direction text is shown in:
|
|
124
|
+
`"ltr"` (the default, UTS #39's `skeleton`), `"rtl"`, `"auto"`, or `None` for
|
|
125
|
+
UTS #39's `internalSkeleton`, which ignores display order. Skeletons are for
|
|
126
|
+
comparison, not display, and change with the Unicode version
|
|
127
|
+
(`confusable.unicode_version`), so recompute any you store when it changes.
|
|
128
|
+
|
|
129
|
+
### Moving from confusable_homoglyphs
|
|
130
|
+
|
|
131
|
+
`confusable.compat.homoglyphs` has the API of `confusable_homoglyphs` 3.3.1, so
|
|
132
|
+
existing code switches by changing one import:
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
from confusable.compat.homoglyphs import categories, confusables
|
|
136
|
+
|
|
137
|
+
confusables.is_dangerous("p\u0430ypal") # True
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
It returns what that package returns, with current Unicode data. It also finds
|
|
141
|
+
Arabic and Hebrew look-alikes, which that package's data misses: its generator
|
|
142
|
+
copies the invisible LEFT-TO-RIGHT MARKs that `confusables.txt` puts around
|
|
143
|
+
right-to-left characters in its comments. For new code, prefer `match` and
|
|
144
|
+
`restriction_level`, which follow UTS #39.
|
|
145
|
+
|
|
146
|
+
## What it is for
|
|
147
|
+
|
|
148
|
+
`confusable` decides whether user-supplied text is safe to accept as an identity: a
|
|
149
|
+
username, display name, project name, slug or filename. Planned for v1:
|
|
150
|
+
|
|
151
|
+
- **Look-alike detection** with the UTS #39 skeleton algorithm, so that `pаypal`
|
|
152
|
+
(with a Cyrillic `а`) and `paypal` compare as confusable.
|
|
153
|
+
- **Restriction levels and mixed-script checks** from UTS #39 section 5, built on the
|
|
154
|
+
Unicode `Scripts` and `Script_Extensions` properties.
|
|
155
|
+
- **Identifier status and type** from the UTS #39 General Security Profile.
|
|
156
|
+
- **Mixed numbers and suspicious combining marks** from UTS #39 sections 5.3 and
|
|
157
|
+
5.4: digits from different decimal systems, repeated or stacked marks, and a dot
|
|
158
|
+
above hidden by the letter under it.
|
|
159
|
+
- **Invisible and control characters**: default-ignorable code points, zero-width
|
|
160
|
+
characters, bidirectional controls, and Tag-block "ASCII smuggling", while still
|
|
161
|
+
accepting valid emoji tag sequences such as subdivision flags.
|
|
162
|
+
- **UAX #31 identifier checks** equivalent to Python 3.15's `unicodedata.isxidstart()`
|
|
163
|
+
and `unicodedata.isxidcontinue()`, on Python 3.10 and later.
|
|
164
|
+
- **Grapheme clusters**: iteration and truncation that never split a user-perceived
|
|
165
|
+
character, equivalent to Python 3.15's `unicodedata.iter_graphemes()`.
|
|
166
|
+
- **A compatibility module**, `confusable.compat.homoglyphs`, with the same API as
|
|
167
|
+
`confusable_homoglyphs` 3.3.1, so existing code can switch by changing one import,
|
|
168
|
+
and that package's tests run against it.
|
|
169
|
+
|
|
170
|
+
The Unicode data is pinned to a single Unicode version (18.0.0) and generated into the
|
|
171
|
+
package, so installing needs no network access and no compiler.
|
|
172
|
+
|
|
173
|
+
## Correctness
|
|
174
|
+
|
|
175
|
+
A look-alike that slips through is a security bug, so the tests come first. Every
|
|
176
|
+
operation is checked against two independent implementations of UTS #39, ICU and
|
|
177
|
+
the Rust `unicode-security` crate, over every Unicode code point and a corpus of
|
|
178
|
+
known attacks and generated strings. Both oracles run at the pinned Unicode
|
|
179
|
+
version, and every place where they differ is explained by a specific rule. See
|
|
180
|
+
[tools/oracles/README.md](tools/oracles/README.md).
|
|
181
|
+
|
|
182
|
+
## Development
|
|
183
|
+
|
|
184
|
+
```bash
|
|
185
|
+
uv sync
|
|
186
|
+
uv run pre-commit install
|
|
187
|
+
uv run pytest
|
|
188
|
+
uv run pre-commit run --all-files
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
[CHARTER.md](CHARTER.md) says what the project is for and the principles every
|
|
192
|
+
change is held to. [AGENTS.md](AGENTS.md) has the conventions, for people and
|
|
193
|
+
coding agents alike.
|
|
194
|
+
|
|
195
|
+
## Security
|
|
196
|
+
|
|
197
|
+
To report a vulnerability, see [SECURITY.md](.github/SECURITY.md).
|
|
198
|
+
|
|
199
|
+
## License
|
|
200
|
+
|
|
201
|
+
MIT. See [LICENSE](LICENSE). The Unicode data the package and its test fixtures are
|
|
202
|
+
generated from is under the Unicode License v3; see
|
|
203
|
+
[LICENSES/Unicode-3.0.txt](LICENSES/Unicode-3.0.txt).
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
# confusable
|
|
2
|
+
|
|
3
|
+
Unicode spoofing and confusable detection for Python, following
|
|
4
|
+
[Unicode Technical Standard #39](https://www.unicode.org/reports/tr39/). Pure Python,
|
|
5
|
+
no runtime dependencies.
|
|
6
|
+
|
|
7
|
+
> **Status: pre-alpha.** Look-alike detection (`skeleton`, `match`), mixed-script
|
|
8
|
+
> detection, restriction levels, the identifier checks, the mixed-number and
|
|
9
|
+
> combining-mark checks, invisible-character detection, grapheme clusters and the
|
|
10
|
+
> compatibility module work and pass their tests. Nothing is on PyPI yet.
|
|
11
|
+
|
|
12
|
+
## Usage
|
|
13
|
+
|
|
14
|
+
```python
|
|
15
|
+
import confusable
|
|
16
|
+
|
|
17
|
+
confusable.skeleton("p\u0430ypal") # 'paypal' (with CYRILLIC SMALL LETTER A)
|
|
18
|
+
confusable.match("paypal", "p\u0430ypal") # Match.MIXED_SCRIPT
|
|
19
|
+
confusable.match("rn", "m") # Match.SINGLE_SCRIPT
|
|
20
|
+
confusable.match("paypal", "paypa1x") # None: not confusable
|
|
21
|
+
|
|
22
|
+
# The skeleton follows display order, so a right-to-left override is seen
|
|
23
|
+
# the way a reader sees it:
|
|
24
|
+
confusable.skeleton("invoice\u202egpj.exe") # 'invoiceexe.jpg'
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
Restriction levels (UTS #39 section 5.2) rank how safely a string mixes scripts,
|
|
28
|
+
from `ASCII_ONLY` to `UNRESTRICTED`, using the General Security Profile:
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
confusable.restriction_level("caf\u00e9") # RestrictionLevel.SINGLE_SCRIPT
|
|
32
|
+
confusable.restriction_level("abc\u3042") # HIGHLY_RESTRICTIVE: Latin with Japanese
|
|
33
|
+
confusable.restriction_level(
|
|
34
|
+
"p\u0430ypal"
|
|
35
|
+
) # MINIMALLY_RESTRICTIVE: Latin with Cyrillic
|
|
36
|
+
confusable.is_mixed_script("p\u0430ypal") # True
|
|
37
|
+
|
|
38
|
+
# Levels are ordered, most restrictive first:
|
|
39
|
+
confusable.restriction_level(name) <= confusable.RestrictionLevel.HIGHLY_RESTRICTIVE
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
The identifier checks follow UTS #39's General Security Profile, under canonical
|
|
43
|
+
equivalence, so the answer is the same however the text is encoded:
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
confusable.is_allowed("paypal") # True
|
|
47
|
+
confusable.is_allowed("pay pal") # False: a space is Restricted
|
|
48
|
+
confusable.is_allowed("\u0114") # True: U+0114 is Restricted, E + U+0306 is not
|
|
49
|
+
confusable.identifier_types("\u200d") # {IdentifierType.DEFAULT_IGNORABLE}
|
|
50
|
+
confusable.isxidstart("_"), confusable.isxidcontinue("_") # (False, True)
|
|
51
|
+
|
|
52
|
+
# Persian, Malayalam and Sinhala need ZERO WIDTH (NON-)JOINER in some words. The
|
|
53
|
+
# profile excludes both; joining_controls=True allows them only where UTS #39
|
|
54
|
+
# section 3.1.1.1 says they change how the word looks:
|
|
55
|
+
confusable.is_allowed("\u0dc1\u0dca\u200d\u0dbb\u0dd3", joining_controls=True) # True
|
|
56
|
+
confusable.is_allowed("pay\u200dpal", joining_controls=True) # False
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Three more checks from UTS #39 sections 5.3 and 5.4 catch digits from different
|
|
60
|
+
number systems and combining marks that pile up or hide:
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
confusable.has_mixed_numbers("1\u0661") # True: ASCII and Arabic-Indic digits
|
|
64
|
+
confusable.has_unlikely_marks("e\u0301\u0301") # True: the same accent twice
|
|
65
|
+
confusable.has_hidden_overlay("i\u0307") # True: a dot above, lost in i's own dot
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Invisible and control characters, for text that is not held to the identifier
|
|
69
|
+
profile, such as display names and file names:
|
|
70
|
+
|
|
71
|
+
```python
|
|
72
|
+
confusable.find_invisible("pay\u200bpal") # [(3, Invisible.DEFAULT_IGNORABLE)]: ZWSP
|
|
73
|
+
confusable.find_invisible("invoice\u202egpj.exe") # [(7, Invisible.BIDI_CONTROL)]
|
|
74
|
+
confusable.find_invisible("hi\U000e0068\U000e0069") # two Invisible.TAG: hidden "hi"
|
|
75
|
+
confusable.has_invisible(
|
|
76
|
+
"\U0001f468\u200d\U0001f469\u200d\U0001f467"
|
|
77
|
+
) # False: an emoji
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Grapheme clusters, the characters a reader sees, and truncation that never cuts
|
|
81
|
+
one apart. A cluster can be any size, so pair a grapheme limit with a size limit:
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
[
|
|
85
|
+
str(g) for g in confusable.iter_graphemes("e\u0301\U0001f1fa\U0001f1f8")
|
|
86
|
+
] # ['e' + accent, a flag]
|
|
87
|
+
confusable.truncate(name, 32) # at most 32 graphemes
|
|
88
|
+
confusable.truncate(bio, 300, max_utf8_bytes=3000) # and at most 3000 bytes
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
`skeleton(text, direction)` takes the paragraph direction text is shown in:
|
|
92
|
+
`"ltr"` (the default, UTS #39's `skeleton`), `"rtl"`, `"auto"`, or `None` for
|
|
93
|
+
UTS #39's `internalSkeleton`, which ignores display order. Skeletons are for
|
|
94
|
+
comparison, not display, and change with the Unicode version
|
|
95
|
+
(`confusable.unicode_version`), so recompute any you store when it changes.
|
|
96
|
+
|
|
97
|
+
### Moving from confusable_homoglyphs
|
|
98
|
+
|
|
99
|
+
`confusable.compat.homoglyphs` has the API of `confusable_homoglyphs` 3.3.1, so
|
|
100
|
+
existing code switches by changing one import:
|
|
101
|
+
|
|
102
|
+
```python
|
|
103
|
+
from confusable.compat.homoglyphs import categories, confusables
|
|
104
|
+
|
|
105
|
+
confusables.is_dangerous("p\u0430ypal") # True
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
It returns what that package returns, with current Unicode data. It also finds
|
|
109
|
+
Arabic and Hebrew look-alikes, which that package's data misses: its generator
|
|
110
|
+
copies the invisible LEFT-TO-RIGHT MARKs that `confusables.txt` puts around
|
|
111
|
+
right-to-left characters in its comments. For new code, prefer `match` and
|
|
112
|
+
`restriction_level`, which follow UTS #39.
|
|
113
|
+
|
|
114
|
+
## What it is for
|
|
115
|
+
|
|
116
|
+
`confusable` decides whether user-supplied text is safe to accept as an identity: a
|
|
117
|
+
username, display name, project name, slug or filename. Planned for v1:
|
|
118
|
+
|
|
119
|
+
- **Look-alike detection** with the UTS #39 skeleton algorithm, so that `pаypal`
|
|
120
|
+
(with a Cyrillic `а`) and `paypal` compare as confusable.
|
|
121
|
+
- **Restriction levels and mixed-script checks** from UTS #39 section 5, built on the
|
|
122
|
+
Unicode `Scripts` and `Script_Extensions` properties.
|
|
123
|
+
- **Identifier status and type** from the UTS #39 General Security Profile.
|
|
124
|
+
- **Mixed numbers and suspicious combining marks** from UTS #39 sections 5.3 and
|
|
125
|
+
5.4: digits from different decimal systems, repeated or stacked marks, and a dot
|
|
126
|
+
above hidden by the letter under it.
|
|
127
|
+
- **Invisible and control characters**: default-ignorable code points, zero-width
|
|
128
|
+
characters, bidirectional controls, and Tag-block "ASCII smuggling", while still
|
|
129
|
+
accepting valid emoji tag sequences such as subdivision flags.
|
|
130
|
+
- **UAX #31 identifier checks** equivalent to Python 3.15's `unicodedata.isxidstart()`
|
|
131
|
+
and `unicodedata.isxidcontinue()`, on Python 3.10 and later.
|
|
132
|
+
- **Grapheme clusters**: iteration and truncation that never split a user-perceived
|
|
133
|
+
character, equivalent to Python 3.15's `unicodedata.iter_graphemes()`.
|
|
134
|
+
- **A compatibility module**, `confusable.compat.homoglyphs`, with the same API as
|
|
135
|
+
`confusable_homoglyphs` 3.3.1, so existing code can switch by changing one import,
|
|
136
|
+
and that package's tests run against it.
|
|
137
|
+
|
|
138
|
+
The Unicode data is pinned to a single Unicode version (18.0.0) and generated into the
|
|
139
|
+
package, so installing needs no network access and no compiler.
|
|
140
|
+
|
|
141
|
+
## Correctness
|
|
142
|
+
|
|
143
|
+
A look-alike that slips through is a security bug, so the tests come first. Every
|
|
144
|
+
operation is checked against two independent implementations of UTS #39, ICU and
|
|
145
|
+
the Rust `unicode-security` crate, over every Unicode code point and a corpus of
|
|
146
|
+
known attacks and generated strings. Both oracles run at the pinned Unicode
|
|
147
|
+
version, and every place where they differ is explained by a specific rule. See
|
|
148
|
+
[tools/oracles/README.md](tools/oracles/README.md).
|
|
149
|
+
|
|
150
|
+
## Development
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
uv sync
|
|
154
|
+
uv run pre-commit install
|
|
155
|
+
uv run pytest
|
|
156
|
+
uv run pre-commit run --all-files
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
[CHARTER.md](CHARTER.md) says what the project is for and the principles every
|
|
160
|
+
change is held to. [AGENTS.md](AGENTS.md) has the conventions, for people and
|
|
161
|
+
coding agents alike.
|
|
162
|
+
|
|
163
|
+
## Security
|
|
164
|
+
|
|
165
|
+
To report a vulnerability, see [SECURITY.md](.github/SECURITY.md).
|
|
166
|
+
|
|
167
|
+
## License
|
|
168
|
+
|
|
169
|
+
MIT. See [LICENSE](LICENSE). The Unicode data the package and its test fixtures are
|
|
170
|
+
generated from is under the Unicode License v3; see
|
|
171
|
+
[LICENSES/Unicode-3.0.txt](LICENSES/Unicode-3.0.txt).
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "confusable"
|
|
3
|
+
version = "0.1.0a1"
|
|
4
|
+
description = "Unicode spoofing and confusable detection (UTS #39) in pure Python."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = [
|
|
8
|
+
"LICENSE",
|
|
9
|
+
"LICENSES/Unicode-3.0.txt",
|
|
10
|
+
]
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
dependencies = []
|
|
13
|
+
keywords = [
|
|
14
|
+
"unicode",
|
|
15
|
+
"confusables",
|
|
16
|
+
"homoglyph",
|
|
17
|
+
"homograph",
|
|
18
|
+
"spoofing",
|
|
19
|
+
"uts39",
|
|
20
|
+
"identifier",
|
|
21
|
+
"security",
|
|
22
|
+
]
|
|
23
|
+
classifiers = [
|
|
24
|
+
"Development Status :: 3 - Alpha",
|
|
25
|
+
"Intended Audience :: Developers",
|
|
26
|
+
"Operating System :: OS Independent",
|
|
27
|
+
"Programming Language :: Python :: 3",
|
|
28
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
29
|
+
"Programming Language :: Python :: 3.10",
|
|
30
|
+
"Programming Language :: Python :: 3.11",
|
|
31
|
+
"Programming Language :: Python :: 3.12",
|
|
32
|
+
"Programming Language :: Python :: 3.13",
|
|
33
|
+
"Programming Language :: Python :: 3.14",
|
|
34
|
+
"Programming Language :: Python :: 3.15",
|
|
35
|
+
"Topic :: Security",
|
|
36
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
37
|
+
"Topic :: Text Processing :: Linguistic",
|
|
38
|
+
"Typing :: Typed",
|
|
39
|
+
]
|
|
40
|
+
|
|
41
|
+
[[project.authors]]
|
|
42
|
+
name = "Atilla Guzel"
|
|
43
|
+
email = "33524244+atillaguzel@users.noreply.github.com"
|
|
44
|
+
|
|
45
|
+
[project.urls]
|
|
46
|
+
Homepage = "https://github.com/blip-box/confusable"
|
|
47
|
+
Repository = "https://github.com/blip-box/confusable"
|
|
48
|
+
Issues = "https://github.com/blip-box/confusable/issues"
|
|
49
|
+
Changelog = "https://github.com/blip-box/confusable/releases"
|
|
50
|
+
|
|
51
|
+
[build-system]
|
|
52
|
+
requires = ["uv_build>=0.12.24,<0.14.0"]
|
|
53
|
+
build-backend = "uv_build"
|
|
54
|
+
|
|
55
|
+
[dependency-groups]
|
|
56
|
+
test = [
|
|
57
|
+
"hypothesis>=6.168.5",
|
|
58
|
+
"pytest>=9.1.1",
|
|
59
|
+
]
|
|
60
|
+
lint = [
|
|
61
|
+
{ include-group = "test" },
|
|
62
|
+
"mypy>=2.4.0",
|
|
63
|
+
"pre-commit>=4.6.2",
|
|
64
|
+
"virtualenv>=21.14.13",
|
|
65
|
+
]
|
|
66
|
+
|
|
67
|
+
[tool.uv]
|
|
68
|
+
default-groups = [
|
|
69
|
+
"test",
|
|
70
|
+
"lint",
|
|
71
|
+
]
|
|
72
|
+
required-version = ">=0.13.4"
|
|
73
|
+
|
|
74
|
+
[tool.pytest.ini_options]
|
|
75
|
+
minversion = "9.0"
|
|
76
|
+
addopts = [
|
|
77
|
+
"-ra",
|
|
78
|
+
"--strict-markers",
|
|
79
|
+
"--strict-config",
|
|
80
|
+
"--import-mode=importlib",
|
|
81
|
+
]
|
|
82
|
+
testpaths = ["tests"]
|
|
83
|
+
pythonpath = ["."]
|
|
84
|
+
xfail_strict = true
|
|
85
|
+
filterwarnings = ["error"]
|
|
86
|
+
|
|
87
|
+
[tool.mypy]
|
|
88
|
+
python_version = "3.10"
|
|
89
|
+
strict = true
|
|
90
|
+
warn_unreachable = true
|
|
91
|
+
files = [
|
|
92
|
+
"src",
|
|
93
|
+
"tests",
|
|
94
|
+
"tools",
|
|
95
|
+
]
|
|
96
|
+
exclude = [
|
|
97
|
+
"^tools/oracles/rust/(vendor|target)/",
|
|
98
|
+
'^tools/oracles/icu/\.build/',
|
|
99
|
+
'^tools/oracles/homoglyphs/\.build/',
|
|
100
|
+
'^tools/\.cache/',
|
|
101
|
+
]
|
|
102
|
+
|
|
103
|
+
[tool.ruff]
|
|
104
|
+
line-length = 88
|
|
105
|
+
extend-exclude = ["src/confusable/_data/"]
|
|
106
|
+
force-exclude = true
|
|
107
|
+
|
|
108
|
+
[tool.ruff.lint]
|
|
109
|
+
select = [
|
|
110
|
+
"E",
|
|
111
|
+
"F",
|
|
112
|
+
"W",
|
|
113
|
+
"D",
|
|
114
|
+
"I",
|
|
115
|
+
"UP",
|
|
116
|
+
"N",
|
|
117
|
+
"B",
|
|
118
|
+
"PT",
|
|
119
|
+
"RUF",
|
|
120
|
+
"PLE25",
|
|
121
|
+
]
|
|
122
|
+
ignore = [
|
|
123
|
+
"D203",
|
|
124
|
+
"D213",
|
|
125
|
+
]
|
|
126
|
+
|
|
127
|
+
[tool.ruff.lint.per-file-ignores]
|
|
128
|
+
"tests/**" = [
|
|
129
|
+
"D",
|
|
130
|
+
"RUF001",
|
|
131
|
+
"RUF002",
|
|
132
|
+
"RUF003",
|
|
133
|
+
]
|
|
134
|
+
"tools/**" = ["D1"]
|
|
135
|
+
|
|
136
|
+
[tool.ruff.format]
|
|
137
|
+
docstring-code-format = true
|