url-normalize 3.0.1__tar.gz → 3.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- url_normalize-3.1.0/PKG-INFO +219 -0
- url_normalize-3.1.0/README.md +188 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/pyproject.toml +8 -7
- {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_cli.py +21 -37
- {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_normalize_host.py +37 -9
- {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_normalize_query_filters.py +64 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_url_humanize.py +4 -2
- {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_url_normalize.py +25 -1
- {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/__init__.py +1 -1
- {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/cli.py +4 -12
- {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/normalize_host.py +5 -7
- {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/normalize_query.py +6 -1
- {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/tools.py +5 -8
- {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/url_humanize.py +1 -4
- {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/url_normalize.py +3 -8
- url_normalize-3.1.0/url_normalize.egg-info/PKG-INFO +219 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize.egg-info/requires.txt +1 -1
- url_normalize-3.0.1/PKG-INFO +0 -215
- url_normalize-3.0.1/README.md +0 -184
- url_normalize-3.0.1/url_normalize.egg-info/PKG-INFO +0 -215
- {url_normalize-3.0.1 → url_normalize-3.1.0}/LICENSE +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/setup.cfg +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_deconstruct_url.py +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_generic_url_cleanup.py +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_normalize_fragment.py +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_normalize_path.py +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_normalize_port.py +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_normalize_query.py +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_normalize_scheme.py +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_normalize_userinfo.py +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_provide_url_domain.py +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_provide_url_scheme.py +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_reconstruct_url.py +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_tools.py +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/generic_url_cleanup.py +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/normalize_fragment.py +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/normalize_path.py +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/normalize_port.py +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/normalize_scheme.py +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/normalize_userinfo.py +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/param_allowlist.py +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/provide_url_domain.py +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/provide_url_scheme.py +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/py.typed +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize.egg-info/SOURCES.txt +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize.egg-info/dependency_links.txt +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize.egg-info/entry_points.txt +0 -0
- {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: url-normalize
|
|
3
|
+
Version: 3.1.0
|
|
4
|
+
Summary: URL normalization for Python
|
|
5
|
+
Author-email: Nikolay Panov <github@npanov.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/niksite/url-normalize
|
|
8
|
+
Project-URL: Repository, https://github.com/niksite/url-normalize
|
|
9
|
+
Project-URL: Issues, https://github.com/niksite/url-normalize/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/niksite/url-normalize/blob/master/CHANGELOG.md
|
|
11
|
+
Keywords: url,normalization,normalize,normalizer
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
19
|
+
Requires-Python: >=3.11
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Requires-Dist: idna>=3.20
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: mypy; extra == "dev"
|
|
25
|
+
Requires-Dist: pre-commit; extra == "dev"
|
|
26
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
27
|
+
Requires-Dist: pytest-socket; extra == "dev"
|
|
28
|
+
Requires-Dist: pytest; extra == "dev"
|
|
29
|
+
Requires-Dist: ruff; extra == "dev"
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
# url-normalize
|
|
33
|
+
|
|
34
|
+
[](https://github.com/niksite/url-normalize/actions/workflows/ci.yml)
|
|
35
|
+
[](https://coveralls.io/r/niksite/url-normalize)
|
|
36
|
+
[](https://pypi.org/project/url-normalize/)
|
|
37
|
+
[](https://pypi.org/project/url-normalize/)
|
|
38
|
+
[](https://github.com/niksite/url-normalize/blob/master/LICENSE)
|
|
39
|
+
[](https://github.com/astral-sh/ruff)
|
|
40
|
+
|
|
41
|
+
Normalize URL spelling consistently for storage, comparison, and display. Use `url_normalize()` for normalized URLs and `url_humanize()` for readable display URLs.
|
|
42
|
+
|
|
43
|
+
Requires **Python 3.11 or newer**. Normalization operates on URL text; it does not fetch pages or follow redirects.
|
|
44
|
+
|
|
45
|
+
## Installation
|
|
46
|
+
|
|
47
|
+
Install the library and CLI:
|
|
48
|
+
|
|
49
|
+
```sh
|
|
50
|
+
python -m pip install url-normalize
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Or install the CLI in its own environment with [uv](https://docs.astral.sh/uv/):
|
|
54
|
+
|
|
55
|
+
```sh
|
|
56
|
+
uv tool install url-normalize
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Quick start
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
from url_normalize import url_normalize
|
|
63
|
+
|
|
64
|
+
print(url_normalize("HTTP://Example.COM:80/a/../b"))
|
|
65
|
+
# Output: http://example.com/b
|
|
66
|
+
|
|
67
|
+
print(url_normalize("bücher.de/résumé"))
|
|
68
|
+
# Output: https://xn--bcher-kva.de/r%C3%A9sum%C3%A9
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
From the terminal:
|
|
72
|
+
|
|
73
|
+
```sh
|
|
74
|
+
url-normalize "HTTP://Example.COM:80/a/../b"
|
|
75
|
+
# Output: http://example.com/b
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
## Normalization behavior
|
|
79
|
+
|
|
80
|
+
| Behavior | Input | Output |
|
|
81
|
+
| --- | --- | --- |
|
|
82
|
+
| Add a missing scheme | `example.com` | `https://example.com/` |
|
|
83
|
+
| Lowercase scheme and host; remove a default port | `HTTP://Example.COM:80/` | `http://example.com/` |
|
|
84
|
+
| Resolve dot segments | `https://example.com/a/../b` | `https://example.com/b` |
|
|
85
|
+
| Encode Unicode hosts and paths | `https://bücher.de/résumé` | `https://xn--bcher-kva.de/r%C3%A9sum%C3%A9` |
|
|
86
|
+
| Remove trailing host dots, including Unicode equivalents | `https://example.com。/` | `https://example.com/` |
|
|
87
|
+
| Preserve encoded path separators; uppercase escape hex | `https://example.com/a%2fb` | `https://example.com/a%2Fb` |
|
|
88
|
+
| Preserve query order and repeated keys | `https://example.com/?b=2&a=1&a=3` | `https://example.com/?b=2&a=1&a=3` |
|
|
89
|
+
| Preserve explicit empty query values | `https://example.com/?flag=&flag` | `https://example.com/?flag=&flag` |
|
|
90
|
+
|
|
91
|
+
In query parameters, `+`, `%2B`, and `%20` remain distinct. Parameter filtering is disabled by default. Fragments are preserved and normalized; hashbang fragments (`#!`) are converted to the `_escaped_fragment_` query parameter.
|
|
92
|
+
|
|
93
|
+
Normalization applies consistent spelling rules; it does not establish whether two URLs serve the same content. Applications using normalized URLs as persistent cache keys or identifiers should review the [changelog](CHANGELOG.md) when upgrading.
|
|
94
|
+
|
|
95
|
+
## Python API
|
|
96
|
+
|
|
97
|
+
Both functions accept a URL as `str` or `None` and return `str` or `None`. An empty string and `None` are returned unchanged. Malformed input can raise `ValueError`, including `UnicodeError` for invalid hostnames.
|
|
98
|
+
|
|
99
|
+
The following options are keyword-only and apply to both functions:
|
|
100
|
+
|
|
101
|
+
| Option | Default | Purpose |
|
|
102
|
+
| --- | --- | --- |
|
|
103
|
+
| `default_scheme` | `"https"` | Scheme to add when the URL has none. |
|
|
104
|
+
| `default_domain` | `None` | Domain to add to an absolute path such as `/images/logo.png`. |
|
|
105
|
+
| `filter_params` | `False` | Retain only allowlisted query parameters. |
|
|
106
|
+
| `param_allowlist` | `None` | Override built-in rules with a list of names or a dictionary mapping domains to lists. Requires `filter_params=True`. |
|
|
107
|
+
|
|
108
|
+
Unicode URL characters always use UTF-8 percent encoding.
|
|
109
|
+
|
|
110
|
+
### Query parameter filtering
|
|
111
|
+
|
|
112
|
+
With `filter_params=True`, only explicitly allowed parameters remain. Built-in rules cover Google, Bing, Baidu, and YouTube; see the [domain rules](url_normalize/param_allowlist.py) for their parameter names.
|
|
113
|
+
|
|
114
|
+
**Unknown domains have an empty default allowlist**, so filtering removes all their query parameters unless you supply a custom rule. A custom allowlist replaces the built-in rules.
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
from url_normalize import url_normalize
|
|
118
|
+
|
|
119
|
+
# Use the built-in Google rule.
|
|
120
|
+
result = url_normalize(
|
|
121
|
+
"www.google.com/search?q=test&utm_source=test", filter_params=True
|
|
122
|
+
)
|
|
123
|
+
print(result)
|
|
124
|
+
# Output: https://www.google.com/search?q=test
|
|
125
|
+
|
|
126
|
+
# Apply a list of allowed names to any domain.
|
|
127
|
+
result = url_normalize(
|
|
128
|
+
"example.com/?page=1&id=123&ref=test",
|
|
129
|
+
filter_params=True,
|
|
130
|
+
param_allowlist=["page", "id"],
|
|
131
|
+
)
|
|
132
|
+
print(result)
|
|
133
|
+
# Output: https://example.com/?page=1&id=123
|
|
134
|
+
|
|
135
|
+
# Use domain-specific rules, including Unicode parameter names.
|
|
136
|
+
result = url_normalize(
|
|
137
|
+
"https://example.com/?r%C3%A9sum%C3%A9=1&drop=2",
|
|
138
|
+
filter_params=True,
|
|
139
|
+
param_allowlist={"example.com": ["résumé"]},
|
|
140
|
+
)
|
|
141
|
+
print(result)
|
|
142
|
+
# Output: https://example.com/?r%C3%A9sum%C3%A9=1
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
Allowed names use the same normalization as query keys, including Unicode and spaces. Domain rules match normalized hostnames and ignore a leading `www.` and the port. Rules do not automatically apply to other subdomains.
|
|
146
|
+
|
|
147
|
+
### Default domain and scheme
|
|
148
|
+
|
|
149
|
+
Use `default_domain` for paths starting with a single `/`. URLs starting with `//` already specify their host. This option does not resolve relative paths such as `../images/logo.png` against a page URL.
|
|
150
|
+
|
|
151
|
+
```python
|
|
152
|
+
from url_normalize import url_normalize
|
|
153
|
+
|
|
154
|
+
result = url_normalize(
|
|
155
|
+
"/images/logo.png", default_domain="example.com", default_scheme="http"
|
|
156
|
+
)
|
|
157
|
+
print(result)
|
|
158
|
+
# Output: http://example.com/images/logo.png
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
### Human-readable URLs
|
|
162
|
+
|
|
163
|
+
`url_humanize()` normalizes its input, then decodes components only when the display URL normalizes back to the same result. Some escapes therefore remain encoded.
|
|
164
|
+
|
|
165
|
+
```python
|
|
166
|
+
from url_normalize import url_humanize, url_normalize
|
|
167
|
+
|
|
168
|
+
normalized = "https://xn--bcher-kva.de/r%C3%A9sum%C3%A9"
|
|
169
|
+
readable = url_humanize(normalized)
|
|
170
|
+
print(readable)
|
|
171
|
+
# Output: https://bücher.de/résumé
|
|
172
|
+
|
|
173
|
+
assert url_normalize(readable) == normalized
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
## Command-line usage
|
|
177
|
+
|
|
178
|
+
The CLI accepts one URL per invocation and prints the result to standard output. Quote URLs containing shell characters such as `&`.
|
|
179
|
+
|
|
180
|
+
```sh
|
|
181
|
+
# Keep selected query parameters.
|
|
182
|
+
url-normalize -f -p page,id "example.com/?page=1&id=123&ref=test"
|
|
183
|
+
# Output: https://example.com/?page=1&id=123
|
|
184
|
+
|
|
185
|
+
# Supply a domain and scheme for an absolute path.
|
|
186
|
+
url-normalize -d example.com -s http "/images/logo.png"
|
|
187
|
+
# Output: http://example.com/images/logo.png
|
|
188
|
+
|
|
189
|
+
# Display a readable URL.
|
|
190
|
+
url-normalize -H "https://xn--bcher-kva.de/r%C3%A9sum%C3%A9"
|
|
191
|
+
# Output: https://bücher.de/résumé
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
Use `url-normalize --help` for all options and `url-normalize --version` for the installed version. Normalization errors are written to standard error with exit status 1; invalid command-line arguments use exit status 2.
|
|
195
|
+
|
|
196
|
+
To run without a persistent tool installation:
|
|
197
|
+
|
|
198
|
+
```sh
|
|
199
|
+
uvx url-normalize "example.com"
|
|
200
|
+
# Output: https://example.com/
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
## Contributing
|
|
204
|
+
|
|
205
|
+
After cloning the repository, install the development dependencies and run the checks with [uv](https://docs.astral.sh/uv/):
|
|
206
|
+
|
|
207
|
+
```sh
|
|
208
|
+
uv sync --locked --extra dev
|
|
209
|
+
uv run --locked --extra dev python -m pytest
|
|
210
|
+
uv run --locked --extra dev pre-commit run --all-files
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
The test suite requires 100% statement coverage. CI tests Python 3.11 through 3.15 on Linux, macOS, and Windows. Please include regression tests with bug fixes.
|
|
214
|
+
|
|
215
|
+
## Changelog and license
|
|
216
|
+
|
|
217
|
+
See [CHANGELOG.md](CHANGELOG.md) for release history and upgrade notes. Distributed under the [MIT License](LICENSE).
|
|
218
|
+
|
|
219
|
+
Inspired by Sam Ruby's [urlnorm.py](http://intertwingly.net/blog/2004/08/04/Urlnorm).
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
# url-normalize
|
|
2
|
+
|
|
3
|
+
[](https://github.com/niksite/url-normalize/actions/workflows/ci.yml)
|
|
4
|
+
[](https://coveralls.io/r/niksite/url-normalize)
|
|
5
|
+
[](https://pypi.org/project/url-normalize/)
|
|
6
|
+
[](https://pypi.org/project/url-normalize/)
|
|
7
|
+
[](https://github.com/niksite/url-normalize/blob/master/LICENSE)
|
|
8
|
+
[](https://github.com/astral-sh/ruff)
|
|
9
|
+
|
|
10
|
+
Normalize URL spelling consistently for storage, comparison, and display. Use `url_normalize()` for normalized URLs and `url_humanize()` for readable display URLs.
|
|
11
|
+
|
|
12
|
+
Requires **Python 3.11 or newer**. Normalization operates on URL text; it does not fetch pages or follow redirects.
|
|
13
|
+
|
|
14
|
+
## Installation
|
|
15
|
+
|
|
16
|
+
Install the library and CLI:
|
|
17
|
+
|
|
18
|
+
```sh
|
|
19
|
+
python -m pip install url-normalize
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Or install the CLI in its own environment with [uv](https://docs.astral.sh/uv/):
|
|
23
|
+
|
|
24
|
+
```sh
|
|
25
|
+
uv tool install url-normalize
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Quick start
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
from url_normalize import url_normalize
|
|
32
|
+
|
|
33
|
+
print(url_normalize("HTTP://Example.COM:80/a/../b"))
|
|
34
|
+
# Output: http://example.com/b
|
|
35
|
+
|
|
36
|
+
print(url_normalize("bücher.de/résumé"))
|
|
37
|
+
# Output: https://xn--bcher-kva.de/r%C3%A9sum%C3%A9
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
From the terminal:
|
|
41
|
+
|
|
42
|
+
```sh
|
|
43
|
+
url-normalize "HTTP://Example.COM:80/a/../b"
|
|
44
|
+
# Output: http://example.com/b
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Normalization behavior
|
|
48
|
+
|
|
49
|
+
| Behavior | Input | Output |
|
|
50
|
+
| --- | --- | --- |
|
|
51
|
+
| Add a missing scheme | `example.com` | `https://example.com/` |
|
|
52
|
+
| Lowercase scheme and host; remove a default port | `HTTP://Example.COM:80/` | `http://example.com/` |
|
|
53
|
+
| Resolve dot segments | `https://example.com/a/../b` | `https://example.com/b` |
|
|
54
|
+
| Encode Unicode hosts and paths | `https://bücher.de/résumé` | `https://xn--bcher-kva.de/r%C3%A9sum%C3%A9` |
|
|
55
|
+
| Remove trailing host dots, including Unicode equivalents | `https://example.com。/` | `https://example.com/` |
|
|
56
|
+
| Preserve encoded path separators; uppercase escape hex | `https://example.com/a%2fb` | `https://example.com/a%2Fb` |
|
|
57
|
+
| Preserve query order and repeated keys | `https://example.com/?b=2&a=1&a=3` | `https://example.com/?b=2&a=1&a=3` |
|
|
58
|
+
| Preserve explicit empty query values | `https://example.com/?flag=&flag` | `https://example.com/?flag=&flag` |
|
|
59
|
+
|
|
60
|
+
In query parameters, `+`, `%2B`, and `%20` remain distinct. Parameter filtering is disabled by default. Fragments are preserved and normalized; hashbang fragments (`#!`) are converted to the `_escaped_fragment_` query parameter.
|
|
61
|
+
|
|
62
|
+
Normalization applies consistent spelling rules; it does not establish whether two URLs serve the same content. Applications using normalized URLs as persistent cache keys or identifiers should review the [changelog](CHANGELOG.md) when upgrading.
|
|
63
|
+
|
|
64
|
+
## Python API
|
|
65
|
+
|
|
66
|
+
Both functions accept a URL as `str` or `None` and return `str` or `None`. An empty string and `None` are returned unchanged. Malformed input can raise `ValueError`, including `UnicodeError` for invalid hostnames.
|
|
67
|
+
|
|
68
|
+
The following options are keyword-only and apply to both functions:
|
|
69
|
+
|
|
70
|
+
| Option | Default | Purpose |
|
|
71
|
+
| --- | --- | --- |
|
|
72
|
+
| `default_scheme` | `"https"` | Scheme to add when the URL has none. |
|
|
73
|
+
| `default_domain` | `None` | Domain to add to an absolute path such as `/images/logo.png`. |
|
|
74
|
+
| `filter_params` | `False` | Retain only allowlisted query parameters. |
|
|
75
|
+
| `param_allowlist` | `None` | Override built-in rules with a list of names or a dictionary mapping domains to lists. Requires `filter_params=True`. |
|
|
76
|
+
|
|
77
|
+
Unicode URL characters always use UTF-8 percent encoding.
|
|
78
|
+
|
|
79
|
+
### Query parameter filtering
|
|
80
|
+
|
|
81
|
+
With `filter_params=True`, only explicitly allowed parameters remain. Built-in rules cover Google, Bing, Baidu, and YouTube; see the [domain rules](url_normalize/param_allowlist.py) for their parameter names.
|
|
82
|
+
|
|
83
|
+
**Unknown domains have an empty default allowlist**, so filtering removes all their query parameters unless you supply a custom rule. A custom allowlist replaces the built-in rules.
|
|
84
|
+
|
|
85
|
+
```python
|
|
86
|
+
from url_normalize import url_normalize
|
|
87
|
+
|
|
88
|
+
# Use the built-in Google rule.
|
|
89
|
+
result = url_normalize(
|
|
90
|
+
"www.google.com/search?q=test&utm_source=test", filter_params=True
|
|
91
|
+
)
|
|
92
|
+
print(result)
|
|
93
|
+
# Output: https://www.google.com/search?q=test
|
|
94
|
+
|
|
95
|
+
# Apply a list of allowed names to any domain.
|
|
96
|
+
result = url_normalize(
|
|
97
|
+
"example.com/?page=1&id=123&ref=test",
|
|
98
|
+
filter_params=True,
|
|
99
|
+
param_allowlist=["page", "id"],
|
|
100
|
+
)
|
|
101
|
+
print(result)
|
|
102
|
+
# Output: https://example.com/?page=1&id=123
|
|
103
|
+
|
|
104
|
+
# Use domain-specific rules, including Unicode parameter names.
|
|
105
|
+
result = url_normalize(
|
|
106
|
+
"https://example.com/?r%C3%A9sum%C3%A9=1&drop=2",
|
|
107
|
+
filter_params=True,
|
|
108
|
+
param_allowlist={"example.com": ["résumé"]},
|
|
109
|
+
)
|
|
110
|
+
print(result)
|
|
111
|
+
# Output: https://example.com/?r%C3%A9sum%C3%A9=1
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
Allowed names use the same normalization as query keys, including Unicode and spaces. Domain rules match normalized hostnames and ignore a leading `www.` and the port. Rules do not automatically apply to other subdomains.
|
|
115
|
+
|
|
116
|
+
### Default domain and scheme
|
|
117
|
+
|
|
118
|
+
Use `default_domain` for paths starting with a single `/`. URLs starting with `//` already specify their host. This option does not resolve relative paths such as `../images/logo.png` against a page URL.
|
|
119
|
+
|
|
120
|
+
```python
|
|
121
|
+
from url_normalize import url_normalize
|
|
122
|
+
|
|
123
|
+
result = url_normalize(
|
|
124
|
+
"/images/logo.png", default_domain="example.com", default_scheme="http"
|
|
125
|
+
)
|
|
126
|
+
print(result)
|
|
127
|
+
# Output: http://example.com/images/logo.png
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
### Human-readable URLs
|
|
131
|
+
|
|
132
|
+
`url_humanize()` normalizes its input, then decodes components only when the display URL normalizes back to the same result. Some escapes therefore remain encoded.
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
from url_normalize import url_humanize, url_normalize
|
|
136
|
+
|
|
137
|
+
normalized = "https://xn--bcher-kva.de/r%C3%A9sum%C3%A9"
|
|
138
|
+
readable = url_humanize(normalized)
|
|
139
|
+
print(readable)
|
|
140
|
+
# Output: https://bücher.de/résumé
|
|
141
|
+
|
|
142
|
+
assert url_normalize(readable) == normalized
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
## Command-line usage
|
|
146
|
+
|
|
147
|
+
The CLI accepts one URL per invocation and prints the result to standard output. Quote URLs containing shell characters such as `&`.
|
|
148
|
+
|
|
149
|
+
```sh
|
|
150
|
+
# Keep selected query parameters.
|
|
151
|
+
url-normalize -f -p page,id "example.com/?page=1&id=123&ref=test"
|
|
152
|
+
# Output: https://example.com/?page=1&id=123
|
|
153
|
+
|
|
154
|
+
# Supply a domain and scheme for an absolute path.
|
|
155
|
+
url-normalize -d example.com -s http "/images/logo.png"
|
|
156
|
+
# Output: http://example.com/images/logo.png
|
|
157
|
+
|
|
158
|
+
# Display a readable URL.
|
|
159
|
+
url-normalize -H "https://xn--bcher-kva.de/r%C3%A9sum%C3%A9"
|
|
160
|
+
# Output: https://bücher.de/résumé
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
Use `url-normalize --help` for all options and `url-normalize --version` for the installed version. Normalization errors are written to standard error with exit status 1; invalid command-line arguments use exit status 2.
|
|
164
|
+
|
|
165
|
+
To run without a persistent tool installation:
|
|
166
|
+
|
|
167
|
+
```sh
|
|
168
|
+
uvx url-normalize "example.com"
|
|
169
|
+
# Output: https://example.com/
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
## Contributing
|
|
173
|
+
|
|
174
|
+
After cloning the repository, install the development dependencies and run the checks with [uv](https://docs.astral.sh/uv/):
|
|
175
|
+
|
|
176
|
+
```sh
|
|
177
|
+
uv sync --locked --extra dev
|
|
178
|
+
uv run --locked --extra dev python -m pytest
|
|
179
|
+
uv run --locked --extra dev pre-commit run --all-files
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
The test suite requires 100% statement coverage. CI tests Python 3.11 through 3.15 on Linux, macOS, and Windows. Please include regression tests with bug fixes.
|
|
183
|
+
|
|
184
|
+
## Changelog and license
|
|
185
|
+
|
|
186
|
+
See [CHANGELOG.md](CHANGELOG.md) for release history and upgrade notes. Distributed under the [MIT License](LICENSE).
|
|
187
|
+
|
|
188
|
+
Inspired by Sam Ruby's [urlnorm.py](http://intertwingly.net/blog/2004/08/04/Urlnorm).
|
|
@@ -1,22 +1,22 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "url-normalize"
|
|
3
|
-
version = "3.0
|
|
3
|
+
version = "3.1.0"
|
|
4
4
|
description = "URL normalization for Python"
|
|
5
5
|
authors = [{ name = "Nikolay Panov", email = "github@npanov.com" }]
|
|
6
6
|
license = "MIT"
|
|
7
7
|
readme = "README.md"
|
|
8
|
-
requires-python = ">=3.
|
|
8
|
+
requires-python = ">=3.11"
|
|
9
9
|
keywords = ["url", "normalization", "normalize", "normalizer"]
|
|
10
10
|
classifiers = [
|
|
11
11
|
"Programming Language :: Python :: 3",
|
|
12
12
|
"Programming Language :: Python :: 3 :: Only",
|
|
13
|
-
"Programming Language :: Python :: 3.10",
|
|
14
13
|
"Programming Language :: Python :: 3.11",
|
|
15
14
|
"Programming Language :: Python :: 3.12",
|
|
16
15
|
"Programming Language :: Python :: 3.13",
|
|
17
16
|
"Programming Language :: Python :: 3.14",
|
|
17
|
+
"Programming Language :: Python :: 3.15",
|
|
18
18
|
]
|
|
19
|
-
dependencies = ["idna>=3.
|
|
19
|
+
dependencies = ["idna>=3.20"]
|
|
20
20
|
|
|
21
21
|
[project.urls]
|
|
22
22
|
Homepage = "https://github.com/niksite/url-normalize"
|
|
@@ -31,7 +31,7 @@ url-normalize = "url_normalize.cli:main"
|
|
|
31
31
|
dev = ["mypy", "pre-commit", "pytest-cov", "pytest-socket", "pytest", "ruff"]
|
|
32
32
|
|
|
33
33
|
[tool.ruff]
|
|
34
|
-
target-version = "
|
|
34
|
+
target-version = "py311"
|
|
35
35
|
line-length = 88
|
|
36
36
|
unsafe-fixes = true
|
|
37
37
|
|
|
@@ -45,6 +45,7 @@ extend-select = [
|
|
|
45
45
|
fixable = ["ALL"]
|
|
46
46
|
ignore = [
|
|
47
47
|
"COM812", # missing-trailing-comma
|
|
48
|
+
"CPY001", # License is documented in LICENSE rather than per-file headers
|
|
48
49
|
"D203", # One blank line before class - we prefer D211 instead
|
|
49
50
|
"D213", # multi-line-summary-second-line - we prefer D212 instead
|
|
50
51
|
]
|
|
@@ -53,7 +54,7 @@ ignore = [
|
|
|
53
54
|
convention = "google"
|
|
54
55
|
|
|
55
56
|
[tool.ruff.lint.per-file-ignores]
|
|
56
|
-
"tests/**" = ["INP001", "ANN001", "ANN201", "S101"
|
|
57
|
+
"tests/**" = ["INP001", "ANN001", "ANN201", "S101"]
|
|
57
58
|
|
|
58
59
|
[tool.ruff.format]
|
|
59
60
|
quote-style = "double"
|
|
@@ -62,7 +63,7 @@ indent-style = "space"
|
|
|
62
63
|
[tool.mypy]
|
|
63
64
|
ignore_missing_imports = true
|
|
64
65
|
exclude = ["tests"]
|
|
65
|
-
python_version = "3.
|
|
66
|
+
python_version = "3.11"
|
|
66
67
|
show_error_codes = true
|
|
67
68
|
|
|
68
69
|
[build-system]
|
|
@@ -35,16 +35,23 @@ def test_cli_help_describes_allowlist_filtering() -> None:
|
|
|
35
35
|
assert "Unknown domains have an empty default allowlist." in help_text
|
|
36
36
|
|
|
37
37
|
|
|
38
|
-
def
|
|
39
|
-
"""
|
|
38
|
+
def test_cli_help_omits_removed_charset_option() -> None:
|
|
39
|
+
"""Remove the obsolete encoding option from CLI help."""
|
|
40
40
|
result = run_cli("--help")
|
|
41
41
|
assert result.returncode == 0
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
42
|
+
assert "charset" not in result.stdout
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@pytest.mark.parametrize("option", ["-c", "--charset"])
|
|
46
|
+
def test_cli_rejects_removed_charset_option(option, monkeypatch):
|
|
47
|
+
"""Report removed flags with UTF-8 diagnostics even under a legacy encoding."""
|
|
48
|
+
monkeypatch.setenv("PYTHONIOENCODING", "cp1252")
|
|
49
|
+
usage_error_status = 2
|
|
50
|
+
result = run_cli(option, "utf-8", "https://example.com/é")
|
|
51
|
+
assert result.returncode == usage_error_status
|
|
52
|
+
assert "unrecognized arguments" in result.stderr
|
|
53
|
+
assert "https://example.com/é" in result.stderr
|
|
54
|
+
assert not result.stdout
|
|
48
55
|
|
|
49
56
|
|
|
50
57
|
def test_cli_successful_main(capsys, monkeypatch):
|
|
@@ -284,35 +291,12 @@ def test_cli_version_short() -> None:
|
|
|
284
291
|
assert not result.stderr
|
|
285
292
|
|
|
286
293
|
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
)
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
url = "http://пример.рф/path"
|
|
294
|
-
expected_idn = "http://xn--e1afmkfd.xn--p1ai/path"
|
|
295
|
-
|
|
296
|
-
# Test with default UTF-8
|
|
297
|
-
result_utf8 = run_cli(url)
|
|
298
|
-
|
|
299
|
-
assert result_utf8.returncode == 0
|
|
300
|
-
assert result_utf8.stdout.strip() == expected_idn
|
|
301
|
-
assert not result_utf8.stderr
|
|
302
|
-
|
|
303
|
-
# Test specifying UTF-8 explicitly
|
|
304
|
-
result_charset = run_cli("--charset", "utf-8", url)
|
|
305
|
-
|
|
306
|
-
assert result_charset.returncode == 0
|
|
307
|
-
assert result_charset.stdout.strip() == expected_idn
|
|
308
|
-
assert not result_charset.stderr
|
|
309
|
-
|
|
310
|
-
# Test specifying UTF-8 explicitly using short arg
|
|
311
|
-
result_charset_short = run_cli("-c", "utf-8", url)
|
|
312
|
-
|
|
313
|
-
assert result_charset_short.returncode == 0
|
|
314
|
-
assert result_charset_short.stdout.strip() == expected_idn
|
|
315
|
-
assert not result_charset_short.stderr
|
|
294
|
+
def test_cli_unicode_host_and_path() -> None:
|
|
295
|
+
"""Encode Unicode hosts with IDNA and paths with UTF-8 on every platform."""
|
|
296
|
+
result = run_cli("http://пример.рф/é")
|
|
297
|
+
assert result.returncode == 0
|
|
298
|
+
assert result.stdout.strip() == "http://xn--e1afmkfd.xn--p1ai/%C3%A9"
|
|
299
|
+
assert not result.stderr
|
|
316
300
|
|
|
317
301
|
|
|
318
302
|
def test_cli_default_domain() -> None:
|
|
@@ -32,7 +32,36 @@ def test_normalize_host_result_is_expected(host: str, expected: str) -> None:
|
|
|
32
32
|
assert result == expected, host
|
|
33
33
|
|
|
34
34
|
|
|
35
|
-
@pytest.mark.parametrize("
|
|
35
|
+
@pytest.mark.parametrize("separator", [".", "\u3002", "\uff0e", "\uff61"])
|
|
36
|
+
@pytest.mark.parametrize(
|
|
37
|
+
("host", "expected"),
|
|
38
|
+
[
|
|
39
|
+
("EXAMPLE.COM", "example.com"),
|
|
40
|
+
("faß.de", "xn--fa-hia.de"),
|
|
41
|
+
("under_score.example", "under_score.example"),
|
|
42
|
+
],
|
|
43
|
+
)
|
|
44
|
+
def test_normalize_host_strips_trailing_idna_separator(host, expected, separator):
|
|
45
|
+
"""Strip equivalent trailing label separators before IDNA conversion."""
|
|
46
|
+
normalized = normalize_host(host + separator)
|
|
47
|
+
assert normalized == expected
|
|
48
|
+
assert normalize_host(normalized) == normalized
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@pytest.mark.parametrize("prefix", ["", "."])
|
|
52
|
+
@pytest.mark.parametrize("separator", ["\u3002", "\uff0e", "\uff61"])
|
|
53
|
+
def test_normalize_host_rejects_leading_unicode_separator(prefix, separator):
|
|
54
|
+
"""Keep rejecting leading Unicode label separators as before."""
|
|
55
|
+
with pytest.raises(UnicodeError):
|
|
56
|
+
normalize_host(prefix + separator + "example.com")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@pytest.mark.parametrize("prefix", [".", ".."])
|
|
60
|
+
def test_normalize_host_strips_leading_ascii_dots(prefix):
|
|
61
|
+
"""Retain the existing normalization of leading ASCII dots."""
|
|
62
|
+
assert normalize_host(prefix + "example.com") == "example.com"
|
|
63
|
+
|
|
64
|
+
|
|
36
65
|
@pytest.mark.parametrize(
|
|
37
66
|
("host", "expected"),
|
|
38
67
|
[
|
|
@@ -42,17 +71,16 @@ def test_normalize_host_result_is_expected(host: str, expected: str) -> None:
|
|
|
42
71
|
("[2001:DB8::1]", "[2001:db8::1]"),
|
|
43
72
|
],
|
|
44
73
|
)
|
|
45
|
-
def
|
|
46
|
-
"""
|
|
47
|
-
assert normalize_host(host
|
|
48
|
-
assert normalize_host(
|
|
74
|
+
def test_normalize_host_accepts_text_and_utf8_bytes(host, expected):
|
|
75
|
+
"""Normalize text and UTF-8 bytes to the same ASCII hostname."""
|
|
76
|
+
assert normalize_host(host) == expected
|
|
77
|
+
assert normalize_host(host.encode("utf-8")) == expected
|
|
49
78
|
|
|
50
79
|
|
|
51
80
|
@pytest.mark.parametrize("zone", ["%25ethA", "%ethA", "%25Interface.10", "%25ETH0"])
|
|
52
|
-
|
|
53
|
-
def test_normalize_host_preserves_ipv6_zone_case(zone, charset):
|
|
81
|
+
def test_normalize_host_preserves_ipv6_zone_case(zone):
|
|
54
82
|
"""Lowercase the IPv6 address without changing its interface identifier."""
|
|
55
83
|
host = f"[FE80::ABCD{zone}]"
|
|
56
84
|
expected = f"[fe80::abcd{zone}]"
|
|
57
|
-
assert normalize_host(host
|
|
58
|
-
assert normalize_host(host.encode(
|
|
85
|
+
assert normalize_host(host) == expected
|
|
86
|
+
assert normalize_host(host.encode("utf-8")) == expected
|
|
@@ -57,6 +57,70 @@ def test_custom_list_allowlist():
|
|
|
57
57
|
)
|
|
58
58
|
|
|
59
59
|
|
|
60
|
+
@pytest.mark.parametrize("separator", [".", "\u3002", "\uff0e", "\uff61"])
|
|
61
|
+
def test_custom_allowlist_matches_trailing_idna_separator(separator):
|
|
62
|
+
"""Match custom domain keys with any IDNA trailing label separator."""
|
|
63
|
+
assert (
|
|
64
|
+
url_normalize(
|
|
65
|
+
"https://example.com/?q=1&drop=2",
|
|
66
|
+
filter_params=True,
|
|
67
|
+
param_allowlist={f"example.com{separator}": ["q"]},
|
|
68
|
+
)
|
|
69
|
+
== "https://example.com/?q=1"
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@pytest.mark.parametrize(
|
|
74
|
+
("name", "query", "expected"),
|
|
75
|
+
[
|
|
76
|
+
("café", "café=1", "caf%C3%A9=1"), # codespell:ignore caf
|
|
77
|
+
("café", "caf%C3%A9=1", "caf%C3%A9=1"), # codespell:ignore caf
|
|
78
|
+
("cafe\u0301", "caf%C3%A9=1", "caf%C3%A9=1"), # codespell:ignore caf
|
|
79
|
+
("search term", "search%20term=1", "search%20term=1"),
|
|
80
|
+
],
|
|
81
|
+
)
|
|
82
|
+
@pytest.mark.parametrize("domain_specific", [False, True])
|
|
83
|
+
def test_custom_allowlist_normalizes_parameter_names(
|
|
84
|
+
name, query, expected, domain_specific
|
|
85
|
+
):
|
|
86
|
+
"""Match parameter names using the same normalization as query keys."""
|
|
87
|
+
allowlist = {"example.com": [name]} if domain_specific else [name]
|
|
88
|
+
assert (
|
|
89
|
+
url_normalize(
|
|
90
|
+
f"https://example.com/?{query}&drop=2",
|
|
91
|
+
filter_params=True,
|
|
92
|
+
param_allowlist=allowlist,
|
|
93
|
+
)
|
|
94
|
+
== f"https://example.com/?{expected}"
|
|
95
|
+
)
|
|
96
|
+
assert allowlist == ({"example.com": [name]} if domain_specific else [name])
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@pytest.mark.parametrize(
|
|
100
|
+
("name", "expected"),
|
|
101
|
+
[
|
|
102
|
+
("a+b", "a+b=1"),
|
|
103
|
+
("a%2Bb", "a%2Bb=2"),
|
|
104
|
+
("a b", "a%20b=3"),
|
|
105
|
+
("a%20b", "a%20b=3"),
|
|
106
|
+
],
|
|
107
|
+
)
|
|
108
|
+
@pytest.mark.parametrize("domain_specific", [False, True])
|
|
109
|
+
def test_custom_allowlist_keeps_plus_and_space_names_distinct(
|
|
110
|
+
name, expected, domain_specific
|
|
111
|
+
):
|
|
112
|
+
"""Keep literal plus, encoded plus, and encoded space keys distinct."""
|
|
113
|
+
allowlist = {"example.com": [name]} if domain_specific else [name]
|
|
114
|
+
assert (
|
|
115
|
+
url_normalize(
|
|
116
|
+
"https://example.com/?a+b=1&a%2Bb=2&a%20b=3&drop=4",
|
|
117
|
+
filter_params=True,
|
|
118
|
+
param_allowlist=allowlist,
|
|
119
|
+
)
|
|
120
|
+
== f"https://example.com/?{expected}"
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
|
|
60
124
|
@pytest.mark.parametrize(
|
|
61
125
|
("url", "expected"),
|
|
62
126
|
[
|