url-normalize 3.0.0__tar.gz → 3.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- url_normalize-3.1.0/PKG-INFO +219 -0
- url_normalize-3.1.0/README.md +188 -0
- {url_normalize-3.0.0 → url_normalize-3.1.0}/pyproject.toml +9 -7
- {url_normalize-3.0.0 → url_normalize-3.1.0}/tests/test_cli.py +45 -29
- url_normalize-3.1.0/tests/test_deconstruct_url.py +87 -0
- url_normalize-3.1.0/tests/test_generic_url_cleanup.py +72 -0
- url_normalize-3.1.0/tests/test_normalize_host.py +86 -0
- {url_normalize-3.0.0 → url_normalize-3.1.0}/tests/test_normalize_path.py +15 -1
- url_normalize-3.1.0/tests/test_normalize_query.py +57 -0
- url_normalize-3.1.0/tests/test_normalize_query_filters.py +290 -0
- url_normalize-3.1.0/tests/test_normalize_userinfo.py +44 -0
- url_normalize-3.1.0/tests/test_provide_url_scheme.py +140 -0
- {url_normalize-3.0.0 → url_normalize-3.1.0}/tests/test_url_humanize.py +50 -5
- {url_normalize-3.0.0 → url_normalize-3.1.0}/tests/test_url_normalize.py +70 -2
- {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/__init__.py +1 -1
- {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/cli.py +8 -11
- url_normalize-3.1.0/url_normalize/generic_url_cleanup.py +27 -0
- {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/normalize_host.py +10 -9
- {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/normalize_path.py +2 -2
- {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/normalize_query.py +14 -8
- {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/normalize_userinfo.py +9 -1
- {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/param_allowlist.py +23 -6
- {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/provide_url_scheme.py +12 -2
- {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/tools.py +36 -15
- {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/url_humanize.py +17 -9
- {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/url_normalize.py +7 -8
- url_normalize-3.1.0/url_normalize.egg-info/PKG-INFO +219 -0
- {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize.egg-info/requires.txt +1 -1
- url_normalize-3.0.0/PKG-INFO +0 -211
- url_normalize-3.0.0/README.md +0 -180
- url_normalize-3.0.0/tests/test_deconstruct_url.py +0 -40
- url_normalize-3.0.0/tests/test_generic_url_cleanup.py +0 -21
- url_normalize-3.0.0/tests/test_normalize_host.py +0 -32
- url_normalize-3.0.0/tests/test_normalize_query.py +0 -24
- url_normalize-3.0.0/tests/test_normalize_query_filters.py +0 -98
- url_normalize-3.0.0/tests/test_normalize_userinfo.py +0 -21
- url_normalize-3.0.0/tests/test_provide_url_scheme.py +0 -32
- url_normalize-3.0.0/url_normalize/generic_url_cleanup.py +0 -19
- url_normalize-3.0.0/url_normalize.egg-info/PKG-INFO +0 -211
- {url_normalize-3.0.0 → url_normalize-3.1.0}/LICENSE +0 -0
- {url_normalize-3.0.0 → url_normalize-3.1.0}/setup.cfg +0 -0
- {url_normalize-3.0.0 → url_normalize-3.1.0}/tests/test_normalize_fragment.py +0 -0
- {url_normalize-3.0.0 → url_normalize-3.1.0}/tests/test_normalize_port.py +0 -0
- {url_normalize-3.0.0 → url_normalize-3.1.0}/tests/test_normalize_scheme.py +0 -0
- {url_normalize-3.0.0 → url_normalize-3.1.0}/tests/test_provide_url_domain.py +0 -0
- {url_normalize-3.0.0 → url_normalize-3.1.0}/tests/test_reconstruct_url.py +0 -0
- {url_normalize-3.0.0 → url_normalize-3.1.0}/tests/test_tools.py +0 -0
- {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/normalize_fragment.py +0 -0
- {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/normalize_port.py +0 -0
- {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/normalize_scheme.py +0 -0
- {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/provide_url_domain.py +0 -0
- {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/py.typed +0 -0
- {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize.egg-info/SOURCES.txt +0 -0
- {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize.egg-info/dependency_links.txt +0 -0
- {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize.egg-info/entry_points.txt +0 -0
- {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: url-normalize
|
|
3
|
+
Version: 3.1.0
|
|
4
|
+
Summary: URL normalization for Python
|
|
5
|
+
Author-email: Nikolay Panov <github@npanov.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/niksite/url-normalize
|
|
8
|
+
Project-URL: Repository, https://github.com/niksite/url-normalize
|
|
9
|
+
Project-URL: Issues, https://github.com/niksite/url-normalize/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/niksite/url-normalize/blob/master/CHANGELOG.md
|
|
11
|
+
Keywords: url,normalization,normalize,normalizer
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
19
|
+
Requires-Python: >=3.11
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Requires-Dist: idna>=3.20
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: mypy; extra == "dev"
|
|
25
|
+
Requires-Dist: pre-commit; extra == "dev"
|
|
26
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
27
|
+
Requires-Dist: pytest-socket; extra == "dev"
|
|
28
|
+
Requires-Dist: pytest; extra == "dev"
|
|
29
|
+
Requires-Dist: ruff; extra == "dev"
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
# url-normalize
|
|
33
|
+
|
|
34
|
+
[](https://github.com/niksite/url-normalize/actions/workflows/ci.yml)
|
|
35
|
+
[](https://coveralls.io/r/niksite/url-normalize)
|
|
36
|
+
[](https://pypi.org/project/url-normalize/)
|
|
37
|
+
[](https://pypi.org/project/url-normalize/)
|
|
38
|
+
[](https://github.com/niksite/url-normalize/blob/master/LICENSE)
|
|
39
|
+
[](https://github.com/astral-sh/ruff)
|
|
40
|
+
|
|
41
|
+
Normalize URL spelling consistently for storage, comparison, and display. Use `url_normalize()` for normalized URLs and `url_humanize()` for readable display URLs.
|
|
42
|
+
|
|
43
|
+
Requires **Python 3.11 or newer**. Normalization operates on URL text; it does not fetch pages or follow redirects.
|
|
44
|
+
|
|
45
|
+
## Installation
|
|
46
|
+
|
|
47
|
+
Install the library and CLI:
|
|
48
|
+
|
|
49
|
+
```sh
|
|
50
|
+
python -m pip install url-normalize
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Or install the CLI in its own environment with [uv](https://docs.astral.sh/uv/):
|
|
54
|
+
|
|
55
|
+
```sh
|
|
56
|
+
uv tool install url-normalize
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Quick start
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
from url_normalize import url_normalize
|
|
63
|
+
|
|
64
|
+
print(url_normalize("HTTP://Example.COM:80/a/../b"))
|
|
65
|
+
# Output: http://example.com/b
|
|
66
|
+
|
|
67
|
+
print(url_normalize("bücher.de/résumé"))
|
|
68
|
+
# Output: https://xn--bcher-kva.de/r%C3%A9sum%C3%A9
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
From the terminal:
|
|
72
|
+
|
|
73
|
+
```sh
|
|
74
|
+
url-normalize "HTTP://Example.COM:80/a/../b"
|
|
75
|
+
# Output: http://example.com/b
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
## Normalization behavior
|
|
79
|
+
|
|
80
|
+
| Behavior | Input | Output |
|
|
81
|
+
| --- | --- | --- |
|
|
82
|
+
| Add a missing scheme | `example.com` | `https://example.com/` |
|
|
83
|
+
| Lowercase scheme and host; remove a default port | `HTTP://Example.COM:80/` | `http://example.com/` |
|
|
84
|
+
| Resolve dot segments | `https://example.com/a/../b` | `https://example.com/b` |
|
|
85
|
+
| Encode Unicode hosts and paths | `https://bücher.de/résumé` | `https://xn--bcher-kva.de/r%C3%A9sum%C3%A9` |
|
|
86
|
+
| Remove trailing host dots, including Unicode equivalents | `https://example.com。/` | `https://example.com/` |
|
|
87
|
+
| Preserve encoded path separators; uppercase escape hex | `https://example.com/a%2fb` | `https://example.com/a%2Fb` |
|
|
88
|
+
| Preserve query order and repeated keys | `https://example.com/?b=2&a=1&a=3` | `https://example.com/?b=2&a=1&a=3` |
|
|
89
|
+
| Preserve explicit empty query values | `https://example.com/?flag=&flag` | `https://example.com/?flag=&flag` |
|
|
90
|
+
|
|
91
|
+
In query parameters, `+`, `%2B`, and `%20` remain distinct. Parameter filtering is disabled by default. Fragments are preserved and normalized; hashbang fragments (`#!`) are converted to the `_escaped_fragment_` query parameter.
|
|
92
|
+
|
|
93
|
+
Normalization applies consistent spelling rules; it does not establish whether two URLs serve the same content. Applications using normalized URLs as persistent cache keys or identifiers should review the [changelog](CHANGELOG.md) when upgrading.
|
|
94
|
+
|
|
95
|
+
## Python API
|
|
96
|
+
|
|
97
|
+
Both functions accept a URL as `str` or `None` and return `str` or `None`. An empty string and `None` are returned unchanged. Malformed input can raise `ValueError`, including `UnicodeError` for invalid hostnames.
|
|
98
|
+
|
|
99
|
+
The following options are keyword-only and apply to both functions:
|
|
100
|
+
|
|
101
|
+
| Option | Default | Purpose |
|
|
102
|
+
| --- | --- | --- |
|
|
103
|
+
| `default_scheme` | `"https"` | Scheme to add when the URL has none. |
|
|
104
|
+
| `default_domain` | `None` | Domain to add to an absolute path such as `/images/logo.png`. |
|
|
105
|
+
| `filter_params` | `False` | Retain only allowlisted query parameters. |
|
|
106
|
+
| `param_allowlist` | `None` | Override built-in rules with a list of names or a dictionary mapping domains to lists. Requires `filter_params=True`. |
|
|
107
|
+
|
|
108
|
+
Unicode URL characters always use UTF-8 percent encoding.
|
|
109
|
+
|
|
110
|
+
### Query parameter filtering
|
|
111
|
+
|
|
112
|
+
With `filter_params=True`, only explicitly allowed parameters remain. Built-in rules cover Google, Bing, Baidu, and YouTube; see the [domain rules](url_normalize/param_allowlist.py) for their parameter names.
|
|
113
|
+
|
|
114
|
+
**Unknown domains have an empty default allowlist**, so filtering removes all their query parameters unless you supply a custom rule. A custom allowlist replaces the built-in rules.
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
from url_normalize import url_normalize
|
|
118
|
+
|
|
119
|
+
# Use the built-in Google rule.
|
|
120
|
+
result = url_normalize(
|
|
121
|
+
"www.google.com/search?q=test&utm_source=test", filter_params=True
|
|
122
|
+
)
|
|
123
|
+
print(result)
|
|
124
|
+
# Output: https://www.google.com/search?q=test
|
|
125
|
+
|
|
126
|
+
# Apply a list of allowed names to any domain.
|
|
127
|
+
result = url_normalize(
|
|
128
|
+
"example.com/?page=1&id=123&ref=test",
|
|
129
|
+
filter_params=True,
|
|
130
|
+
param_allowlist=["page", "id"],
|
|
131
|
+
)
|
|
132
|
+
print(result)
|
|
133
|
+
# Output: https://example.com/?page=1&id=123
|
|
134
|
+
|
|
135
|
+
# Use domain-specific rules, including Unicode parameter names.
|
|
136
|
+
result = url_normalize(
|
|
137
|
+
"https://example.com/?r%C3%A9sum%C3%A9=1&drop=2",
|
|
138
|
+
filter_params=True,
|
|
139
|
+
param_allowlist={"example.com": ["résumé"]},
|
|
140
|
+
)
|
|
141
|
+
print(result)
|
|
142
|
+
# Output: https://example.com/?r%C3%A9sum%C3%A9=1
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
Allowed names use the same normalization as query keys, including Unicode and spaces. Domain rules match normalized hostnames and ignore a leading `www.` and the port. Rules do not automatically apply to other subdomains.
|
|
146
|
+
|
|
147
|
+
### Default domain and scheme
|
|
148
|
+
|
|
149
|
+
Use `default_domain` for paths starting with a single `/`. URLs starting with `//` already specify their host. This option does not resolve relative paths such as `../images/logo.png` against a page URL.
|
|
150
|
+
|
|
151
|
+
```python
|
|
152
|
+
from url_normalize import url_normalize
|
|
153
|
+
|
|
154
|
+
result = url_normalize(
|
|
155
|
+
"/images/logo.png", default_domain="example.com", default_scheme="http"
|
|
156
|
+
)
|
|
157
|
+
print(result)
|
|
158
|
+
# Output: http://example.com/images/logo.png
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
### Human-readable URLs
|
|
162
|
+
|
|
163
|
+
`url_humanize()` normalizes its input, then decodes components only when the display URL normalizes back to the same result. Some escapes therefore remain encoded.
|
|
164
|
+
|
|
165
|
+
```python
|
|
166
|
+
from url_normalize import url_humanize, url_normalize
|
|
167
|
+
|
|
168
|
+
normalized = "https://xn--bcher-kva.de/r%C3%A9sum%C3%A9"
|
|
169
|
+
readable = url_humanize(normalized)
|
|
170
|
+
print(readable)
|
|
171
|
+
# Output: https://bücher.de/résumé
|
|
172
|
+
|
|
173
|
+
assert url_normalize(readable) == normalized
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
## Command-line usage
|
|
177
|
+
|
|
178
|
+
The CLI accepts one URL per invocation and prints the result to standard output. Quote URLs containing shell characters such as `&`.
|
|
179
|
+
|
|
180
|
+
```sh
|
|
181
|
+
# Keep selected query parameters.
|
|
182
|
+
url-normalize -f -p page,id "example.com/?page=1&id=123&ref=test"
|
|
183
|
+
# Output: https://example.com/?page=1&id=123
|
|
184
|
+
|
|
185
|
+
# Supply a domain and scheme for an absolute path.
|
|
186
|
+
url-normalize -d example.com -s http "/images/logo.png"
|
|
187
|
+
# Output: http://example.com/images/logo.png
|
|
188
|
+
|
|
189
|
+
# Display a readable URL.
|
|
190
|
+
url-normalize -H "https://xn--bcher-kva.de/r%C3%A9sum%C3%A9"
|
|
191
|
+
# Output: https://bücher.de/résumé
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
Use `url-normalize --help` for all options and `url-normalize --version` for the installed version. Normalization errors are written to standard error with exit status 1; invalid command-line arguments use exit status 2.
|
|
195
|
+
|
|
196
|
+
To run without a persistent tool installation:
|
|
197
|
+
|
|
198
|
+
```sh
|
|
199
|
+
uvx url-normalize "example.com"
|
|
200
|
+
# Output: https://example.com/
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
## Contributing
|
|
204
|
+
|
|
205
|
+
After cloning the repository, install the development dependencies and run the checks with [uv](https://docs.astral.sh/uv/):
|
|
206
|
+
|
|
207
|
+
```sh
|
|
208
|
+
uv sync --locked --extra dev
|
|
209
|
+
uv run --locked --extra dev python -m pytest
|
|
210
|
+
uv run --locked --extra dev pre-commit run --all-files
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
The test suite requires 100% statement coverage. CI tests Python 3.11 through 3.15 on Linux, macOS, and Windows. Please include regression tests with bug fixes.
|
|
214
|
+
|
|
215
|
+
## Changelog and license
|
|
216
|
+
|
|
217
|
+
See [CHANGELOG.md](CHANGELOG.md) for release history and upgrade notes. Distributed under the [MIT License](LICENSE).
|
|
218
|
+
|
|
219
|
+
Inspired by Sam Ruby's [urlnorm.py](http://intertwingly.net/blog/2004/08/04/Urlnorm).
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
# url-normalize
|
|
2
|
+
|
|
3
|
+
[](https://github.com/niksite/url-normalize/actions/workflows/ci.yml)
|
|
4
|
+
[](https://coveralls.io/r/niksite/url-normalize)
|
|
5
|
+
[](https://pypi.org/project/url-normalize/)
|
|
6
|
+
[](https://pypi.org/project/url-normalize/)
|
|
7
|
+
[](https://github.com/niksite/url-normalize/blob/master/LICENSE)
|
|
8
|
+
[](https://github.com/astral-sh/ruff)
|
|
9
|
+
|
|
10
|
+
Normalize URL spelling consistently for storage, comparison, and display. Use `url_normalize()` for normalized URLs and `url_humanize()` for readable display URLs.
|
|
11
|
+
|
|
12
|
+
Requires **Python 3.11 or newer**. Normalization operates on URL text; it does not fetch pages or follow redirects.
|
|
13
|
+
|
|
14
|
+
## Installation
|
|
15
|
+
|
|
16
|
+
Install the library and CLI:
|
|
17
|
+
|
|
18
|
+
```sh
|
|
19
|
+
python -m pip install url-normalize
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Or install the CLI in its own environment with [uv](https://docs.astral.sh/uv/):
|
|
23
|
+
|
|
24
|
+
```sh
|
|
25
|
+
uv tool install url-normalize
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Quick start
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
from url_normalize import url_normalize
|
|
32
|
+
|
|
33
|
+
print(url_normalize("HTTP://Example.COM:80/a/../b"))
|
|
34
|
+
# Output: http://example.com/b
|
|
35
|
+
|
|
36
|
+
print(url_normalize("bücher.de/résumé"))
|
|
37
|
+
# Output: https://xn--bcher-kva.de/r%C3%A9sum%C3%A9
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
From the terminal:
|
|
41
|
+
|
|
42
|
+
```sh
|
|
43
|
+
url-normalize "HTTP://Example.COM:80/a/../b"
|
|
44
|
+
# Output: http://example.com/b
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Normalization behavior
|
|
48
|
+
|
|
49
|
+
| Behavior | Input | Output |
|
|
50
|
+
| --- | --- | --- |
|
|
51
|
+
| Add a missing scheme | `example.com` | `https://example.com/` |
|
|
52
|
+
| Lowercase scheme and host; remove a default port | `HTTP://Example.COM:80/` | `http://example.com/` |
|
|
53
|
+
| Resolve dot segments | `https://example.com/a/../b` | `https://example.com/b` |
|
|
54
|
+
| Encode Unicode hosts and paths | `https://bücher.de/résumé` | `https://xn--bcher-kva.de/r%C3%A9sum%C3%A9` |
|
|
55
|
+
| Remove trailing host dots, including Unicode equivalents | `https://example.com。/` | `https://example.com/` |
|
|
56
|
+
| Preserve encoded path separators; uppercase escape hex | `https://example.com/a%2fb` | `https://example.com/a%2Fb` |
|
|
57
|
+
| Preserve query order and repeated keys | `https://example.com/?b=2&a=1&a=3` | `https://example.com/?b=2&a=1&a=3` |
|
|
58
|
+
| Preserve explicit empty query values | `https://example.com/?flag=&flag` | `https://example.com/?flag=&flag` |
|
|
59
|
+
|
|
60
|
+
In query parameters, `+`, `%2B`, and `%20` remain distinct. Parameter filtering is disabled by default. Fragments are preserved and normalized; hashbang fragments (`#!`) are converted to the `_escaped_fragment_` query parameter.
|
|
61
|
+
|
|
62
|
+
Normalization applies consistent spelling rules; it does not establish whether two URLs serve the same content. Applications using normalized URLs as persistent cache keys or identifiers should review the [changelog](CHANGELOG.md) when upgrading.
|
|
63
|
+
|
|
64
|
+
## Python API
|
|
65
|
+
|
|
66
|
+
Both functions accept a URL as `str` or `None` and return `str` or `None`. An empty string and `None` are returned unchanged. Malformed input can raise `ValueError`, including `UnicodeError` for invalid hostnames.
|
|
67
|
+
|
|
68
|
+
The following options are keyword-only and apply to both functions:
|
|
69
|
+
|
|
70
|
+
| Option | Default | Purpose |
|
|
71
|
+
| --- | --- | --- |
|
|
72
|
+
| `default_scheme` | `"https"` | Scheme to add when the URL has none. |
|
|
73
|
+
| `default_domain` | `None` | Domain to add to an absolute path such as `/images/logo.png`. |
|
|
74
|
+
| `filter_params` | `False` | Retain only allowlisted query parameters. |
|
|
75
|
+
| `param_allowlist` | `None` | Override built-in rules with a list of names or a dictionary mapping domains to lists. Requires `filter_params=True`. |
|
|
76
|
+
|
|
77
|
+
Unicode URL characters always use UTF-8 percent encoding.
|
|
78
|
+
|
|
79
|
+
### Query parameter filtering
|
|
80
|
+
|
|
81
|
+
With `filter_params=True`, only explicitly allowed parameters remain. Built-in rules cover Google, Bing, Baidu, and YouTube; see the [domain rules](url_normalize/param_allowlist.py) for their parameter names.
|
|
82
|
+
|
|
83
|
+
**Unknown domains have an empty default allowlist**, so filtering removes all their query parameters unless you supply a custom rule. A custom allowlist replaces the built-in rules.
|
|
84
|
+
|
|
85
|
+
```python
|
|
86
|
+
from url_normalize import url_normalize
|
|
87
|
+
|
|
88
|
+
# Use the built-in Google rule.
|
|
89
|
+
result = url_normalize(
|
|
90
|
+
"www.google.com/search?q=test&utm_source=test", filter_params=True
|
|
91
|
+
)
|
|
92
|
+
print(result)
|
|
93
|
+
# Output: https://www.google.com/search?q=test
|
|
94
|
+
|
|
95
|
+
# Apply a list of allowed names to any domain.
|
|
96
|
+
result = url_normalize(
|
|
97
|
+
"example.com/?page=1&id=123&ref=test",
|
|
98
|
+
filter_params=True,
|
|
99
|
+
param_allowlist=["page", "id"],
|
|
100
|
+
)
|
|
101
|
+
print(result)
|
|
102
|
+
# Output: https://example.com/?page=1&id=123
|
|
103
|
+
|
|
104
|
+
# Use domain-specific rules, including Unicode parameter names.
|
|
105
|
+
result = url_normalize(
|
|
106
|
+
"https://example.com/?r%C3%A9sum%C3%A9=1&drop=2",
|
|
107
|
+
filter_params=True,
|
|
108
|
+
param_allowlist={"example.com": ["résumé"]},
|
|
109
|
+
)
|
|
110
|
+
print(result)
|
|
111
|
+
# Output: https://example.com/?r%C3%A9sum%C3%A9=1
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
Allowed names use the same normalization as query keys, including Unicode and spaces. Domain rules match normalized hostnames and ignore a leading `www.` and the port. Rules do not automatically apply to other subdomains.
|
|
115
|
+
|
|
116
|
+
### Default domain and scheme
|
|
117
|
+
|
|
118
|
+
Use `default_domain` for paths starting with a single `/`. URLs starting with `//` already specify their host. This option does not resolve relative paths such as `../images/logo.png` against a page URL.
|
|
119
|
+
|
|
120
|
+
```python
|
|
121
|
+
from url_normalize import url_normalize
|
|
122
|
+
|
|
123
|
+
result = url_normalize(
|
|
124
|
+
"/images/logo.png", default_domain="example.com", default_scheme="http"
|
|
125
|
+
)
|
|
126
|
+
print(result)
|
|
127
|
+
# Output: http://example.com/images/logo.png
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
### Human-readable URLs
|
|
131
|
+
|
|
132
|
+
`url_humanize()` normalizes its input, then decodes components only when the display URL normalizes back to the same result. Some escapes therefore remain encoded.
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
from url_normalize import url_humanize, url_normalize
|
|
136
|
+
|
|
137
|
+
normalized = "https://xn--bcher-kva.de/r%C3%A9sum%C3%A9"
|
|
138
|
+
readable = url_humanize(normalized)
|
|
139
|
+
print(readable)
|
|
140
|
+
# Output: https://bücher.de/résumé
|
|
141
|
+
|
|
142
|
+
assert url_normalize(readable) == normalized
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
## Command-line usage
|
|
146
|
+
|
|
147
|
+
The CLI accepts one URL per invocation and prints the result to standard output. Quote URLs containing shell characters such as `&`.
|
|
148
|
+
|
|
149
|
+
```sh
|
|
150
|
+
# Keep selected query parameters.
|
|
151
|
+
url-normalize -f -p page,id "example.com/?page=1&id=123&ref=test"
|
|
152
|
+
# Output: https://example.com/?page=1&id=123
|
|
153
|
+
|
|
154
|
+
# Supply a domain and scheme for an absolute path.
|
|
155
|
+
url-normalize -d example.com -s http "/images/logo.png"
|
|
156
|
+
# Output: http://example.com/images/logo.png
|
|
157
|
+
|
|
158
|
+
# Display a readable URL.
|
|
159
|
+
url-normalize -H "https://xn--bcher-kva.de/r%C3%A9sum%C3%A9"
|
|
160
|
+
# Output: https://bücher.de/résumé
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
Use `url-normalize --help` for all options and `url-normalize --version` for the installed version. Normalization errors are written to standard error with exit status 1; invalid command-line arguments use exit status 2.
|
|
164
|
+
|
|
165
|
+
To run without a persistent tool installation:
|
|
166
|
+
|
|
167
|
+
```sh
|
|
168
|
+
uvx url-normalize "example.com"
|
|
169
|
+
# Output: https://example.com/
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
## Contributing
|
|
173
|
+
|
|
174
|
+
After cloning the repository, install the development dependencies and run the checks with [uv](https://docs.astral.sh/uv/):
|
|
175
|
+
|
|
176
|
+
```sh
|
|
177
|
+
uv sync --locked --extra dev
|
|
178
|
+
uv run --locked --extra dev python -m pytest
|
|
179
|
+
uv run --locked --extra dev pre-commit run --all-files
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
The test suite requires 100% statement coverage. CI tests Python 3.11 through 3.15 on Linux, macOS, and Windows. Please include regression tests with bug fixes.
|
|
183
|
+
|
|
184
|
+
## Changelog and license
|
|
185
|
+
|
|
186
|
+
See [CHANGELOG.md](CHANGELOG.md) for release history and upgrade notes. Distributed under the [MIT License](LICENSE).
|
|
187
|
+
|
|
188
|
+
Inspired by Sam Ruby's [urlnorm.py](http://intertwingly.net/blog/2004/08/04/Urlnorm).
|
|
@@ -1,22 +1,22 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "url-normalize"
|
|
3
|
-
version = "3.
|
|
3
|
+
version = "3.1.0"
|
|
4
4
|
description = "URL normalization for Python"
|
|
5
5
|
authors = [{ name = "Nikolay Panov", email = "github@npanov.com" }]
|
|
6
6
|
license = "MIT"
|
|
7
7
|
readme = "README.md"
|
|
8
|
-
requires-python = ">=3.
|
|
8
|
+
requires-python = ">=3.11"
|
|
9
9
|
keywords = ["url", "normalization", "normalize", "normalizer"]
|
|
10
10
|
classifiers = [
|
|
11
11
|
"Programming Language :: Python :: 3",
|
|
12
12
|
"Programming Language :: Python :: 3 :: Only",
|
|
13
|
-
"Programming Language :: Python :: 3.10",
|
|
14
13
|
"Programming Language :: Python :: 3.11",
|
|
15
14
|
"Programming Language :: Python :: 3.12",
|
|
16
15
|
"Programming Language :: Python :: 3.13",
|
|
17
16
|
"Programming Language :: Python :: 3.14",
|
|
17
|
+
"Programming Language :: Python :: 3.15",
|
|
18
18
|
]
|
|
19
|
-
dependencies = ["idna>=3.
|
|
19
|
+
dependencies = ["idna>=3.20"]
|
|
20
20
|
|
|
21
21
|
[project.urls]
|
|
22
22
|
Homepage = "https://github.com/niksite/url-normalize"
|
|
@@ -31,7 +31,7 @@ url-normalize = "url_normalize.cli:main"
|
|
|
31
31
|
dev = ["mypy", "pre-commit", "pytest-cov", "pytest-socket", "pytest", "ruff"]
|
|
32
32
|
|
|
33
33
|
[tool.ruff]
|
|
34
|
-
target-version = "
|
|
34
|
+
target-version = "py311"
|
|
35
35
|
line-length = 88
|
|
36
36
|
unsafe-fixes = true
|
|
37
37
|
|
|
@@ -45,6 +45,7 @@ extend-select = [
|
|
|
45
45
|
fixable = ["ALL"]
|
|
46
46
|
ignore = [
|
|
47
47
|
"COM812", # missing-trailing-comma
|
|
48
|
+
"CPY001", # License is documented in LICENSE rather than per-file headers
|
|
48
49
|
"D203", # One blank line before class - we prefer D211 instead
|
|
49
50
|
"D213", # multi-line-summary-second-line - we prefer D212 instead
|
|
50
51
|
]
|
|
@@ -53,7 +54,7 @@ ignore = [
|
|
|
53
54
|
convention = "google"
|
|
54
55
|
|
|
55
56
|
[tool.ruff.lint.per-file-ignores]
|
|
56
|
-
"tests/**" = ["INP001", "ANN001", "ANN201", "S101"
|
|
57
|
+
"tests/**" = ["INP001", "ANN001", "ANN201", "S101"]
|
|
57
58
|
|
|
58
59
|
[tool.ruff.format]
|
|
59
60
|
quote-style = "double"
|
|
@@ -62,7 +63,7 @@ indent-style = "space"
|
|
|
62
63
|
[tool.mypy]
|
|
63
64
|
ignore_missing_imports = true
|
|
64
65
|
exclude = ["tests"]
|
|
65
|
-
python_version = "3.
|
|
66
|
+
python_version = "3.11"
|
|
66
67
|
show_error_codes = true
|
|
67
68
|
|
|
68
69
|
[build-system]
|
|
@@ -73,6 +74,7 @@ build-backend = "setuptools.build_meta"
|
|
|
73
74
|
addopts = [
|
|
74
75
|
"--cov-report=term-missing:skip-covered",
|
|
75
76
|
"--cov=url_normalize",
|
|
77
|
+
"--cov-fail-under=100",
|
|
76
78
|
"--disable-socket",
|
|
77
79
|
"-v",
|
|
78
80
|
]
|
|
@@ -26,6 +26,45 @@ def run_cli(*args: str) -> subprocess.CompletedProcess:
|
|
|
26
26
|
)
|
|
27
27
|
|
|
28
28
|
|
|
29
|
+
def test_cli_help_describes_allowlist_filtering() -> None:
|
|
30
|
+
"""Describe filtering as an allowlist, not a tracking-parameter blocklist."""
|
|
31
|
+
result = run_cli("--help")
|
|
32
|
+
assert result.returncode == 0
|
|
33
|
+
help_text = " ".join(result.stdout.split())
|
|
34
|
+
assert "Keep only allowlisted query parameters." in help_text
|
|
35
|
+
assert "Unknown domains have an empty default allowlist." in help_text
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def test_cli_help_omits_removed_charset_option() -> None:
|
|
39
|
+
"""Remove the obsolete encoding option from CLI help."""
|
|
40
|
+
result = run_cli("--help")
|
|
41
|
+
assert result.returncode == 0
|
|
42
|
+
assert "charset" not in result.stdout
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@pytest.mark.parametrize("option", ["-c", "--charset"])
|
|
46
|
+
def test_cli_rejects_removed_charset_option(option, monkeypatch):
|
|
47
|
+
"""Report removed flags with UTF-8 diagnostics even under a legacy encoding."""
|
|
48
|
+
monkeypatch.setenv("PYTHONIOENCODING", "cp1252")
|
|
49
|
+
usage_error_status = 2
|
|
50
|
+
result = run_cli(option, "utf-8", "https://example.com/é")
|
|
51
|
+
assert result.returncode == usage_error_status
|
|
52
|
+
assert "unrecognized arguments" in result.stderr
|
|
53
|
+
assert "https://example.com/é" in result.stderr
|
|
54
|
+
assert not result.stdout
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def test_cli_successful_main(capsys, monkeypatch):
|
|
58
|
+
"""Exercise successful CLI output in the measured process."""
|
|
59
|
+
monkeypatch.setattr(
|
|
60
|
+
sys, "argv", ["url-normalize", "http://EXAMPLE.com/./path/../other/"]
|
|
61
|
+
)
|
|
62
|
+
main()
|
|
63
|
+
captured = capsys.readouterr()
|
|
64
|
+
assert captured.out == "http://example.com/other/\n"
|
|
65
|
+
assert not captured.err
|
|
66
|
+
|
|
67
|
+
|
|
29
68
|
def test_cli_error_handling(capsys, monkeypatch):
|
|
30
69
|
"""Test CLI error handling when URL normalization fails."""
|
|
31
70
|
with patch("url_normalize.cli.url_normalize") as mock_normalize:
|
|
@@ -252,35 +291,12 @@ def test_cli_version_short() -> None:
|
|
|
252
291
|
assert not result.stderr
|
|
253
292
|
|
|
254
293
|
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
)
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
url = "http://пример.рф/path"
|
|
262
|
-
expected_idn = "http://xn--e1afmkfd.xn--p1ai/path"
|
|
263
|
-
|
|
264
|
-
# Test with default UTF-8
|
|
265
|
-
result_utf8 = run_cli(url)
|
|
266
|
-
|
|
267
|
-
assert result_utf8.returncode == 0
|
|
268
|
-
assert result_utf8.stdout.strip() == expected_idn
|
|
269
|
-
assert not result_utf8.stderr
|
|
270
|
-
|
|
271
|
-
# Test specifying UTF-8 explicitly
|
|
272
|
-
result_charset = run_cli("--charset", "utf-8", url)
|
|
273
|
-
|
|
274
|
-
assert result_charset.returncode == 0
|
|
275
|
-
assert result_charset.stdout.strip() == expected_idn
|
|
276
|
-
assert not result_charset.stderr
|
|
277
|
-
|
|
278
|
-
# Test specifying UTF-8 explicitly using short arg
|
|
279
|
-
result_charset_short = run_cli("-c", "utf-8", url)
|
|
280
|
-
|
|
281
|
-
assert result_charset_short.returncode == 0
|
|
282
|
-
assert result_charset_short.stdout.strip() == expected_idn
|
|
283
|
-
assert not result_charset_short.stderr
|
|
294
|
+
def test_cli_unicode_host_and_path() -> None:
|
|
295
|
+
"""Encode Unicode hosts with IDNA and paths with UTF-8 on every platform."""
|
|
296
|
+
result = run_cli("http://пример.рф/é")
|
|
297
|
+
assert result.returncode == 0
|
|
298
|
+
assert result.stdout.strip() == "http://xn--e1afmkfd.xn--p1ai/%C3%A9"
|
|
299
|
+
assert not result.stderr
|
|
284
300
|
|
|
285
301
|
|
|
286
302
|
def test_cli_default_domain() -> None:
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""Deconstruct url tests."""
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from url_normalize import url_normalize
|
|
6
|
+
from url_normalize.tools import URL, deconstruct_url
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@pytest.mark.parametrize(
|
|
10
|
+
("url", "expected"),
|
|
11
|
+
[
|
|
12
|
+
(
|
|
13
|
+
"http://site.com",
|
|
14
|
+
URL(
|
|
15
|
+
fragment="",
|
|
16
|
+
host="site.com",
|
|
17
|
+
path="",
|
|
18
|
+
port="",
|
|
19
|
+
query="",
|
|
20
|
+
scheme="http",
|
|
21
|
+
userinfo="",
|
|
22
|
+
),
|
|
23
|
+
),
|
|
24
|
+
(
|
|
25
|
+
"http://user@www.example.com:8080/path/index.html?param=val#fragment",
|
|
26
|
+
URL(
|
|
27
|
+
fragment="fragment",
|
|
28
|
+
host="www.example.com",
|
|
29
|
+
path="/path/index.html",
|
|
30
|
+
port="8080",
|
|
31
|
+
query="param=val",
|
|
32
|
+
scheme="http",
|
|
33
|
+
userinfo="user@",
|
|
34
|
+
),
|
|
35
|
+
),
|
|
36
|
+
],
|
|
37
|
+
)
|
|
38
|
+
def test_deconstruct_url_result_is_expected(url: str, expected: URL) -> None:
|
|
39
|
+
"""Assert we got expected results from the deconstruct_url function."""
|
|
40
|
+
result = deconstruct_url(url)
|
|
41
|
+
assert result == expected, url
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@pytest.mark.parametrize(
|
|
45
|
+
("authority", "userinfo", "host", "port", "normalized_authority"),
|
|
46
|
+
[
|
|
47
|
+
("[2001:DB8::1]:080", "", "[2001:DB8::1]", "080", "[2001:db8::1]"),
|
|
48
|
+
("[2001:DB8::1]:0081", "", "[2001:DB8::1]", "0081", "[2001:db8::1]:81"),
|
|
49
|
+
("[::1]", "", "[::1]", "", "[::1]"),
|
|
50
|
+
("[::1]:", "", "[::1]", "", "[::1]"),
|
|
51
|
+
("user:pass@[::1]:080", "user:pass@", "[::1]", "080", "user:pass@[::1]"),
|
|
52
|
+
("EXAMPLE.com:080", "", "EXAMPLE.com", "080", "example.com"),
|
|
53
|
+
("127.0.0.1:0081", "", "127.0.0.1", "0081", "127.0.0.1:81"),
|
|
54
|
+
],
|
|
55
|
+
)
|
|
56
|
+
def test_deconstruct_url_separates_bracketed_host_and_port(
|
|
57
|
+
authority, userinfo, host, port, normalized_authority
|
|
58
|
+
):
|
|
59
|
+
"""Parse the entire bracketed host before considering a port separator."""
|
|
60
|
+
url = f"http://{authority}/path?q=1#fragment"
|
|
61
|
+
assert deconstruct_url(url) == URL(
|
|
62
|
+
"http", userinfo, host, port, "/path", "q=1", "fragment"
|
|
63
|
+
)
|
|
64
|
+
assert url_normalize(url) == f"http://{normalized_authority}/path?q=1#fragment"
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@pytest.mark.parametrize("whitespace", [" ", "\u00a0", " \t\r\n"])
|
|
68
|
+
@pytest.mark.parametrize("suffix", ["", "/path ", "?q=x ", "#part "])
|
|
69
|
+
@pytest.mark.parametrize(
|
|
70
|
+
("authority", "normalized"),
|
|
71
|
+
[
|
|
72
|
+
("EXAMPLE.com", "example.com"),
|
|
73
|
+
("example.com:443", "example.com"),
|
|
74
|
+
("user:pass@example.com:443", "user:pass@example.com"),
|
|
75
|
+
("[FE80::1%25ethA]:443", "[fe80::1%25ethA]"),
|
|
76
|
+
],
|
|
77
|
+
)
|
|
78
|
+
def test_deconstruct_url_trims_authority_whitespace_only(
|
|
79
|
+
authority, normalized, whitespace, suffix
|
|
80
|
+
):
|
|
81
|
+
"""Trim the authority without deleting whitespace from other components."""
|
|
82
|
+
value = f"https://{authority}{whitespace}{suffix}"
|
|
83
|
+
assert deconstruct_url(value) == deconstruct_url(f"https://{authority}{suffix}")
|
|
84
|
+
path_suffix = suffix if suffix.startswith("/") else "/" + suffix
|
|
85
|
+
assert (
|
|
86
|
+
url_normalize(value) == f"https://{normalized}{path_suffix.replace(' ', '%20')}"
|
|
87
|
+
)
|