url-normalize 3.0.1__tar.gz → 3.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. url_normalize-3.1.0/PKG-INFO +219 -0
  2. url_normalize-3.1.0/README.md +188 -0
  3. {url_normalize-3.0.1 → url_normalize-3.1.0}/pyproject.toml +8 -7
  4. {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_cli.py +21 -37
  5. {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_normalize_host.py +37 -9
  6. {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_normalize_query_filters.py +64 -0
  7. {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_url_humanize.py +4 -2
  8. {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_url_normalize.py +25 -1
  9. {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/__init__.py +1 -1
  10. {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/cli.py +4 -12
  11. {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/normalize_host.py +5 -7
  12. {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/normalize_query.py +6 -1
  13. {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/tools.py +5 -8
  14. {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/url_humanize.py +1 -4
  15. {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/url_normalize.py +3 -8
  16. url_normalize-3.1.0/url_normalize.egg-info/PKG-INFO +219 -0
  17. {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize.egg-info/requires.txt +1 -1
  18. url_normalize-3.0.1/PKG-INFO +0 -215
  19. url_normalize-3.0.1/README.md +0 -184
  20. url_normalize-3.0.1/url_normalize.egg-info/PKG-INFO +0 -215
  21. {url_normalize-3.0.1 → url_normalize-3.1.0}/LICENSE +0 -0
  22. {url_normalize-3.0.1 → url_normalize-3.1.0}/setup.cfg +0 -0
  23. {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_deconstruct_url.py +0 -0
  24. {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_generic_url_cleanup.py +0 -0
  25. {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_normalize_fragment.py +0 -0
  26. {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_normalize_path.py +0 -0
  27. {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_normalize_port.py +0 -0
  28. {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_normalize_query.py +0 -0
  29. {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_normalize_scheme.py +0 -0
  30. {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_normalize_userinfo.py +0 -0
  31. {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_provide_url_domain.py +0 -0
  32. {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_provide_url_scheme.py +0 -0
  33. {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_reconstruct_url.py +0 -0
  34. {url_normalize-3.0.1 → url_normalize-3.1.0}/tests/test_tools.py +0 -0
  35. {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/generic_url_cleanup.py +0 -0
  36. {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/normalize_fragment.py +0 -0
  37. {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/normalize_path.py +0 -0
  38. {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/normalize_port.py +0 -0
  39. {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/normalize_scheme.py +0 -0
  40. {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/normalize_userinfo.py +0 -0
  41. {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/param_allowlist.py +0 -0
  42. {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/provide_url_domain.py +0 -0
  43. {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/provide_url_scheme.py +0 -0
  44. {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize/py.typed +0 -0
  45. {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize.egg-info/SOURCES.txt +0 -0
  46. {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize.egg-info/dependency_links.txt +0 -0
  47. {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize.egg-info/entry_points.txt +0 -0
  48. {url_normalize-3.0.1 → url_normalize-3.1.0}/url_normalize.egg-info/top_level.txt +0 -0
@@ -0,0 +1,219 @@
1
+ Metadata-Version: 2.4
2
+ Name: url-normalize
3
+ Version: 3.1.0
4
+ Summary: URL normalization for Python
5
+ Author-email: Nikolay Panov <github@npanov.com>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/niksite/url-normalize
8
+ Project-URL: Repository, https://github.com/niksite/url-normalize
9
+ Project-URL: Issues, https://github.com/niksite/url-normalize/issues
10
+ Project-URL: Changelog, https://github.com/niksite/url-normalize/blob/master/CHANGELOG.md
11
+ Keywords: url,normalization,normalize,normalizer
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Programming Language :: Python :: 3 :: Only
14
+ Classifier: Programming Language :: Python :: 3.11
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Programming Language :: Python :: 3.13
17
+ Classifier: Programming Language :: Python :: 3.14
18
+ Classifier: Programming Language :: Python :: 3.15
19
+ Requires-Python: >=3.11
20
+ Description-Content-Type: text/markdown
21
+ License-File: LICENSE
22
+ Requires-Dist: idna>=3.20
23
+ Provides-Extra: dev
24
+ Requires-Dist: mypy; extra == "dev"
25
+ Requires-Dist: pre-commit; extra == "dev"
26
+ Requires-Dist: pytest-cov; extra == "dev"
27
+ Requires-Dist: pytest-socket; extra == "dev"
28
+ Requires-Dist: pytest; extra == "dev"
29
+ Requires-Dist: ruff; extra == "dev"
30
+ Dynamic: license-file
31
+
32
+ # url-normalize
33
+
34
+ [![tests](https://github.com/niksite/url-normalize/actions/workflows/ci.yml/badge.svg)](https://github.com/niksite/url-normalize/actions/workflows/ci.yml)
35
+ [![Coveralls](https://img.shields.io/coveralls/github/niksite/url-normalize/master.svg)](https://coveralls.io/r/niksite/url-normalize)
36
+ [![PyPI](https://img.shields.io/pypi/v/url-normalize.svg)](https://pypi.org/project/url-normalize/)
37
+ [![Python Versions](https://img.shields.io/pypi/pyversions/url-normalize.svg)](https://pypi.org/project/url-normalize/)
38
+ [![License](https://img.shields.io/pypi/l/url-normalize.svg)](https://github.com/niksite/url-normalize/blob/master/LICENSE)
39
+ [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
40
+
41
+ Normalize URL spelling consistently for storage, comparison, and display. Use `url_normalize()` for normalized URLs and `url_humanize()` for readable display URLs.
42
+
43
+ Requires **Python 3.11 or newer**. Normalization operates on URL text; it does not fetch pages or follow redirects.
44
+
45
+ ## Installation
46
+
47
+ Install the library and CLI:
48
+
49
+ ```sh
50
+ python -m pip install url-normalize
51
+ ```
52
+
53
+ Or install the CLI in its own environment with [uv](https://docs.astral.sh/uv/):
54
+
55
+ ```sh
56
+ uv tool install url-normalize
57
+ ```
58
+
59
+ ## Quick start
60
+
61
+ ```python
62
+ from url_normalize import url_normalize
63
+
64
+ print(url_normalize("HTTP://Example.COM:80/a/../b"))
65
+ # Output: http://example.com/b
66
+
67
+ print(url_normalize("bücher.de/résumé"))
68
+ # Output: https://xn--bcher-kva.de/r%C3%A9sum%C3%A9
69
+ ```
70
+
71
+ From the terminal:
72
+
73
+ ```sh
74
+ url-normalize "HTTP://Example.COM:80/a/../b"
75
+ # Output: http://example.com/b
76
+ ```
77
+
78
+ ## Normalization behavior
79
+
80
+ | Behavior | Input | Output |
81
+ | --- | --- | --- |
82
+ | Add a missing scheme | `example.com` | `https://example.com/` |
83
+ | Lowercase scheme and host; remove a default port | `HTTP://Example.COM:80/` | `http://example.com/` |
84
+ | Resolve dot segments | `https://example.com/a/../b` | `https://example.com/b` |
85
+ | Encode Unicode hosts and paths | `https://bücher.de/résumé` | `https://xn--bcher-kva.de/r%C3%A9sum%C3%A9` |
86
+ | Remove trailing host dots, including Unicode equivalents | `https://example.com。/` | `https://example.com/` |
87
+ | Preserve encoded path separators; uppercase escape hex | `https://example.com/a%2fb` | `https://example.com/a%2Fb` |
88
+ | Preserve query order and repeated keys | `https://example.com/?b=2&a=1&a=3` | `https://example.com/?b=2&a=1&a=3` |
89
+ | Preserve explicit empty query values | `https://example.com/?flag=&flag` | `https://example.com/?flag=&flag` |
90
+
91
+ In query parameters, `+`, `%2B`, and `%20` remain distinct. Parameter filtering is disabled by default. Fragments are preserved and normalized; hashbang fragments (`#!`) are converted to the `_escaped_fragment_` query parameter.
92
+
93
+ Normalization applies consistent spelling rules; it does not establish whether two URLs serve the same content. Applications using normalized URLs as persistent cache keys or identifiers should review the [changelog](CHANGELOG.md) when upgrading.
94
+
95
+ ## Python API
96
+
97
+ Both functions accept a URL as `str` or `None` and return `str` or `None`. An empty string and `None` are returned unchanged. Malformed input can raise `ValueError`, including `UnicodeError` for invalid hostnames.
98
+
99
+ The following options are keyword-only and apply to both functions:
100
+
101
+ | Option | Default | Purpose |
102
+ | --- | --- | --- |
103
+ | `default_scheme` | `"https"` | Scheme to add when the URL has none. |
104
+ | `default_domain` | `None` | Domain to add to an absolute path such as `/images/logo.png`. |
105
+ | `filter_params` | `False` | Retain only allowlisted query parameters. |
106
+ | `param_allowlist` | `None` | Override built-in rules with a list of names or a dictionary mapping domains to lists. Requires `filter_params=True`. |
107
+
108
+ Unicode URL characters always use UTF-8 percent encoding.
109
+
110
+ ### Query parameter filtering
111
+
112
+ With `filter_params=True`, only explicitly allowed parameters remain. Built-in rules cover Google, Bing, Baidu, and YouTube; see the [domain rules](url_normalize/param_allowlist.py) for their parameter names.
113
+
114
+ **Unknown domains have an empty default allowlist**, so filtering removes all their query parameters unless you supply a custom rule. A custom allowlist replaces the built-in rules.
115
+
116
+ ```python
117
+ from url_normalize import url_normalize
118
+
119
+ # Use the built-in Google rule.
120
+ result = url_normalize(
121
+ "www.google.com/search?q=test&utm_source=test", filter_params=True
122
+ )
123
+ print(result)
124
+ # Output: https://www.google.com/search?q=test
125
+
126
+ # Apply a list of allowed names to any domain.
127
+ result = url_normalize(
128
+ "example.com/?page=1&id=123&ref=test",
129
+ filter_params=True,
130
+ param_allowlist=["page", "id"],
131
+ )
132
+ print(result)
133
+ # Output: https://example.com/?page=1&id=123
134
+
135
+ # Use domain-specific rules, including Unicode parameter names.
136
+ result = url_normalize(
137
+ "https://example.com/?r%C3%A9sum%C3%A9=1&drop=2",
138
+ filter_params=True,
139
+ param_allowlist={"example.com": ["résumé"]},
140
+ )
141
+ print(result)
142
+ # Output: https://example.com/?r%C3%A9sum%C3%A9=1
143
+ ```
144
+
145
+ Allowed names use the same normalization as query keys, including Unicode and spaces. Domain rules match normalized hostnames and ignore a leading `www.` and the port. Rules do not automatically apply to other subdomains.
146
+
147
+ ### Default domain and scheme
148
+
149
+ Use `default_domain` for paths starting with a single `/`. URLs starting with `//` already specify their host. This option does not resolve relative paths such as `../images/logo.png` against a page URL.
150
+
151
+ ```python
152
+ from url_normalize import url_normalize
153
+
154
+ result = url_normalize(
155
+ "/images/logo.png", default_domain="example.com", default_scheme="http"
156
+ )
157
+ print(result)
158
+ # Output: http://example.com/images/logo.png
159
+ ```
160
+
161
+ ### Human-readable URLs
162
+
163
+ `url_humanize()` normalizes its input, then decodes components only when the display URL normalizes back to the same result. Some escapes therefore remain encoded.
164
+
165
+ ```python
166
+ from url_normalize import url_humanize, url_normalize
167
+
168
+ normalized = "https://xn--bcher-kva.de/r%C3%A9sum%C3%A9"
169
+ readable = url_humanize(normalized)
170
+ print(readable)
171
+ # Output: https://bücher.de/résumé
172
+
173
+ assert url_normalize(readable) == normalized
174
+ ```
175
+
176
+ ## Command-line usage
177
+
178
+ The CLI accepts one URL per invocation and prints the result to standard output. Quote URLs containing shell characters such as `&`.
179
+
180
+ ```sh
181
+ # Keep selected query parameters.
182
+ url-normalize -f -p page,id "example.com/?page=1&id=123&ref=test"
183
+ # Output: https://example.com/?page=1&id=123
184
+
185
+ # Supply a domain and scheme for an absolute path.
186
+ url-normalize -d example.com -s http "/images/logo.png"
187
+ # Output: http://example.com/images/logo.png
188
+
189
+ # Display a readable URL.
190
+ url-normalize -H "https://xn--bcher-kva.de/r%C3%A9sum%C3%A9"
191
+ # Output: https://bücher.de/résumé
192
+ ```
193
+
194
+ Use `url-normalize --help` for all options and `url-normalize --version` for the installed version. Normalization errors are written to standard error with exit status 1; invalid command-line arguments use exit status 2.
195
+
196
+ To run without a persistent tool installation:
197
+
198
+ ```sh
199
+ uvx url-normalize "example.com"
200
+ # Output: https://example.com/
201
+ ```
202
+
203
+ ## Contributing
204
+
205
+ After cloning the repository, install the development dependencies and run the checks with [uv](https://docs.astral.sh/uv/):
206
+
207
+ ```sh
208
+ uv sync --locked --extra dev
209
+ uv run --locked --extra dev python -m pytest
210
+ uv run --locked --extra dev pre-commit run --all-files
211
+ ```
212
+
213
+ The test suite requires 100% statement coverage. CI tests Python 3.11 through 3.15 on Linux, macOS, and Windows. Please include regression tests with bug fixes.
214
+
215
+ ## Changelog and license
216
+
217
+ See [CHANGELOG.md](CHANGELOG.md) for release history and upgrade notes. Distributed under the [MIT License](LICENSE).
218
+
219
+ Inspired by Sam Ruby's [urlnorm.py](http://intertwingly.net/blog/2004/08/04/Urlnorm).
@@ -0,0 +1,188 @@
1
+ # url-normalize
2
+
3
+ [![tests](https://github.com/niksite/url-normalize/actions/workflows/ci.yml/badge.svg)](https://github.com/niksite/url-normalize/actions/workflows/ci.yml)
4
+ [![Coveralls](https://img.shields.io/coveralls/github/niksite/url-normalize/master.svg)](https://coveralls.io/r/niksite/url-normalize)
5
+ [![PyPI](https://img.shields.io/pypi/v/url-normalize.svg)](https://pypi.org/project/url-normalize/)
6
+ [![Python Versions](https://img.shields.io/pypi/pyversions/url-normalize.svg)](https://pypi.org/project/url-normalize/)
7
+ [![License](https://img.shields.io/pypi/l/url-normalize.svg)](https://github.com/niksite/url-normalize/blob/master/LICENSE)
8
+ [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
9
+
10
+ Normalize URL spelling consistently for storage, comparison, and display. Use `url_normalize()` for normalized URLs and `url_humanize()` for readable display URLs.
11
+
12
+ Requires **Python 3.11 or newer**. Normalization operates on URL text; it does not fetch pages or follow redirects.
13
+
14
+ ## Installation
15
+
16
+ Install the library and CLI:
17
+
18
+ ```sh
19
+ python -m pip install url-normalize
20
+ ```
21
+
22
+ Or install the CLI in its own environment with [uv](https://docs.astral.sh/uv/):
23
+
24
+ ```sh
25
+ uv tool install url-normalize
26
+ ```
27
+
28
+ ## Quick start
29
+
30
+ ```python
31
+ from url_normalize import url_normalize
32
+
33
+ print(url_normalize("HTTP://Example.COM:80/a/../b"))
34
+ # Output: http://example.com/b
35
+
36
+ print(url_normalize("bücher.de/résumé"))
37
+ # Output: https://xn--bcher-kva.de/r%C3%A9sum%C3%A9
38
+ ```
39
+
40
+ From the terminal:
41
+
42
+ ```sh
43
+ url-normalize "HTTP://Example.COM:80/a/../b"
44
+ # Output: http://example.com/b
45
+ ```
46
+
47
+ ## Normalization behavior
48
+
49
+ | Behavior | Input | Output |
50
+ | --- | --- | --- |
51
+ | Add a missing scheme | `example.com` | `https://example.com/` |
52
+ | Lowercase scheme and host; remove a default port | `HTTP://Example.COM:80/` | `http://example.com/` |
53
+ | Resolve dot segments | `https://example.com/a/../b` | `https://example.com/b` |
54
+ | Encode Unicode hosts and paths | `https://bücher.de/résumé` | `https://xn--bcher-kva.de/r%C3%A9sum%C3%A9` |
55
+ | Remove trailing host dots, including Unicode equivalents | `https://example.com。/` | `https://example.com/` |
56
+ | Preserve encoded path separators; uppercase escape hex | `https://example.com/a%2fb` | `https://example.com/a%2Fb` |
57
+ | Preserve query order and repeated keys | `https://example.com/?b=2&a=1&a=3` | `https://example.com/?b=2&a=1&a=3` |
58
+ | Preserve explicit empty query values | `https://example.com/?flag=&flag` | `https://example.com/?flag=&flag` |
59
+
60
+ In query parameters, `+`, `%2B`, and `%20` remain distinct. Parameter filtering is disabled by default. Fragments are preserved and normalized; hashbang fragments (`#!`) are converted to the `_escaped_fragment_` query parameter.
61
+
62
+ Normalization applies consistent spelling rules; it does not establish whether two URLs serve the same content. Applications using normalized URLs as persistent cache keys or identifiers should review the [changelog](CHANGELOG.md) when upgrading.
63
+
64
+ ## Python API
65
+
66
+ Both functions accept a URL as `str` or `None` and return `str` or `None`. An empty string and `None` are returned unchanged. Malformed input can raise `ValueError`, including `UnicodeError` for invalid hostnames.
67
+
68
+ The following options are keyword-only and apply to both functions:
69
+
70
+ | Option | Default | Purpose |
71
+ | --- | --- | --- |
72
+ | `default_scheme` | `"https"` | Scheme to add when the URL has none. |
73
+ | `default_domain` | `None` | Domain to add to an absolute path such as `/images/logo.png`. |
74
+ | `filter_params` | `False` | Retain only allowlisted query parameters. |
75
+ | `param_allowlist` | `None` | Override built-in rules with a list of names or a dictionary mapping domains to lists. Requires `filter_params=True`. |
76
+
77
+ Unicode URL characters always use UTF-8 percent encoding.
78
+
79
+ ### Query parameter filtering
80
+
81
+ With `filter_params=True`, only explicitly allowed parameters remain. Built-in rules cover Google, Bing, Baidu, and YouTube; see the [domain rules](url_normalize/param_allowlist.py) for their parameter names.
82
+
83
+ **Unknown domains have an empty default allowlist**, so filtering removes all their query parameters unless you supply a custom rule. A custom allowlist replaces the built-in rules.
84
+
85
+ ```python
86
+ from url_normalize import url_normalize
87
+
88
+ # Use the built-in Google rule.
89
+ result = url_normalize(
90
+ "www.google.com/search?q=test&utm_source=test", filter_params=True
91
+ )
92
+ print(result)
93
+ # Output: https://www.google.com/search?q=test
94
+
95
+ # Apply a list of allowed names to any domain.
96
+ result = url_normalize(
97
+ "example.com/?page=1&id=123&ref=test",
98
+ filter_params=True,
99
+ param_allowlist=["page", "id"],
100
+ )
101
+ print(result)
102
+ # Output: https://example.com/?page=1&id=123
103
+
104
+ # Use domain-specific rules, including Unicode parameter names.
105
+ result = url_normalize(
106
+ "https://example.com/?r%C3%A9sum%C3%A9=1&drop=2",
107
+ filter_params=True,
108
+ param_allowlist={"example.com": ["résumé"]},
109
+ )
110
+ print(result)
111
+ # Output: https://example.com/?r%C3%A9sum%C3%A9=1
112
+ ```
113
+
114
+ Allowed names use the same normalization as query keys, including Unicode and spaces. Domain rules match normalized hostnames and ignore a leading `www.` and the port. Rules do not automatically apply to other subdomains.
115
+
116
+ ### Default domain and scheme
117
+
118
+ Use `default_domain` for paths starting with a single `/`. URLs starting with `//` already specify their host. This option does not resolve relative paths such as `../images/logo.png` against a page URL.
119
+
120
+ ```python
121
+ from url_normalize import url_normalize
122
+
123
+ result = url_normalize(
124
+ "/images/logo.png", default_domain="example.com", default_scheme="http"
125
+ )
126
+ print(result)
127
+ # Output: http://example.com/images/logo.png
128
+ ```
129
+
130
+ ### Human-readable URLs
131
+
132
+ `url_humanize()` normalizes its input, then decodes components only when the display URL normalizes back to the same result. Some escapes therefore remain encoded.
133
+
134
+ ```python
135
+ from url_normalize import url_humanize, url_normalize
136
+
137
+ normalized = "https://xn--bcher-kva.de/r%C3%A9sum%C3%A9"
138
+ readable = url_humanize(normalized)
139
+ print(readable)
140
+ # Output: https://bücher.de/résumé
141
+
142
+ assert url_normalize(readable) == normalized
143
+ ```
144
+
145
+ ## Command-line usage
146
+
147
+ The CLI accepts one URL per invocation and prints the result to standard output. Quote URLs containing shell characters such as `&`.
148
+
149
+ ```sh
150
+ # Keep selected query parameters.
151
+ url-normalize -f -p page,id "example.com/?page=1&id=123&ref=test"
152
+ # Output: https://example.com/?page=1&id=123
153
+
154
+ # Supply a domain and scheme for an absolute path.
155
+ url-normalize -d example.com -s http "/images/logo.png"
156
+ # Output: http://example.com/images/logo.png
157
+
158
+ # Display a readable URL.
159
+ url-normalize -H "https://xn--bcher-kva.de/r%C3%A9sum%C3%A9"
160
+ # Output: https://bücher.de/résumé
161
+ ```
162
+
163
+ Use `url-normalize --help` for all options and `url-normalize --version` for the installed version. Normalization errors are written to standard error with exit status 1; invalid command-line arguments use exit status 2.
164
+
165
+ To run without a persistent tool installation:
166
+
167
+ ```sh
168
+ uvx url-normalize "example.com"
169
+ # Output: https://example.com/
170
+ ```
171
+
172
+ ## Contributing
173
+
174
+ After cloning the repository, install the development dependencies and run the checks with [uv](https://docs.astral.sh/uv/):
175
+
176
+ ```sh
177
+ uv sync --locked --extra dev
178
+ uv run --locked --extra dev python -m pytest
179
+ uv run --locked --extra dev pre-commit run --all-files
180
+ ```
181
+
182
+ The test suite requires 100% statement coverage. CI tests Python 3.11 through 3.15 on Linux, macOS, and Windows. Please include regression tests with bug fixes.
183
+
184
+ ## Changelog and license
185
+
186
+ See [CHANGELOG.md](CHANGELOG.md) for release history and upgrade notes. Distributed under the [MIT License](LICENSE).
187
+
188
+ Inspired by Sam Ruby's [urlnorm.py](http://intertwingly.net/blog/2004/08/04/Urlnorm).
@@ -1,22 +1,22 @@
1
1
  [project]
2
2
  name = "url-normalize"
3
- version = "3.0.1"
3
+ version = "3.1.0"
4
4
  description = "URL normalization for Python"
5
5
  authors = [{ name = "Nikolay Panov", email = "github@npanov.com" }]
6
6
  license = "MIT"
7
7
  readme = "README.md"
8
- requires-python = ">=3.10"
8
+ requires-python = ">=3.11"
9
9
  keywords = ["url", "normalization", "normalize", "normalizer"]
10
10
  classifiers = [
11
11
  "Programming Language :: Python :: 3",
12
12
  "Programming Language :: Python :: 3 :: Only",
13
- "Programming Language :: Python :: 3.10",
14
13
  "Programming Language :: Python :: 3.11",
15
14
  "Programming Language :: Python :: 3.12",
16
15
  "Programming Language :: Python :: 3.13",
17
16
  "Programming Language :: Python :: 3.14",
17
+ "Programming Language :: Python :: 3.15",
18
18
  ]
19
- dependencies = ["idna>=3.3"]
19
+ dependencies = ["idna>=3.20"]
20
20
 
21
21
  [project.urls]
22
22
  Homepage = "https://github.com/niksite/url-normalize"
@@ -31,7 +31,7 @@ url-normalize = "url_normalize.cli:main"
31
31
  dev = ["mypy", "pre-commit", "pytest-cov", "pytest-socket", "pytest", "ruff"]
32
32
 
33
33
  [tool.ruff]
34
- target-version = "py310"
34
+ target-version = "py311"
35
35
  line-length = 88
36
36
  unsafe-fixes = true
37
37
 
@@ -45,6 +45,7 @@ extend-select = [
45
45
  fixable = ["ALL"]
46
46
  ignore = [
47
47
  "COM812", # missing-trailing-comma
48
+ "CPY001", # License is documented in LICENSE rather than per-file headers
48
49
  "D203", # One blank line before class - we prefer D211 instead
49
50
  "D213", # multi-line-summary-second-line - we prefer D212 instead
50
51
  ]
@@ -53,7 +54,7 @@ ignore = [
53
54
  convention = "google"
54
55
 
55
56
  [tool.ruff.lint.per-file-ignores]
56
- "tests/**" = ["INP001", "ANN001", "ANN201", "S101", "CPY001"]
57
+ "tests/**" = ["INP001", "ANN001", "ANN201", "S101"]
57
58
 
58
59
  [tool.ruff.format]
59
60
  quote-style = "double"
@@ -62,7 +63,7 @@ indent-style = "space"
62
63
  [tool.mypy]
63
64
  ignore_missing_imports = true
64
65
  exclude = ["tests"]
65
- python_version = "3.10"
66
+ python_version = "3.11"
66
67
  show_error_codes = true
67
68
 
68
69
  [build-system]
@@ -35,16 +35,23 @@ def test_cli_help_describes_allowlist_filtering() -> None:
35
35
  assert "Unknown domains have an empty default allowlist." in help_text
36
36
 
37
37
 
38
- def test_cli_help_describes_fixed_utf8_encoding() -> None:
39
- """Describe charset as a compatibility option for Unicode URL input."""
38
+ def test_cli_help_omits_removed_charset_option() -> None:
39
+ """Remove the obsolete encoding option from CLI help."""
40
40
  result = run_cli("--help")
41
41
  assert result.returncode == 0
42
- help_text = " ".join(result.stdout.split())
43
- assert "Retained for compatibility." in help_text
44
- assert "Unicode characters use UTF-8 percent encoding." in help_text
45
- normalized = run_cli("--charset", "iso-8859-1", "https://example.com/é")
46
- assert normalized.returncode == 0
47
- assert normalized.stdout.strip() == "https://example.com/%C3%A9"
42
+ assert "charset" not in result.stdout
43
+
44
+
45
+ @pytest.mark.parametrize("option", ["-c", "--charset"])
46
+ def test_cli_rejects_removed_charset_option(option, monkeypatch):
47
+ """Report removed flags with UTF-8 diagnostics even under a legacy encoding."""
48
+ monkeypatch.setenv("PYTHONIOENCODING", "cp1252")
49
+ usage_error_status = 2
50
+ result = run_cli(option, "utf-8", "https://example.com/é")
51
+ assert result.returncode == usage_error_status
52
+ assert "unrecognized arguments" in result.stderr
53
+ assert "https://example.com/é" in result.stderr
54
+ assert not result.stdout
48
55
 
49
56
 
50
57
  def test_cli_successful_main(capsys, monkeypatch):
@@ -284,35 +291,12 @@ def test_cli_version_short() -> None:
284
291
  assert not result.stderr
285
292
 
286
293
 
287
- @pytest.mark.skipif(
288
- sys.platform == "win32", reason="Charset handling differs on Windows CLI"
289
- )
290
- def test_cli_charset() -> None:
291
- """Test charset handling via CLI (might be platform-dependent)."""
292
- # Example using Cyrillic characters which need correct encoding
293
- url = "http://пример.рф/path"
294
- expected_idn = "http://xn--e1afmkfd.xn--p1ai/path"
295
-
296
- # Test with default UTF-8
297
- result_utf8 = run_cli(url)
298
-
299
- assert result_utf8.returncode == 0
300
- assert result_utf8.stdout.strip() == expected_idn
301
- assert not result_utf8.stderr
302
-
303
- # Test specifying UTF-8 explicitly
304
- result_charset = run_cli("--charset", "utf-8", url)
305
-
306
- assert result_charset.returncode == 0
307
- assert result_charset.stdout.strip() == expected_idn
308
- assert not result_charset.stderr
309
-
310
- # Test specifying UTF-8 explicitly using short arg
311
- result_charset_short = run_cli("-c", "utf-8", url)
312
-
313
- assert result_charset_short.returncode == 0
314
- assert result_charset_short.stdout.strip() == expected_idn
315
- assert not result_charset_short.stderr
294
+ def test_cli_unicode_host_and_path() -> None:
295
+ """Encode Unicode hosts with IDNA and paths with UTF-8 on every platform."""
296
+ result = run_cli("http://пример.рф/é")
297
+ assert result.returncode == 0
298
+ assert result.stdout.strip() == "http://xn--e1afmkfd.xn--p1ai/%C3%A9"
299
+ assert not result.stderr
316
300
 
317
301
 
318
302
  def test_cli_default_domain() -> None:
@@ -32,7 +32,36 @@ def test_normalize_host_result_is_expected(host: str, expected: str) -> None:
32
32
  assert result == expected, host
33
33
 
34
34
 
35
- @pytest.mark.parametrize("charset", ["utf-8", "utf-16", "utf-32", "iso-8859-1"])
35
+ @pytest.mark.parametrize("separator", [".", "\u3002", "\uff0e", "\uff61"])
36
+ @pytest.mark.parametrize(
37
+ ("host", "expected"),
38
+ [
39
+ ("EXAMPLE.COM", "example.com"),
40
+ ("faß.de", "xn--fa-hia.de"),
41
+ ("under_score.example", "under_score.example"),
42
+ ],
43
+ )
44
+ def test_normalize_host_strips_trailing_idna_separator(host, expected, separator):
45
+ """Strip equivalent trailing label separators before IDNA conversion."""
46
+ normalized = normalize_host(host + separator)
47
+ assert normalized == expected
48
+ assert normalize_host(normalized) == normalized
49
+
50
+
51
+ @pytest.mark.parametrize("prefix", ["", "."])
52
+ @pytest.mark.parametrize("separator", ["\u3002", "\uff0e", "\uff61"])
53
+ def test_normalize_host_rejects_leading_unicode_separator(prefix, separator):
54
+ """Keep rejecting leading Unicode label separators as before."""
55
+ with pytest.raises(UnicodeError):
56
+ normalize_host(prefix + separator + "example.com")
57
+
58
+
59
+ @pytest.mark.parametrize("prefix", [".", ".."])
60
+ def test_normalize_host_strips_leading_ascii_dots(prefix):
61
+ """Retain the existing normalization of leading ASCII dots."""
62
+ assert normalize_host(prefix + "example.com") == "example.com"
63
+
64
+
36
65
  @pytest.mark.parametrize(
37
66
  ("host", "expected"),
38
67
  [
@@ -42,17 +71,16 @@ def test_normalize_host_result_is_expected(host: str, expected: str) -> None:
42
71
  ("[2001:DB8::1]", "[2001:db8::1]"),
43
72
  ],
44
73
  )
45
- def test_normalize_host_decodes_idna_output_as_ascii(host, expected, charset):
46
- """Decode ASCII IDNA output independently of the input byte encoding."""
47
- assert normalize_host(host, charset) == expected
48
- assert normalize_host(expected.encode(charset), charset) == expected
74
+ def test_normalize_host_accepts_text_and_utf8_bytes(host, expected):
75
+ """Normalize text and UTF-8 bytes to the same ASCII hostname."""
76
+ assert normalize_host(host) == expected
77
+ assert normalize_host(host.encode("utf-8")) == expected
49
78
 
50
79
 
51
80
  @pytest.mark.parametrize("zone", ["%25ethA", "%ethA", "%25Interface.10", "%25ETH0"])
52
- @pytest.mark.parametrize("charset", ["utf-8", "utf-16"])
53
- def test_normalize_host_preserves_ipv6_zone_case(zone, charset):
81
+ def test_normalize_host_preserves_ipv6_zone_case(zone):
54
82
  """Lowercase the IPv6 address without changing its interface identifier."""
55
83
  host = f"[FE80::ABCD{zone}]"
56
84
  expected = f"[fe80::abcd{zone}]"
57
- assert normalize_host(host, charset) == expected
58
- assert normalize_host(host.encode(charset), charset) == expected
85
+ assert normalize_host(host) == expected
86
+ assert normalize_host(host.encode("utf-8")) == expected
@@ -57,6 +57,70 @@ def test_custom_list_allowlist():
57
57
  )
58
58
 
59
59
 
60
+ @pytest.mark.parametrize("separator", [".", "\u3002", "\uff0e", "\uff61"])
61
+ def test_custom_allowlist_matches_trailing_idna_separator(separator):
62
+ """Match custom domain keys with any IDNA trailing label separator."""
63
+ assert (
64
+ url_normalize(
65
+ "https://example.com/?q=1&drop=2",
66
+ filter_params=True,
67
+ param_allowlist={f"example.com{separator}": ["q"]},
68
+ )
69
+ == "https://example.com/?q=1"
70
+ )
71
+
72
+
73
+ @pytest.mark.parametrize(
74
+ ("name", "query", "expected"),
75
+ [
76
+ ("café", "café=1", "caf%C3%A9=1"), # codespell:ignore caf
77
+ ("café", "caf%C3%A9=1", "caf%C3%A9=1"), # codespell:ignore caf
78
+ ("cafe\u0301", "caf%C3%A9=1", "caf%C3%A9=1"), # codespell:ignore caf
79
+ ("search term", "search%20term=1", "search%20term=1"),
80
+ ],
81
+ )
82
+ @pytest.mark.parametrize("domain_specific", [False, True])
83
+ def test_custom_allowlist_normalizes_parameter_names(
84
+ name, query, expected, domain_specific
85
+ ):
86
+ """Match parameter names using the same normalization as query keys."""
87
+ allowlist = {"example.com": [name]} if domain_specific else [name]
88
+ assert (
89
+ url_normalize(
90
+ f"https://example.com/?{query}&drop=2",
91
+ filter_params=True,
92
+ param_allowlist=allowlist,
93
+ )
94
+ == f"https://example.com/?{expected}"
95
+ )
96
+ assert allowlist == ({"example.com": [name]} if domain_specific else [name])
97
+
98
+
99
+ @pytest.mark.parametrize(
100
+ ("name", "expected"),
101
+ [
102
+ ("a+b", "a+b=1"),
103
+ ("a%2Bb", "a%2Bb=2"),
104
+ ("a b", "a%20b=3"),
105
+ ("a%20b", "a%20b=3"),
106
+ ],
107
+ )
108
+ @pytest.mark.parametrize("domain_specific", [False, True])
109
+ def test_custom_allowlist_keeps_plus_and_space_names_distinct(
110
+ name, expected, domain_specific
111
+ ):
112
+ """Keep literal plus, encoded plus, and encoded space keys distinct."""
113
+ allowlist = {"example.com": [name]} if domain_specific else [name]
114
+ assert (
115
+ url_normalize(
116
+ "https://example.com/?a+b=1&a%2Bb=2&a%20b=3&drop=4",
117
+ filter_params=True,
118
+ param_allowlist=allowlist,
119
+ )
120
+ == f"https://example.com/?{expected}"
121
+ )
122
+
123
+
60
124
  @pytest.mark.parametrize(
61
125
  ("url", "expected"),
62
126
  [