url-normalize 3.0.0__tar.gz → 3.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. url_normalize-3.1.0/PKG-INFO +219 -0
  2. url_normalize-3.1.0/README.md +188 -0
  3. {url_normalize-3.0.0 → url_normalize-3.1.0}/pyproject.toml +9 -7
  4. {url_normalize-3.0.0 → url_normalize-3.1.0}/tests/test_cli.py +45 -29
  5. url_normalize-3.1.0/tests/test_deconstruct_url.py +87 -0
  6. url_normalize-3.1.0/tests/test_generic_url_cleanup.py +72 -0
  7. url_normalize-3.1.0/tests/test_normalize_host.py +86 -0
  8. {url_normalize-3.0.0 → url_normalize-3.1.0}/tests/test_normalize_path.py +15 -1
  9. url_normalize-3.1.0/tests/test_normalize_query.py +57 -0
  10. url_normalize-3.1.0/tests/test_normalize_query_filters.py +290 -0
  11. url_normalize-3.1.0/tests/test_normalize_userinfo.py +44 -0
  12. url_normalize-3.1.0/tests/test_provide_url_scheme.py +140 -0
  13. {url_normalize-3.0.0 → url_normalize-3.1.0}/tests/test_url_humanize.py +50 -5
  14. {url_normalize-3.0.0 → url_normalize-3.1.0}/tests/test_url_normalize.py +70 -2
  15. {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/__init__.py +1 -1
  16. {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/cli.py +8 -11
  17. url_normalize-3.1.0/url_normalize/generic_url_cleanup.py +27 -0
  18. {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/normalize_host.py +10 -9
  19. {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/normalize_path.py +2 -2
  20. {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/normalize_query.py +14 -8
  21. {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/normalize_userinfo.py +9 -1
  22. {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/param_allowlist.py +23 -6
  23. {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/provide_url_scheme.py +12 -2
  24. {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/tools.py +36 -15
  25. {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/url_humanize.py +17 -9
  26. {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/url_normalize.py +7 -8
  27. url_normalize-3.1.0/url_normalize.egg-info/PKG-INFO +219 -0
  28. {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize.egg-info/requires.txt +1 -1
  29. url_normalize-3.0.0/PKG-INFO +0 -211
  30. url_normalize-3.0.0/README.md +0 -180
  31. url_normalize-3.0.0/tests/test_deconstruct_url.py +0 -40
  32. url_normalize-3.0.0/tests/test_generic_url_cleanup.py +0 -21
  33. url_normalize-3.0.0/tests/test_normalize_host.py +0 -32
  34. url_normalize-3.0.0/tests/test_normalize_query.py +0 -24
  35. url_normalize-3.0.0/tests/test_normalize_query_filters.py +0 -98
  36. url_normalize-3.0.0/tests/test_normalize_userinfo.py +0 -21
  37. url_normalize-3.0.0/tests/test_provide_url_scheme.py +0 -32
  38. url_normalize-3.0.0/url_normalize/generic_url_cleanup.py +0 -19
  39. url_normalize-3.0.0/url_normalize.egg-info/PKG-INFO +0 -211
  40. {url_normalize-3.0.0 → url_normalize-3.1.0}/LICENSE +0 -0
  41. {url_normalize-3.0.0 → url_normalize-3.1.0}/setup.cfg +0 -0
  42. {url_normalize-3.0.0 → url_normalize-3.1.0}/tests/test_normalize_fragment.py +0 -0
  43. {url_normalize-3.0.0 → url_normalize-3.1.0}/tests/test_normalize_port.py +0 -0
  44. {url_normalize-3.0.0 → url_normalize-3.1.0}/tests/test_normalize_scheme.py +0 -0
  45. {url_normalize-3.0.0 → url_normalize-3.1.0}/tests/test_provide_url_domain.py +0 -0
  46. {url_normalize-3.0.0 → url_normalize-3.1.0}/tests/test_reconstruct_url.py +0 -0
  47. {url_normalize-3.0.0 → url_normalize-3.1.0}/tests/test_tools.py +0 -0
  48. {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/normalize_fragment.py +0 -0
  49. {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/normalize_port.py +0 -0
  50. {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/normalize_scheme.py +0 -0
  51. {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/provide_url_domain.py +0 -0
  52. {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize/py.typed +0 -0
  53. {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize.egg-info/SOURCES.txt +0 -0
  54. {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize.egg-info/dependency_links.txt +0 -0
  55. {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize.egg-info/entry_points.txt +0 -0
  56. {url_normalize-3.0.0 → url_normalize-3.1.0}/url_normalize.egg-info/top_level.txt +0 -0
@@ -0,0 +1,219 @@
1
+ Metadata-Version: 2.4
2
+ Name: url-normalize
3
+ Version: 3.1.0
4
+ Summary: URL normalization for Python
5
+ Author-email: Nikolay Panov <github@npanov.com>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/niksite/url-normalize
8
+ Project-URL: Repository, https://github.com/niksite/url-normalize
9
+ Project-URL: Issues, https://github.com/niksite/url-normalize/issues
10
+ Project-URL: Changelog, https://github.com/niksite/url-normalize/blob/master/CHANGELOG.md
11
+ Keywords: url,normalization,normalize,normalizer
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Programming Language :: Python :: 3 :: Only
14
+ Classifier: Programming Language :: Python :: 3.11
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Programming Language :: Python :: 3.13
17
+ Classifier: Programming Language :: Python :: 3.14
18
+ Classifier: Programming Language :: Python :: 3.15
19
+ Requires-Python: >=3.11
20
+ Description-Content-Type: text/markdown
21
+ License-File: LICENSE
22
+ Requires-Dist: idna>=3.20
23
+ Provides-Extra: dev
24
+ Requires-Dist: mypy; extra == "dev"
25
+ Requires-Dist: pre-commit; extra == "dev"
26
+ Requires-Dist: pytest-cov; extra == "dev"
27
+ Requires-Dist: pytest-socket; extra == "dev"
28
+ Requires-Dist: pytest; extra == "dev"
29
+ Requires-Dist: ruff; extra == "dev"
30
+ Dynamic: license-file
31
+
32
+ # url-normalize
33
+
34
+ [![tests](https://github.com/niksite/url-normalize/actions/workflows/ci.yml/badge.svg)](https://github.com/niksite/url-normalize/actions/workflows/ci.yml)
35
+ [![Coveralls](https://img.shields.io/coveralls/github/niksite/url-normalize/master.svg)](https://coveralls.io/r/niksite/url-normalize)
36
+ [![PyPI](https://img.shields.io/pypi/v/url-normalize.svg)](https://pypi.org/project/url-normalize/)
37
+ [![Python Versions](https://img.shields.io/pypi/pyversions/url-normalize.svg)](https://pypi.org/project/url-normalize/)
38
+ [![License](https://img.shields.io/pypi/l/url-normalize.svg)](https://github.com/niksite/url-normalize/blob/master/LICENSE)
39
+ [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
40
+
41
+ Normalize URL spelling consistently for storage, comparison, and display. Use `url_normalize()` for normalized URLs and `url_humanize()` for readable display URLs.
42
+
43
+ Requires **Python 3.11 or newer**. Normalization operates on URL text; it does not fetch pages or follow redirects.
44
+
45
+ ## Installation
46
+
47
+ Install the library and CLI:
48
+
49
+ ```sh
50
+ python -m pip install url-normalize
51
+ ```
52
+
53
+ Or install the CLI in its own environment with [uv](https://docs.astral.sh/uv/):
54
+
55
+ ```sh
56
+ uv tool install url-normalize
57
+ ```
58
+
59
+ ## Quick start
60
+
61
+ ```python
62
+ from url_normalize import url_normalize
63
+
64
+ print(url_normalize("HTTP://Example.COM:80/a/../b"))
65
+ # Output: http://example.com/b
66
+
67
+ print(url_normalize("bücher.de/résumé"))
68
+ # Output: https://xn--bcher-kva.de/r%C3%A9sum%C3%A9
69
+ ```
70
+
71
+ From the terminal:
72
+
73
+ ```sh
74
+ url-normalize "HTTP://Example.COM:80/a/../b"
75
+ # Output: http://example.com/b
76
+ ```
77
+
78
+ ## Normalization behavior
79
+
80
+ | Behavior | Input | Output |
81
+ | --- | --- | --- |
82
+ | Add a missing scheme | `example.com` | `https://example.com/` |
83
+ | Lowercase scheme and host; remove a default port | `HTTP://Example.COM:80/` | `http://example.com/` |
84
+ | Resolve dot segments | `https://example.com/a/../b` | `https://example.com/b` |
85
+ | Encode Unicode hosts and paths | `https://bücher.de/résumé` | `https://xn--bcher-kva.de/r%C3%A9sum%C3%A9` |
86
+ | Remove trailing host dots, including Unicode equivalents | `https://example.com。/` | `https://example.com/` |
87
+ | Preserve encoded path separators; uppercase escape hex | `https://example.com/a%2fb` | `https://example.com/a%2Fb` |
88
+ | Preserve query order and repeated keys | `https://example.com/?b=2&a=1&a=3` | `https://example.com/?b=2&a=1&a=3` |
89
+ | Preserve explicit empty query values | `https://example.com/?flag=&flag` | `https://example.com/?flag=&flag` |
90
+
91
+ In query parameters, `+`, `%2B`, and `%20` remain distinct. Parameter filtering is disabled by default. Fragments are preserved and normalized; hashbang fragments (`#!`) are converted to the `_escaped_fragment_` query parameter.
92
+
93
+ Normalization applies consistent spelling rules; it does not establish whether two URLs serve the same content. Applications using normalized URLs as persistent cache keys or identifiers should review the [changelog](CHANGELOG.md) when upgrading.
94
+
95
+ ## Python API
96
+
97
+ Both functions accept a URL as `str` or `None` and return `str` or `None`. An empty string and `None` are returned unchanged. Malformed input can raise `ValueError`, including `UnicodeError` for invalid hostnames.
98
+
99
+ The following options are keyword-only and apply to both functions:
100
+
101
+ | Option | Default | Purpose |
102
+ | --- | --- | --- |
103
+ | `default_scheme` | `"https"` | Scheme to add when the URL has none. |
104
+ | `default_domain` | `None` | Domain to add to an absolute path such as `/images/logo.png`. |
105
+ | `filter_params` | `False` | Retain only allowlisted query parameters. |
106
+ | `param_allowlist` | `None` | Override built-in rules with a list of names or a dictionary mapping domains to lists. Requires `filter_params=True`. |
107
+
108
+ Unicode URL characters always use UTF-8 percent encoding.
109
+
110
+ ### Query parameter filtering
111
+
112
+ With `filter_params=True`, only explicitly allowed parameters remain. Built-in rules cover Google, Bing, Baidu, and YouTube; see the [domain rules](url_normalize/param_allowlist.py) for their parameter names.
113
+
114
+ **Unknown domains have an empty default allowlist**, so filtering removes all their query parameters unless you supply a custom rule. A custom allowlist replaces the built-in rules.
115
+
116
+ ```python
117
+ from url_normalize import url_normalize
118
+
119
+ # Use the built-in Google rule.
120
+ result = url_normalize(
121
+ "www.google.com/search?q=test&utm_source=test", filter_params=True
122
+ )
123
+ print(result)
124
+ # Output: https://www.google.com/search?q=test
125
+
126
+ # Apply a list of allowed names to any domain.
127
+ result = url_normalize(
128
+ "example.com/?page=1&id=123&ref=test",
129
+ filter_params=True,
130
+ param_allowlist=["page", "id"],
131
+ )
132
+ print(result)
133
+ # Output: https://example.com/?page=1&id=123
134
+
135
+ # Use domain-specific rules, including Unicode parameter names.
136
+ result = url_normalize(
137
+ "https://example.com/?r%C3%A9sum%C3%A9=1&drop=2",
138
+ filter_params=True,
139
+ param_allowlist={"example.com": ["résumé"]},
140
+ )
141
+ print(result)
142
+ # Output: https://example.com/?r%C3%A9sum%C3%A9=1
143
+ ```
144
+
145
+ Allowed names use the same normalization as query keys, including Unicode and spaces. Domain rules match normalized hostnames and ignore a leading `www.` and the port. Rules do not automatically apply to other subdomains.
146
+
147
+ ### Default domain and scheme
148
+
149
+ Use `default_domain` for paths starting with a single `/`. URLs starting with `//` already specify their host. This option does not resolve relative paths such as `../images/logo.png` against a page URL.
150
+
151
+ ```python
152
+ from url_normalize import url_normalize
153
+
154
+ result = url_normalize(
155
+ "/images/logo.png", default_domain="example.com", default_scheme="http"
156
+ )
157
+ print(result)
158
+ # Output: http://example.com/images/logo.png
159
+ ```
160
+
161
+ ### Human-readable URLs
162
+
163
+ `url_humanize()` normalizes its input, then decodes components only when the display URL normalizes back to the same result. Some escapes therefore remain encoded.
164
+
165
+ ```python
166
+ from url_normalize import url_humanize, url_normalize
167
+
168
+ normalized = "https://xn--bcher-kva.de/r%C3%A9sum%C3%A9"
169
+ readable = url_humanize(normalized)
170
+ print(readable)
171
+ # Output: https://bücher.de/résumé
172
+
173
+ assert url_normalize(readable) == normalized
174
+ ```
175
+
176
+ ## Command-line usage
177
+
178
+ The CLI accepts one URL per invocation and prints the result to standard output. Quote URLs containing shell characters such as `&`.
179
+
180
+ ```sh
181
+ # Keep selected query parameters.
182
+ url-normalize -f -p page,id "example.com/?page=1&id=123&ref=test"
183
+ # Output: https://example.com/?page=1&id=123
184
+
185
+ # Supply a domain and scheme for an absolute path.
186
+ url-normalize -d example.com -s http "/images/logo.png"
187
+ # Output: http://example.com/images/logo.png
188
+
189
+ # Display a readable URL.
190
+ url-normalize -H "https://xn--bcher-kva.de/r%C3%A9sum%C3%A9"
191
+ # Output: https://bücher.de/résumé
192
+ ```
193
+
194
+ Use `url-normalize --help` for all options and `url-normalize --version` for the installed version. Normalization errors are written to standard error with exit status 1; invalid command-line arguments use exit status 2.
195
+
196
+ To run without a persistent tool installation:
197
+
198
+ ```sh
199
+ uvx url-normalize "example.com"
200
+ # Output: https://example.com/
201
+ ```
202
+
203
+ ## Contributing
204
+
205
+ After cloning the repository, install the development dependencies and run the checks with [uv](https://docs.astral.sh/uv/):
206
+
207
+ ```sh
208
+ uv sync --locked --extra dev
209
+ uv run --locked --extra dev python -m pytest
210
+ uv run --locked --extra dev pre-commit run --all-files
211
+ ```
212
+
213
+ The test suite requires 100% statement coverage. CI tests Python 3.11 through 3.15 on Linux, macOS, and Windows. Please include regression tests with bug fixes.
214
+
215
+ ## Changelog and license
216
+
217
+ See [CHANGELOG.md](CHANGELOG.md) for release history and upgrade notes. Distributed under the [MIT License](LICENSE).
218
+
219
+ Inspired by Sam Ruby's [urlnorm.py](http://intertwingly.net/blog/2004/08/04/Urlnorm).
@@ -0,0 +1,188 @@
1
+ # url-normalize
2
+
3
+ [![tests](https://github.com/niksite/url-normalize/actions/workflows/ci.yml/badge.svg)](https://github.com/niksite/url-normalize/actions/workflows/ci.yml)
4
+ [![Coveralls](https://img.shields.io/coveralls/github/niksite/url-normalize/master.svg)](https://coveralls.io/r/niksite/url-normalize)
5
+ [![PyPI](https://img.shields.io/pypi/v/url-normalize.svg)](https://pypi.org/project/url-normalize/)
6
+ [![Python Versions](https://img.shields.io/pypi/pyversions/url-normalize.svg)](https://pypi.org/project/url-normalize/)
7
+ [![License](https://img.shields.io/pypi/l/url-normalize.svg)](https://github.com/niksite/url-normalize/blob/master/LICENSE)
8
+ [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
9
+
10
+ Normalize URL spelling consistently for storage, comparison, and display. Use `url_normalize()` for normalized URLs and `url_humanize()` for readable display URLs.
11
+
12
+ Requires **Python 3.11 or newer**. Normalization operates on URL text; it does not fetch pages or follow redirects.
13
+
14
+ ## Installation
15
+
16
+ Install the library and CLI:
17
+
18
+ ```sh
19
+ python -m pip install url-normalize
20
+ ```
21
+
22
+ Or install the CLI in its own environment with [uv](https://docs.astral.sh/uv/):
23
+
24
+ ```sh
25
+ uv tool install url-normalize
26
+ ```
27
+
28
+ ## Quick start
29
+
30
+ ```python
31
+ from url_normalize import url_normalize
32
+
33
+ print(url_normalize("HTTP://Example.COM:80/a/../b"))
34
+ # Output: http://example.com/b
35
+
36
+ print(url_normalize("bücher.de/résumé"))
37
+ # Output: https://xn--bcher-kva.de/r%C3%A9sum%C3%A9
38
+ ```
39
+
40
+ From the terminal:
41
+
42
+ ```sh
43
+ url-normalize "HTTP://Example.COM:80/a/../b"
44
+ # Output: http://example.com/b
45
+ ```
46
+
47
+ ## Normalization behavior
48
+
49
+ | Behavior | Input | Output |
50
+ | --- | --- | --- |
51
+ | Add a missing scheme | `example.com` | `https://example.com/` |
52
+ | Lowercase scheme and host; remove a default port | `HTTP://Example.COM:80/` | `http://example.com/` |
53
+ | Resolve dot segments | `https://example.com/a/../b` | `https://example.com/b` |
54
+ | Encode Unicode hosts and paths | `https://bücher.de/résumé` | `https://xn--bcher-kva.de/r%C3%A9sum%C3%A9` |
55
+ | Remove trailing host dots, including Unicode equivalents | `https://example.com。/` | `https://example.com/` |
56
+ | Preserve encoded path separators; uppercase escape hex | `https://example.com/a%2fb` | `https://example.com/a%2Fb` |
57
+ | Preserve query order and repeated keys | `https://example.com/?b=2&a=1&a=3` | `https://example.com/?b=2&a=1&a=3` |
58
+ | Preserve explicit empty query values | `https://example.com/?flag=&flag` | `https://example.com/?flag=&flag` |
59
+
60
+ In query parameters, `+`, `%2B`, and `%20` remain distinct. Parameter filtering is disabled by default. Fragments are preserved and normalized; hashbang fragments (`#!`) are converted to the `_escaped_fragment_` query parameter.
61
+
62
+ Normalization applies consistent spelling rules; it does not establish whether two URLs serve the same content. Applications using normalized URLs as persistent cache keys or identifiers should review the [changelog](CHANGELOG.md) when upgrading.
63
+
64
+ ## Python API
65
+
66
+ Both functions accept a URL as `str` or `None` and return `str` or `None`. An empty string and `None` are returned unchanged. Malformed input can raise `ValueError`, including `UnicodeError` for invalid hostnames.
67
+
68
+ The following options are keyword-only and apply to both functions:
69
+
70
+ | Option | Default | Purpose |
71
+ | --- | --- | --- |
72
+ | `default_scheme` | `"https"` | Scheme to add when the URL has none. |
73
+ | `default_domain` | `None` | Domain to add to an absolute path such as `/images/logo.png`. |
74
+ | `filter_params` | `False` | Retain only allowlisted query parameters. |
75
+ | `param_allowlist` | `None` | Override built-in rules with a list of names or a dictionary mapping domains to lists. Requires `filter_params=True`. |
76
+
77
+ Unicode URL characters always use UTF-8 percent encoding.
78
+
79
+ ### Query parameter filtering
80
+
81
+ With `filter_params=True`, only explicitly allowed parameters remain. Built-in rules cover Google, Bing, Baidu, and YouTube; see the [domain rules](url_normalize/param_allowlist.py) for their parameter names.
82
+
83
+ **Unknown domains have an empty default allowlist**, so filtering removes all their query parameters unless you supply a custom rule. A custom allowlist replaces the built-in rules.
84
+
85
+ ```python
86
+ from url_normalize import url_normalize
87
+
88
+ # Use the built-in Google rule.
89
+ result = url_normalize(
90
+ "www.google.com/search?q=test&utm_source=test", filter_params=True
91
+ )
92
+ print(result)
93
+ # Output: https://www.google.com/search?q=test
94
+
95
+ # Apply a list of allowed names to any domain.
96
+ result = url_normalize(
97
+ "example.com/?page=1&id=123&ref=test",
98
+ filter_params=True,
99
+ param_allowlist=["page", "id"],
100
+ )
101
+ print(result)
102
+ # Output: https://example.com/?page=1&id=123
103
+
104
+ # Use domain-specific rules, including Unicode parameter names.
105
+ result = url_normalize(
106
+ "https://example.com/?r%C3%A9sum%C3%A9=1&drop=2",
107
+ filter_params=True,
108
+ param_allowlist={"example.com": ["résumé"]},
109
+ )
110
+ print(result)
111
+ # Output: https://example.com/?r%C3%A9sum%C3%A9=1
112
+ ```
113
+
114
+ Allowed names use the same normalization as query keys, including Unicode and spaces. Domain rules match normalized hostnames and ignore a leading `www.` and the port. Rules do not automatically apply to other subdomains.
115
+
116
+ ### Default domain and scheme
117
+
118
+ Use `default_domain` for paths starting with a single `/`. URLs starting with `//` already specify their host. This option does not resolve relative paths such as `../images/logo.png` against a page URL.
119
+
120
+ ```python
121
+ from url_normalize import url_normalize
122
+
123
+ result = url_normalize(
124
+ "/images/logo.png", default_domain="example.com", default_scheme="http"
125
+ )
126
+ print(result)
127
+ # Output: http://example.com/images/logo.png
128
+ ```
129
+
130
+ ### Human-readable URLs
131
+
132
+ `url_humanize()` normalizes its input, then decodes components only when the display URL normalizes back to the same result. Some escapes therefore remain encoded.
133
+
134
+ ```python
135
+ from url_normalize import url_humanize, url_normalize
136
+
137
+ normalized = "https://xn--bcher-kva.de/r%C3%A9sum%C3%A9"
138
+ readable = url_humanize(normalized)
139
+ print(readable)
140
+ # Output: https://bücher.de/résumé
141
+
142
+ assert url_normalize(readable) == normalized
143
+ ```
144
+
145
+ ## Command-line usage
146
+
147
+ The CLI accepts one URL per invocation and prints the result to standard output. Quote URLs containing shell characters such as `&`.
148
+
149
+ ```sh
150
+ # Keep selected query parameters.
151
+ url-normalize -f -p page,id "example.com/?page=1&id=123&ref=test"
152
+ # Output: https://example.com/?page=1&id=123
153
+
154
+ # Supply a domain and scheme for an absolute path.
155
+ url-normalize -d example.com -s http "/images/logo.png"
156
+ # Output: http://example.com/images/logo.png
157
+
158
+ # Display a readable URL.
159
+ url-normalize -H "https://xn--bcher-kva.de/r%C3%A9sum%C3%A9"
160
+ # Output: https://bücher.de/résumé
161
+ ```
162
+
163
+ Use `url-normalize --help` for all options and `url-normalize --version` for the installed version. Normalization errors are written to standard error with exit status 1; invalid command-line arguments use exit status 2.
164
+
165
+ To run without a persistent tool installation:
166
+
167
+ ```sh
168
+ uvx url-normalize "example.com"
169
+ # Output: https://example.com/
170
+ ```
171
+
172
+ ## Contributing
173
+
174
+ After cloning the repository, install the development dependencies and run the checks with [uv](https://docs.astral.sh/uv/):
175
+
176
+ ```sh
177
+ uv sync --locked --extra dev
178
+ uv run --locked --extra dev python -m pytest
179
+ uv run --locked --extra dev pre-commit run --all-files
180
+ ```
181
+
182
+ The test suite requires 100% statement coverage. CI tests Python 3.11 through 3.15 on Linux, macOS, and Windows. Please include regression tests with bug fixes.
183
+
184
+ ## Changelog and license
185
+
186
+ See [CHANGELOG.md](CHANGELOG.md) for release history and upgrade notes. Distributed under the [MIT License](LICENSE).
187
+
188
+ Inspired by Sam Ruby's [urlnorm.py](http://intertwingly.net/blog/2004/08/04/Urlnorm).
@@ -1,22 +1,22 @@
1
1
  [project]
2
2
  name = "url-normalize"
3
- version = "3.0.0"
3
+ version = "3.1.0"
4
4
  description = "URL normalization for Python"
5
5
  authors = [{ name = "Nikolay Panov", email = "github@npanov.com" }]
6
6
  license = "MIT"
7
7
  readme = "README.md"
8
- requires-python = ">=3.10"
8
+ requires-python = ">=3.11"
9
9
  keywords = ["url", "normalization", "normalize", "normalizer"]
10
10
  classifiers = [
11
11
  "Programming Language :: Python :: 3",
12
12
  "Programming Language :: Python :: 3 :: Only",
13
- "Programming Language :: Python :: 3.10",
14
13
  "Programming Language :: Python :: 3.11",
15
14
  "Programming Language :: Python :: 3.12",
16
15
  "Programming Language :: Python :: 3.13",
17
16
  "Programming Language :: Python :: 3.14",
17
+ "Programming Language :: Python :: 3.15",
18
18
  ]
19
- dependencies = ["idna>=3.3"]
19
+ dependencies = ["idna>=3.20"]
20
20
 
21
21
  [project.urls]
22
22
  Homepage = "https://github.com/niksite/url-normalize"
@@ -31,7 +31,7 @@ url-normalize = "url_normalize.cli:main"
31
31
  dev = ["mypy", "pre-commit", "pytest-cov", "pytest-socket", "pytest", "ruff"]
32
32
 
33
33
  [tool.ruff]
34
- target-version = "py310"
34
+ target-version = "py311"
35
35
  line-length = 88
36
36
  unsafe-fixes = true
37
37
 
@@ -45,6 +45,7 @@ extend-select = [
45
45
  fixable = ["ALL"]
46
46
  ignore = [
47
47
  "COM812", # missing-trailing-comma
48
+ "CPY001", # License is documented in LICENSE rather than per-file headers
48
49
  "D203", # One blank line before class - we prefer D211 instead
49
50
  "D213", # multi-line-summary-second-line - we prefer D212 instead
50
51
  ]
@@ -53,7 +54,7 @@ ignore = [
53
54
  convention = "google"
54
55
 
55
56
  [tool.ruff.lint.per-file-ignores]
56
- "tests/**" = ["INP001", "ANN001", "ANN201", "S101", "CPY001"]
57
+ "tests/**" = ["INP001", "ANN001", "ANN201", "S101"]
57
58
 
58
59
  [tool.ruff.format]
59
60
  quote-style = "double"
@@ -62,7 +63,7 @@ indent-style = "space"
62
63
  [tool.mypy]
63
64
  ignore_missing_imports = true
64
65
  exclude = ["tests"]
65
- python_version = "3.10"
66
+ python_version = "3.11"
66
67
  show_error_codes = true
67
68
 
68
69
  [build-system]
@@ -73,6 +74,7 @@ build-backend = "setuptools.build_meta"
73
74
  addopts = [
74
75
  "--cov-report=term-missing:skip-covered",
75
76
  "--cov=url_normalize",
77
+ "--cov-fail-under=100",
76
78
  "--disable-socket",
77
79
  "-v",
78
80
  ]
@@ -26,6 +26,45 @@ def run_cli(*args: str) -> subprocess.CompletedProcess:
26
26
  )
27
27
 
28
28
 
29
+ def test_cli_help_describes_allowlist_filtering() -> None:
30
+ """Describe filtering as an allowlist, not a tracking-parameter blocklist."""
31
+ result = run_cli("--help")
32
+ assert result.returncode == 0
33
+ help_text = " ".join(result.stdout.split())
34
+ assert "Keep only allowlisted query parameters." in help_text
35
+ assert "Unknown domains have an empty default allowlist." in help_text
36
+
37
+
38
+ def test_cli_help_omits_removed_charset_option() -> None:
39
+ """Remove the obsolete encoding option from CLI help."""
40
+ result = run_cli("--help")
41
+ assert result.returncode == 0
42
+ assert "charset" not in result.stdout
43
+
44
+
45
+ @pytest.mark.parametrize("option", ["-c", "--charset"])
46
+ def test_cli_rejects_removed_charset_option(option, monkeypatch):
47
+ """Report removed flags with UTF-8 diagnostics even under a legacy encoding."""
48
+ monkeypatch.setenv("PYTHONIOENCODING", "cp1252")
49
+ usage_error_status = 2
50
+ result = run_cli(option, "utf-8", "https://example.com/é")
51
+ assert result.returncode == usage_error_status
52
+ assert "unrecognized arguments" in result.stderr
53
+ assert "https://example.com/é" in result.stderr
54
+ assert not result.stdout
55
+
56
+
57
+ def test_cli_successful_main(capsys, monkeypatch):
58
+ """Exercise successful CLI output in the measured process."""
59
+ monkeypatch.setattr(
60
+ sys, "argv", ["url-normalize", "http://EXAMPLE.com/./path/../other/"]
61
+ )
62
+ main()
63
+ captured = capsys.readouterr()
64
+ assert captured.out == "http://example.com/other/\n"
65
+ assert not captured.err
66
+
67
+
29
68
  def test_cli_error_handling(capsys, monkeypatch):
30
69
  """Test CLI error handling when URL normalization fails."""
31
70
  with patch("url_normalize.cli.url_normalize") as mock_normalize:
@@ -252,35 +291,12 @@ def test_cli_version_short() -> None:
252
291
  assert not result.stderr
253
292
 
254
293
 
255
- @pytest.mark.skipif(
256
- sys.platform == "win32", reason="Charset handling differs on Windows CLI"
257
- )
258
- def test_cli_charset() -> None:
259
- """Test charset handling via CLI (might be platform-dependent)."""
260
- # Example using Cyrillic characters which need correct encoding
261
- url = "http://пример.рф/path"
262
- expected_idn = "http://xn--e1afmkfd.xn--p1ai/path"
263
-
264
- # Test with default UTF-8
265
- result_utf8 = run_cli(url)
266
-
267
- assert result_utf8.returncode == 0
268
- assert result_utf8.stdout.strip() == expected_idn
269
- assert not result_utf8.stderr
270
-
271
- # Test specifying UTF-8 explicitly
272
- result_charset = run_cli("--charset", "utf-8", url)
273
-
274
- assert result_charset.returncode == 0
275
- assert result_charset.stdout.strip() == expected_idn
276
- assert not result_charset.stderr
277
-
278
- # Test specifying UTF-8 explicitly using short arg
279
- result_charset_short = run_cli("-c", "utf-8", url)
280
-
281
- assert result_charset_short.returncode == 0
282
- assert result_charset_short.stdout.strip() == expected_idn
283
- assert not result_charset_short.stderr
294
+ def test_cli_unicode_host_and_path() -> None:
295
+ """Encode Unicode hosts with IDNA and paths with UTF-8 on every platform."""
296
+ result = run_cli("http://пример.рф/é")
297
+ assert result.returncode == 0
298
+ assert result.stdout.strip() == "http://xn--e1afmkfd.xn--p1ai/%C3%A9"
299
+ assert not result.stderr
284
300
 
285
301
 
286
302
  def test_cli_default_domain() -> None:
@@ -0,0 +1,87 @@
1
+ """Deconstruct url tests."""
2
+
3
+ import pytest
4
+
5
+ from url_normalize import url_normalize
6
+ from url_normalize.tools import URL, deconstruct_url
7
+
8
+
9
+ @pytest.mark.parametrize(
10
+ ("url", "expected"),
11
+ [
12
+ (
13
+ "http://site.com",
14
+ URL(
15
+ fragment="",
16
+ host="site.com",
17
+ path="",
18
+ port="",
19
+ query="",
20
+ scheme="http",
21
+ userinfo="",
22
+ ),
23
+ ),
24
+ (
25
+ "http://user@www.example.com:8080/path/index.html?param=val#fragment",
26
+ URL(
27
+ fragment="fragment",
28
+ host="www.example.com",
29
+ path="/path/index.html",
30
+ port="8080",
31
+ query="param=val",
32
+ scheme="http",
33
+ userinfo="user@",
34
+ ),
35
+ ),
36
+ ],
37
+ )
38
+ def test_deconstruct_url_result_is_expected(url: str, expected: URL) -> None:
39
+ """Assert we got expected results from the deconstruct_url function."""
40
+ result = deconstruct_url(url)
41
+ assert result == expected, url
42
+
43
+
44
+ @pytest.mark.parametrize(
45
+ ("authority", "userinfo", "host", "port", "normalized_authority"),
46
+ [
47
+ ("[2001:DB8::1]:080", "", "[2001:DB8::1]", "080", "[2001:db8::1]"),
48
+ ("[2001:DB8::1]:0081", "", "[2001:DB8::1]", "0081", "[2001:db8::1]:81"),
49
+ ("[::1]", "", "[::1]", "", "[::1]"),
50
+ ("[::1]:", "", "[::1]", "", "[::1]"),
51
+ ("user:pass@[::1]:080", "user:pass@", "[::1]", "080", "user:pass@[::1]"),
52
+ ("EXAMPLE.com:080", "", "EXAMPLE.com", "080", "example.com"),
53
+ ("127.0.0.1:0081", "", "127.0.0.1", "0081", "127.0.0.1:81"),
54
+ ],
55
+ )
56
+ def test_deconstruct_url_separates_bracketed_host_and_port(
57
+ authority, userinfo, host, port, normalized_authority
58
+ ):
59
+ """Parse the entire bracketed host before considering a port separator."""
60
+ url = f"http://{authority}/path?q=1#fragment"
61
+ assert deconstruct_url(url) == URL(
62
+ "http", userinfo, host, port, "/path", "q=1", "fragment"
63
+ )
64
+ assert url_normalize(url) == f"http://{normalized_authority}/path?q=1#fragment"
65
+
66
+
67
+ @pytest.mark.parametrize("whitespace", [" ", "\u00a0", " \t\r\n"])
68
+ @pytest.mark.parametrize("suffix", ["", "/path ", "?q=x ", "#part "])
69
+ @pytest.mark.parametrize(
70
+ ("authority", "normalized"),
71
+ [
72
+ ("EXAMPLE.com", "example.com"),
73
+ ("example.com:443", "example.com"),
74
+ ("user:pass@example.com:443", "user:pass@example.com"),
75
+ ("[FE80::1%25ethA]:443", "[fe80::1%25ethA]"),
76
+ ],
77
+ )
78
+ def test_deconstruct_url_trims_authority_whitespace_only(
79
+ authority, normalized, whitespace, suffix
80
+ ):
81
+ """Trim the authority without deleting whitespace from other components."""
82
+ value = f"https://{authority}{whitespace}{suffix}"
83
+ assert deconstruct_url(value) == deconstruct_url(f"https://{authority}{suffix}")
84
+ path_suffix = suffix if suffix.startswith("/") else "/" + suffix
85
+ assert (
86
+ url_normalize(value) == f"https://{normalized}{path_suffix.replace(' ', '%20')}"
87
+ )