url-normalize 2.2.0__tar.gz → 3.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of url-normalize might be problematic. Click here for more details.

Files changed (57) hide show
  1. url_normalize-3.0.0/PKG-INFO +211 -0
  2. url_normalize-3.0.0/README.md +180 -0
  3. {url_normalize-2.2.0 → url_normalize-3.0.0}/pyproject.toml +17 -19
  4. {url_normalize-2.2.0 → url_normalize-3.0.0}/tests/test_cli.py +51 -1
  5. url_normalize-3.0.0/tests/test_deconstruct_url.py +40 -0
  6. url_normalize-3.0.0/tests/test_normalize_fragment.py +23 -0
  7. url_normalize-3.0.0/tests/test_normalize_host.py +32 -0
  8. url_normalize-3.0.0/tests/test_normalize_path.py +44 -0
  9. url_normalize-3.0.0/tests/test_normalize_port.py +26 -0
  10. url_normalize-3.0.0/tests/test_normalize_scheme.py +18 -0
  11. url_normalize-3.0.0/tests/test_normalize_userinfo.py +21 -0
  12. url_normalize-3.0.0/tests/test_provide_url_scheme.py +32 -0
  13. url_normalize-3.0.0/tests/test_reconstruct_url.py +42 -0
  14. url_normalize-3.0.0/tests/test_url_humanize.py +122 -0
  15. {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/__init__.py +3 -2
  16. {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/cli.py +14 -2
  17. {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/normalize_host.py +2 -6
  18. {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/param_allowlist.py +1 -2
  19. url_normalize-3.0.0/url_normalize/py.typed +0 -0
  20. url_normalize-3.0.0/url_normalize/url_humanize.py +128 -0
  21. url_normalize-3.0.0/url_normalize.egg-info/PKG-INFO +211 -0
  22. {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize.egg-info/SOURCES.txt +3 -0
  23. {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize.egg-info/requires.txt +0 -2
  24. url_normalize-2.2.0/PKG-INFO +0 -150
  25. url_normalize-2.2.0/README.md +0 -124
  26. url_normalize-2.2.0/tests/test_deconstruct_url.py +0 -35
  27. url_normalize-2.2.0/tests/test_normalize_fragment.py +0 -21
  28. url_normalize-2.2.0/tests/test_normalize_host.py +0 -29
  29. url_normalize-2.2.0/tests/test_normalize_path.py +0 -42
  30. url_normalize-2.2.0/tests/test_normalize_port.py +0 -13
  31. url_normalize-2.2.0/tests/test_normalize_scheme.py +0 -13
  32. url_normalize-2.2.0/tests/test_normalize_userinfo.py +0 -19
  33. url_normalize-2.2.0/tests/test_provide_url_scheme.py +0 -30
  34. url_normalize-2.2.0/tests/test_reconstruct_url.py +0 -41
  35. url_normalize-2.2.0/url_normalize.egg-info/PKG-INFO +0 -150
  36. {url_normalize-2.2.0 → url_normalize-3.0.0}/LICENSE +0 -0
  37. {url_normalize-2.2.0 → url_normalize-3.0.0}/setup.cfg +0 -0
  38. {url_normalize-2.2.0 → url_normalize-3.0.0}/tests/test_generic_url_cleanup.py +0 -0
  39. {url_normalize-2.2.0 → url_normalize-3.0.0}/tests/test_normalize_query.py +0 -0
  40. {url_normalize-2.2.0 → url_normalize-3.0.0}/tests/test_normalize_query_filters.py +0 -0
  41. {url_normalize-2.2.0 → url_normalize-3.0.0}/tests/test_provide_url_domain.py +0 -0
  42. {url_normalize-2.2.0 → url_normalize-3.0.0}/tests/test_tools.py +0 -0
  43. {url_normalize-2.2.0 → url_normalize-3.0.0}/tests/test_url_normalize.py +0 -0
  44. {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/generic_url_cleanup.py +0 -0
  45. {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/normalize_fragment.py +0 -0
  46. {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/normalize_path.py +0 -0
  47. {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/normalize_port.py +0 -0
  48. {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/normalize_query.py +0 -0
  49. {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/normalize_scheme.py +0 -0
  50. {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/normalize_userinfo.py +0 -0
  51. {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/provide_url_domain.py +0 -0
  52. {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/provide_url_scheme.py +0 -0
  53. {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/tools.py +0 -0
  54. {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/url_normalize.py +0 -0
  55. {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize.egg-info/dependency_links.txt +0 -0
  56. {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize.egg-info/entry_points.txt +0 -0
  57. {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize.egg-info/top_level.txt +0 -0
@@ -0,0 +1,211 @@
1
+ Metadata-Version: 2.4
2
+ Name: url-normalize
3
+ Version: 3.0.0
4
+ Summary: URL normalization for Python
5
+ Author-email: Nikolay Panov <github@npanov.com>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/niksite/url-normalize
8
+ Project-URL: Repository, https://github.com/niksite/url-normalize
9
+ Project-URL: Issues, https://github.com/niksite/url-normalize/issues
10
+ Project-URL: Changelog, https://github.com/niksite/url-normalize/blob/master/CHANGELOG.md
11
+ Keywords: url,normalization,normalize,normalizer
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Programming Language :: Python :: 3 :: Only
14
+ Classifier: Programming Language :: Python :: 3.10
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Programming Language :: Python :: 3.13
18
+ Classifier: Programming Language :: Python :: 3.14
19
+ Requires-Python: >=3.10
20
+ Description-Content-Type: text/markdown
21
+ License-File: LICENSE
22
+ Requires-Dist: idna>=3.3
23
+ Provides-Extra: dev
24
+ Requires-Dist: mypy; extra == "dev"
25
+ Requires-Dist: pre-commit; extra == "dev"
26
+ Requires-Dist: pytest-cov; extra == "dev"
27
+ Requires-Dist: pytest-socket; extra == "dev"
28
+ Requires-Dist: pytest; extra == "dev"
29
+ Requires-Dist: ruff; extra == "dev"
30
+ Dynamic: license-file
31
+
32
+ # url-normalize
33
+
34
+ [![tests](https://github.com/niksite/url-normalize/actions/workflows/ci.yml/badge.svg)](https://github.com/niksite/url-normalize/actions/workflows/ci.yml)
35
+ [![Coveralls](https://img.shields.io/coveralls/github/niksite/url-normalize/master.svg)](https://coveralls.io/r/niksite/url-normalize)
36
+ [![PyPI](https://img.shields.io/pypi/v/url-normalize.svg)](https://pypi.org/project/url-normalize/)
37
+ [![Python Versions](https://img.shields.io/pypi/pyversions/url-normalize.svg)](https://pypi.org/project/url-normalize/)
38
+ [![License](https://img.shields.io/pypi/l/url-normalize.svg)](https://github.com/niksite/url-normalize/blob/master/LICENSE)
39
+ [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
40
+
41
+ A Python library for standardizing and normalizing URLs. Ideal for database deduplication, caching, web crawling, and anywhere you need to ensure that equivalent URLs resolve to the exact same string.
42
+
43
+ ```python
44
+ from url_normalize import url_normalize
45
+
46
+ # Fixes IDN, lowercases host/scheme, removes default ports, resolves path segments
47
+ url_normalize("HTTP://User:Pass@www.FOO.com:80///foo/../bar/./baz?q=1#frag")
48
+ # -> 'http://User:Pass@www.foo.com/bar/baz?q=1#frag'
49
+ ```
50
+
51
+ ## Features
52
+
53
+ url-normalize provides a robust URI normalization function that handles IDN domains, scheme/host lowercasing, and RFC-compliant path normalization.
54
+
55
+ - **IDN Support**: Full internationalized domain name handling (using IDNA2008 with UTS46).
56
+ - **Humanization**: Convert normalized URLs to a readable display format while preserving round-trip normalization.
57
+ - **RFC Compliance**:
58
+ - Proper percent-encoding (minimal, uppercase hex).
59
+ - Dot-segment removal in paths.
60
+ - Default port and authority handling.
61
+ - UTF-8 NFC normalization.
62
+ - **Configurable Defaults**:
63
+ - Customizable default scheme (https by default).
64
+ - Configurable default domain for absolute paths.
65
+ - **Query Parameter Control**:
66
+ - Parameter filtering with allowlists.
67
+ - Support for domain-specific parameter rules.
68
+ - **Versatile URL Handling**: Handles empty strings, double-slash URLs (//domain.tld), and shebang (#!) URLs.
69
+ - **Developer Friendly**:
70
+ - Python 3.10+ compatibility.
71
+ - 100% test coverage.
72
+ - Modern type hints and string handling.
73
+
74
+ Inspired by Sam Ruby's [urlnorm.py](http://intertwingly.net/blog/2004/08/04/Urlnorm).
75
+
76
+ ## Installation
77
+
78
+ Install as a library:
79
+
80
+ ```sh
81
+ pip install url-normalize
82
+ ```
83
+
84
+ Or install as a standalone CLI tool using [uv](https://docs.astral.sh/uv/):
85
+
86
+ ```sh
87
+ uv tool install url-normalize
88
+ ```
89
+
90
+ ## Usage
91
+
92
+ ### Python API
93
+
94
+ #### Basic Normalization
95
+
96
+ ```python
97
+ from url_normalize import url_normalize
98
+
99
+ # Basic normalization (uses https by default)
100
+ print(url_normalize("www.foo.com:80/foo"))
101
+ # Output: https://www.foo.com/foo
102
+
103
+ # With custom default scheme
104
+ print(url_normalize("www.foo.com/foo", default_scheme="http"))
105
+ # Output: http://www.foo.com/foo
106
+ ```
107
+
108
+ #### Query Parameter Filtering
109
+
110
+ You can strip out tracking parameters and only keep the ones you care about using allowlists.
111
+
112
+ ```python
113
+ # With query parameter filtering enabled (strips all params by default)
114
+ print(url_normalize("www.google.com/search?q=test&utm_source=test", filter_params=True))
115
+ # Output: https://www.google.com/search?q=test
116
+
117
+ # With custom parameter allowlist as a list
118
+ print(url_normalize(
119
+ "example.com?page=1&id=123&ref=test",
120
+ filter_params=True,
121
+ param_allowlist=["page", "id"]
122
+ ))
123
+ # Output: https://example.com?page=1&id=123
124
+
125
+ # With domain-specific parameter allowlists
126
+ print(url_normalize(
127
+ "example.com?page=1&id=123&ref=test",
128
+ filter_params=True,
129
+ param_allowlist={"example.com": ["page", "id"]}
130
+ ))
131
+ # Output: https://example.com?page=1&id=123
132
+ ```
133
+
134
+ #### Default Domain & Scheme
135
+
136
+ Useful for resolving relative URLs found on a specific page.
137
+
138
+ ```python
139
+ # With default domain for absolute paths
140
+ print(url_normalize("/images/logo.png", default_domain="example.com"))
141
+ # Output: https://example.com/images/logo.png
142
+
143
+ # With default domain and custom scheme
144
+ print(url_normalize("/images/logo.png", default_scheme="http", default_domain="example.com"))
145
+ # Output: http://example.com/images/logo.png
146
+ ```
147
+
148
+ #### Humanizing URLs
149
+
150
+ Convert normalized URLs back into a user-friendly format for display, particularly useful for IDN domains and percent-encoded paths.
151
+
152
+ ```python
153
+ from url_normalize import url_humanize
154
+
155
+ # Human-readable display form that still normalizes back to the same URL
156
+ print(url_humanize("https://xn--e1afmkfd.xn--80akhbyknj4f/%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F"))
157
+ # Output: https://пример.испытание/Служебная
158
+
159
+ # Humanization accepts the same normalization options
160
+ print(url_humanize("/%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F", default_domain="xn--e1afmkfd.xn--80akhbyknj4f"))
161
+ # Output: https://пример.испытание/Служебная
162
+ ```
163
+
164
+ ### Command-line Usage
165
+
166
+ You can also use `url-normalize` directly from the terminal to process URLs.
167
+
168
+ ```bash
169
+ $ url-normalize "www.foo.com:80/foo"
170
+ # Output: https://www.foo.com/foo
171
+
172
+ # With custom default scheme
173
+ $ url-normalize -s http "www.foo.com/foo"
174
+ # Output: http://www.foo.com/foo
175
+
176
+ # With query parameter filtering
177
+ $ url-normalize -f "www.google.com/search?q=test&utm_source=test"
178
+ # Output: https://www.google.com/search?q=test
179
+
180
+ # With custom allowlist
181
+ $ url-normalize -f -p page,id "example.com?page=1&id=123&ref=test"
182
+ # Output: https://example.com/?page=1&id=123
183
+
184
+ # With default domain for absolute paths
185
+ $ url-normalize -d example.com "/images/logo.png"
186
+ # Output: https://example.com/images/logo.png
187
+
188
+ # With default domain and custom scheme
189
+ $ url-normalize -d example.com -s http "/images/logo.png"
190
+ # Output: http://example.com/images/logo.png
191
+
192
+ # Human-readable display form
193
+ $ url-normalize -H "https://xn--e1afmkfd.xn--80akhbyknj4f/%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F"
194
+ # Output: https://пример.испытание/Служебная
195
+
196
+ # Via uv tool/uvx
197
+ $ uvx url-normalize www.foo.com:80/foo
198
+ # Output: https://www.foo.com:80/foo
199
+ ```
200
+
201
+ ## Documentation
202
+
203
+ For a complete history of changes, see [CHANGELOG.md](CHANGELOG.md).
204
+
205
+ ## Contributing
206
+
207
+ Contributions are welcome! Please feel free to submit a Pull Request.
208
+
209
+ ## License
210
+
211
+ MIT License
@@ -0,0 +1,180 @@
1
+ # url-normalize
2
+
3
+ [![tests](https://github.com/niksite/url-normalize/actions/workflows/ci.yml/badge.svg)](https://github.com/niksite/url-normalize/actions/workflows/ci.yml)
4
+ [![Coveralls](https://img.shields.io/coveralls/github/niksite/url-normalize/master.svg)](https://coveralls.io/r/niksite/url-normalize)
5
+ [![PyPI](https://img.shields.io/pypi/v/url-normalize.svg)](https://pypi.org/project/url-normalize/)
6
+ [![Python Versions](https://img.shields.io/pypi/pyversions/url-normalize.svg)](https://pypi.org/project/url-normalize/)
7
+ [![License](https://img.shields.io/pypi/l/url-normalize.svg)](https://github.com/niksite/url-normalize/blob/master/LICENSE)
8
+ [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
9
+
10
+ A Python library for standardizing and normalizing URLs. Ideal for database deduplication, caching, web crawling, and anywhere you need to ensure that equivalent URLs resolve to the exact same string.
11
+
12
+ ```python
13
+ from url_normalize import url_normalize
14
+
15
+ # Fixes IDN, lowercases host/scheme, removes default ports, resolves path segments
16
+ url_normalize("HTTP://User:Pass@www.FOO.com:80///foo/../bar/./baz?q=1#frag")
17
+ # -> 'http://User:Pass@www.foo.com/bar/baz?q=1#frag'
18
+ ```
19
+
20
+ ## Features
21
+
22
+ url-normalize provides a robust URI normalization function that handles IDN domains, scheme/host lowercasing, and RFC-compliant path normalization.
23
+
24
+ - **IDN Support**: Full internationalized domain name handling (using IDNA2008 with UTS46).
25
+ - **Humanization**: Convert normalized URLs to a readable display format while preserving round-trip normalization.
26
+ - **RFC Compliance**:
27
+ - Proper percent-encoding (minimal, uppercase hex).
28
+ - Dot-segment removal in paths.
29
+ - Default port and authority handling.
30
+ - UTF-8 NFC normalization.
31
+ - **Configurable Defaults**:
32
+ - Customizable default scheme (https by default).
33
+ - Configurable default domain for absolute paths.
34
+ - **Query Parameter Control**:
35
+ - Parameter filtering with allowlists.
36
+ - Support for domain-specific parameter rules.
37
+ - **Versatile URL Handling**: Handles empty strings, double-slash URLs (//domain.tld), and shebang (#!) URLs.
38
+ - **Developer Friendly**:
39
+ - Python 3.10+ compatibility.
40
+ - 100% test coverage.
41
+ - Modern type hints and string handling.
42
+
43
+ Inspired by Sam Ruby's [urlnorm.py](http://intertwingly.net/blog/2004/08/04/Urlnorm).
44
+
45
+ ## Installation
46
+
47
+ Install as a library:
48
+
49
+ ```sh
50
+ pip install url-normalize
51
+ ```
52
+
53
+ Or install as a standalone CLI tool using [uv](https://docs.astral.sh/uv/):
54
+
55
+ ```sh
56
+ uv tool install url-normalize
57
+ ```
58
+
59
+ ## Usage
60
+
61
+ ### Python API
62
+
63
+ #### Basic Normalization
64
+
65
+ ```python
66
+ from url_normalize import url_normalize
67
+
68
+ # Basic normalization (uses https by default)
69
+ print(url_normalize("www.foo.com:80/foo"))
70
+ # Output: https://www.foo.com/foo
71
+
72
+ # With custom default scheme
73
+ print(url_normalize("www.foo.com/foo", default_scheme="http"))
74
+ # Output: http://www.foo.com/foo
75
+ ```
76
+
77
+ #### Query Parameter Filtering
78
+
79
+ You can strip out tracking parameters and only keep the ones you care about using allowlists.
80
+
81
+ ```python
82
+ # With query parameter filtering enabled (strips all params by default)
83
+ print(url_normalize("www.google.com/search?q=test&utm_source=test", filter_params=True))
84
+ # Output: https://www.google.com/search?q=test
85
+
86
+ # With custom parameter allowlist as a list
87
+ print(url_normalize(
88
+ "example.com?page=1&id=123&ref=test",
89
+ filter_params=True,
90
+ param_allowlist=["page", "id"]
91
+ ))
92
+ # Output: https://example.com?page=1&id=123
93
+
94
+ # With domain-specific parameter allowlists
95
+ print(url_normalize(
96
+ "example.com?page=1&id=123&ref=test",
97
+ filter_params=True,
98
+ param_allowlist={"example.com": ["page", "id"]}
99
+ ))
100
+ # Output: https://example.com?page=1&id=123
101
+ ```
102
+
103
+ #### Default Domain & Scheme
104
+
105
+ Useful for resolving relative URLs found on a specific page.
106
+
107
+ ```python
108
+ # With default domain for absolute paths
109
+ print(url_normalize("/images/logo.png", default_domain="example.com"))
110
+ # Output: https://example.com/images/logo.png
111
+
112
+ # With default domain and custom scheme
113
+ print(url_normalize("/images/logo.png", default_scheme="http", default_domain="example.com"))
114
+ # Output: http://example.com/images/logo.png
115
+ ```
116
+
117
+ #### Humanizing URLs
118
+
119
+ Convert normalized URLs back into a user-friendly format for display, particularly useful for IDN domains and percent-encoded paths.
120
+
121
+ ```python
122
+ from url_normalize import url_humanize
123
+
124
+ # Human-readable display form that still normalizes back to the same URL
125
+ print(url_humanize("https://xn--e1afmkfd.xn--80akhbyknj4f/%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F"))
126
+ # Output: https://пример.испытание/Служебная
127
+
128
+ # Humanization accepts the same normalization options
129
+ print(url_humanize("/%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F", default_domain="xn--e1afmkfd.xn--80akhbyknj4f"))
130
+ # Output: https://пример.испытание/Служебная
131
+ ```
132
+
133
+ ### Command-line Usage
134
+
135
+ You can also use `url-normalize` directly from the terminal to process URLs.
136
+
137
+ ```bash
138
+ $ url-normalize "www.foo.com:80/foo"
139
+ # Output: https://www.foo.com/foo
140
+
141
+ # With custom default scheme
142
+ $ url-normalize -s http "www.foo.com/foo"
143
+ # Output: http://www.foo.com/foo
144
+
145
+ # With query parameter filtering
146
+ $ url-normalize -f "www.google.com/search?q=test&utm_source=test"
147
+ # Output: https://www.google.com/search?q=test
148
+
149
+ # With custom allowlist
150
+ $ url-normalize -f -p page,id "example.com?page=1&id=123&ref=test"
151
+ # Output: https://example.com/?page=1&id=123
152
+
153
+ # With default domain for absolute paths
154
+ $ url-normalize -d example.com "/images/logo.png"
155
+ # Output: https://example.com/images/logo.png
156
+
157
+ # With default domain and custom scheme
158
+ $ url-normalize -d example.com -s http "/images/logo.png"
159
+ # Output: http://example.com/images/logo.png
160
+
161
+ # Human-readable display form
162
+ $ url-normalize -H "https://xn--e1afmkfd.xn--80akhbyknj4f/%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F"
163
+ # Output: https://пример.испытание/Служебная
164
+
165
+ # Via uv tool/uvx
166
+ $ uvx url-normalize www.foo.com:80/foo
167
+ # Output: https://www.foo.com:80/foo
168
+ ```
169
+
170
+ ## Documentation
171
+
172
+ For a complete history of changes, see [CHANGELOG.md](CHANGELOG.md).
173
+
174
+ ## Contributing
175
+
176
+ Contributions are welcome! Please feel free to submit a Pull Request.
177
+
178
+ ## License
179
+
180
+ MIT License
@@ -1,12 +1,21 @@
1
1
  [project]
2
2
  name = "url-normalize"
3
- version = "2.2.0"
3
+ version = "3.0.0"
4
4
  description = "URL normalization for Python"
5
5
  authors = [{ name = "Nikolay Panov", email = "github@npanov.com" }]
6
- license = { text = "MIT" }
6
+ license = "MIT"
7
7
  readme = "README.md"
8
- requires-python = ">=3.8"
9
- keywords = ["url", "normalization", "normalize"]
8
+ requires-python = ">=3.10"
9
+ keywords = ["url", "normalization", "normalize", "normalizer"]
10
+ classifiers = [
11
+ "Programming Language :: Python :: 3",
12
+ "Programming Language :: Python :: 3 :: Only",
13
+ "Programming Language :: Python :: 3.10",
14
+ "Programming Language :: Python :: 3.11",
15
+ "Programming Language :: Python :: 3.12",
16
+ "Programming Language :: Python :: 3.13",
17
+ "Programming Language :: Python :: 3.14",
18
+ ]
10
19
  dependencies = ["idna>=3.3"]
11
20
 
12
21
  [project.urls]
@@ -19,19 +28,10 @@ Changelog = "https://github.com/niksite/url-normalize/blob/master/CHANGELOG.md"
19
28
  url-normalize = "url_normalize.cli:main"
20
29
 
21
30
  [project.optional-dependencies]
22
- dev = [
23
- "mypy",
24
- "pre-commit",
25
- "pytest-cov",
26
- "pytest-ruff",
27
- "pytest-socket",
28
- "pytest",
29
- "ruff",
30
- "tox",
31
- ]
31
+ dev = ["mypy", "pre-commit", "pytest-cov", "pytest-socket", "pytest", "ruff"]
32
32
 
33
33
  [tool.ruff]
34
- target-version = "py38"
34
+ target-version = "py310"
35
35
  line-length = 88
36
36
  unsafe-fixes = true
37
37
 
@@ -62,20 +62,18 @@ indent-style = "space"
62
62
  [tool.mypy]
63
63
  ignore_missing_imports = true
64
64
  exclude = ["tests"]
65
- python_version = "3.8"
65
+ python_version = "3.10"
66
66
  show_error_codes = true
67
67
 
68
68
  [build-system]
69
- requires = ["setuptools>=42", "wheel"]
69
+ requires = ["setuptools>=77", "wheel"]
70
70
  build-backend = "setuptools.build_meta"
71
71
 
72
72
  [tool.pytest.ini_options]
73
73
  addopts = [
74
- "--cov-fail-under=100",
75
74
  "--cov-report=term-missing:skip-covered",
76
75
  "--cov=url_normalize",
77
76
  "--disable-socket",
78
- "--ruff",
79
77
  "-v",
80
78
  ]
81
79
  python_files = ["tests.py", "test_*.py", "*_tests.py"]
@@ -22,7 +22,7 @@ def run_cli(*args: str) -> subprocess.CompletedProcess:
22
22
  """
23
23
  command = [sys.executable, "-m", "url_normalize.cli", *list(args)]
24
24
  return subprocess.run( # noqa: S603
25
- command, capture_output=True, text=True, check=False
25
+ command, capture_output=True, text=True, encoding="utf-8", check=False
26
26
  )
27
27
 
28
28
 
@@ -66,6 +66,56 @@ def test_cli_basic_normalization_short_args() -> None:
66
66
  assert not result.stderr
67
67
 
68
68
 
69
+ def test_cli_humanize() -> None:
70
+ """Test human-readable URL output via CLI."""
71
+ url = (
72
+ "https://xn--e1afmkfd.xn--80akhbyknj4f/"
73
+ "%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F"
74
+ )
75
+ expected = "https://пример.испытание/Служебная"
76
+
77
+ result = run_cli("--humanize", url)
78
+
79
+ assert result.returncode == 0
80
+ assert result.stdout.strip() == expected
81
+ assert not result.stderr
82
+
83
+
84
+ def test_cli_humanize_short_arg() -> None:
85
+ """Test human-readable URL output via CLI using short argument."""
86
+ url = (
87
+ "https://xn--e1afmkfd.xn--80akhbyknj4f/"
88
+ "%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F"
89
+ )
90
+ expected = "https://пример.испытание/Служебная"
91
+
92
+ result = run_cli("-H", url)
93
+
94
+ assert result.returncode == 0
95
+ assert result.stdout.strip() == expected
96
+ assert not result.stderr
97
+
98
+
99
+ def test_cli_humanize_respects_normalization_options() -> None:
100
+ """Test humanized output keeps the existing CLI normalization options."""
101
+ url = "/%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F"
102
+ expected = "https://пример.испытание/Служебная?keep=Ç"
103
+
104
+ result = run_cli(
105
+ "--humanize",
106
+ "--default-domain",
107
+ "xn--e1afmkfd.xn--80akhbyknj4f",
108
+ "--filter-params",
109
+ "--param-allowlist",
110
+ "keep",
111
+ f"{url}?utm_source=ignored&keep=%C3%87",
112
+ )
113
+
114
+ assert result.returncode == 0
115
+ assert result.stdout.strip() == expected
116
+ assert not result.stderr
117
+
118
+
69
119
  def test_cli_default_scheme() -> None:
70
120
  """Test default scheme addition via CLI."""
71
121
  url = "//example.com"
@@ -0,0 +1,40 @@
1
+ """Deconstruct url tests."""
2
+
3
+ import pytest
4
+
5
+ from url_normalize.tools import URL, deconstruct_url
6
+
7
+
8
+ @pytest.mark.parametrize(
9
+ ("url", "expected"),
10
+ [
11
+ (
12
+ "http://site.com",
13
+ URL(
14
+ fragment="",
15
+ host="site.com",
16
+ path="",
17
+ port="",
18
+ query="",
19
+ scheme="http",
20
+ userinfo="",
21
+ ),
22
+ ),
23
+ (
24
+ "http://user@www.example.com:8080/path/index.html?param=val#fragment",
25
+ URL(
26
+ fragment="fragment",
27
+ host="www.example.com",
28
+ path="/path/index.html",
29
+ port="8080",
30
+ query="param=val",
31
+ scheme="http",
32
+ userinfo="user@",
33
+ ),
34
+ ),
35
+ ],
36
+ )
37
+ def test_deconstruct_url_result_is_expected(url: str, expected: URL) -> None:
38
+ """Assert we got expected results from the deconstruct_url function."""
39
+ result = deconstruct_url(url)
40
+ assert result == expected, url
@@ -0,0 +1,23 @@
1
+ """Tests for normalize_fragment function."""
2
+
3
+ import pytest
4
+
5
+ from url_normalize.url_normalize import normalize_fragment
6
+
7
+
8
+ @pytest.mark.parametrize(
9
+ ("fragment", "expected"),
10
+ [
11
+ ("", ""),
12
+ ("fragment", "fragment"),
13
+ ("пример", "%D0%BF%D1%80%D0%B8%D0%BC%D0%B5%D1%80"),
14
+ ("!fragment", "%21fragment"),
15
+ ("~fragment", "~fragment"),
16
+ # Issue #36: Equal sign should not be encoded
17
+ ("gid=1234", "gid=1234"),
18
+ ],
19
+ )
20
+ def test_normalize_fragment_result_is_expected(fragment: str, expected: str) -> None:
21
+ """Assert we got expected results from the normalize_fragment function."""
22
+ result = normalize_fragment(fragment)
23
+ assert result == expected, fragment
@@ -0,0 +1,32 @@
1
+ """Tests for normalize_host function."""
2
+
3
+ import pytest
4
+
5
+ from url_normalize.url_normalize import normalize_host
6
+
7
+
8
+ @pytest.mark.parametrize(
9
+ ("host", "expected"),
10
+ [
11
+ # Basic cases
12
+ ("site.com", "site.com"),
13
+ ("SITE.COM", "site.com"),
14
+ ("site.com.", "site.com"),
15
+ # Cyrillic domains
16
+ ("пример.испытание", "xn--e1afmkfd.xn--80akhbyknj4f"),
17
+ # Mixed case with Cyrillic
18
+ ("ExAmPle.РФ", "example.xn--p1ai"),
19
+ # IDNA2008 with UTS46
20
+ ("faß.de", "xn--fa-hia.de"),
21
+ # Edge cases
22
+ ("ドメイン.テスト", "xn--eckwd4c7c.xn--zckzah"), # Japanese
23
+ ("domain.café", "domain.xn--caf-dma"), # Latin with diacritic
24
+ # Normalization tests
25
+ ("über.example", "xn--ber-goa.example"), # IDNA 2008 for umlaut
26
+ ("example。com", "example.com"), # Normalize full-width punctuation
27
+ ],
28
+ )
29
+ def test_normalize_host_result_is_expected(host: str, expected: str) -> None:
30
+ """Assert we got expected results from the normalize_host function."""
31
+ result = normalize_host(host)
32
+ assert result == expected, host
@@ -0,0 +1,44 @@
1
+ """Tests for normalize_path function."""
2
+
3
+ import pytest
4
+
5
+ from url_normalize.url_normalize import normalize_path
6
+
7
+
8
+ @pytest.mark.parametrize(
9
+ ("path", "expected"),
10
+ [
11
+ ("..", "/"),
12
+ ("", "/"),
13
+ ("/../foo", "/foo"),
14
+ ("/..foo", "/..foo"),
15
+ ("/./../foo", "/foo"),
16
+ ("/./foo", "/foo"),
17
+ ("/./foo/.", "/foo/"),
18
+ ("/.foo", "/.foo"),
19
+ ("/", "/"),
20
+ ("/foo..", "/foo.."),
21
+ ("/foo.", "/foo."),
22
+ ("/FOO", "/FOO"),
23
+ ("/foo/../bar", "/bar"),
24
+ ("/foo/./bar", "/foo/bar"),
25
+ ("/foo//", "/foo/"),
26
+ ("/foo///bar//", "/foo/bar/"),
27
+ ("/foo/bar/..", "/foo/"),
28
+ ("/foo/bar/../..", "/"),
29
+ ("/foo/bar/../../../../baz", "/baz"),
30
+ ("/foo/bar/../../../baz", "/baz"),
31
+ ("/foo/bar/../../", "/"),
32
+ ("/foo/bar/../../baz", "/baz"),
33
+ ("/foo/bar/../", "/foo/"),
34
+ ("/foo/bar/../baz", "/foo/baz"),
35
+ ("/foo/bar/.", "/foo/bar/"),
36
+ ("/foo/bar/./", "/foo/bar/"),
37
+ # Issue #25: we should preserve ? in the path
38
+ ("/More+Tea+Vicar%3F/discussion", "/More+Tea+Vicar%3F/discussion"),
39
+ ],
40
+ )
41
+ def test_normalize_path_result_is_expected(path: str, expected: str) -> None:
42
+ """Assert we got expected results from the normalize_path function."""
43
+ result = normalize_path(path, "http")
44
+ assert result == expected, path