url-normalize 2.2.0__tar.gz → 3.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of url-normalize might be problematic. Click here for more details.
- url_normalize-3.0.0/PKG-INFO +211 -0
- url_normalize-3.0.0/README.md +180 -0
- {url_normalize-2.2.0 → url_normalize-3.0.0}/pyproject.toml +17 -19
- {url_normalize-2.2.0 → url_normalize-3.0.0}/tests/test_cli.py +51 -1
- url_normalize-3.0.0/tests/test_deconstruct_url.py +40 -0
- url_normalize-3.0.0/tests/test_normalize_fragment.py +23 -0
- url_normalize-3.0.0/tests/test_normalize_host.py +32 -0
- url_normalize-3.0.0/tests/test_normalize_path.py +44 -0
- url_normalize-3.0.0/tests/test_normalize_port.py +26 -0
- url_normalize-3.0.0/tests/test_normalize_scheme.py +18 -0
- url_normalize-3.0.0/tests/test_normalize_userinfo.py +21 -0
- url_normalize-3.0.0/tests/test_provide_url_scheme.py +32 -0
- url_normalize-3.0.0/tests/test_reconstruct_url.py +42 -0
- url_normalize-3.0.0/tests/test_url_humanize.py +122 -0
- {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/__init__.py +3 -2
- {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/cli.py +14 -2
- {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/normalize_host.py +2 -6
- {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/param_allowlist.py +1 -2
- url_normalize-3.0.0/url_normalize/py.typed +0 -0
- url_normalize-3.0.0/url_normalize/url_humanize.py +128 -0
- url_normalize-3.0.0/url_normalize.egg-info/PKG-INFO +211 -0
- {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize.egg-info/SOURCES.txt +3 -0
- {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize.egg-info/requires.txt +0 -2
- url_normalize-2.2.0/PKG-INFO +0 -150
- url_normalize-2.2.0/README.md +0 -124
- url_normalize-2.2.0/tests/test_deconstruct_url.py +0 -35
- url_normalize-2.2.0/tests/test_normalize_fragment.py +0 -21
- url_normalize-2.2.0/tests/test_normalize_host.py +0 -29
- url_normalize-2.2.0/tests/test_normalize_path.py +0 -42
- url_normalize-2.2.0/tests/test_normalize_port.py +0 -13
- url_normalize-2.2.0/tests/test_normalize_scheme.py +0 -13
- url_normalize-2.2.0/tests/test_normalize_userinfo.py +0 -19
- url_normalize-2.2.0/tests/test_provide_url_scheme.py +0 -30
- url_normalize-2.2.0/tests/test_reconstruct_url.py +0 -41
- url_normalize-2.2.0/url_normalize.egg-info/PKG-INFO +0 -150
- {url_normalize-2.2.0 → url_normalize-3.0.0}/LICENSE +0 -0
- {url_normalize-2.2.0 → url_normalize-3.0.0}/setup.cfg +0 -0
- {url_normalize-2.2.0 → url_normalize-3.0.0}/tests/test_generic_url_cleanup.py +0 -0
- {url_normalize-2.2.0 → url_normalize-3.0.0}/tests/test_normalize_query.py +0 -0
- {url_normalize-2.2.0 → url_normalize-3.0.0}/tests/test_normalize_query_filters.py +0 -0
- {url_normalize-2.2.0 → url_normalize-3.0.0}/tests/test_provide_url_domain.py +0 -0
- {url_normalize-2.2.0 → url_normalize-3.0.0}/tests/test_tools.py +0 -0
- {url_normalize-2.2.0 → url_normalize-3.0.0}/tests/test_url_normalize.py +0 -0
- {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/generic_url_cleanup.py +0 -0
- {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/normalize_fragment.py +0 -0
- {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/normalize_path.py +0 -0
- {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/normalize_port.py +0 -0
- {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/normalize_query.py +0 -0
- {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/normalize_scheme.py +0 -0
- {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/normalize_userinfo.py +0 -0
- {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/provide_url_domain.py +0 -0
- {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/provide_url_scheme.py +0 -0
- {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/tools.py +0 -0
- {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize/url_normalize.py +0 -0
- {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize.egg-info/dependency_links.txt +0 -0
- {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize.egg-info/entry_points.txt +0 -0
- {url_normalize-2.2.0 → url_normalize-3.0.0}/url_normalize.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: url-normalize
|
|
3
|
+
Version: 3.0.0
|
|
4
|
+
Summary: URL normalization for Python
|
|
5
|
+
Author-email: Nikolay Panov <github@npanov.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/niksite/url-normalize
|
|
8
|
+
Project-URL: Repository, https://github.com/niksite/url-normalize
|
|
9
|
+
Project-URL: Issues, https://github.com/niksite/url-normalize/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/niksite/url-normalize/blob/master/CHANGELOG.md
|
|
11
|
+
Keywords: url,normalization,normalize,normalizer
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Requires-Dist: idna>=3.3
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: mypy; extra == "dev"
|
|
25
|
+
Requires-Dist: pre-commit; extra == "dev"
|
|
26
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
27
|
+
Requires-Dist: pytest-socket; extra == "dev"
|
|
28
|
+
Requires-Dist: pytest; extra == "dev"
|
|
29
|
+
Requires-Dist: ruff; extra == "dev"
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
# url-normalize
|
|
33
|
+
|
|
34
|
+
[](https://github.com/niksite/url-normalize/actions/workflows/ci.yml)
|
|
35
|
+
[](https://coveralls.io/r/niksite/url-normalize)
|
|
36
|
+
[](https://pypi.org/project/url-normalize/)
|
|
37
|
+
[](https://pypi.org/project/url-normalize/)
|
|
38
|
+
[](https://github.com/niksite/url-normalize/blob/master/LICENSE)
|
|
39
|
+
[](https://github.com/astral-sh/ruff)
|
|
40
|
+
|
|
41
|
+
A Python library for standardizing and normalizing URLs. Ideal for database deduplication, caching, web crawling, and anywhere you need to ensure that equivalent URLs resolve to the exact same string.
|
|
42
|
+
|
|
43
|
+
```python
|
|
44
|
+
from url_normalize import url_normalize
|
|
45
|
+
|
|
46
|
+
# Fixes IDN, lowercases host/scheme, removes default ports, resolves path segments
|
|
47
|
+
url_normalize("HTTP://User:Pass@www.FOO.com:80///foo/../bar/./baz?q=1#frag")
|
|
48
|
+
# -> 'http://User:Pass@www.foo.com/bar/baz?q=1#frag'
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
## Features
|
|
52
|
+
|
|
53
|
+
url-normalize provides a robust URI normalization function that handles IDN domains, scheme/host lowercasing, and RFC-compliant path normalization.
|
|
54
|
+
|
|
55
|
+
- **IDN Support**: Full internationalized domain name handling (using IDNA2008 with UTS46).
|
|
56
|
+
- **Humanization**: Convert normalized URLs to a readable display format while preserving round-trip normalization.
|
|
57
|
+
- **RFC Compliance**:
|
|
58
|
+
- Proper percent-encoding (minimal, uppercase hex).
|
|
59
|
+
- Dot-segment removal in paths.
|
|
60
|
+
- Default port and authority handling.
|
|
61
|
+
- UTF-8 NFC normalization.
|
|
62
|
+
- **Configurable Defaults**:
|
|
63
|
+
- Customizable default scheme (https by default).
|
|
64
|
+
- Configurable default domain for absolute paths.
|
|
65
|
+
- **Query Parameter Control**:
|
|
66
|
+
- Parameter filtering with allowlists.
|
|
67
|
+
- Support for domain-specific parameter rules.
|
|
68
|
+
- **Versatile URL Handling**: Handles empty strings, double-slash URLs (//domain.tld), and shebang (#!) URLs.
|
|
69
|
+
- **Developer Friendly**:
|
|
70
|
+
- Python 3.10+ compatibility.
|
|
71
|
+
- 100% test coverage.
|
|
72
|
+
- Modern type hints and string handling.
|
|
73
|
+
|
|
74
|
+
Inspired by Sam Ruby's [urlnorm.py](http://intertwingly.net/blog/2004/08/04/Urlnorm).
|
|
75
|
+
|
|
76
|
+
## Installation
|
|
77
|
+
|
|
78
|
+
Install as a library:
|
|
79
|
+
|
|
80
|
+
```sh
|
|
81
|
+
pip install url-normalize
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Or install as a standalone CLI tool using [uv](https://docs.astral.sh/uv/):
|
|
85
|
+
|
|
86
|
+
```sh
|
|
87
|
+
uv tool install url-normalize
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
## Usage
|
|
91
|
+
|
|
92
|
+
### Python API
|
|
93
|
+
|
|
94
|
+
#### Basic Normalization
|
|
95
|
+
|
|
96
|
+
```python
|
|
97
|
+
from url_normalize import url_normalize
|
|
98
|
+
|
|
99
|
+
# Basic normalization (uses https by default)
|
|
100
|
+
print(url_normalize("www.foo.com:80/foo"))
|
|
101
|
+
# Output: https://www.foo.com/foo
|
|
102
|
+
|
|
103
|
+
# With custom default scheme
|
|
104
|
+
print(url_normalize("www.foo.com/foo", default_scheme="http"))
|
|
105
|
+
# Output: http://www.foo.com/foo
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
#### Query Parameter Filtering
|
|
109
|
+
|
|
110
|
+
You can strip out tracking parameters and only keep the ones you care about using allowlists.
|
|
111
|
+
|
|
112
|
+
```python
|
|
113
|
+
# With query parameter filtering enabled (strips all params by default)
|
|
114
|
+
print(url_normalize("www.google.com/search?q=test&utm_source=test", filter_params=True))
|
|
115
|
+
# Output: https://www.google.com/search?q=test
|
|
116
|
+
|
|
117
|
+
# With custom parameter allowlist as a list
|
|
118
|
+
print(url_normalize(
|
|
119
|
+
"example.com?page=1&id=123&ref=test",
|
|
120
|
+
filter_params=True,
|
|
121
|
+
param_allowlist=["page", "id"]
|
|
122
|
+
))
|
|
123
|
+
# Output: https://example.com?page=1&id=123
|
|
124
|
+
|
|
125
|
+
# With domain-specific parameter allowlists
|
|
126
|
+
print(url_normalize(
|
|
127
|
+
"example.com?page=1&id=123&ref=test",
|
|
128
|
+
filter_params=True,
|
|
129
|
+
param_allowlist={"example.com": ["page", "id"]}
|
|
130
|
+
))
|
|
131
|
+
# Output: https://example.com?page=1&id=123
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
#### Default Domain & Scheme
|
|
135
|
+
|
|
136
|
+
Useful for resolving relative URLs found on a specific page.
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
# With default domain for absolute paths
|
|
140
|
+
print(url_normalize("/images/logo.png", default_domain="example.com"))
|
|
141
|
+
# Output: https://example.com/images/logo.png
|
|
142
|
+
|
|
143
|
+
# With default domain and custom scheme
|
|
144
|
+
print(url_normalize("/images/logo.png", default_scheme="http", default_domain="example.com"))
|
|
145
|
+
# Output: http://example.com/images/logo.png
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
#### Humanizing URLs
|
|
149
|
+
|
|
150
|
+
Convert normalized URLs back into a user-friendly format for display, particularly useful for IDN domains and percent-encoded paths.
|
|
151
|
+
|
|
152
|
+
```python
|
|
153
|
+
from url_normalize import url_humanize
|
|
154
|
+
|
|
155
|
+
# Human-readable display form that still normalizes back to the same URL
|
|
156
|
+
print(url_humanize("https://xn--e1afmkfd.xn--80akhbyknj4f/%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F"))
|
|
157
|
+
# Output: https://пример.испытание/Служебная
|
|
158
|
+
|
|
159
|
+
# Humanization accepts the same normalization options
|
|
160
|
+
print(url_humanize("/%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F", default_domain="xn--e1afmkfd.xn--80akhbyknj4f"))
|
|
161
|
+
# Output: https://пример.испытание/Служебная
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
### Command-line Usage
|
|
165
|
+
|
|
166
|
+
You can also use `url-normalize` directly from the terminal to process URLs.
|
|
167
|
+
|
|
168
|
+
```bash
|
|
169
|
+
$ url-normalize "www.foo.com:80/foo"
|
|
170
|
+
# Output: https://www.foo.com/foo
|
|
171
|
+
|
|
172
|
+
# With custom default scheme
|
|
173
|
+
$ url-normalize -s http "www.foo.com/foo"
|
|
174
|
+
# Output: http://www.foo.com/foo
|
|
175
|
+
|
|
176
|
+
# With query parameter filtering
|
|
177
|
+
$ url-normalize -f "www.google.com/search?q=test&utm_source=test"
|
|
178
|
+
# Output: https://www.google.com/search?q=test
|
|
179
|
+
|
|
180
|
+
# With custom allowlist
|
|
181
|
+
$ url-normalize -f -p page,id "example.com?page=1&id=123&ref=test"
|
|
182
|
+
# Output: https://example.com/?page=1&id=123
|
|
183
|
+
|
|
184
|
+
# With default domain for absolute paths
|
|
185
|
+
$ url-normalize -d example.com "/images/logo.png"
|
|
186
|
+
# Output: https://example.com/images/logo.png
|
|
187
|
+
|
|
188
|
+
# With default domain and custom scheme
|
|
189
|
+
$ url-normalize -d example.com -s http "/images/logo.png"
|
|
190
|
+
# Output: http://example.com/images/logo.png
|
|
191
|
+
|
|
192
|
+
# Human-readable display form
|
|
193
|
+
$ url-normalize -H "https://xn--e1afmkfd.xn--80akhbyknj4f/%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F"
|
|
194
|
+
# Output: https://пример.испытание/Служебная
|
|
195
|
+
|
|
196
|
+
# Via uv tool/uvx
|
|
197
|
+
$ uvx url-normalize www.foo.com:80/foo
|
|
198
|
+
# Output: https://www.foo.com:80/foo
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
## Documentation
|
|
202
|
+
|
|
203
|
+
For a complete history of changes, see [CHANGELOG.md](CHANGELOG.md).
|
|
204
|
+
|
|
205
|
+
## Contributing
|
|
206
|
+
|
|
207
|
+
Contributions are welcome! Please feel free to submit a Pull Request.
|
|
208
|
+
|
|
209
|
+
## License
|
|
210
|
+
|
|
211
|
+
MIT License
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
# url-normalize
|
|
2
|
+
|
|
3
|
+
[](https://github.com/niksite/url-normalize/actions/workflows/ci.yml)
|
|
4
|
+
[](https://coveralls.io/r/niksite/url-normalize)
|
|
5
|
+
[](https://pypi.org/project/url-normalize/)
|
|
6
|
+
[](https://pypi.org/project/url-normalize/)
|
|
7
|
+
[](https://github.com/niksite/url-normalize/blob/master/LICENSE)
|
|
8
|
+
[](https://github.com/astral-sh/ruff)
|
|
9
|
+
|
|
10
|
+
A Python library for standardizing and normalizing URLs. Ideal for database deduplication, caching, web crawling, and anywhere you need to ensure that equivalent URLs resolve to the exact same string.
|
|
11
|
+
|
|
12
|
+
```python
|
|
13
|
+
from url_normalize import url_normalize
|
|
14
|
+
|
|
15
|
+
# Fixes IDN, lowercases host/scheme, removes default ports, resolves path segments
|
|
16
|
+
url_normalize("HTTP://User:Pass@www.FOO.com:80///foo/../bar/./baz?q=1#frag")
|
|
17
|
+
# -> 'http://User:Pass@www.foo.com/bar/baz?q=1#frag'
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## Features
|
|
21
|
+
|
|
22
|
+
url-normalize provides a robust URI normalization function that handles IDN domains, scheme/host lowercasing, and RFC-compliant path normalization.
|
|
23
|
+
|
|
24
|
+
- **IDN Support**: Full internationalized domain name handling (using IDNA2008 with UTS46).
|
|
25
|
+
- **Humanization**: Convert normalized URLs to a readable display format while preserving round-trip normalization.
|
|
26
|
+
- **RFC Compliance**:
|
|
27
|
+
- Proper percent-encoding (minimal, uppercase hex).
|
|
28
|
+
- Dot-segment removal in paths.
|
|
29
|
+
- Default port and authority handling.
|
|
30
|
+
- UTF-8 NFC normalization.
|
|
31
|
+
- **Configurable Defaults**:
|
|
32
|
+
- Customizable default scheme (https by default).
|
|
33
|
+
- Configurable default domain for absolute paths.
|
|
34
|
+
- **Query Parameter Control**:
|
|
35
|
+
- Parameter filtering with allowlists.
|
|
36
|
+
- Support for domain-specific parameter rules.
|
|
37
|
+
- **Versatile URL Handling**: Handles empty strings, double-slash URLs (//domain.tld), and shebang (#!) URLs.
|
|
38
|
+
- **Developer Friendly**:
|
|
39
|
+
- Python 3.10+ compatibility.
|
|
40
|
+
- 100% test coverage.
|
|
41
|
+
- Modern type hints and string handling.
|
|
42
|
+
|
|
43
|
+
Inspired by Sam Ruby's [urlnorm.py](http://intertwingly.net/blog/2004/08/04/Urlnorm).
|
|
44
|
+
|
|
45
|
+
## Installation
|
|
46
|
+
|
|
47
|
+
Install as a library:
|
|
48
|
+
|
|
49
|
+
```sh
|
|
50
|
+
pip install url-normalize
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Or install as a standalone CLI tool using [uv](https://docs.astral.sh/uv/):
|
|
54
|
+
|
|
55
|
+
```sh
|
|
56
|
+
uv tool install url-normalize
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Usage
|
|
60
|
+
|
|
61
|
+
### Python API
|
|
62
|
+
|
|
63
|
+
#### Basic Normalization
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
from url_normalize import url_normalize
|
|
67
|
+
|
|
68
|
+
# Basic normalization (uses https by default)
|
|
69
|
+
print(url_normalize("www.foo.com:80/foo"))
|
|
70
|
+
# Output: https://www.foo.com/foo
|
|
71
|
+
|
|
72
|
+
# With custom default scheme
|
|
73
|
+
print(url_normalize("www.foo.com/foo", default_scheme="http"))
|
|
74
|
+
# Output: http://www.foo.com/foo
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
#### Query Parameter Filtering
|
|
78
|
+
|
|
79
|
+
You can strip out tracking parameters and only keep the ones you care about using allowlists.
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
# With query parameter filtering enabled (strips all params by default)
|
|
83
|
+
print(url_normalize("www.google.com/search?q=test&utm_source=test", filter_params=True))
|
|
84
|
+
# Output: https://www.google.com/search?q=test
|
|
85
|
+
|
|
86
|
+
# With custom parameter allowlist as a list
|
|
87
|
+
print(url_normalize(
|
|
88
|
+
"example.com?page=1&id=123&ref=test",
|
|
89
|
+
filter_params=True,
|
|
90
|
+
param_allowlist=["page", "id"]
|
|
91
|
+
))
|
|
92
|
+
# Output: https://example.com?page=1&id=123
|
|
93
|
+
|
|
94
|
+
# With domain-specific parameter allowlists
|
|
95
|
+
print(url_normalize(
|
|
96
|
+
"example.com?page=1&id=123&ref=test",
|
|
97
|
+
filter_params=True,
|
|
98
|
+
param_allowlist={"example.com": ["page", "id"]}
|
|
99
|
+
))
|
|
100
|
+
# Output: https://example.com?page=1&id=123
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
#### Default Domain & Scheme
|
|
104
|
+
|
|
105
|
+
Useful for resolving relative URLs found on a specific page.
|
|
106
|
+
|
|
107
|
+
```python
|
|
108
|
+
# With default domain for absolute paths
|
|
109
|
+
print(url_normalize("/images/logo.png", default_domain="example.com"))
|
|
110
|
+
# Output: https://example.com/images/logo.png
|
|
111
|
+
|
|
112
|
+
# With default domain and custom scheme
|
|
113
|
+
print(url_normalize("/images/logo.png", default_scheme="http", default_domain="example.com"))
|
|
114
|
+
# Output: http://example.com/images/logo.png
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
#### Humanizing URLs
|
|
118
|
+
|
|
119
|
+
Convert normalized URLs back into a user-friendly format for display, particularly useful for IDN domains and percent-encoded paths.
|
|
120
|
+
|
|
121
|
+
```python
|
|
122
|
+
from url_normalize import url_humanize
|
|
123
|
+
|
|
124
|
+
# Human-readable display form that still normalizes back to the same URL
|
|
125
|
+
print(url_humanize("https://xn--e1afmkfd.xn--80akhbyknj4f/%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F"))
|
|
126
|
+
# Output: https://пример.испытание/Служебная
|
|
127
|
+
|
|
128
|
+
# Humanization accepts the same normalization options
|
|
129
|
+
print(url_humanize("/%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F", default_domain="xn--e1afmkfd.xn--80akhbyknj4f"))
|
|
130
|
+
# Output: https://пример.испытание/Служебная
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
### Command-line Usage
|
|
134
|
+
|
|
135
|
+
You can also use `url-normalize` directly from the terminal to process URLs.
|
|
136
|
+
|
|
137
|
+
```bash
|
|
138
|
+
$ url-normalize "www.foo.com:80/foo"
|
|
139
|
+
# Output: https://www.foo.com/foo
|
|
140
|
+
|
|
141
|
+
# With custom default scheme
|
|
142
|
+
$ url-normalize -s http "www.foo.com/foo"
|
|
143
|
+
# Output: http://www.foo.com/foo
|
|
144
|
+
|
|
145
|
+
# With query parameter filtering
|
|
146
|
+
$ url-normalize -f "www.google.com/search?q=test&utm_source=test"
|
|
147
|
+
# Output: https://www.google.com/search?q=test
|
|
148
|
+
|
|
149
|
+
# With custom allowlist
|
|
150
|
+
$ url-normalize -f -p page,id "example.com?page=1&id=123&ref=test"
|
|
151
|
+
# Output: https://example.com/?page=1&id=123
|
|
152
|
+
|
|
153
|
+
# With default domain for absolute paths
|
|
154
|
+
$ url-normalize -d example.com "/images/logo.png"
|
|
155
|
+
# Output: https://example.com/images/logo.png
|
|
156
|
+
|
|
157
|
+
# With default domain and custom scheme
|
|
158
|
+
$ url-normalize -d example.com -s http "/images/logo.png"
|
|
159
|
+
# Output: http://example.com/images/logo.png
|
|
160
|
+
|
|
161
|
+
# Human-readable display form
|
|
162
|
+
$ url-normalize -H "https://xn--e1afmkfd.xn--80akhbyknj4f/%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F"
|
|
163
|
+
# Output: https://пример.испытание/Служебная
|
|
164
|
+
|
|
165
|
+
# Via uv tool/uvx
|
|
166
|
+
$ uvx url-normalize www.foo.com:80/foo
|
|
167
|
+
# Output: https://www.foo.com:80/foo
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
## Documentation
|
|
171
|
+
|
|
172
|
+
For a complete history of changes, see [CHANGELOG.md](CHANGELOG.md).
|
|
173
|
+
|
|
174
|
+
## Contributing
|
|
175
|
+
|
|
176
|
+
Contributions are welcome! Please feel free to submit a Pull Request.
|
|
177
|
+
|
|
178
|
+
## License
|
|
179
|
+
|
|
180
|
+
MIT License
|
|
@@ -1,12 +1,21 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "url-normalize"
|
|
3
|
-
version = "
|
|
3
|
+
version = "3.0.0"
|
|
4
4
|
description = "URL normalization for Python"
|
|
5
5
|
authors = [{ name = "Nikolay Panov", email = "github@npanov.com" }]
|
|
6
|
-
license =
|
|
6
|
+
license = "MIT"
|
|
7
7
|
readme = "README.md"
|
|
8
|
-
requires-python = ">=3.
|
|
9
|
-
keywords = ["url", "normalization", "normalize"]
|
|
8
|
+
requires-python = ">=3.10"
|
|
9
|
+
keywords = ["url", "normalization", "normalize", "normalizer"]
|
|
10
|
+
classifiers = [
|
|
11
|
+
"Programming Language :: Python :: 3",
|
|
12
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
13
|
+
"Programming Language :: Python :: 3.10",
|
|
14
|
+
"Programming Language :: Python :: 3.11",
|
|
15
|
+
"Programming Language :: Python :: 3.12",
|
|
16
|
+
"Programming Language :: Python :: 3.13",
|
|
17
|
+
"Programming Language :: Python :: 3.14",
|
|
18
|
+
]
|
|
10
19
|
dependencies = ["idna>=3.3"]
|
|
11
20
|
|
|
12
21
|
[project.urls]
|
|
@@ -19,19 +28,10 @@ Changelog = "https://github.com/niksite/url-normalize/blob/master/CHANGELOG.md"
|
|
|
19
28
|
url-normalize = "url_normalize.cli:main"
|
|
20
29
|
|
|
21
30
|
[project.optional-dependencies]
|
|
22
|
-
dev = [
|
|
23
|
-
"mypy",
|
|
24
|
-
"pre-commit",
|
|
25
|
-
"pytest-cov",
|
|
26
|
-
"pytest-ruff",
|
|
27
|
-
"pytest-socket",
|
|
28
|
-
"pytest",
|
|
29
|
-
"ruff",
|
|
30
|
-
"tox",
|
|
31
|
-
]
|
|
31
|
+
dev = ["mypy", "pre-commit", "pytest-cov", "pytest-socket", "pytest", "ruff"]
|
|
32
32
|
|
|
33
33
|
[tool.ruff]
|
|
34
|
-
target-version = "
|
|
34
|
+
target-version = "py310"
|
|
35
35
|
line-length = 88
|
|
36
36
|
unsafe-fixes = true
|
|
37
37
|
|
|
@@ -62,20 +62,18 @@ indent-style = "space"
|
|
|
62
62
|
[tool.mypy]
|
|
63
63
|
ignore_missing_imports = true
|
|
64
64
|
exclude = ["tests"]
|
|
65
|
-
python_version = "3.
|
|
65
|
+
python_version = "3.10"
|
|
66
66
|
show_error_codes = true
|
|
67
67
|
|
|
68
68
|
[build-system]
|
|
69
|
-
requires = ["setuptools>=
|
|
69
|
+
requires = ["setuptools>=77", "wheel"]
|
|
70
70
|
build-backend = "setuptools.build_meta"
|
|
71
71
|
|
|
72
72
|
[tool.pytest.ini_options]
|
|
73
73
|
addopts = [
|
|
74
|
-
"--cov-fail-under=100",
|
|
75
74
|
"--cov-report=term-missing:skip-covered",
|
|
76
75
|
"--cov=url_normalize",
|
|
77
76
|
"--disable-socket",
|
|
78
|
-
"--ruff",
|
|
79
77
|
"-v",
|
|
80
78
|
]
|
|
81
79
|
python_files = ["tests.py", "test_*.py", "*_tests.py"]
|
|
@@ -22,7 +22,7 @@ def run_cli(*args: str) -> subprocess.CompletedProcess:
|
|
|
22
22
|
"""
|
|
23
23
|
command = [sys.executable, "-m", "url_normalize.cli", *list(args)]
|
|
24
24
|
return subprocess.run( # noqa: S603
|
|
25
|
-
command, capture_output=True, text=True, check=False
|
|
25
|
+
command, capture_output=True, text=True, encoding="utf-8", check=False
|
|
26
26
|
)
|
|
27
27
|
|
|
28
28
|
|
|
@@ -66,6 +66,56 @@ def test_cli_basic_normalization_short_args() -> None:
|
|
|
66
66
|
assert not result.stderr
|
|
67
67
|
|
|
68
68
|
|
|
69
|
+
def test_cli_humanize() -> None:
|
|
70
|
+
"""Test human-readable URL output via CLI."""
|
|
71
|
+
url = (
|
|
72
|
+
"https://xn--e1afmkfd.xn--80akhbyknj4f/"
|
|
73
|
+
"%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F"
|
|
74
|
+
)
|
|
75
|
+
expected = "https://пример.испытание/Служебная"
|
|
76
|
+
|
|
77
|
+
result = run_cli("--humanize", url)
|
|
78
|
+
|
|
79
|
+
assert result.returncode == 0
|
|
80
|
+
assert result.stdout.strip() == expected
|
|
81
|
+
assert not result.stderr
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_cli_humanize_short_arg() -> None:
|
|
85
|
+
"""Test human-readable URL output via CLI using short argument."""
|
|
86
|
+
url = (
|
|
87
|
+
"https://xn--e1afmkfd.xn--80akhbyknj4f/"
|
|
88
|
+
"%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F"
|
|
89
|
+
)
|
|
90
|
+
expected = "https://пример.испытание/Служебная"
|
|
91
|
+
|
|
92
|
+
result = run_cli("-H", url)
|
|
93
|
+
|
|
94
|
+
assert result.returncode == 0
|
|
95
|
+
assert result.stdout.strip() == expected
|
|
96
|
+
assert not result.stderr
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def test_cli_humanize_respects_normalization_options() -> None:
|
|
100
|
+
"""Test humanized output keeps the existing CLI normalization options."""
|
|
101
|
+
url = "/%D0%A1%D0%BB%D1%83%D0%B6%D0%B5%D0%B1%D0%BD%D0%B0%D1%8F"
|
|
102
|
+
expected = "https://пример.испытание/Служебная?keep=Ç"
|
|
103
|
+
|
|
104
|
+
result = run_cli(
|
|
105
|
+
"--humanize",
|
|
106
|
+
"--default-domain",
|
|
107
|
+
"xn--e1afmkfd.xn--80akhbyknj4f",
|
|
108
|
+
"--filter-params",
|
|
109
|
+
"--param-allowlist",
|
|
110
|
+
"keep",
|
|
111
|
+
f"{url}?utm_source=ignored&keep=%C3%87",
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
assert result.returncode == 0
|
|
115
|
+
assert result.stdout.strip() == expected
|
|
116
|
+
assert not result.stderr
|
|
117
|
+
|
|
118
|
+
|
|
69
119
|
def test_cli_default_scheme() -> None:
|
|
70
120
|
"""Test default scheme addition via CLI."""
|
|
71
121
|
url = "//example.com"
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Deconstruct url tests."""
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from url_normalize.tools import URL, deconstruct_url
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@pytest.mark.parametrize(
|
|
9
|
+
("url", "expected"),
|
|
10
|
+
[
|
|
11
|
+
(
|
|
12
|
+
"http://site.com",
|
|
13
|
+
URL(
|
|
14
|
+
fragment="",
|
|
15
|
+
host="site.com",
|
|
16
|
+
path="",
|
|
17
|
+
port="",
|
|
18
|
+
query="",
|
|
19
|
+
scheme="http",
|
|
20
|
+
userinfo="",
|
|
21
|
+
),
|
|
22
|
+
),
|
|
23
|
+
(
|
|
24
|
+
"http://user@www.example.com:8080/path/index.html?param=val#fragment",
|
|
25
|
+
URL(
|
|
26
|
+
fragment="fragment",
|
|
27
|
+
host="www.example.com",
|
|
28
|
+
path="/path/index.html",
|
|
29
|
+
port="8080",
|
|
30
|
+
query="param=val",
|
|
31
|
+
scheme="http",
|
|
32
|
+
userinfo="user@",
|
|
33
|
+
),
|
|
34
|
+
),
|
|
35
|
+
],
|
|
36
|
+
)
|
|
37
|
+
def test_deconstruct_url_result_is_expected(url: str, expected: URL) -> None:
|
|
38
|
+
"""Assert we got expected results from the deconstruct_url function."""
|
|
39
|
+
result = deconstruct_url(url)
|
|
40
|
+
assert result == expected, url
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Tests for normalize_fragment function."""
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from url_normalize.url_normalize import normalize_fragment
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@pytest.mark.parametrize(
|
|
9
|
+
("fragment", "expected"),
|
|
10
|
+
[
|
|
11
|
+
("", ""),
|
|
12
|
+
("fragment", "fragment"),
|
|
13
|
+
("пример", "%D0%BF%D1%80%D0%B8%D0%BC%D0%B5%D1%80"),
|
|
14
|
+
("!fragment", "%21fragment"),
|
|
15
|
+
("~fragment", "~fragment"),
|
|
16
|
+
# Issue #36: Equal sign should not be encoded
|
|
17
|
+
("gid=1234", "gid=1234"),
|
|
18
|
+
],
|
|
19
|
+
)
|
|
20
|
+
def test_normalize_fragment_result_is_expected(fragment: str, expected: str) -> None:
|
|
21
|
+
"""Assert we got expected results from the normalize_fragment function."""
|
|
22
|
+
result = normalize_fragment(fragment)
|
|
23
|
+
assert result == expected, fragment
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""Tests for normalize_host function."""
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from url_normalize.url_normalize import normalize_host
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@pytest.mark.parametrize(
|
|
9
|
+
("host", "expected"),
|
|
10
|
+
[
|
|
11
|
+
# Basic cases
|
|
12
|
+
("site.com", "site.com"),
|
|
13
|
+
("SITE.COM", "site.com"),
|
|
14
|
+
("site.com.", "site.com"),
|
|
15
|
+
# Cyrillic domains
|
|
16
|
+
("пример.испытание", "xn--e1afmkfd.xn--80akhbyknj4f"),
|
|
17
|
+
# Mixed case with Cyrillic
|
|
18
|
+
("ExAmPle.РФ", "example.xn--p1ai"),
|
|
19
|
+
# IDNA2008 with UTS46
|
|
20
|
+
("faß.de", "xn--fa-hia.de"),
|
|
21
|
+
# Edge cases
|
|
22
|
+
("ドメイン.テスト", "xn--eckwd4c7c.xn--zckzah"), # Japanese
|
|
23
|
+
("domain.café", "domain.xn--caf-dma"), # Latin with diacritic
|
|
24
|
+
# Normalization tests
|
|
25
|
+
("über.example", "xn--ber-goa.example"), # IDNA 2008 for umlaut
|
|
26
|
+
("example。com", "example.com"), # Normalize full-width punctuation
|
|
27
|
+
],
|
|
28
|
+
)
|
|
29
|
+
def test_normalize_host_result_is_expected(host: str, expected: str) -> None:
|
|
30
|
+
"""Assert we got expected results from the normalize_host function."""
|
|
31
|
+
result = normalize_host(host)
|
|
32
|
+
assert result == expected, host
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Tests for normalize_path function."""
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from url_normalize.url_normalize import normalize_path
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@pytest.mark.parametrize(
|
|
9
|
+
("path", "expected"),
|
|
10
|
+
[
|
|
11
|
+
("..", "/"),
|
|
12
|
+
("", "/"),
|
|
13
|
+
("/../foo", "/foo"),
|
|
14
|
+
("/..foo", "/..foo"),
|
|
15
|
+
("/./../foo", "/foo"),
|
|
16
|
+
("/./foo", "/foo"),
|
|
17
|
+
("/./foo/.", "/foo/"),
|
|
18
|
+
("/.foo", "/.foo"),
|
|
19
|
+
("/", "/"),
|
|
20
|
+
("/foo..", "/foo.."),
|
|
21
|
+
("/foo.", "/foo."),
|
|
22
|
+
("/FOO", "/FOO"),
|
|
23
|
+
("/foo/../bar", "/bar"),
|
|
24
|
+
("/foo/./bar", "/foo/bar"),
|
|
25
|
+
("/foo//", "/foo/"),
|
|
26
|
+
("/foo///bar//", "/foo/bar/"),
|
|
27
|
+
("/foo/bar/..", "/foo/"),
|
|
28
|
+
("/foo/bar/../..", "/"),
|
|
29
|
+
("/foo/bar/../../../../baz", "/baz"),
|
|
30
|
+
("/foo/bar/../../../baz", "/baz"),
|
|
31
|
+
("/foo/bar/../../", "/"),
|
|
32
|
+
("/foo/bar/../../baz", "/baz"),
|
|
33
|
+
("/foo/bar/../", "/foo/"),
|
|
34
|
+
("/foo/bar/../baz", "/foo/baz"),
|
|
35
|
+
("/foo/bar/.", "/foo/bar/"),
|
|
36
|
+
("/foo/bar/./", "/foo/bar/"),
|
|
37
|
+
# Issue #25: we should preserve ? in the path
|
|
38
|
+
("/More+Tea+Vicar%3F/discussion", "/More+Tea+Vicar%3F/discussion"),
|
|
39
|
+
],
|
|
40
|
+
)
|
|
41
|
+
def test_normalize_path_result_is_expected(path: str, expected: str) -> None:
|
|
42
|
+
"""Assert we got expected results from the normalize_path function."""
|
|
43
|
+
result = normalize_path(path, "http")
|
|
44
|
+
assert result == expected, path
|