url-normalize 1.4.3__tar.gz → 2.0.0__tar.gz

url_normalize-2.0.0/PKG-INFO

@@ -0,0 +1,105 @@
+Metadata-Version: 2.4
+Name: url-normalize
+Version: 2.0.0
+Summary: URL normalization for Python
+Author-email: Nikolay Panov <github@npanov.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/niksite/url-normalize
+Project-URL: Repository, https://github.com/niksite/url-normalize
+Project-URL: Issues, https://github.com/niksite/url-normalize/issues
+Project-URL: Changelog, https://github.com/niksite/url-normalize/blob/master/CHANGELOG.md
+Keywords: url,normalization,normalize
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: idna>=3.3
+Provides-Extra: dev
+Requires-Dist: mypy; extra == "dev"
+Requires-Dist: pre-commit; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: pytest-ruff; extra == "dev"
+Requires-Dist: pytest-socket; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: tox; extra == "dev"
+Dynamic: license-file
+
+# url-normalize
+
+URI Normalization function:
+
+* Take care of IDN domains.
+* Always provide the URI scheme in lowercase characters.
+* Always provide the host, if any, in lowercase characters.
+* Only perform percent-encoding where it is essential.
+* Always use uppercase A-through-F characters when percent-encoding.
+* Prevent dot-segments appearing in non-relative URI paths.
+* For schemes that define a default authority, use an empty authority if the
+  default is desired.
+* For schemes that define an empty path to be equivalent to a path of "/",
+  use "/".
+* For schemes that define a port, use an empty port if the default is desired
+* All portions of the URI must be utf-8 encoded NFC from Unicode strings
+
+Inspired by Sam Ruby's [urlnorm.py](<http://intertwingly.net/blog/2004/08/04/Urlnorm>)
+
+## Features
+
+* IDN (Internationalized Domain Name) support
+* Configurable default scheme (https by default)
+* Query parameter filtering with allowlists
+* Support for various URL formats including:
+  * Empty string URLs
+  * Double slash URLs (//domain.tld)
+  * Shebang (#!) URLs
+* Cross-version Python compatibility (3.8+)
+* 100% test coverage
+* Modern type hints and string handling
+
+## Installation
+
+```sh
+pip install url-normalize
+```
+
+## Usage
+
+Basic usage:
+
+```python
+from url_normalize import url_normalize
+
+# Basic normalization (uses https by default)
+print(url_normalize("www.foo.com:80/foo"))
+# Output: https://www.foo.com/foo
+
+# With custom default scheme
+print(url_normalize("www.foo.com/foo", default_scheme="http"))
+# Output: http://www.foo.com/foo
+
+# With query parameter filtering enabled
+print(url_normalize("www.google.com/search?q=test&utm_source=test", filter_params=True))
+# Output: https://www.google.com/search?q=test
+
+# With custom parameter allowlist
+print(url_normalize(
+    "example.com?page=1&id=123&ref=test",
+    filter_params=True,
+    param_allowlist={"example.com": ["page", "id"]}
+))
+# Output: https://example.com?page=1&id=123
+print(url_normalize(
+    "example.com?page=1&id=123&ref=test",
+    filter_params=True,
+    param_allowlist=["page", "id"]
+))
+# Output: https://example.com?page=1&id=123
+```
+
+## Documentation
+
+For a complete history of changes, see [CHANGELOG.md](CHANGELOG.md).
+
+## License
+
+MIT License

url_normalize-2.0.0/README.md

@@ -0,0 +1,79 @@
+# url-normalize
+
+URI Normalization function:
+
+* Take care of IDN domains.
+* Always provide the URI scheme in lowercase characters.
+* Always provide the host, if any, in lowercase characters.
+* Only perform percent-encoding where it is essential.
+* Always use uppercase A-through-F characters when percent-encoding.
+* Prevent dot-segments appearing in non-relative URI paths.
+* For schemes that define a default authority, use an empty authority if the
+  default is desired.
+* For schemes that define an empty path to be equivalent to a path of "/",
+  use "/".
+* For schemes that define a port, use an empty port if the default is desired
+* All portions of the URI must be utf-8 encoded NFC from Unicode strings
+
+Inspired by Sam Ruby's [urlnorm.py](<http://intertwingly.net/blog/2004/08/04/Urlnorm>)
+
+## Features
+
+* IDN (Internationalized Domain Name) support
+* Configurable default scheme (https by default)
+* Query parameter filtering with allowlists
+* Support for various URL formats including:
+  * Empty string URLs
+  * Double slash URLs (//domain.tld)
+  * Shebang (#!) URLs
+* Cross-version Python compatibility (3.8+)
+* 100% test coverage
+* Modern type hints and string handling
+
+## Installation
+
+```sh
+pip install url-normalize
+```
+
+## Usage
+
+Basic usage:
+
+```python
+from url_normalize import url_normalize
+
+# Basic normalization (uses https by default)
+print(url_normalize("www.foo.com:80/foo"))
+# Output: https://www.foo.com/foo
+
+# With custom default scheme
+print(url_normalize("www.foo.com/foo", default_scheme="http"))
+# Output: http://www.foo.com/foo
+
+# With query parameter filtering enabled
+print(url_normalize("www.google.com/search?q=test&utm_source=test", filter_params=True))
+# Output: https://www.google.com/search?q=test
+
+# With custom parameter allowlist
+print(url_normalize(
+    "example.com?page=1&id=123&ref=test",
+    filter_params=True,
+    param_allowlist={"example.com": ["page", "id"]}
+))
+# Output: https://example.com?page=1&id=123
+print(url_normalize(
+    "example.com?page=1&id=123&ref=test",
+    filter_params=True,
+    param_allowlist=["page", "id"]
+))
+# Output: https://example.com?page=1&id=123
+```
+
+## Documentation
+
+For a complete history of changes, see [CHANGELOG.md](CHANGELOG.md).
+
+## License
+
+MIT License

url_normalize-2.0.0/pyproject.toml

@@ -0,0 +1,78 @@
+[project]
+name = "url-normalize"
+version = "2.0.0"
+description = "URL normalization for Python"
+authors = [{ name = "Nikolay Panov", email = "github@npanov.com" }]
+license = "MIT"
+readme = "README.md"
+requires-python = ">=3.8"
+keywords = ["url", "normalization", "normalize"]
+dependencies = ["idna>=3.3"]
+
+[project.urls]
+Homepage = "https://github.com/niksite/url-normalize"
+Repository = "https://github.com/niksite/url-normalize"
+Issues = "https://github.com/niksite/url-normalize/issues"
+Changelog = "https://github.com/niksite/url-normalize/blob/master/CHANGELOG.md"
+
+[project.optional-dependencies]
+dev = [
+  "mypy",
+  "pre-commit",
+  "pytest-cov",
+  "pytest-ruff",
+  "pytest-socket",
+  "pytest",
+  "ruff",
+  "tox",
+]
+
+[tool.ruff]
+target-version = "py38"
+line-length = 88
+unsafe-fixes = true
+
+[tool.ruff.lint]
+select = ["ALL"]
+extend-select = [
+  "D400", # First line should end with a period
+  "D401", # First line should be in imperative mood
+  "D413", # Missing blank line after the last section of a multiline docstring
+]
+fixable = ["ALL"]
+ignore = [
+  "COM812", # missing-trailing-comma
+  "D203",   # One blank line before class - we prefer D211 instead
+  "D213",   # multi-line-summary-second-line - we prefer D212 instead
+]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**" = ["INP001", "ANN001", "ANN201", "S101", "CPY001"]
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+
+[tool.mypy]
+ignore_missing_imports = true
+exclude = ["tests"]
+python_version = "3.8"
+show_error_codes = true
+
+[build-system]
+requires = ["setuptools>=42", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.pytest.ini_options]
+addopts = [
+  "--cov-fail-under=100",
+  "--cov-report=term-missing:skip-covered",
+  "--cov=url_normalize",
+  "--disable-socket",
+  "--ruff",
+  "-v",
+]
+python_files = ["tests.py", "test_*.py", "*_tests.py"]

url_normalize-2.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

url_normalize-2.0.0/tests/test_deconstruct_url.py

@@ -0,0 +1,35 @@
+"""Deconstruct url tests."""
+
+from __future__ import annotations
+
+from typing import Final
+
+from url_normalize.tools import URL, deconstruct_url
+
+EXPECTED_DATA: Final[dict[str, URL]] = {
+    "http://site.com": URL(
+        fragment="",
+        host="site.com",
+        path="",
+        port="",
+        query="",
+        scheme="http",
+        userinfo="",
+    ),
+    "http://user@www.example.com:8080/path/index.html?param=val#fragment": URL(
+        fragment="fragment",
+        host="www.example.com",
+        path="/path/index.html",
+        port="8080",
+        query="param=val",
+        scheme="http",
+        userinfo="user@",
+    ),
+}
+
+
+def test_deconstruct_url_result_is_expected() -> None:
+    """Assert we got expected results from the deconstruct_url function."""
+    for url, expected in EXPECTED_DATA.items():
+        result = deconstruct_url(url)
+        assert result == expected, url

url_normalize-2.0.0/tests/test_generic_url_cleanup.py

@@ -0,0 +1,21 @@
+"""Tests for generic_url_cleanup function."""
+
+from __future__ import annotations
+
+import pytest
+
+from url_normalize.url_normalize import generic_url_cleanup
+
+
+@pytest.mark.parametrize(
+    ("url", "expected"),
+    [
+        ("//site/#!fragment", "//site/?_escaped_fragment_=fragment"),
+        ("//site/page", "//site/page"),
+        ("//site/?& ", "//site/"),
+    ],
+)
+def test_generic_url_cleanup_result_is_expected(url: str, expected: str) -> None:
+    """Assert we got expected results from the generic_url_cleanup function."""
+    result = generic_url_cleanup(url)
+    assert result == expected

url_normalize-2.0.0/tests/test_normalize_fragment.py

@@ -0,0 +1,19 @@
+"""Tests for normalize_fragment function."""
+
+from url_normalize.url_normalize import normalize_fragment
+
+EXPECTED_DATA = {
+    "": "",
+    "fragment": "fragment",
+    "пример": "%D0%BF%D1%80%D0%B8%D0%BC%D0%B5%D1%80",
+    "!fragment": "%21fragment",
+    "~fragment": "~fragment",
+}
+
+
+def test_normalize_fragment_result_is_expected():
+    """Assert we got expected results from the normalize_fragment function."""
+    for url, expected in EXPECTED_DATA.items():
+        result = normalize_fragment(url)
+
+        assert result == expected, url

url_normalize-2.0.0/tests/test_normalize_host.py

@@ -0,0 +1,29 @@
+"""Tests for normalize_host function."""
+
+from url_normalize.url_normalize import normalize_host
+
+EXPECTED_DATA = {
+    # Basic cases
+    "site.com": "site.com",
+    "SITE.COM": "site.com",
+    "site.com.": "site.com",
+    # Cyrillic domains
+    "пример.испытание": "xn--e1afmkfd.xn--80akhbyknj4f",
+    # Mixed case with Cyrillic
+    "ExAmPle.РФ": "example.xn--p1ai",
+    # IDNA2008 with UTS46
+    "faß.de": "fass.de",  # Normalize using transitional rules
+    # Edge cases
+    "ドメイン.テスト": "xn--eckwd4c7c.xn--zckzah",  # Japanese
+    "domain.café": "domain.xn--caf-dma",  # Latin with diacritic
+    # Normalization tests
+    "über.example": "xn--ber-goa.example",  # IDNA 2008 for umlaut
+    "example。com": "example.com",  # Normalize full-width punctuation
+}
+
+
+def test_normalize_host_result_is_expected() -> None:
+    """Assert we got expected results from the normalize_host function."""
+    for url, expected in EXPECTED_DATA.items():
+        result = normalize_host(url)
+        assert result == expected, url

url_normalize-2.0.0/tests/test_normalize_path.py

@@ -0,0 +1,40 @@
+"""Tests for normalize_path function."""
+
+from url_normalize.url_normalize import normalize_path
+
+EXPECTED_DATA = {
+    "..": "/",
+    "": "/",
+    "/../foo": "/foo",
+    "/..foo": "/..foo",
+    "/./../foo": "/foo",
+    "/./foo": "/foo",
+    "/./foo/.": "/foo/",
+    "/.foo": "/.foo",
+    "/": "/",
+    "/foo..": "/foo..",
+    "/foo.": "/foo.",
+    "/FOO": "/FOO",
+    "/foo/../bar": "/bar",
+    "/foo/./bar": "/foo/bar",
+    "/foo//": "/foo/",
+    "/foo///bar//": "/foo/bar/",
+    "/foo/bar/..": "/foo/",
+    "/foo/bar/../..": "/",
+    "/foo/bar/../../../../baz": "/baz",
+    "/foo/bar/../../../baz": "/baz",
+    "/foo/bar/../../": "/",
+    "/foo/bar/../../baz": "/baz",
+    "/foo/bar/../": "/foo/",
+    "/foo/bar/../baz": "/foo/baz",
+    "/foo/bar/.": "/foo/bar/",
+    "/foo/bar/./": "/foo/bar/",
+}
+
+
+def test_normalize_host_result_is_expected():
+    """Assert we got expected results from the normalize_path function."""
+    for url, expected in EXPECTED_DATA.items():
+        result = normalize_path(url, "http")
+
+        assert result == expected, url

url_normalize-2.0.0/tests/test_normalize_port.py

@@ -0,0 +1,13 @@
+"""Tests for normalize_port function."""
+
+from url_normalize.url_normalize import normalize_port
+
+EXPECTED_DATA = {"8080": "8080", "": "", "80": "", "string": "string"}
+
+
+def test_normalize_port_result_is_expected():
+    """Assert we got expected results from the normalize_port function."""
+    for url, expected in EXPECTED_DATA.items():
+        result = normalize_port(url, "http")
+
+        assert result == expected, url

url_normalize-2.0.0/tests/test_normalize_query.py

@@ -0,0 +1,22 @@
+"""Tests for normalize_query function."""
+
+import pytest
+
+from url_normalize.url_normalize import normalize_query
+
+
+@pytest.mark.parametrize(
+    ("query", "expected"),
+    [
+        ("", ""),
+        ("&&&", ""),
+        ("param1=val1&param2=val2", "param1=val1&param2=val2"),
+        ("Ç=Ç", "%C3%87=%C3%87"),
+        ("%C3%87=%C3%87", "%C3%87=%C3%87"),
+        ("q=C%CC%A7", "q=%C3%87"),
+    ],
+)
+def test_normalize_query_result_is_expected(query, expected):
+    """Assert we got expected results from the normalize_query function."""
+    result = normalize_query(query)
+    assert result == expected, query

url_normalize-2.0.0/tests/test_normalize_query_filters.py

@@ -0,0 +1,98 @@
+"""URL parameter filtering test module."""
+
+from __future__ import annotations
+
+import pytest
+
+from url_normalize import url_normalize
+
+
+def test_param_filtering_disabled_by_default():
+    """Test that parameter filtering is disabled by default."""
+    url = "https://www.google.com/search?q=test&utm_source=test"
+    assert url_normalize(url) == url
+
+
+def test_empty_query():
+    """Test handling empty query strings."""
+    assert url_normalize("https://example.com/page?") == "https://example.com/page"
+
+
+def test_custom_allowlist():
+    """Test custom allowlist functionality with preserved order."""
+    custom_allowlist = {"example.com": ["page", "id"], "google.com": ["q", "lang"]}
+
+    # Order should match input query string order
+    assert (
+        url_normalize(
+            "https://example.com/search?page=1&id=123&utm_source=test",
+            filter_params=True,
+            param_allowlist=custom_allowlist,
+        )
+        == "https://example.com/search?page=1&id=123"
+    )
+
+    assert (
+        url_normalize(
+            "https://google.com/search?q=test&ie=utf8&lang=en",
+            filter_params=True,
+            param_allowlist=custom_allowlist,
+        )
+        == "https://google.com/search?q=test&lang=en"
+    )
+
+
+def test_custom_list_allowlist():
+    """Test custom list allowlist functionality."""
+    assert (
+        url_normalize(
+            "https://google.com/search?qq=test&ie=utf8&utm_source=test",
+            filter_params=True,
+            param_allowlist=["ie", "qq"],
+        )
+        == "https://google.com/search?qq=test&ie=utf8"
+    )
+
+
+@pytest.mark.parametrize(
+    ("url", "expected"),
+    [
+        # Basic parameter filtering
+        (
+            "https://www.google.com/search?q=test&utm_source=test",
+            "https://www.google.com/search?q=test",
+        ),
+        (
+            "https://www.youtube.com/watch?v=12345&utm_source=share",
+            "https://www.youtube.com/watch?v=12345",
+        ),
+        # With www subdomain
+        (
+            "https://www.google.com/search?q=test&ref=test",
+            "https://www.google.com/search?q=test",
+        ),
+        # With port number
+        (
+            "https://google.com:8080/search?q=test&ref=test",
+            "https://google.com:8080/search?q=test",
+        ),
+        # Default allowlist cases
+        (
+            "https://www.google.com/search?q=test&utm_source=test&ie=utf8",
+            "https://www.google.com/search?q=test&ie=utf8",
+        ),
+        (
+            "https://www.baidu.com/s?wd=test&utm_source=test&ie=utf8",
+            "https://www.baidu.com/s?wd=test&ie=utf8",
+        ),
+        (
+            "https://youtube.com/watch?v=12345&utm_source=test&search_query=test",
+            "https://youtube.com/watch?v=12345&search_query=test",
+        ),
+        # Non-allowlisted domain
+        ("https://example.org/page?a=1&b=2", "https://example.org/page"),
+    ],
+)
+def test_parameter_filtering(url: str, expected: str):
+    """Test URL parameter filtering functionality with various scenarios."""
+    assert url_normalize(url, filter_params=True) == expected

url_normalize-2.0.0/tests/test_normalize_scheme.py

@@ -0,0 +1,13 @@
+"""Tests for normalize_scheme function."""
+
+from url_normalize.url_normalize import normalize_scheme
+
+EXPECTED_DATA = {"http": "http", "HTTP": "http"}
+
+
+def test_normalize_scheme_result_is_expected():
+    """Assert we got expected results from the normalize_scheme function."""
+    for url, expected in EXPECTED_DATA.items():
+        result = normalize_scheme(url)
+
+        assert result == expected, url

url_normalize-2.0.0/tests/test_normalize_userinfo.py

@@ -0,0 +1,19 @@
+"""Tests for normalize_userinfo function."""
+
+from url_normalize.url_normalize import normalize_userinfo
+
+EXPECTED_DATA = {
+    ":@": "",
+    "": "",
+    "@": "",
+    "user:password@": "user:password@",
+    "user@": "user@",
+}
+
+
+def test_normalize_userinfo_result_is_expected():
+    """Assert we got expected results from the normalize_userinfo function."""
+    for url, expected in EXPECTED_DATA.items():
+        result = normalize_userinfo(url)
+
+        assert result == expected, url

url_normalize-2.0.0/tests/test_provide_url_scheme.py

@@ -0,0 +1,30 @@
+"""Tests for provide_url_scheme function."""
+
+from url_normalize.url_normalize import provide_url_scheme
+
+EXPECTED_DATA = {
+    "": "",
+    "-": "-",
+    "/file/path": "/file/path",
+    "//site/path": "https://site/path",
+    "ftp://site/": "ftp://site/",
+    "site/page": "https://site/page",
+}
+
+
+def test_provide_url_scheme_result_is_expected():
+    """Assert we got expected results from the provide_url_scheme function."""
+    for url, expected in EXPECTED_DATA.items():
+        result = provide_url_scheme(url)
+
+        assert result == expected, url
+
+
+def test_provide_url_scheme_accept_default_scheme_param():
+    """Assert we could provide default_scheme param other than https."""
+    url = "//site/path"
+    expected = "http://site/path"
+
+    actual = provide_url_scheme(url, default_scheme="http")
+
+    assert actual == expected

url_normalize-2.0.0/tests/test_reconstruct_url.py

@@ -0,0 +1,41 @@
+"""Reconstruct url tests."""
+
+from __future__ import annotations
+
+from typing import Final
+
+from url_normalize.tools import URL, reconstruct_url
+
+EXPECTED_DATA: Final[tuple[tuple[URL, str], ...]] = (
+    (
+        URL(
+            fragment="",
+            host="site.com",
+            path="",
+            port="",
+            query="",
+            scheme="http",
+            userinfo="",
+        ),
+        "http://site.com",
+    ),
+    (
+        URL(
+            fragment="fragment",
+            host="www.example.com",
+            path="/path/index.html",
+            port="8080",
+            query="param=val",
+            scheme="http",
+            userinfo="user@",
+        ),
+        "http://user@www.example.com:8080/path/index.html?param=val#fragment",
+    ),
+)
+
+
+def test_deconstruct_url_result_is_expected() -> None:
+    """Assert we got expected results from the deconstruct_url function."""
+    for url, expected in EXPECTED_DATA:
+        result = reconstruct_url(url)
+        assert result == expected, url

url_normalize-2.0.0/tests/test_tools.py

@@ -0,0 +1,12 @@
+"""Tools module tests."""
+
+from __future__ import annotations
+
+from url_normalize.tools import force_unicode
+
+
+def test_force_unicode_with_bytes() -> None:
+    """Test force_unicode handles bytes input correctly."""
+    test_bytes = b"hello world"
+    result = force_unicode(test_bytes)
+    assert result == "hello world"