url-normalize 2.1.0__tar.gz → 2.2.1__tar.gz

url_normalize-2.2.1/PKG-INFO

@@ -0,0 +1,175 @@
+Metadata-Version: 2.4
+Name: url-normalize
+Version: 2.2.1
+Summary: URL normalization for Python
+Author-email: Nikolay Panov <github@npanov.com>
+License: MIT
+Project-URL: Homepage, https://github.com/niksite/url-normalize
+Project-URL: Repository, https://github.com/niksite/url-normalize
+Project-URL: Issues, https://github.com/niksite/url-normalize/issues
+Project-URL: Changelog, https://github.com/niksite/url-normalize/blob/master/CHANGELOG.md
+Keywords: url,normalization,normalize,normalizer
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: idna>=3.3
+Provides-Extra: dev
+Requires-Dist: mypy; extra == "dev"
+Requires-Dist: pre-commit; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: pytest-socket; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Dynamic: license-file
+
+# url-normalize
+
+[![tests](https://github.com/niksite/url-normalize/actions/workflows/ci.yml/badge.svg)](https://github.com/niksite/url-normalize/actions/workflows/ci.yml)
+[![Coveralls](https://img.shields.io/coveralls/github/niksite/url-normalize/master.svg)](https://coveralls.io/r/niksite/url-normalize)
+[![PyPI](https://img.shields.io/pypi/v/url-normalize.svg)](https://pypi.org/project/url-normalize/)
+
+A Python library for standardizing and normalizing URLs with support for internationalized domain names (IDN).
+
+## Table of Contents
+
+- [Introduction](#introduction)
+- [Features](#features)
+- [Installation](#installation)
+- [Usage](#usage)
+  - [Python API](#python-api)
+  - [Command Line](#command-line-usage)
+- [Documentation](#documentation)
+- [Contributing](#contributing)
+- [License](#license)
+
+## Introduction
+
+url-normalize provides a robust URI normalization function that:
+
+- Takes care of IDN domains.
+- Always provides the URI scheme in lowercase characters.
+- Always provides the host, if any, in lowercase characters.
+- Only performs percent-encoding where it is essential.
+- Always uses uppercase A-through-F characters when percent-encoding.
+- Prevents dot-segments appearing in non-relative URI paths.
+- For schemes that define a default authority, uses an empty authority if the
+  default is desired.
+- For schemes that define an empty path to be equivalent to a path of "/",
+  uses "/".
+- For schemes that define a port, uses an empty port if the default is desired
+- Ensures all portions of the URI are utf-8 encoded NFC from Unicode strings
+
+Inspired by Sam Ruby's [urlnorm.py](http://intertwingly.net/blog/2004/08/04/Urlnorm)
+
+## Features
+
+- **IDN Support**: Full internationalized domain name handling
+- **Configurable Defaults**:
+  - Customizable default scheme (https by default)
+  - Configurable default domain for absolute paths
+- **Query Parameter Control**:
+  - Parameter filtering with allowlists
+  - Support for domain-specific parameter rules
+- **Versatile URL Handling**:
+  - Empty string URLs
+  - Double slash URLs (//domain.tld)
+  - Shebang (#!) URLs
+- **Developer Friendly**:
+  - Cross-version Python compatibility (3.8+)
+  - 100% test coverage
+  - Modern type hints and string handling
+
+## Installation
+
+```sh
+pip install url-normalize
+```
+
+## Usage
+
+### Python API
+
+```python
+from url_normalize import url_normalize
+
+# Basic normalization (uses https by default)
+print(url_normalize("www.foo.com:80/foo"))
+# Output: https://www.foo.com/foo
+
+# With custom default scheme
+print(url_normalize("www.foo.com/foo", default_scheme="http"))
+# Output: http://www.foo.com/foo
+
+# With query parameter filtering enabled
+print(url_normalize("www.google.com/search?q=test&utm_source=test", filter_params=True))
+# Output: https://www.google.com/search?q=test
+
+# With custom parameter allowlist as a dict
+print(url_normalize(
+    "example.com?page=1&id=123&ref=test",
+    filter_params=True,
+    param_allowlist={"example.com": ["page", "id"]}
+))
+# Output: https://example.com?page=1&id=123
+
+# With custom parameter allowlist as a list
+print(url_normalize(
+    "example.com?page=1&id=123&ref=test",
+    filter_params=True,
+    param_allowlist=["page", "id"]
+))
+# Output: https://example.com?page=1&id=123
+
+# With default domain for absolute paths
+print(url_normalize("/images/logo.png", default_domain="example.com"))
+# Output: https://example.com/images/logo.png
+
+# With default domain and custom scheme
+print(url_normalize("/images/logo.png", default_scheme="http", default_domain="example.com"))
+# Output: http://example.com/images/logo.png
+```
+
+### Command-line Usage
+
+You can also use `url-normalize` from the command line:
+
+```bash
+$ url-normalize "www.foo.com:80/foo"
+# Output: https://www.foo.com/foo
+
+# With custom default scheme
+$ url-normalize -s http "www.foo.com/foo"
+# Output: http://www.foo.com/foo
+
+# With query parameter filtering
+$ url-normalize -f "www.google.com/search?q=test&utm_source=test"
+# Output: https://www.google.com/search?q=test
+
+# With custom allowlist
+$ url-normalize -f -p page,id "example.com?page=1&id=123&ref=test"
+# Output: https://example.com/?page=1&id=123
+
+# With default domain for absolute paths
+$ url-normalize -d example.com "/images/logo.png"
+# Output: https://example.com/images/logo.png
+
+# With default domain and custom scheme
+$ url-normalize -d example.com -s http "/images/logo.png"
+# Output: http://example.com/images/logo.png
+
+# Via uv tool/uvx
+$ uvx url-normalize www.foo.com:80/foo
+# Output: https://www.foo.com:80/foo
+```
+
+## Documentation
+
+For a complete history of changes, see [CHANGELOG.md](CHANGELOG.md).
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## License
+
+MIT License

url_normalize-2.2.1/README.md

@@ -0,0 +1,151 @@
+# url-normalize
+
+[![tests](https://github.com/niksite/url-normalize/actions/workflows/ci.yml/badge.svg)](https://github.com/niksite/url-normalize/actions/workflows/ci.yml)
+[![Coveralls](https://img.shields.io/coveralls/github/niksite/url-normalize/master.svg)](https://coveralls.io/r/niksite/url-normalize)
+[![PyPI](https://img.shields.io/pypi/v/url-normalize.svg)](https://pypi.org/project/url-normalize/)
+
+A Python library for standardizing and normalizing URLs with support for internationalized domain names (IDN).
+
+## Table of Contents
+
+- [Introduction](#introduction)
+- [Features](#features)
+- [Installation](#installation)
+- [Usage](#usage)
+  - [Python API](#python-api)
+  - [Command Line](#command-line-usage)
+- [Documentation](#documentation)
+- [Contributing](#contributing)
+- [License](#license)
+
+## Introduction
+
+url-normalize provides a robust URI normalization function that:
+
+- Takes care of IDN domains.
+- Always provides the URI scheme in lowercase characters.
+- Always provides the host, if any, in lowercase characters.
+- Only performs percent-encoding where it is essential.
+- Always uses uppercase A-through-F characters when percent-encoding.
+- Prevents dot-segments appearing in non-relative URI paths.
+- For schemes that define a default authority, uses an empty authority if the
+  default is desired.
+- For schemes that define an empty path to be equivalent to a path of "/",
+  uses "/".
+- For schemes that define a port, uses an empty port if the default is desired
+- Ensures all portions of the URI are utf-8 encoded NFC from Unicode strings
+
+Inspired by Sam Ruby's [urlnorm.py](http://intertwingly.net/blog/2004/08/04/Urlnorm)
+
+## Features
+
+- **IDN Support**: Full internationalized domain name handling
+- **Configurable Defaults**:
+  - Customizable default scheme (https by default)
+  - Configurable default domain for absolute paths
+- **Query Parameter Control**:
+  - Parameter filtering with allowlists
+  - Support for domain-specific parameter rules
+- **Versatile URL Handling**:
+  - Empty string URLs
+  - Double slash URLs (//domain.tld)
+  - Shebang (#!) URLs
+- **Developer Friendly**:
+  - Cross-version Python compatibility (3.8+)
+  - 100% test coverage
+  - Modern type hints and string handling
+
+## Installation
+
+```sh
+pip install url-normalize
+```
+
+## Usage
+
+### Python API
+
+```python
+from url_normalize import url_normalize
+
+# Basic normalization (uses https by default)
+print(url_normalize("www.foo.com:80/foo"))
+# Output: https://www.foo.com/foo
+
+# With custom default scheme
+print(url_normalize("www.foo.com/foo", default_scheme="http"))
+# Output: http://www.foo.com/foo
+
+# With query parameter filtering enabled
+print(url_normalize("www.google.com/search?q=test&utm_source=test", filter_params=True))
+# Output: https://www.google.com/search?q=test
+
+# With custom parameter allowlist as a dict
+print(url_normalize(
+    "example.com?page=1&id=123&ref=test",
+    filter_params=True,
+    param_allowlist={"example.com": ["page", "id"]}
+))
+# Output: https://example.com?page=1&id=123
+
+# With custom parameter allowlist as a list
+print(url_normalize(
+    "example.com?page=1&id=123&ref=test",
+    filter_params=True,
+    param_allowlist=["page", "id"]
+))
+# Output: https://example.com?page=1&id=123
+
+# With default domain for absolute paths
+print(url_normalize("/images/logo.png", default_domain="example.com"))
+# Output: https://example.com/images/logo.png
+
+# With default domain and custom scheme
+print(url_normalize("/images/logo.png", default_scheme="http", default_domain="example.com"))
+# Output: http://example.com/images/logo.png
+```
+
+### Command-line Usage
+
+You can also use `url-normalize` from the command line:
+
+```bash
+$ url-normalize "www.foo.com:80/foo"
+# Output: https://www.foo.com/foo
+
+# With custom default scheme
+$ url-normalize -s http "www.foo.com/foo"
+# Output: http://www.foo.com/foo
+
+# With query parameter filtering
+$ url-normalize -f "www.google.com/search?q=test&utm_source=test"
+# Output: https://www.google.com/search?q=test
+
+# With custom allowlist
+$ url-normalize -f -p page,id "example.com?page=1&id=123&ref=test"
+# Output: https://example.com/?page=1&id=123
+
+# With default domain for absolute paths
+$ url-normalize -d example.com "/images/logo.png"
+# Output: https://example.com/images/logo.png
+
+# With default domain and custom scheme
+$ url-normalize -d example.com -s http "/images/logo.png"
+# Output: http://example.com/images/logo.png
+
+# Via uv tool/uvx
+$ uvx url-normalize www.foo.com:80/foo
+# Output: https://www.foo.com:80/foo
+```
+
+## Documentation
+
+For a complete history of changes, see [CHANGELOG.md](CHANGELOG.md).
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## License
+
+MIT License

{url_normalize-2.1.0 → url_normalize-2.2.1}/pyproject.toml

@@ -1,12 +1,12 @@
 [project]
 name = "url-normalize"
-version = "2.1.0"
+version = "2.2.1"
 description = "URL normalization for Python"
 authors = [{ name = "Nikolay Panov", email = "github@npanov.com" }]
 license = { text = "MIT" }
 readme = "README.md"
 requires-python = ">=3.8"
-keywords = ["url", "normalization", "normalize"]
+keywords = ["url", "normalization", "normalize", "normalizer"]
 dependencies = ["idna>=3.3"]
 
 [project.urls]
@@ -19,16 +19,7 @@ Changelog = "https://github.com/niksite/url-normalize/blob/master/CHANGELOG.md"
 url-normalize = "url_normalize.cli:main"
 
 [project.optional-dependencies]
-dev = [
-  "mypy",
-  "pre-commit",
-  "pytest-cov",
-  "pytest-ruff",
-  "pytest-socket",
-  "pytest",
-  "ruff",
-  "tox",
-]
+dev = ["mypy", "pre-commit", "pytest-cov", "pytest-socket", "pytest", "ruff"]
 
 [tool.ruff]
 target-version = "py38"
@@ -71,11 +62,9 @@ build-backend = "setuptools.build_meta"
 
 [tool.pytest.ini_options]
 addopts = [
-  "--cov-fail-under=100",
   "--cov-report=term-missing:skip-covered",
   "--cov=url_normalize",
   "--disable-socket",
-  "--ruff",
   "-v",
 ]
 python_files = ["tests.py", "test_*.py", "*_tests.py"]

{url_normalize-2.1.0 → url_normalize-2.2.1}/tests/test_cli.py

@@ -231,3 +231,64 @@ def test_cli_charset() -> None:
     assert result_charset_short.returncode == 0
     assert result_charset_short.stdout.strip() == expected_idn
     assert not result_charset_short.stderr
+
+
+def test_cli_default_domain() -> None:
+    """Test adding default domain to absolute path via CLI."""
+    url = "/path/to/image.png"
+    expected = "https://example.com/path/to/image.png"
+
+    result = run_cli("--default-domain", "example.com", url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_default_domain_short_arg() -> None:
+    """Test adding default domain using short argument."""
+    url = "/path/to/image.png"
+    expected = "https://example.com/path/to/image.png"
+
+    result = run_cli("-d", "example.com", url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_default_domain_with_scheme() -> None:
+    """Test adding default domain with custom scheme."""
+    url = "/path/to/image.png"
+    expected = "http://example.com/path/to/image.png"
+
+    result = run_cli("-d", "example.com", "-s", "http", url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_default_domain_no_effect_on_absolute_urls() -> None:
+    """Test default domain has no effect on absolute URLs."""
+    url = "http://original-domain.com/path"
+    expected = "http://original-domain.com/path"
+
+    result = run_cli("-d", "example.com", url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_default_domain_no_effect_on_relative_paths() -> None:
+    """Test default domain has no effect on relative paths."""
+    url = "path/to/file.html"
+    # This becomes a regular URL with the default scheme
+    expected = "https://path/to/file.html"
+
+    result = run_cli("-d", "example.com", url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr

url_normalize-2.2.1/tests/test_deconstruct_url.py

@@ -0,0 +1,40 @@
+"""Deconstruct url tests."""
+
+import pytest
+
+from url_normalize.tools import URL, deconstruct_url
+
+
+@pytest.mark.parametrize(
+    ("url", "expected"),
+    [
+        (
+            "http://site.com",
+            URL(
+                fragment="",
+                host="site.com",
+                path="",
+                port="",
+                query="",
+                scheme="http",
+                userinfo="",
+            ),
+        ),
+        (
+            "http://user@www.example.com:8080/path/index.html?param=val#fragment",
+            URL(
+                fragment="fragment",
+                host="www.example.com",
+                path="/path/index.html",
+                port="8080",
+                query="param=val",
+                scheme="http",
+                userinfo="user@",
+            ),
+        ),
+    ],
+)
+def test_deconstruct_url_result_is_expected(url: str, expected: URL) -> None:
+    """Assert we got expected results from the deconstruct_url function."""
+    result = deconstruct_url(url)
+    assert result == expected, url

url_normalize-2.2.1/tests/test_normalize_fragment.py

@@ -0,0 +1,23 @@
+"""Tests for normalize_fragment function."""
+
+import pytest
+
+from url_normalize.url_normalize import normalize_fragment
+
+
+@pytest.mark.parametrize(
+    ("fragment", "expected"),
+    [
+        ("", ""),
+        ("fragment", "fragment"),
+        ("пример", "%D0%BF%D1%80%D0%B8%D0%BC%D0%B5%D1%80"),
+        ("!fragment", "%21fragment"),
+        ("~fragment", "~fragment"),
+        # Issue #36: Equal sign should not be encoded
+        ("gid=1234", "gid=1234"),
+    ],
+)
+def test_normalize_fragment_result_is_expected(fragment: str, expected: str) -> None:
+    """Assert we got expected results from the normalize_fragment function."""
+    result = normalize_fragment(fragment)
+    assert result == expected, fragment

url_normalize-2.2.1/tests/test_normalize_host.py

@@ -0,0 +1,32 @@
+"""Tests for normalize_host function."""
+
+import pytest
+
+from url_normalize.url_normalize import normalize_host
+
+
+@pytest.mark.parametrize(
+    ("host", "expected"),
+    [
+        # Basic cases
+        ("site.com", "site.com"),
+        ("SITE.COM", "site.com"),
+        ("site.com.", "site.com"),
+        # Cyrillic domains
+        ("пример.испытание", "xn--e1afmkfd.xn--80akhbyknj4f"),
+        # Mixed case with Cyrillic
+        ("ExAmPle.РФ", "example.xn--p1ai"),
+        # IDNA2008 with UTS46
+        ("faß.de", "fass.de"),  # Normalize using transitional rules
+        # Edge cases
+        ("ドメイン.テスト", "xn--eckwd4c7c.xn--zckzah"),  # Japanese
+        ("domain.café", "domain.xn--caf-dma"),  # Latin with diacritic
+        # Normalization tests
+        ("über.example", "xn--ber-goa.example"),  # IDNA 2008 for umlaut
+        ("example。com", "example.com"),  # Normalize full-width punctuation
+    ],
+)
+def test_normalize_host_result_is_expected(host: str, expected: str) -> None:
+    """Assert we got expected results from the normalize_host function."""
+    result = normalize_host(host)
+    assert result == expected, host

url_normalize-2.2.1/tests/test_normalize_path.py

@@ -0,0 +1,44 @@
+"""Tests for normalize_path function."""
+
+import pytest
+
+from url_normalize.url_normalize import normalize_path
+
+
+@pytest.mark.parametrize(
+    ("path", "expected"),
+    [
+        ("..", "/"),
+        ("", "/"),
+        ("/../foo", "/foo"),
+        ("/..foo", "/..foo"),
+        ("/./../foo", "/foo"),
+        ("/./foo", "/foo"),
+        ("/./foo/.", "/foo/"),
+        ("/.foo", "/.foo"),
+        ("/", "/"),
+        ("/foo..", "/foo.."),
+        ("/foo.", "/foo."),
+        ("/FOO", "/FOO"),
+        ("/foo/../bar", "/bar"),
+        ("/foo/./bar", "/foo/bar"),
+        ("/foo//", "/foo/"),
+        ("/foo///bar//", "/foo/bar/"),
+        ("/foo/bar/..", "/foo/"),
+        ("/foo/bar/../..", "/"),
+        ("/foo/bar/../../../../baz", "/baz"),
+        ("/foo/bar/../../../baz", "/baz"),
+        ("/foo/bar/../../", "/"),
+        ("/foo/bar/../../baz", "/baz"),
+        ("/foo/bar/../", "/foo/"),
+        ("/foo/bar/../baz", "/foo/baz"),
+        ("/foo/bar/.", "/foo/bar/"),
+        ("/foo/bar/./", "/foo/bar/"),
+        # Issue #25: we should preserve ? in the path
+        ("/More+Tea+Vicar%3F/discussion", "/More+Tea+Vicar%3F/discussion"),
+    ],
+)
+def test_normalize_path_result_is_expected(path: str, expected: str) -> None:
+    """Assert we got expected results from the normalize_path function."""
+    result = normalize_path(path, "http")
+    assert result == expected, path

url_normalize-2.2.1/tests/test_normalize_port.py

@@ -0,0 +1,26 @@
+"""Tests for normalize_port function."""
+
+import pytest
+
+from url_normalize.url_normalize import normalize_port
+
+
+@pytest.mark.parametrize(
+    ("port", "expected"),
+    [
+        ("8080", "8080"),  # Non-default port
+        ("", ""),  # Empty port
+        ("80", ""),  # Default HTTP port
+        ("string", "string"),  # Non-numeric port (should pass through)
+        # Add more cases as needed, e.g., for HTTPS
+        pytest.param("443", "", id="https_default_port"),
+    ],
+)
+def test_normalize_port_result_is_expected(port: str, expected: str):
+    """Assert we got expected results from the normalize_port function."""
+    # Test with 'http' scheme for most cases
+    scheme = "https" if port == "443" else "http"
+
+    result = normalize_port(port, scheme)
+
+    assert result == expected

{url_normalize-2.1.0 → url_normalize-2.2.1}/tests/test_normalize_query.py

@@ -14,6 +14,8 @@ from url_normalize.url_normalize import normalize_query
         ("Ç=Ç", "%C3%87=%C3%87"),
         ("%C3%87=%C3%87", "%C3%87=%C3%87"),
         ("q=C%CC%A7", "q=%C3%87"),
+        ("q=%23test", "q=%23test"),  # Preserve encoded # in value, #31
+        ("where=code%3D123", "where=code%3D123"),  # Preserve encoded = in value, #25
     ],
 )
 def test_normalize_query_result_is_expected(query, expected):

url_normalize-2.2.1/tests/test_normalize_scheme.py

@@ -0,0 +1,18 @@
+"""Tests for normalize_scheme function."""
+
+import pytest
+
+from url_normalize.url_normalize import normalize_scheme
+
+
+@pytest.mark.parametrize(
+    ("scheme", "expected"),
+    [
+        ("http", "http"),
+        ("HTTP", "http"),
+    ],
+)
+def test_normalize_scheme_result_is_expected(scheme: str, expected: str) -> None:
+    """Assert we got expected results from the normalize_scheme function."""
+    result = normalize_scheme(scheme)
+    assert result == expected, scheme

url_normalize-2.2.1/tests/test_normalize_userinfo.py

@@ -0,0 +1,21 @@
+"""Tests for normalize_userinfo function."""
+
+import pytest
+
+from url_normalize.url_normalize import normalize_userinfo
+
+
+@pytest.mark.parametrize(
+    ("userinfo", "expected"),
+    [
+        (":@", ""),
+        ("", ""),
+        ("@", ""),
+        ("user:password@", "user:password@"),
+        ("user@", "user@"),
+    ],
+)
+def test_normalize_userinfo_result_is_expected(userinfo: str, expected: str) -> None:
+    """Assert we got expected results from the normalize_userinfo function."""
+    result = normalize_userinfo(userinfo)
+    assert result == expected, userinfo