url-normalize 2.0.1__tar.gz → 2.2.0__tar.gz

{url_normalize-2.0.1 → url_normalize-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: url-normalize
-Version: 2.0.1
+Version: 2.2.0
 Summary: URL normalization for Python
 Author-email: Nikolay Panov <github@npanov.com>
 License: MIT
@@ -26,6 +26,9 @@ Dynamic: license-file
 
 # url-normalize
 
+[![CI](https://github.com/niksite/url-normalize/actions/workflows/ci.yml/badge.svg)](https://github.com/niksite/url-normalize/actions/workflows/ci.yml)
+[![PyPI](https://github.com/niksite/url-normalize/actions/workflows/publish.yml/badge.svg)](https://github.com/niksite/url-normalize/actions/workflows/publish.yml)
+
 URI Normalization function:
 
 * Take care of IDN domains.
@@ -47,6 +50,7 @@ Inspired by Sam Ruby's [urlnorm.py](<http://intertwingly.net/blog/2004/08/04/Url
 
 * IDN (Internationalized Domain Name) support
 * Configurable default scheme (https by default)
+* Configurable default domain for absolute paths
 * Query parameter filtering with allowlists
 * Support for various URL formats including:
   * Empty string URLs
@@ -64,8 +68,6 @@ pip install url-normalize
 
 ## Usage
 
-Basic usage:
-
 ```python
 from url_normalize import url_normalize
 
@@ -81,19 +83,62 @@ print(url_normalize("www.foo.com/foo", default_scheme="http"))
 print(url_normalize("www.google.com/search?q=test&utm_source=test", filter_params=True))
 # Output: https://www.google.com/search?q=test
 
-# With custom parameter allowlist
+# With custom parameter allowlist as a dict
 print(url_normalize(
     "example.com?page=1&id=123&ref=test",
     filter_params=True,
     param_allowlist={"example.com": ["page", "id"]}
 ))
 # Output: https://example.com?page=1&id=123
+
+# With custom parameter allowlist as a list
 print(url_normalize(
     "example.com?page=1&id=123&ref=test",
     filter_params=True,
     param_allowlist=["page", "id"]
 ))
 # Output: https://example.com?page=1&id=123
+
+# With default domain for absolute paths
+print(url_normalize("/images/logo.png", default_domain="example.com"))
+# Output: https://example.com/images/logo.png
+
+# With default domain and custom scheme
+print(url_normalize("/images/logo.png", default_scheme="http", default_domain="example.com"))
+# Output: http://example.com/images/logo.png
+```
+
+### Command-line usage
+
+You can also use `url-normalize` from the command line:
+
+```bash
+$ url-normalize "www.foo.com:80/foo"
+# Output: https://www.foo.com/foo
+
+# With custom default scheme
+$ url-normalize -s http "www.foo.com/foo"
+# Output: http://www.foo.com/foo
+
+# With query parameter filtering
+$ url-normalize -f "www.google.com/search?q=test&utm_source=test"
+# Output: https://www.google.com/search?q=test
+
+# With custom allowlist
+$ url-normalize -f -p page,id "example.com?page=1&id=123&ref=test"
+# Output: https://example.com/?page=1&id=123
+
+# With default domain for absolute paths
+$ url-normalize -d example.com "/images/logo.png"
+# Output: https://example.com/images/logo.png
+
+# With default domain and custom scheme
+$ url-normalize -d example.com -s http "/images/logo.png"
+# Output: http://example.com/images/logo.png
+
+# Via uv tool/uvx
+$ uvx url-normalize www.foo.com:80/foo
+# Output: https://www.foo.com:80/foo
 ```
 
 ## Documentation

{url_normalize-2.0.1 → url_normalize-2.2.0}/README.md

@@ -1,5 +1,8 @@
 # url-normalize
 
+[![CI](https://github.com/niksite/url-normalize/actions/workflows/ci.yml/badge.svg)](https://github.com/niksite/url-normalize/actions/workflows/ci.yml)
+[![PyPI](https://github.com/niksite/url-normalize/actions/workflows/publish.yml/badge.svg)](https://github.com/niksite/url-normalize/actions/workflows/publish.yml)
+
 URI Normalization function:
 
 * Take care of IDN domains.
@@ -21,6 +24,7 @@ Inspired by Sam Ruby's [urlnorm.py](<http://intertwingly.net/blog/2004/08/04/Url
 
 * IDN (Internationalized Domain Name) support
 * Configurable default scheme (https by default)
+* Configurable default domain for absolute paths
 * Query parameter filtering with allowlists
 * Support for various URL formats including:
   * Empty string URLs
@@ -38,8 +42,6 @@ pip install url-normalize
 
 ## Usage
 
-Basic usage:
-
 ```python
 from url_normalize import url_normalize
 
@@ -55,19 +57,62 @@ print(url_normalize("www.foo.com/foo", default_scheme="http"))
 print(url_normalize("www.google.com/search?q=test&utm_source=test", filter_params=True))
 # Output: https://www.google.com/search?q=test
 
-# With custom parameter allowlist
+# With custom parameter allowlist as a dict
 print(url_normalize(
     "example.com?page=1&id=123&ref=test",
     filter_params=True,
     param_allowlist={"example.com": ["page", "id"]}
 ))
 # Output: https://example.com?page=1&id=123
+
+# With custom parameter allowlist as a list
 print(url_normalize(
     "example.com?page=1&id=123&ref=test",
     filter_params=True,
     param_allowlist=["page", "id"]
 ))
 # Output: https://example.com?page=1&id=123
+
+# With default domain for absolute paths
+print(url_normalize("/images/logo.png", default_domain="example.com"))
+# Output: https://example.com/images/logo.png
+
+# With default domain and custom scheme
+print(url_normalize("/images/logo.png", default_scheme="http", default_domain="example.com"))
+# Output: http://example.com/images/logo.png
+```
+
+### Command-line usage
+
+You can also use `url-normalize` from the command line:
+
+```bash
+$ url-normalize "www.foo.com:80/foo"
+# Output: https://www.foo.com/foo
+
+# With custom default scheme
+$ url-normalize -s http "www.foo.com/foo"
+# Output: http://www.foo.com/foo
+
+# With query parameter filtering
+$ url-normalize -f "www.google.com/search?q=test&utm_source=test"
+# Output: https://www.google.com/search?q=test
+
+# With custom allowlist
+$ url-normalize -f -p page,id "example.com?page=1&id=123&ref=test"
+# Output: https://example.com/?page=1&id=123
+
+# With default domain for absolute paths
+$ url-normalize -d example.com "/images/logo.png"
+# Output: https://example.com/images/logo.png
+
+# With default domain and custom scheme
+$ url-normalize -d example.com -s http "/images/logo.png"
+# Output: http://example.com/images/logo.png
+
+# Via uv tool/uvx
+$ uvx url-normalize www.foo.com:80/foo
+# Output: https://www.foo.com:80/foo
 ```
 
 ## Documentation

{url_normalize-2.0.1 → url_normalize-2.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "url-normalize"
-version = "2.0.1"
+version = "2.2.0"
 description = "URL normalization for Python"
 authors = [{ name = "Nikolay Panov", email = "github@npanov.com" }]
 license = { text = "MIT" }
@@ -15,6 +15,9 @@ Repository = "https://github.com/niksite/url-normalize"
 Issues = "https://github.com/niksite/url-normalize/issues"
 Changelog = "https://github.com/niksite/url-normalize/blob/master/CHANGELOG.md"
 
+[project.scripts]
+url-normalize = "url_normalize.cli:main"
+
 [project.optional-dependencies]
 dev = [
   "mypy",

url_normalize-2.2.0/tests/test_cli.py

@@ -0,0 +1,294 @@
+"""Tests for the command line interface."""
+
+import subprocess
+import sys
+from unittest.mock import patch
+
+import pytest
+
+from url_normalize import __version__
+from url_normalize.cli import main
+
+
+def run_cli(*args: str) -> subprocess.CompletedProcess:
+    """Run the CLI command with given arguments.
+
+    Params:
+        *args: Command line arguments to pass to the CLI.
+
+    Returns:
+        A completed process with stdout, stderr, and return code.
+
+    """
+    command = [sys.executable, "-m", "url_normalize.cli", *list(args)]
+    return subprocess.run(  # noqa: S603
+        command, capture_output=True, text=True, check=False
+    )
+
+
+def test_cli_error_handling(capsys, monkeypatch):
+    """Test CLI error handling when URL normalization fails."""
+    with patch("url_normalize.cli.url_normalize") as mock_normalize:
+        mock_normalize.side_effect = Exception("Simulated error")
+        monkeypatch.setattr("sys.argv", ["url-normalize", "http://example.com"])
+
+        with pytest.raises(SystemExit) as excinfo:
+            main()
+
+        assert excinfo.value.code == 1
+        captured = capsys.readouterr()
+        assert "Error normalizing URL: Simulated error" in captured.err
+        assert not captured.out
+
+
+def test_cli_basic_normalization() -> None:
+    """Test basic URL normalization via CLI."""
+    url = "http://EXAMPLE.com/./path/../other/"
+    expected = "http://example.com/other/"
+
+    result = run_cli(url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_basic_normalization_short_args() -> None:
+    """Test basic URL normalization via CLI using short arguments."""
+    url = "http://EXAMPLE.com/./path/../other/"
+    expected = "http://example.com/other/"
+    # Using short args where applicable (none for the URL itself)
+
+    result = run_cli(url)  # No short args needed for basic case
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_default_scheme() -> None:
+    """Test default scheme addition via CLI."""
+    url = "//example.com"
+    expected = "https://example.com/"
+
+    result = run_cli(url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_default_scheme_short_arg() -> None:
+    """Test default scheme addition via CLI using short argument."""
+    url = "//example.com"
+    expected = "https://example.com/"
+
+    result = run_cli(url)  # Default scheme is implicit, no arg needed
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_custom_default_scheme() -> None:
+    """Test custom default scheme via CLI."""
+    url = "//example.com"
+    expected = "ftp://example.com/"
+
+    result = run_cli("--default-scheme", "ftp", url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_custom_default_scheme_short_arg() -> None:
+    """Test custom default scheme via CLI using short argument."""
+    url = "//example.com"
+    expected = "ftp://example.com/"
+
+    result = run_cli("-s", "ftp", url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_filter_params() -> None:
+    """Test parameter filtering via CLI."""
+    url = "http://google.com?utm_source=test&q=1"
+    expected = "http://google.com/?q=1"
+
+    result = run_cli("--filter-params", url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_filter_params_short_arg() -> None:
+    """Test parameter filtering via CLI using short argument."""
+    url = "http://google.com?utm_source=test&q=1"
+    expected = "http://google.com/?q=1"
+
+    result = run_cli("-f", url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_param_allowlist() -> None:
+    """Test parameter allowlist via CLI."""
+    url = "http://example.com?remove=me&keep=this&remove_too=true"
+    expected = "http://example.com/?keep=this"
+    # Use filter_params to enable filtering, then allowlist to keep specific ones
+
+    result = run_cli("-f", "-p", "keep", url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_param_allowlist_multiple() -> None:
+    """Test parameter allowlist with multiple params via CLI."""
+    url = "http://example.com?remove=me&keep=this&keep_too=yes&remove_too=true"
+    expected = "http://example.com/?keep=this&keep_too=yes"
+
+    result = run_cli("-f", "-p", "keep,keep_too", url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_param_allowlist_without_filtering() -> None:
+    """Test allowlist has no effect if filtering is not enabled."""
+    url = "http://example.com?remove=me&keep=this&remove_too=true"
+    expected = "http://example.com/?remove=me&keep=this&remove_too=true"
+    # Not using -f, so allowlist should be ignored
+
+    result = run_cli("-p", "keep", url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_no_url() -> None:
+    """Test CLI error when no URL is provided."""
+    result = run_cli()
+
+    assert result.returncode != 0
+    assert "the following arguments are required: url" in result.stderr
+
+
+def test_cli_version_long() -> None:
+    """Test version output with --version flag."""
+    result = run_cli("--version")
+
+    assert result.returncode == 0
+    assert __version__ in result.stdout
+    assert not result.stderr
+
+
+def test_cli_version_short() -> None:
+    """Test version output with -v flag."""
+    result = run_cli("-v")
+
+    assert result.returncode == 0
+    assert __version__ in result.stdout
+    assert not result.stderr
+
+
+@pytest.mark.skipif(
+    sys.platform == "win32", reason="Charset handling differs on Windows CLI"
+)
+def test_cli_charset() -> None:
+    """Test charset handling via CLI (might be platform-dependent)."""
+    # Example using Cyrillic characters which need correct encoding
+    url = "http://пример.рф/path"
+    expected_idn = "http://xn--e1afmkfd.xn--p1ai/path"
+
+    # Test with default UTF-8
+    result_utf8 = run_cli(url)
+
+    assert result_utf8.returncode == 0
+    assert result_utf8.stdout.strip() == expected_idn
+    assert not result_utf8.stderr
+
+    # Test specifying UTF-8 explicitly
+    result_charset = run_cli("--charset", "utf-8", url)
+
+    assert result_charset.returncode == 0
+    assert result_charset.stdout.strip() == expected_idn
+    assert not result_charset.stderr
+
+    # Test specifying UTF-8 explicitly using short arg
+    result_charset_short = run_cli("-c", "utf-8", url)
+
+    assert result_charset_short.returncode == 0
+    assert result_charset_short.stdout.strip() == expected_idn
+    assert not result_charset_short.stderr
+
+
+def test_cli_default_domain() -> None:
+    """Test adding default domain to absolute path via CLI."""
+    url = "/path/to/image.png"
+    expected = "https://example.com/path/to/image.png"
+
+    result = run_cli("--default-domain", "example.com", url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_default_domain_short_arg() -> None:
+    """Test adding default domain using short argument."""
+    url = "/path/to/image.png"
+    expected = "https://example.com/path/to/image.png"
+
+    result = run_cli("-d", "example.com", url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_default_domain_with_scheme() -> None:
+    """Test adding default domain with custom scheme."""
+    url = "/path/to/image.png"
+    expected = "http://example.com/path/to/image.png"
+
+    result = run_cli("-d", "example.com", "-s", "http", url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_default_domain_no_effect_on_absolute_urls() -> None:
+    """Test default domain has no effect on absolute URLs."""
+    url = "http://original-domain.com/path"
+    expected = "http://original-domain.com/path"
+
+    result = run_cli("-d", "example.com", url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_default_domain_no_effect_on_relative_paths() -> None:
+    """Test default domain has no effect on relative paths."""
+    url = "path/to/file.html"
+    # This becomes a regular URL with the default scheme
+    expected = "https://path/to/file.html"
+
+    result = run_cli("-d", "example.com", url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr

{url_normalize-2.0.1 → url_normalize-2.2.0}/tests/test_normalize_fragment.py

@@ -8,6 +8,8 @@ EXPECTED_DATA = {
     "пример": "%D0%BF%D1%80%D0%B8%D0%BC%D0%B5%D1%80",
     "!fragment": "%21fragment",
     "~fragment": "~fragment",
+    # Issue #36: Equal sign should not be encoded
+    "gid=1234": "gid=1234",
 }
 
 

{url_normalize-2.0.1 → url_normalize-2.2.0}/tests/test_normalize_path.py

@@ -29,10 +29,12 @@ EXPECTED_DATA = {
     "/foo/bar/../baz": "/foo/baz",
     "/foo/bar/.": "/foo/bar/",
     "/foo/bar/./": "/foo/bar/",
+    # Issue #25: we should preserve ? in the path
+    "/More+Tea+Vicar%3F/discussion": "/More+Tea+Vicar%3F/discussion",
 }
 
 
-def test_normalize_host_result_is_expected():
+def test_normalize_path_result_is_expected():
     """Assert we got expected results from the normalize_path function."""
     for url, expected in EXPECTED_DATA.items():
         result = normalize_path(url, "http")

{url_normalize-2.0.1 → url_normalize-2.2.0}/tests/test_normalize_query.py

@@ -14,6 +14,8 @@ from url_normalize.url_normalize import normalize_query
         ("Ç=Ç", "%C3%87=%C3%87"),
         ("%C3%87=%C3%87", "%C3%87=%C3%87"),
         ("q=C%CC%A7", "q=%C3%87"),
+        ("q=%23test", "q=%23test"),  # Preserve encoded # in value, #31
+        ("where=code%3D123", "where=code%3D123"),  # Preserve encoded = in value, #25
     ],
 )
 def test_normalize_query_result_is_expected(query, expected):

url_normalize-2.2.0/tests/test_provide_url_domain.py

@@ -0,0 +1,31 @@
+"""Tests for provide_url_domain function."""
+
+import pytest
+
+from url_normalize.provide_url_domain import provide_url_domain
+
+
+@pytest.mark.parametrize(
+    ("url", "expected"),
+    [
+        ("", ""),
+        ("-", "-"),
+        ("http://example.com/", "http://example.com/"),
+        ("/file/path", "//example.com/file/path"),
+        ("site/page", "site/page"),  # No change for relative paths
+    ],
+)
+def test_provide_url_domain_result_is_expected(url: str, expected: str) -> None:
+    """Assert we get expected results from provide_url_domain function."""
+    result = provide_url_domain(url, default_domain="example.com")
+    assert result == expected
+
+
+def test_provide_url_domain_accept_different_domains():
+    """Assert we could provide different default_domain values."""
+    url = "/file/path"
+    expected = "//custom-domain.org/file/path"
+
+    actual = provide_url_domain(url, default_domain="custom-domain.org")
+
+    assert actual == expected