url-normalize 2.0.0__tar.gz → 2.1.0__tar.gz

{url_normalize-2.0.0/url_normalize.egg-info → url_normalize-2.1.0}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: url-normalize
-Version: 2.0.0
+Version: 2.1.0
 Summary: URL normalization for Python
 Author-email: Nikolay Panov <github@npanov.com>
-License-Expression: MIT
+License: MIT
 Project-URL: Homepage, https://github.com/niksite/url-normalize
 Project-URL: Repository, https://github.com/niksite/url-normalize
 Project-URL: Issues, https://github.com/niksite/url-normalize/issues
@@ -26,6 +26,9 @@ Dynamic: license-file
 
 # url-normalize
 
+[![CI](https://github.com/niksite/url-normalize/actions/workflows/ci.yml/badge.svg)](https://github.com/niksite/url-normalize/actions/workflows/ci.yml)
+[![PyPI](https://github.com/niksite/url-normalize/actions/workflows/publish.yml/badge.svg)](https://github.com/niksite/url-normalize/actions/workflows/publish.yml)
+
 URI Normalization function:
 
 * Take care of IDN domains.
@@ -64,8 +67,6 @@ pip install url-normalize
 
 ## Usage
 
-Basic usage:
-
 ```python
 from url_normalize import url_normalize
 
@@ -81,13 +82,15 @@ print(url_normalize("www.foo.com/foo", default_scheme="http"))
 print(url_normalize("www.google.com/search?q=test&utm_source=test", filter_params=True))
 # Output: https://www.google.com/search?q=test
 
-# With custom parameter allowlist
+# With custom parameter allowlist as a dict
 print(url_normalize(
     "example.com?page=1&id=123&ref=test",
     filter_params=True,
     param_allowlist={"example.com": ["page", "id"]}
 ))
 # Output: https://example.com?page=1&id=123
+
+# With custom parameter allowlist as a list
 print(url_normalize(
     "example.com?page=1&id=123&ref=test",
     filter_params=True,
@@ -96,6 +99,31 @@ print(url_normalize(
 # Output: https://example.com?page=1&id=123
 ```
 
+### Command-line usage
+
+You can also use `url-normalize` from the command line:
+
+```bash
+$ url-normalize "www.foo.com:80/foo"
+# Output: https://www.foo.com/foo
+
+# With custom default scheme
+$ url-normalize -s http "www.foo.com/foo"
+# Output: http://www.foo.com/foo
+
+# With query parameter filtering
+$ url-normalize -f "www.google.com/search?q=test&utm_source=test"
+# Output: https://www.google.com/search?q=test
+
+# With custom allowlist
+$ url-normalize -f -p page,id "example.com?page=1&id=123&ref=test"
+# Output: https://example.com/?page=1&id=123
+
+# Via uv tool/uvx
+$ uvx url-normalize www.foo.com:80/foo
+# Output: https://www.foo.com:80/foo
+```
+
 ## Documentation
 
 For a complete history of changes, see [CHANGELOG.md](CHANGELOG.md).

{url_normalize-2.0.0 → url_normalize-2.1.0}/README.md

@@ -1,5 +1,8 @@
 # url-normalize
 
+[![CI](https://github.com/niksite/url-normalize/actions/workflows/ci.yml/badge.svg)](https://github.com/niksite/url-normalize/actions/workflows/ci.yml)
+[![PyPI](https://github.com/niksite/url-normalize/actions/workflows/publish.yml/badge.svg)](https://github.com/niksite/url-normalize/actions/workflows/publish.yml)
+
 URI Normalization function:
 
 * Take care of IDN domains.
@@ -38,8 +41,6 @@ pip install url-normalize
 
 ## Usage
 
-Basic usage:
-
 ```python
 from url_normalize import url_normalize
 
@@ -55,13 +56,15 @@ print(url_normalize("www.foo.com/foo", default_scheme="http"))
 print(url_normalize("www.google.com/search?q=test&utm_source=test", filter_params=True))
 # Output: https://www.google.com/search?q=test
 
-# With custom parameter allowlist
+# With custom parameter allowlist as a dict
 print(url_normalize(
     "example.com?page=1&id=123&ref=test",
     filter_params=True,
     param_allowlist={"example.com": ["page", "id"]}
 ))
 # Output: https://example.com?page=1&id=123
+
+# With custom parameter allowlist as a list
 print(url_normalize(
     "example.com?page=1&id=123&ref=test",
     filter_params=True,
@@ -70,6 +73,31 @@ print(url_normalize(
 # Output: https://example.com?page=1&id=123
 ```
 
+### Command-line usage
+
+You can also use `url-normalize` from the command line:
+
+```bash
+$ url-normalize "www.foo.com:80/foo"
+# Output: https://www.foo.com/foo
+
+# With custom default scheme
+$ url-normalize -s http "www.foo.com/foo"
+# Output: http://www.foo.com/foo
+
+# With query parameter filtering
+$ url-normalize -f "www.google.com/search?q=test&utm_source=test"
+# Output: https://www.google.com/search?q=test
+
+# With custom allowlist
+$ url-normalize -f -p page,id "example.com?page=1&id=123&ref=test"
+# Output: https://example.com/?page=1&id=123
+
+# Via uv tool/uvx
+$ uvx url-normalize www.foo.com:80/foo
+# Output: https://www.foo.com:80/foo
+```
+
 ## Documentation
 
 For a complete history of changes, see [CHANGELOG.md](CHANGELOG.md).

{url_normalize-2.0.0 → url_normalize-2.1.0}/pyproject.toml

@@ -1,9 +1,9 @@
 [project]
 name = "url-normalize"
-version = "2.0.0"
+version = "2.1.0"
 description = "URL normalization for Python"
 authors = [{ name = "Nikolay Panov", email = "github@npanov.com" }]
-license = "MIT"
+license = { text = "MIT" }
 readme = "README.md"
 requires-python = ">=3.8"
 keywords = ["url", "normalization", "normalize"]
@@ -15,6 +15,9 @@ Repository = "https://github.com/niksite/url-normalize"
 Issues = "https://github.com/niksite/url-normalize/issues"
 Changelog = "https://github.com/niksite/url-normalize/blob/master/CHANGELOG.md"
 
+[project.scripts]
+url-normalize = "url_normalize.cli:main"
+
 [project.optional-dependencies]
 dev = [
   "mypy",

url_normalize-2.1.0/tests/test_cli.py

@@ -0,0 +1,233 @@
+"""Tests for the command line interface."""
+
+import subprocess
+import sys
+from unittest.mock import patch
+
+import pytest
+
+from url_normalize import __version__
+from url_normalize.cli import main
+
+
+def run_cli(*args: str) -> subprocess.CompletedProcess:
+    """Run the CLI command with given arguments.
+
+    Params:
+        *args: Command line arguments to pass to the CLI.
+
+    Returns:
+        A completed process with stdout, stderr, and return code.
+
+    """
+    command = [sys.executable, "-m", "url_normalize.cli", *list(args)]
+    return subprocess.run(  # noqa: S603
+        command, capture_output=True, text=True, check=False
+    )
+
+
+def test_cli_error_handling(capsys, monkeypatch):
+    """Test CLI error handling when URL normalization fails."""
+    with patch("url_normalize.cli.url_normalize") as mock_normalize:
+        mock_normalize.side_effect = Exception("Simulated error")
+        monkeypatch.setattr("sys.argv", ["url-normalize", "http://example.com"])
+
+        with pytest.raises(SystemExit) as excinfo:
+            main()
+
+        assert excinfo.value.code == 1
+        captured = capsys.readouterr()
+        assert "Error normalizing URL: Simulated error" in captured.err
+        assert not captured.out
+
+
+def test_cli_basic_normalization() -> None:
+    """Test basic URL normalization via CLI."""
+    url = "http://EXAMPLE.com/./path/../other/"
+    expected = "http://example.com/other/"
+
+    result = run_cli(url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_basic_normalization_short_args() -> None:
+    """Test basic URL normalization via CLI using short arguments."""
+    url = "http://EXAMPLE.com/./path/../other/"
+    expected = "http://example.com/other/"
+    # Using short args where applicable (none for the URL itself)
+
+    result = run_cli(url)  # No short args needed for basic case
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_default_scheme() -> None:
+    """Test default scheme addition via CLI."""
+    url = "//example.com"
+    expected = "https://example.com/"
+
+    result = run_cli(url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_default_scheme_short_arg() -> None:
+    """Test default scheme addition via CLI using short argument."""
+    url = "//example.com"
+    expected = "https://example.com/"
+
+    result = run_cli(url)  # Default scheme is implicit, no arg needed
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_custom_default_scheme() -> None:
+    """Test custom default scheme via CLI."""
+    url = "//example.com"
+    expected = "ftp://example.com/"
+
+    result = run_cli("--default-scheme", "ftp", url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_custom_default_scheme_short_arg() -> None:
+    """Test custom default scheme via CLI using short argument."""
+    url = "//example.com"
+    expected = "ftp://example.com/"
+
+    result = run_cli("-s", "ftp", url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_filter_params() -> None:
+    """Test parameter filtering via CLI."""
+    url = "http://google.com?utm_source=test&q=1"
+    expected = "http://google.com/?q=1"
+
+    result = run_cli("--filter-params", url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_filter_params_short_arg() -> None:
+    """Test parameter filtering via CLI using short argument."""
+    url = "http://google.com?utm_source=test&q=1"
+    expected = "http://google.com/?q=1"
+
+    result = run_cli("-f", url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_param_allowlist() -> None:
+    """Test parameter allowlist via CLI."""
+    url = "http://example.com?remove=me&keep=this&remove_too=true"
+    expected = "http://example.com/?keep=this"
+    # Use filter_params to enable filtering, then allowlist to keep specific ones
+
+    result = run_cli("-f", "-p", "keep", url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_param_allowlist_multiple() -> None:
+    """Test parameter allowlist with multiple params via CLI."""
+    url = "http://example.com?remove=me&keep=this&keep_too=yes&remove_too=true"
+    expected = "http://example.com/?keep=this&keep_too=yes"
+
+    result = run_cli("-f", "-p", "keep,keep_too", url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_param_allowlist_without_filtering() -> None:
+    """Test allowlist has no effect if filtering is not enabled."""
+    url = "http://example.com?remove=me&keep=this&remove_too=true"
+    expected = "http://example.com/?remove=me&keep=this&remove_too=true"
+    # Not using -f, so allowlist should be ignored
+
+    result = run_cli("-p", "keep", url)
+
+    assert result.returncode == 0
+    assert result.stdout.strip() == expected
+    assert not result.stderr
+
+
+def test_cli_no_url() -> None:
+    """Test CLI error when no URL is provided."""
+    result = run_cli()
+
+    assert result.returncode != 0
+    assert "the following arguments are required: url" in result.stderr
+
+
+def test_cli_version_long() -> None:
+    """Test version output with --version flag."""
+    result = run_cli("--version")
+
+    assert result.returncode == 0
+    assert __version__ in result.stdout
+    assert not result.stderr
+
+
+def test_cli_version_short() -> None:
+    """Test version output with -v flag."""
+    result = run_cli("-v")
+
+    assert result.returncode == 0
+    assert __version__ in result.stdout
+    assert not result.stderr
+
+
+@pytest.mark.skipif(
+    sys.platform == "win32", reason="Charset handling differs on Windows CLI"
+)
+def test_cli_charset() -> None:
+    """Test charset handling via CLI (might be platform-dependent)."""
+    # Example using Cyrillic characters which need correct encoding
+    url = "http://пример.рф/path"
+    expected_idn = "http://xn--e1afmkfd.xn--p1ai/path"
+
+    # Test with default UTF-8
+    result_utf8 = run_cli(url)
+
+    assert result_utf8.returncode == 0
+    assert result_utf8.stdout.strip() == expected_idn
+    assert not result_utf8.stderr
+
+    # Test specifying UTF-8 explicitly
+    result_charset = run_cli("--charset", "utf-8", url)
+
+    assert result_charset.returncode == 0
+    assert result_charset.stdout.strip() == expected_idn
+    assert not result_charset.stderr
+
+    # Test specifying UTF-8 explicitly using short arg
+    result_charset_short = run_cli("-c", "utf-8", url)
+
+    assert result_charset_short.returncode == 0
+    assert result_charset_short.stdout.strip() == expected_idn
+    assert not result_charset_short.stderr

{url_normalize-2.0.0 → url_normalize-2.1.0}/tests/test_normalize_fragment.py

@@ -8,6 +8,8 @@ EXPECTED_DATA = {
     "пример": "%D0%BF%D1%80%D0%B8%D0%BC%D0%B5%D1%80",
     "!fragment": "%21fragment",
     "~fragment": "~fragment",
+    # Issue #36: Equal sign should not be encoded
+    "gid=1234": "gid=1234",
 }
 
 

{url_normalize-2.0.0 → url_normalize-2.1.0}/tests/test_url_normalize.py

@@ -87,6 +87,8 @@ NO_CHANGES_EXPECTED: Final[tuple[str, ...]] = (
     "tel:+1-816-555-1212",
     "telnet://192.0.2.16:80/",
     "urn:oasis:names:specification:docbook:dtd:xml:4.1.2",
+    # Issue #36: Fragment with '=' should not be encoded
+    "https://docs.google.com/spreadsheets/d/abcd/edit#gid=1234",
 )
 
 

{url_normalize-2.0.0 → url_normalize-2.1.0}/url_normalize/__init__.py

@@ -8,6 +8,6 @@ SPDX-License-Identifier: MIT
 from .url_normalize import url_normalize
 
 __license__ = "MIT"
-__version__ = "2.0.0"
+__version__ = "2.1.0"
 
 __all__ = ["url_normalize"]

url_normalize-2.1.0/url_normalize/cli.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+"""Command line interface for url-normalize."""
+
+import argparse
+import sys
+from importlib.metadata import version
+
+from .url_normalize import url_normalize
+
+
+def main() -> None:
+    """Parse arguments and run url_normalize."""
+    parser = argparse.ArgumentParser(description="Normalize a URL.")
+    parser.add_argument(
+        "-v",
+        "--version",
+        action="version",
+        version=f"%(prog)s {version('url-normalize')}",
+    )
+    parser.add_argument("url", help="The URL to normalize.")
+    parser.add_argument(
+        "-c",
+        "--charset",
+        default="utf-8",
+        help="The charset of the URL. Default: utf-8",
+    )
+    parser.add_argument(
+        "-s",
+        "--default-scheme",
+        default="https",
+        help="The default scheme to use if missing. Default: https",
+    )
+    parser.add_argument(
+        "-f",
+        "--filter-params",
+        action="store_true",
+        help="Filter common tracking parameters.",
+    )
+    parser.add_argument(
+        "-p",
+        "--param-allowlist",
+        type=str,
+        help="Comma-separated list of query parameters to allow (e.g., 'q,id').",
+    )
+
+    args = parser.parse_args()
+
+    allowlist = args.param_allowlist.split(",") if args.param_allowlist else None
+
+    try:
+        normalized_url = url_normalize(
+            args.url,
+            charset=args.charset,
+            default_scheme=args.default_scheme,
+            filter_params=args.filter_params,
+            param_allowlist=allowlist,
+        )
+    except Exception as e:  # noqa: BLE001
+        print(f"Error normalizing URL: {e}", file=sys.stderr)  # noqa: T201
+        sys.exit(1)
+    else:
+        print(normalized_url)  # noqa: T201
+
+
+if __name__ == "__main__":
+    main()  # pragma: no cover

url_normalize-2.1.0/url_normalize/normalize_fragment.py

@@ -0,0 +1,27 @@
+"""URL fragment normalization."""
+
+from __future__ import annotations
+
+from .tools import quote, unquote
+
+
+def normalize_fragment(fragment: str) -> str:
+    """Normalize fragment part of the url.
+
+    Params:
+        fragment : string : url fragment, e.g., 'fragment'
+
+    Returns:
+        string : normalized fragment data.
+
+    Notes:
+        According to RFC 3986, the following characters are allowed in a fragment:
+        fragment    = *( pchar / "/" / "?" )
+        pchar       = unreserved / pct-encoded / sub-delims / ":" / "@"
+        unreserved  = ALPHA / DIGIT / "-" / "." / "_" / "~"
+        sub-delims  = "!" / "$" / "&" / "'" / "(" / ")" / "*" / "+" / "," / ";" / "="
+        We specifically allow "~" and "=" as safe characters during normalization.
+        Other sub-delimiters could potentially be added to the `safe` list if needed.
+
+    """
+    return quote(unquote(fragment), safe="~=")

{url_normalize-2.0.0 → url_normalize-2.1.0/url_normalize.egg-info}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: url-normalize
-Version: 2.0.0
+Version: 2.1.0
 Summary: URL normalization for Python
 Author-email: Nikolay Panov <github@npanov.com>
-License-Expression: MIT
+License: MIT
 Project-URL: Homepage, https://github.com/niksite/url-normalize
 Project-URL: Repository, https://github.com/niksite/url-normalize
 Project-URL: Issues, https://github.com/niksite/url-normalize/issues
@@ -26,6 +26,9 @@ Dynamic: license-file
 
 # url-normalize
 
+[![CI](https://github.com/niksite/url-normalize/actions/workflows/ci.yml/badge.svg)](https://github.com/niksite/url-normalize/actions/workflows/ci.yml)
+[![PyPI](https://github.com/niksite/url-normalize/actions/workflows/publish.yml/badge.svg)](https://github.com/niksite/url-normalize/actions/workflows/publish.yml)
+
 URI Normalization function:
 
 * Take care of IDN domains.
@@ -64,8 +67,6 @@ pip install url-normalize
 
 ## Usage
 
-Basic usage:
-
 ```python
 from url_normalize import url_normalize
 
@@ -81,13 +82,15 @@ print(url_normalize("www.foo.com/foo", default_scheme="http"))
 print(url_normalize("www.google.com/search?q=test&utm_source=test", filter_params=True))
 # Output: https://www.google.com/search?q=test
 
-# With custom parameter allowlist
+# With custom parameter allowlist as a dict
 print(url_normalize(
     "example.com?page=1&id=123&ref=test",
     filter_params=True,
     param_allowlist={"example.com": ["page", "id"]}
 ))
 # Output: https://example.com?page=1&id=123
+
+# With custom parameter allowlist as a list
 print(url_normalize(
     "example.com?page=1&id=123&ref=test",
     filter_params=True,
@@ -96,6 +99,31 @@ print(url_normalize(
 # Output: https://example.com?page=1&id=123
 ```
 
+### Command-line usage
+
+You can also use `url-normalize` from the command line:
+
+```bash
+$ url-normalize "www.foo.com:80/foo"
+# Output: https://www.foo.com/foo
+
+# With custom default scheme
+$ url-normalize -s http "www.foo.com/foo"
+# Output: http://www.foo.com/foo
+
+# With query parameter filtering
+$ url-normalize -f "www.google.com/search?q=test&utm_source=test"
+# Output: https://www.google.com/search?q=test
+
+# With custom allowlist
+$ url-normalize -f -p page,id "example.com?page=1&id=123&ref=test"
+# Output: https://example.com/?page=1&id=123
+
+# Via uv tool/uvx
+$ uvx url-normalize www.foo.com:80/foo
+# Output: https://www.foo.com:80/foo
+```
+
 ## Documentation
 
 For a complete history of changes, see [CHANGELOG.md](CHANGELOG.md).

{url_normalize-2.0.0 → url_normalize-2.1.0}/url_normalize.egg-info/SOURCES.txt

@@ -1,6 +1,7 @@
 LICENSE
 README.md
 pyproject.toml
+tests/test_cli.py
 tests/test_deconstruct_url.py
 tests/test_generic_url_cleanup.py
 tests/test_normalize_fragment.py
@@ -16,6 +17,7 @@ tests/test_reconstruct_url.py
 tests/test_tools.py
 tests/test_url_normalize.py
 url_normalize/__init__.py
+url_normalize/cli.py
 url_normalize/generic_url_cleanup.py
 url_normalize/normalize_fragment.py
 url_normalize/normalize_host.py
@@ -31,5 +33,6 @@ url_normalize/url_normalize.py
 url_normalize.egg-info/PKG-INFO
 url_normalize.egg-info/SOURCES.txt
 url_normalize.egg-info/dependency_links.txt
+url_normalize.egg-info/entry_points.txt
 url_normalize.egg-info/requires.txt
 url_normalize.egg-info/top_level.txt

url_normalize-2.1.0/url_normalize.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+url-normalize = url_normalize.cli:main

url_normalize-2.0.0/url_normalize/normalize_fragment.py

@@ -1,18 +0,0 @@
-"""URL fragment normalization."""
-
-from __future__ import annotations
-
-from .tools import quote, unquote
-
-
-def normalize_fragment(fragment: str) -> str:
-    """Normalize fragment part of the url.
-
-    Params:
-        fragment : string : url fragment, e.g., 'fragment'
-
-    Returns:
-        string : normalized fragment data.
-
-    """
-    return quote(unquote(fragment), "~")