tk-normalizer 1.0.0__py3-none-any.whl

tk_normalizer/__init__.py

@@ -0,0 +1,37 @@
+"""
+tk-normalizer: URL normalization library for consistent URL representation.
+
+This library provides URL normalization functionality to create normalized
+representations of URLs, handling variations in protocols, subdomains,
+query parameters, and more.
+"""
+
+from .normalizer import InvalidUrlException, TkNormalizer
+
+__version__ = "0.1.0"
+__all__ = ["TkNormalizer", "InvalidUrlException", "normalize_url"]
+
+
+def normalize_url(url: str) -> str:
+    """
+    Normalize a URL to its normalized form.
+
+    This is a convenience function that creates a TkNormalizer instance
+    and returns the normalized URL string.
+
+    Args:
+        url: The URL string to normalize.
+
+    Returns:
+        The normalized URL string.
+
+    Raises:
+        InvalidUrlException: If the URL is invalid or cannot be normalized.
+
+    Example:
+        >>> from tk_normalizer import normalize_url
+        >>> normalize_url("http://www.Example.com/path?b=2&a=1&utm_source=test")
+        'example.com/path?a=1&b=2'
+    """
+    normalizer = TkNormalizer(url)
+    return normalizer.normalized_url

tk_normalizer/normalizer.py

@@ -0,0 +1,241 @@
+import fnmatch
+import hashlib
+import logging
+import re
+from urllib.parse import ParseResult, parse_qsl, urlencode, urlparse
+
+# Uses https://gist.github.com/Integralist/edcfb88c925658a13fc3e51f581fe4bc as a starting point
+# Modified for more current rules regarding host/domain/tld naming.
+
+ip_middle_octet = r"(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5]))"
+ip_last_octet = r"(?:\.(?:[0-9]\d?|1\d\d|2[0-4]\d|25[0-4]))"
+
+regex = re.compile(
+    r"^"
+    # protocol identifier
+    r"(?:(?:https?|ftp)://)"
+    # user:pass authentication (updated to avoid catastrophic backtracking)
+    r"(?:[a-zA-Z0-9._%+-]+(?::[^\s@]*)?@)?"
+    r"(?:"
+    r"(?P<private_ip>"
+    # IP address exclusion
+    # private & local networks
+    r"(?:(?:10|127)" + ip_middle_octet + "{2}" + ip_last_octet + ")|"
+    r"(?:(?:169\.254|192\.168)" + ip_middle_octet + ip_last_octet + ")|"
+    r"(?:172\.(?:1[6-9]|2\d|3[0-1])" + ip_middle_octet + ip_last_octet + "))"
+    r"|"
+    # IP address dotted notation octets
+    # excludes loopback network 0.0.0.0
+    # excludes reserved space >= 224.0.0.0
+    # excludes network & broadcast addresses
+    # (first & last IP address of each class)
+    r"(?P<public_ip>"
+    r"(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])"
+    r"" + ip_middle_octet + "{2}"
+    r"" + ip_last_octet + ")"
+    r"|"
+    # host name (modified: handles multiple hyphens, underscores in hostnames, and trailing hyphens)
+    r"(?:(?:[a-z\-_\u00a1-\uffff0-9]-?)*[a-z_\u00a1-\uffff0-9\-]+)"
+    # domain name (modified: handles multiple hyphens, and also underscores in domain names, and trailing hyphens)
+    r"(?:\.(?:[a-z\-_\u00a1-\uffff0-9]-?)*[a-z_\u00a1-\uffff0-9\-]+)*"
+    # TLD identifier (modified: handles oddities like site.xn--p1ai/)
+    r"(?:\.(?:[a-z0-9\-\u00a1-\uffff]{2,}))"
+    r")"
+    # port number
+    r"(?::\d{2,5})?"
+    # resource path
+    r"(?:/\S*)?"
+    # query string
+    r"(?:\?\S*)?"
+    r"$",
+    re.UNICODE | re.IGNORECASE,
+)
+
+pattern = re.compile(regex)
+
+
+class InvalidUrlException(Exception):
+    def __init__(self, message: str, original_exception: Exception) -> None:
+        super().__init__(message)
+        self.original_exception = original_exception
+
+
+class TkNormalizer:
+    # Define the list of query parameters to remove
+    query_params_to_remove: list[str] = [
+        "utm_*",
+        "gclid",
+        "fbclid",
+        "dclid",
+        "_ga",
+        "_gid",
+        "_fbp",
+        "_hjid",
+        "msclkid",
+        "aff_id",
+        "affid",
+        "referrer",
+        "adgroupid",
+        "srsltid",
+    ]
+
+    def __init__(self, url: str, log_errors: bool = True) -> None:
+        try:
+            url = url.strip()
+            self.log_errors: bool = log_errors
+            self.logger = logging.getLogger(__name__)
+            self.original_url: str = url
+            self.normalized_url: str
+            self.parent_normal_url: str
+            self.root_normal_url: str
+            self.normalized_url, self.parent_normal_url, self.root_normal_url = self.normalize_url(url)
+            self.url_hashes: dict[str, str] = self.compute_hashes()
+        except InvalidUrlException as e:
+            if self.log_errors:
+                self.logger.warning(f"{e}")
+            raise e
+        except Exception as e:
+            m = f"Invalid URL (exception): {url}"
+            if self.log_errors:
+                self.logger.error(m)
+            raise InvalidUrlException(m, e) from e
+
+    def normalize_url(self, url: str) -> tuple[str, str, str]:
+        url = self.lowercase_url(url)
+        parsed_url = self.parse_url(url)
+        netloc, path, query = self.validate_url(parsed_url)
+        netloc = self.remove_www_subdomain(netloc)
+        path = self.remove_trailing_slash(path)
+        query_params = self.parse_query_params(query)
+        query_params = self.remove_unwanted_params(query_params)
+        query_params = self.sort_query_params(query_params)
+        unique_params = self.remove_duplicate_params(query_params)
+        normalized_url = self.rebuild_url(netloc, path, unique_params)
+        parent_normal_url = self.get_parent_normal_url(netloc)
+        root_normal_url = self.get_root_normal_url(netloc)
+        return normalized_url, parent_normal_url, root_normal_url
+
+    @staticmethod
+    def lowercase_url(url: str) -> str:
+        return url.lower() if url else url
+
+    @staticmethod
+    def parse_url(url: str) -> ParseResult:
+        parsed_url: ParseResult = urlparse(url)
+
+        # Handle URLs with neither scheme nor netloc
+        if not parsed_url.scheme and not parsed_url.netloc:
+            parsed_url = urlparse(f"http://{url}")
+
+        return parsed_url
+
+    @staticmethod
+    def validate_url(parsed_url: ParseResult) -> tuple[str, str, str]:
+        # This will also capture localhost by its nature
+        if "." not in parsed_url.netloc:
+            raise InvalidUrlException(
+                f"Invalid URL provided (no dots) '{parsed_url.geturl()}'",
+                ValueError(f"URLs without a tld are forbidden. '{parsed_url.netloc}'"),
+            )
+
+        # Only http(s) urls are currently allowed; this means no file, ftp, etc.
+        if not str(parsed_url.scheme).startswith("http"):
+            raise InvalidUrlException(
+                f"Invalid URL provided (non-HTTP) '{parsed_url.geturl()}'",
+                ValueError(f"Only http(s) URLs are currently allowed, received: '{parsed_url.scheme}'"),
+            )
+
+        # Some oddball or broken URLs will be caught here
+        if not parsed_url.scheme or not parsed_url.netloc:
+            raise InvalidUrlException(
+                f"Invalid URL provided: (empty) '{parsed_url.geturl()}'",
+                ValueError(f"URL could not be parsed. '{parsed_url.netloc}'"),
+            )
+
+        # Check again the mega regular-expression
+        if not pattern.match(parsed_url.geturl()):
+            raise InvalidUrlException(
+                f"Invalid URL provided (regex.fail) '{parsed_url.geturl()}'", ValueError("URL failed regex check.")
+            )
+
+        return parsed_url.netloc, parsed_url.path, parsed_url.query
+
+    @staticmethod
+    def remove_www_subdomain(netloc: str) -> str:
+        return netloc[4:] if str(netloc).startswith("www.") else netloc
+
+    @staticmethod
+    def remove_trailing_slash(path: str) -> str:
+        return path.rstrip("/")
+
+    @staticmethod
+    def parse_query_params(query: str) -> list[tuple[str, str]]:
+        return parse_qsl(query, keep_blank_values=True)
+
+    def remove_unwanted_params(self, query_params: list[tuple[str, str]]) -> list[tuple[str, str]]:
+        def is_unwanted_param(param: str) -> bool:
+            return any(fnmatch.fnmatch(param, pattern) for pattern in self.query_params_to_remove)
+
+        return [(k, v) for k, v in query_params if not is_unwanted_param(k)]
+
+    @staticmethod
+    def sort_query_params(query_params: list[tuple[str, str]]) -> list[tuple[str, str]]:
+        return sorted(query_params, key=lambda x: (x[0], x[1]))
+
+    @staticmethod
+    def remove_duplicate_params(query_params: list[tuple[str, str]]) -> list[tuple[str, str]]:
+        seen_params: set = set()
+        unique_params: list[tuple[str, str]] = []
+        for param in query_params:
+            if param not in seen_params:
+                seen_params.add(param)
+                unique_params.append(param)
+        return unique_params
+
+    @staticmethod
+    def rebuild_url(netloc: str, path: str, query_params: list[tuple[str, str]]) -> str:
+        query_string: str = urlencode(query_params)
+        return f"{netloc}{path}?{query_string}" if query_string else f"{netloc}{path}"
+
+    @staticmethod
+    def get_parent_normal_url(netloc: str) -> str:
+        return netloc
+
+    @staticmethod
+    def get_root_normal_url(netloc: str) -> str:
+        return ".".join(netloc.split(".")[-2:])
+
+    def compute_hashes(self) -> dict[str, str]:
+        def sha256_hash(value: str) -> str:
+            return hashlib.sha256(value.encode()).hexdigest()
+
+        return {
+            "normalized_url_hash": sha256_hash(self.normalized_url),
+            "parent_normal_url_hash": sha256_hash(self.parent_normal_url),
+            "root_normal_url_hash": sha256_hash(self.root_normal_url),
+        }
+
+    def get_normalized_url(self) -> dict[str, str | dict[str, str]]:
+        return {
+            "normalized_url": self.normalized_url,
+            "parent_normal_url": self.parent_normal_url,
+            "root_normal_url": self.root_normal_url,
+            **self.url_hashes,
+        }
+
+
+if __name__ == "__main__":  # pragma: no cover
+    url1 = "http://www.Example.com/some-sub-folder/or_page.html?b=2&a=1&a=1&b=2&c=3&bad_param=some_value"
+    url2 = "http://blog.example.com/some-folder/some-page.html?b=2&a=1&a=1&b=2&c=3&bad_param=another_value"
+
+    try:
+        normalizer1 = TkNormalizer(url1)
+        print(normalizer1.get_normalized_url())
+    except InvalidUrlException as e:
+        print(f"Error: {e}")
+
+    try:
+        normalizer2 = TkNormalizer(url2)
+        print(normalizer2.get_normalized_url())
+    except InvalidUrlException as e:
+        print(f"Error: {e}")

tk_normalizer-1.0.0.dist-info/METADATA

@@ -0,0 +1,231 @@
+Metadata-Version: 2.4
+Name: tk-normalizer
+Version: 1.0.0
+Summary: URL normalization library for consistent URL representation
+Author-email: Terakeet <engineering@terakeet.com>
+Maintainer-email: Terakeet <engineering@terakeet.com>
+License: MIT
+Project-URL: Homepage, https://github.com/terakeet/tk-normalizer
+Project-URL: Repository, https://github.com/terakeet/tk-normalizer.git
+Project-URL: Issues, https://github.com/terakeet/tk-normalizer/issues
+Project-URL: Documentation, https://github.com/terakeet/tk-normalizer/blob/main/docs/ARCHITECTURE.md
+Keywords: url,normalization,canonicalization,web,utilities
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing :: Filters
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-mock>=3.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: pre-commit>=3.0.0; extra == "dev"
+Requires-Dist: build>=0.10.0; extra == "dev"
+Requires-Dist: twine>=4.0.0; extra == "dev"
+Dynamic: license-file
+
+# tk-normalizer
+
+[![Python](https://img.shields.io/pypi/pyversions/tk-normalizer.svg)](https://pypi.org/project/tk-normalizer/)
+[![PyPI](https://img.shields.io/pypi/v/tk-normalizer.svg)](https://pypi.org/project/tk-normalizer/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+URL normalization library for creating consistent URL representations.
+
+## Purpose
+
+The URL normalization process creates a mechanism to provide equivalence between URLs with varying string, protocol, scheme, and query parameter ordering. This library helps create normalized representations of URLs for consistent storage, comparison, and analysis.
+
+## Installation
+
+```bash
+pip install tk-normalizer
+```
+
+## Quick Start
+
+```python
+from tk_normalizer import normalize_url
+
+# Simple usage with the convenience function
+normalized = normalize_url("http://www.Example.com/path?b=2&a=1&utm_source=test")
+print(normalized)  # Output: example.com/path?a=1&b=2
+
+# Using the class directly for more control
+from tk_normalizer import TkNormalizer
+
+normalizer = TkNormalizer("http://www.Example.com/path?b=2&a=1&utm_source=test")
+print(normalizer.normalized_url)  # example.com/path?a=1&b=2
+print(normalizer.get_normalized_url())  # Full details including hashes
+```
+
+## Features
+
+### URL Normalization
+
+The following URLs all normalize to the same normalized form:
+
+```
+https://example.com/
+http://www.example.com/
+http://www.example.com
+http://www.example.com/#my_search_engine_is_great
+https://www.example.com/?utm_campaign=SomeGoogleCampaign
+https://www.example.com/?utm_source=because&utm_campaign=SomeGoogleCampaign
+```
+
+All normalize to: `example.com`
+
+### Normalization Process
+
+URLs are normalized through the following steps:
+
+- ✅ Protocol and www subdomains removed
+- ✅ Lowercased
+- ✅ Trailing slashes removed
+- ✅ Query parameters reordered alphabetically by key
+- ✅ Duplicate query parameter key/value pairs removed
+- ✅ Common tracking parameters removed (utm_*, gclid, fbclid, etc.)
+- ✅ Non-HTTP(S) protocols rejected
+- ✅ Localhost URLs rejected
+
+### Tracking Parameters Removed
+
+The following tracking parameters are automatically removed during normalization:
+
+- `utm_*` (all utm parameters)
+- `gclid`, `fbclid`, `dclid` (click identifiers)
+- `_ga`, `_gid`, `_fbp`, `_hjid` (analytics cookies)
+- `msclkid` (Microsoft Ads)
+- `aff_id`, `affid` (affiliate tracking)
+- `referrer`, `adgroupid`, `srsltid`
+
+## Advanced Usage
+
+### Getting Full Normalization Details
+
+```python
+from tk_normalizer import TkNormalizer
+
+normalizer = TkNormalizer("http://blog.example.com/page?b=2&a=1")
+result = normalizer.get_normalized_url()
+
+print(result)
+# {
+#   'normalized_url': 'blog.example.com/page?a=1&b=2',
+#   'parent_normal_url': 'blog.example.com',
+#   'root_normal_url': 'example.com',
+#   'normalized_url_hash': '...',
+#   'parent_normal_url_hash': '...',
+#   'root_normal_url_hash': '...'
+# }
+```
+
+### Error Handling
+
+```python
+from tk_normalizer import normalize_url, InvalidUrlException
+
+try:
+    normalized = normalize_url("not a valid url")
+except InvalidUrlException as e:
+    print(f"Invalid URL: {e}")
+```
+
+### Accessing Individual Components
+
+```python
+from tk_normalizer import TkNormalizer
+
+normalizer = TkNormalizer("https://blog.example.com/path?a=1")
+
+# Access individual normalized components
+print(normalizer.normalized_url)         # blog.example.com/path?a=1
+print(normalizer.parent_normal_url)   # blog.example.com
+print(normalizer.root_normal_url)     # example.com
+```
+
+## Hashing
+
+For efficient storage and comparison, SHA-256 hashes are computed for:
+- The normalized URL
+- The parent normal URL (domain without path)
+- The root normal URL (root domain without subdomains)
+
+This provides fixed-length representations suitable for database indexing.
+
+## Important Caveats
+
+While this normalization process works well for most use cases, there are some limitations:
+
+1. **www subdomain removal**: Technically, `www.example.com` and `example.com` could serve different content, though this is rare in practice.
+
+2. **Case sensitivity**: URLs are lowercased, but some servers are case-sensitive for paths.
+
+3. **Tracking parameters**: New tracking parameters emerge over time and may not be in the removal list.
+
+4. **Fragment removal**: URL fragments (#anchors) are removed, which may affect single-page applications.
+
+## Development
+
+### Setting Up Development Environment
+
+```bash
+# Clone the repository
+git clone https://github.com/terakeet/tk-normalizer.git
+cd tk-normalizer
+
+# Install development dependencies
+pip install -e ".[dev]"
+
+# Run tests
+pytest
+
+# Run tests with coverage
+pytest --cov=tk_normalizer
+
+# Run linting
+ruff check src tests
+```
+
+### Running Tests
+
+```bash
+# Run all tests
+pytest
+
+# Run with verbose output
+pytest -v
+
+# Run specific test file
+pytest tests/test_normalizer.py
+
+# Run with coverage report
+pytest --cov=tk_normalizer --cov-report=html
+```
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## License
+
+This project is licensed under the MIT License - see the LICENSE file for details.
+
+## Support
+
+For issues and questions, please use the [GitHub issue tracker](https://github.com/terakeet/tk-normalizer/issues).
+
+## Credits
+
+Based on the URL normalization functionality from [tk-core](https://github.com/terakeet/tk-core), extracted and packaged for standalone use.

tk_normalizer-1.0.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+tk_normalizer/__init__.py,sha256=yUNyiw5QMY_r766L6RebLL-el1rnKLZrrC_1W4LnPK4,1067
+tk_normalizer/normalizer.py,sha256=ytLYdGQlqcEwrpuN4eZS72Dnt53moHkR0P2zGUW7iOU,9059
+tk_normalizer-1.0.0.dist-info/licenses/LICENSE,sha256=7rybr3ioo2HAWA784BIwucFSrECCwJB4mSztAF3krtc,1065
+tk_normalizer-1.0.0.dist-info/METADATA,sha256=p5zaBitT681qCSapFF3WjIyDBlWksAsLFZ_eFuia18k,7022
+tk_normalizer-1.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+tk_normalizer-1.0.0.dist-info/top_level.txt,sha256=YFjFkT1gCcO4_rGcNtcTI0CO4KDutwzQpvw0FcqEMnQ,14
+tk_normalizer-1.0.0.dist-info/RECORD,,

tk_normalizer-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

tk_normalizer-1.0.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Terakeet
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tk_normalizer-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+tk_normalizer