tk-normalizer 1.0.0__tar.gz

tk_normalizer-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Terakeet
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tk_normalizer-1.0.0/PKG-INFO

@@ -0,0 +1,231 @@
+Metadata-Version: 2.4
+Name: tk-normalizer
+Version: 1.0.0
+Summary: URL normalization library for consistent URL representation
+Author-email: Terakeet <engineering@terakeet.com>
+Maintainer-email: Terakeet <engineering@terakeet.com>
+License: MIT
+Project-URL: Homepage, https://github.com/terakeet/tk-normalizer
+Project-URL: Repository, https://github.com/terakeet/tk-normalizer.git
+Project-URL: Issues, https://github.com/terakeet/tk-normalizer/issues
+Project-URL: Documentation, https://github.com/terakeet/tk-normalizer/blob/main/docs/ARCHITECTURE.md
+Keywords: url,normalization,canonicalization,web,utilities
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing :: Filters
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-mock>=3.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: pre-commit>=3.0.0; extra == "dev"
+Requires-Dist: build>=0.10.0; extra == "dev"
+Requires-Dist: twine>=4.0.0; extra == "dev"
+Dynamic: license-file
+
+# tk-normalizer
+
+[![Python](https://img.shields.io/pypi/pyversions/tk-normalizer.svg)](https://pypi.org/project/tk-normalizer/)
+[![PyPI](https://img.shields.io/pypi/v/tk-normalizer.svg)](https://pypi.org/project/tk-normalizer/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+URL normalization library for creating consistent URL representations.
+
+## Purpose
+
+The URL normalization process creates a mechanism to provide equivalence between URLs with varying string, protocol, scheme, and query parameter ordering. This library helps create normalized representations of URLs for consistent storage, comparison, and analysis.
+
+## Installation
+
+```bash
+pip install tk-normalizer
+```
+
+## Quick Start
+
+```python
+from tk_normalizer import normalize_url
+
+# Simple usage with the convenience function
+normalized = normalize_url("http://www.Example.com/path?b=2&a=1&utm_source=test")
+print(normalized)  # Output: example.com/path?a=1&b=2
+
+# Using the class directly for more control
+from tk_normalizer import TkNormalizer
+
+normalizer = TkNormalizer("http://www.Example.com/path?b=2&a=1&utm_source=test")
+print(normalizer.normalized_url)  # example.com/path?a=1&b=2
+print(normalizer.get_normalized_url())  # Full details including hashes
+```
+
+## Features
+
+### URL Normalization
+
+The following URLs all normalize to the same normalized form:
+
+```
+https://example.com/
+http://www.example.com/
+http://www.example.com
+http://www.example.com/#my_search_engine_is_great
+https://www.example.com/?utm_campaign=SomeGoogleCampaign
+https://www.example.com/?utm_source=because&utm_campaign=SomeGoogleCampaign
+```
+
+All normalize to: `example.com`
+
+### Normalization Process
+
+URLs are normalized through the following steps:
+
+- ✅ Protocol and www subdomains removed
+- ✅ Lowercased
+- ✅ Trailing slashes removed
+- ✅ Query parameters reordered alphabetically by key
+- ✅ Duplicate query parameter key/value pairs removed
+- ✅ Common tracking parameters removed (utm_*, gclid, fbclid, etc.)
+- ✅ Non-HTTP(S) protocols rejected
+- ✅ Localhost URLs rejected
+
+### Tracking Parameters Removed
+
+The following tracking parameters are automatically removed during normalization:
+
+- `utm_*` (all utm parameters)
+- `gclid`, `fbclid`, `dclid` (click identifiers)
+- `_ga`, `_gid`, `_fbp`, `_hjid` (analytics cookies)
+- `msclkid` (Microsoft Ads)
+- `aff_id`, `affid` (affiliate tracking)
+- `referrer`, `adgroupid`, `srsltid`
+
+## Advanced Usage
+
+### Getting Full Normalization Details
+
+```python
+from tk_normalizer import TkNormalizer
+
+normalizer = TkNormalizer("http://blog.example.com/page?b=2&a=1")
+result = normalizer.get_normalized_url()
+
+print(result)
+# {
+#   'normalized_url': 'blog.example.com/page?a=1&b=2',
+#   'parent_normal_url': 'blog.example.com',
+#   'root_normal_url': 'example.com',
+#   'normalized_url_hash': '...',
+#   'parent_normal_url_hash': '...',
+#   'root_normal_url_hash': '...'
+# }
+```
+
+### Error Handling
+
+```python
+from tk_normalizer import normalize_url, InvalidUrlException
+
+try:
+    normalized = normalize_url("not a valid url")
+except InvalidUrlException as e:
+    print(f"Invalid URL: {e}")
+```
+
+### Accessing Individual Components
+
+```python
+from tk_normalizer import TkNormalizer
+
+normalizer = TkNormalizer("https://blog.example.com/path?a=1")
+
+# Access individual normalized components
+print(normalizer.normalized_url)         # blog.example.com/path?a=1
+print(normalizer.parent_normal_url)   # blog.example.com
+print(normalizer.root_normal_url)     # example.com
+```
+
+## Hashing
+
+For efficient storage and comparison, SHA-256 hashes are computed for:
+- The normalized URL
+- The parent normal URL (domain without path)
+- The root normal URL (root domain without subdomains)
+
+This provides fixed-length representations suitable for database indexing.
+
+## Important Caveats
+
+While this normalization process works well for most use cases, there are some limitations:
+
+1. **www subdomain removal**: Technically, `www.example.com` and `example.com` could serve different content, though this is rare in practice.
+
+2. **Case sensitivity**: URLs are lowercased, but some servers are case-sensitive for paths.
+
+3. **Tracking parameters**: New tracking parameters emerge over time and may not be in the removal list.
+
+4. **Fragment removal**: URL fragments (#anchors) are removed, which may affect single-page applications.
+
+## Development
+
+### Setting Up Development Environment
+
+```bash
+# Clone the repository
+git clone https://github.com/terakeet/tk-normalizer.git
+cd tk-normalizer
+
+# Install development dependencies
+pip install -e ".[dev]"
+
+# Run tests
+pytest
+
+# Run tests with coverage
+pytest --cov=tk_normalizer
+
+# Run linting
+ruff check src tests
+```
+
+### Running Tests
+
+```bash
+# Run all tests
+pytest
+
+# Run with verbose output
+pytest -v
+
+# Run specific test file
+pytest tests/test_normalizer.py
+
+# Run with coverage report
+pytest --cov=tk_normalizer --cov-report=html
+```
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## License
+
+This project is licensed under the MIT License - see the LICENSE file for details.
+
+## Support
+
+For issues and questions, please use the [GitHub issue tracker](https://github.com/terakeet/tk-normalizer/issues).
+
+## Credits
+
+Based on the URL normalization functionality from [tk-core](https://github.com/terakeet/tk-core), extracted and packaged for standalone use.

tk_normalizer-1.0.0/README.md

@@ -0,0 +1,195 @@
+# tk-normalizer
+
+[![Python](https://img.shields.io/pypi/pyversions/tk-normalizer.svg)](https://pypi.org/project/tk-normalizer/)
+[![PyPI](https://img.shields.io/pypi/v/tk-normalizer.svg)](https://pypi.org/project/tk-normalizer/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+URL normalization library for creating consistent URL representations.
+
+## Purpose
+
+The URL normalization process creates a mechanism to provide equivalence between URLs with varying string, protocol, scheme, and query parameter ordering. This library helps create normalized representations of URLs for consistent storage, comparison, and analysis.
+
+## Installation
+
+```bash
+pip install tk-normalizer
+```
+
+## Quick Start
+
+```python
+from tk_normalizer import normalize_url
+
+# Simple usage with the convenience function
+normalized = normalize_url("http://www.Example.com/path?b=2&a=1&utm_source=test")
+print(normalized)  # Output: example.com/path?a=1&b=2
+
+# Using the class directly for more control
+from tk_normalizer import TkNormalizer
+
+normalizer = TkNormalizer("http://www.Example.com/path?b=2&a=1&utm_source=test")
+print(normalizer.normalized_url)  # example.com/path?a=1&b=2
+print(normalizer.get_normalized_url())  # Full details including hashes
+```
+
+## Features
+
+### URL Normalization
+
+The following URLs all normalize to the same normalized form:
+
+```
+https://example.com/
+http://www.example.com/
+http://www.example.com
+http://www.example.com/#my_search_engine_is_great
+https://www.example.com/?utm_campaign=SomeGoogleCampaign
+https://www.example.com/?utm_source=because&utm_campaign=SomeGoogleCampaign
+```
+
+All normalize to: `example.com`
+
+### Normalization Process
+
+URLs are normalized through the following steps:
+
+- ✅ Protocol and www subdomains removed
+- ✅ Lowercased
+- ✅ Trailing slashes removed
+- ✅ Query parameters reordered alphabetically by key
+- ✅ Duplicate query parameter key/value pairs removed
+- ✅ Common tracking parameters removed (utm_*, gclid, fbclid, etc.)
+- ✅ Non-HTTP(S) protocols rejected
+- ✅ Localhost URLs rejected
+
+### Tracking Parameters Removed
+
+The following tracking parameters are automatically removed during normalization:
+
+- `utm_*` (all utm parameters)
+- `gclid`, `fbclid`, `dclid` (click identifiers)
+- `_ga`, `_gid`, `_fbp`, `_hjid` (analytics cookies)
+- `msclkid` (Microsoft Ads)
+- `aff_id`, `affid` (affiliate tracking)
+- `referrer`, `adgroupid`, `srsltid`
+
+## Advanced Usage
+
+### Getting Full Normalization Details
+
+```python
+from tk_normalizer import TkNormalizer
+
+normalizer = TkNormalizer("http://blog.example.com/page?b=2&a=1")
+result = normalizer.get_normalized_url()
+
+print(result)
+# {
+#   'normalized_url': 'blog.example.com/page?a=1&b=2',
+#   'parent_normal_url': 'blog.example.com',
+#   'root_normal_url': 'example.com',
+#   'normalized_url_hash': '...',
+#   'parent_normal_url_hash': '...',
+#   'root_normal_url_hash': '...'
+# }
+```
+
+### Error Handling
+
+```python
+from tk_normalizer import normalize_url, InvalidUrlException
+
+try:
+    normalized = normalize_url("not a valid url")
+except InvalidUrlException as e:
+    print(f"Invalid URL: {e}")
+```
+
+### Accessing Individual Components
+
+```python
+from tk_normalizer import TkNormalizer
+
+normalizer = TkNormalizer("https://blog.example.com/path?a=1")
+
+# Access individual normalized components
+print(normalizer.normalized_url)         # blog.example.com/path?a=1
+print(normalizer.parent_normal_url)   # blog.example.com
+print(normalizer.root_normal_url)     # example.com
+```
+
+## Hashing
+
+For efficient storage and comparison, SHA-256 hashes are computed for:
+- The normalized URL
+- The parent normal URL (domain without path)
+- The root normal URL (root domain without subdomains)
+
+This provides fixed-length representations suitable for database indexing.
+
+## Important Caveats
+
+While this normalization process works well for most use cases, there are some limitations:
+
+1. **www subdomain removal**: Technically, `www.example.com` and `example.com` could serve different content, though this is rare in practice.
+
+2. **Case sensitivity**: URLs are lowercased, but some servers are case-sensitive for paths.
+
+3. **Tracking parameters**: New tracking parameters emerge over time and may not be in the removal list.
+
+4. **Fragment removal**: URL fragments (#anchors) are removed, which may affect single-page applications.
+
+## Development
+
+### Setting Up Development Environment
+
+```bash
+# Clone the repository
+git clone https://github.com/terakeet/tk-normalizer.git
+cd tk-normalizer
+
+# Install development dependencies
+pip install -e ".[dev]"
+
+# Run tests
+pytest
+
+# Run tests with coverage
+pytest --cov=tk_normalizer
+
+# Run linting
+ruff check src tests
+```
+
+### Running Tests
+
+```bash
+# Run all tests
+pytest
+
+# Run with verbose output
+pytest -v
+
+# Run specific test file
+pytest tests/test_normalizer.py
+
+# Run with coverage report
+pytest --cov=tk_normalizer --cov-report=html
+```
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## License
+
+This project is licensed under the MIT License - see the LICENSE file for details.
+
+## Support
+
+For issues and questions, please use the [GitHub issue tracker](https://github.com/terakeet/tk-normalizer/issues).
+
+## Credits
+
+Based on the URL normalization functionality from [tk-core](https://github.com/terakeet/tk-core), extracted and packaged for standalone use.

tk_normalizer-1.0.0/pyproject.toml

@@ -0,0 +1,162 @@
+[build-system]
+requires = ["setuptools>=61.0.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "tk-normalizer"
+version = "1.0.0"
+description = "URL normalization library for consistent URL representation"
+readme = "README.md"
+requires-python = ">=3.11"
+license = {text = "MIT"}
+authors = [
+    {name = "Terakeet", email = "engineering@terakeet.com"}
+]
+maintainers = [
+    {name = "Terakeet", email = "engineering@terakeet.com"}
+]
+keywords = ["url", "normalization", "canonicalization", "web", "utilities"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Internet :: WWW/HTTP",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: Text Processing :: Filters",
+    "Operating System :: OS Independent",
+]
+
+[project.urls]
+Homepage = "https://github.com/terakeet/tk-normalizer"
+Repository = "https://github.com/terakeet/tk-normalizer.git"
+Issues = "https://github.com/terakeet/tk-normalizer/issues"
+Documentation = "https://github.com/terakeet/tk-normalizer/blob/main/docs/ARCHITECTURE.md"
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0.0",
+    "pytest-mock>=3.0.0",
+    "pytest-cov>=4.0.0",
+    "ruff>=0.1.0",
+    "pre-commit>=3.0.0",
+    "build>=0.10.0",
+    "twine>=4.0.0",
+]
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+"*" = ["py.typed"]
+
+[tool.pytest.ini_options]
+pythonpath = ["src"]
+testpaths = ["tests"]
+python_files = ["test_*.py", "*_test.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = [
+    "-ra",
+    "--strict-markers",
+    "--strict-config",
+    "--cov=tk_normalizer",
+    "--cov-report=term-missing",
+    "--cov-branch",
+    "-vv",
+]
+log_cli = true
+log_cli_level = "INFO"
+log_cli_format = "%(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)"
+log_cli_date_format = "%Y-%m-%d %H:%M:%S"
+filterwarnings = [
+    "ignore::DeprecationWarning",
+    "ignore::PendingDeprecationWarning",
+]
+markers = [
+    "unit: Unit tests",
+    "integration: Integration tests",
+    "slow: Slow tests",
+]
+
+[tool.ruff]
+line-length = 120
+target-version = "py311"
+exclude = [
+    ".git",
+    ".venv",
+    "venv",
+    "__pycache__",
+    ".ruff_cache",
+    "build",
+    "dist",
+    "*.egg-info",
+    "htmlcov",
+]
+
+[tool.ruff.lint]
+select = [
+    "E",     # pycodestyle errors
+    "W",     # pycodestyle warnings
+    "F",     # pyflakes
+    "I",     # isort
+    "B",     # flake8-bugbear
+    "C4",    # flake8-comprehensions
+    "UP",    # pyupgrade
+    "ARG",   # flake8-unused-arguments
+    "SIM",   # flake8-simplify
+    "S",     # flake8-bandit
+    "ANN",   # annotations
+]
+ignore = [
+    "E501",   # line too long (handled by formatter)
+    "B008",   # do not perform function calls in argument defaults
+    "S101",   # use of assert in tests is fine
+    "ANN002", # Args type hint skip
+    "ANN003", # Kwargs type hint skip
+    "ANN204", # Missing return type annotation for special method `__init__`
+    "ANN401", # Allow Any type declaration
+    "SIM108", # ternary operator required
+]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/*" = ["S101", "ARG001", "ARG002", "S105", "S106", "ANN201"]
+"test_*.py" = ["S101", "ARG001", "ARG002", "S105", "S106", "ANN201"]
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+skip-magic-trailing-comma = false
+line-ending = "lf"
+
+[tool.coverage.run]
+branch = true
+source = ["src/tk_normalizer"]
+omit = [
+    "tests/*",
+    "test_*.py",
+    ".venv/*",
+    "venv/*",
+    "*/site-packages/*",
+    "*/__init__.py",
+]
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "if self.debug:",
+    "if __name__ == .__main__.:",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "pass",
+    "except ImportError:",
+    "if TYPE_CHECKING:",
+]
+show_missing = true
+skip_covered = false
+precision = 2
+fail_under = 0

tk_normalizer-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

tk_normalizer-1.0.0/src/tk_normalizer/__init__.py

@@ -0,0 +1,37 @@
+"""
+tk-normalizer: URL normalization library for consistent URL representation.
+
+This library provides URL normalization functionality to create normalized
+representations of URLs, handling variations in protocols, subdomains,
+query parameters, and more.
+"""
+
+from .normalizer import InvalidUrlException, TkNormalizer
+
+__version__ = "0.1.0"
+__all__ = ["TkNormalizer", "InvalidUrlException", "normalize_url"]
+
+
+def normalize_url(url: str) -> str:
+    """
+    Normalize a URL to its normalized form.
+
+    This is a convenience function that creates a TkNormalizer instance
+    and returns the normalized URL string.
+
+    Args:
+        url: The URL string to normalize.
+
+    Returns:
+        The normalized URL string.
+
+    Raises:
+        InvalidUrlException: If the URL is invalid or cannot be normalized.
+
+    Example:
+        >>> from tk_normalizer import normalize_url
+        >>> normalize_url("http://www.Example.com/path?b=2&a=1&utm_source=test")
+        'example.com/path?a=1&b=2'
+    """
+    normalizer = TkNormalizer(url)
+    return normalizer.normalized_url