tk-normalizer 1.4.0__tar.gz → 1.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tk_normalizer-1.4.0/src/tk_normalizer.egg-info → tk_normalizer-1.4.1}/PKG-INFO +1 -1
- {tk_normalizer-1.4.0 → tk_normalizer-1.4.1}/pyproject.toml +1 -1
- {tk_normalizer-1.4.0 → tk_normalizer-1.4.1}/src/tk_normalizer/__init__.py +1 -1
- {tk_normalizer-1.4.0 → tk_normalizer-1.4.1}/src/tk_normalizer/normalizer.py +5 -2
- {tk_normalizer-1.4.0 → tk_normalizer-1.4.1/src/tk_normalizer.egg-info}/PKG-INFO +1 -1
- {tk_normalizer-1.4.0 → tk_normalizer-1.4.1}/tests/test_normalizer.py +3 -0
- {tk_normalizer-1.4.0 → tk_normalizer-1.4.1}/LICENSE +0 -0
- {tk_normalizer-1.4.0 → tk_normalizer-1.4.1}/README.md +0 -0
- {tk_normalizer-1.4.0 → tk_normalizer-1.4.1}/setup.cfg +0 -0
- {tk_normalizer-1.4.0 → tk_normalizer-1.4.1}/src/tk_normalizer.egg-info/SOURCES.txt +0 -0
- {tk_normalizer-1.4.0 → tk_normalizer-1.4.1}/src/tk_normalizer.egg-info/dependency_links.txt +0 -0
- {tk_normalizer-1.4.0 → tk_normalizer-1.4.1}/src/tk_normalizer.egg-info/requires.txt +0 -0
- {tk_normalizer-1.4.0 → tk_normalizer-1.4.1}/src/tk_normalizer.egg-info/top_level.txt +0 -0
|
@@ -10,6 +10,8 @@ import tldextract
|
|
|
10
10
|
# The bundled Public Suffix List snapshot (ICANN section): no network fetch and no cache writes,
|
|
11
11
|
# so it behaves the same in a Snowflake UDF, a worker pool, or a read-only filesystem.
|
|
12
12
|
_PUBLIC_SUFFIXES = tldextract.TLDExtract(suffix_list_urls=(), cache_dir=None)
|
|
13
|
+
# A trailing port (":8080", or a bare ":") on a netloc.
|
|
14
|
+
_PORT_RE = re.compile(r":\d*$")
|
|
13
15
|
|
|
14
16
|
# Uses https://gist.github.com/Integralist/edcfb88c925658a13fc3e51f581fe4bc as a starting point
|
|
15
17
|
# Modified for more current rules regarding host/domain/tld naming.
|
|
@@ -231,9 +233,10 @@ class TkNormalizer:
|
|
|
231
233
|
def get_registrable_domain(netloc: str) -> str:
|
|
232
234
|
"""The host one label below its public suffix: bbc.co.uk for news.bbc.co.uk, where the root is co.uk.
|
|
233
235
|
|
|
234
|
-
Falls back to the host when there is no such label (an IP address, or a bare suffix like
|
|
236
|
+
Falls back to the host, without its port, when there is no such label (an IP address, or a bare suffix like
|
|
237
|
+
gov.uk).
|
|
235
238
|
"""
|
|
236
|
-
return _PUBLIC_SUFFIXES(netloc).top_domain_under_public_suffix or netloc
|
|
239
|
+
return _PUBLIC_SUFFIXES(netloc).top_domain_under_public_suffix or _PORT_RE.sub("", netloc)
|
|
237
240
|
|
|
238
241
|
@staticmethod
|
|
239
242
|
def get_root_normalized_url(netloc: str) -> str:
|
|
@@ -422,6 +422,9 @@ def test_registrable_domain(url: str, registrable: str, root: str) -> None:
|
|
|
422
422
|
# A bare public suffix has no label under it: fall back to the host.
|
|
423
423
|
("https://www.gov.uk/browse/tax", "gov.uk"),
|
|
424
424
|
("http://93.184.216.34/path", "93.184.216.34"),
|
|
425
|
+
# The fallback drops the port too, like the Public Suffix List path does.
|
|
426
|
+
("http://93.184.216.34:8080/path", "93.184.216.34"),
|
|
427
|
+
("https://www.gov.uk:443/browse/tax", "gov.uk"),
|
|
425
428
|
],
|
|
426
429
|
)
|
|
427
430
|
def test_registrable_domain_falls_back_to_host(url: str, registrable: str) -> None:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|