tldextract 3.4.1__py3-none-any.whl → 3.4.3__py3-none-any.whl

tldextract/_version.py

@@ -1,4 +1,4 @@
 # file generated by setuptools_scm
 # don't change, don't track in version control
-__version__ = version = '3.4.1'
-__version_tuple__ = version_tuple = (3, 4, 1)
+__version__ = version = '3.4.3'
+__version_tuple__ = version_tuple = (3, 4, 3)

tldextract/cache.py

@@ -1,4 +1,4 @@
-"""Helpers """
+"""Helpers."""
 import errno
 import hashlib
 import json
@@ -30,7 +30,7 @@ T = TypeVar("T")  # pylint: disable=invalid-name
 
 def get_pkg_unique_identifier() -> str:
     """
-    Generate an identifier unique to the python version, tldextract version, and python instance
+    Generate an identifier unique to the python version, tldextract version, and python instance.
 
     This will prevent interference between virtualenvs and issues that might arise when installing
     a new version of tldextract
@@ -61,7 +61,7 @@ def get_pkg_unique_identifier() -> str:
 
 def get_cache_dir() -> str:
     """
-    Get a cache dir that we have permission to write to
+    Get a cache dir that we have permission to write to.
 
     Try to follow the XDG standard, but if that doesn't work fallback to the package directory
     http://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html
@@ -86,7 +86,7 @@ def get_cache_dir() -> str:
 
 
 class DiskCache:
-    """Disk _cache that only works for jsonable values"""
+    """Disk _cache that only works for jsonable values."""
 
     def __init__(self, cache_dir: Optional[str], lock_timeout: int = 20):
         self.enabled = bool(cache_dir)
@@ -115,7 +115,7 @@ class DiskCache:
     def set(
         self, namespace: str, key: Union[str, Dict[str, Hashable]], value: object
     ) -> None:
-        """Set a value in the disk cache"""
+        """Set a value in the disk cache."""
         if not self.enabled:
             return
 
@@ -142,7 +142,7 @@ class DiskCache:
                 _DID_LOG_UNABLE_TO_CACHE = True
 
     def clear(self) -> None:
-        """Clear the disk cache"""
+        """Clear the disk cache."""
         for root, _, files in os.walk(self.cache_dir):
             for filename in files:
                 if filename.endswith(self.file_ext) or filename.endswith(
@@ -175,7 +175,7 @@ class DiskCache:
         kwargs: Dict[str, Hashable],
         hashed_argnames: Iterable[str],
     ) -> T:
-        """Get a url but cache the response"""
+        """Get a url but cache the response."""
         if not self.enabled:
             return func(**kwargs)
 
@@ -215,7 +215,7 @@ class DiskCache:
     def cached_fetch_url(
         self, session: requests.Session, url: str, timeout: Union[float, int, None]
     ) -> str:
-        """Get a url but cache the response"""
+        """Get a url but cache the response."""
         return self.run_and_cache(
             func=_fetch_url,
             namespace="urls",
@@ -241,7 +241,7 @@ def _make_cache_key(inputs: Union[str, Dict[str, Hashable]]) -> str:
 
 
 def _make_dir(filename: str) -> None:
-    """Make a directory if it doesn't already exist"""
+    """Make a directory if it doesn't already exist."""
     if not os.path.exists(os.path.dirname(filename)):
         try:
             os.makedirs(os.path.dirname(filename))

tldextract/cli.py

@@ -1,4 +1,4 @@
-"""tldextract CLI"""
+"""tldextract CLI."""
 
 
 import argparse
@@ -12,7 +12,7 @@ from .tldextract import TLDExtract
 
 
 def main() -> None:
-    """tldextract CLI main command."""
+    """Tldextract CLI main command."""
     logging.basicConfig()
 
     parser = argparse.ArgumentParser(

tldextract/remote.py

@@ -1,36 +1,51 @@
-"tldextract helpers for testing and fetching remote resources."
+"""tldextract helpers for testing and fetching remote resources."""
 
 import re
 import socket
 from urllib.parse import scheme_chars
 
 IP_RE = re.compile(
-    # pylint: disable-next=line-too-long
-    r"^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$"
+    r"^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.)"
+    r"{3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$"
 )
 
-SCHEME_RE = re.compile(r"^([" + scheme_chars + "]+:)?//")
+scheme_chars_set = set(scheme_chars)
 
 
 def lenient_netloc(url: str) -> str:
-    """Extract the netloc of a URL-like string, similar to the netloc attribute
-    returned by urllib.parse.{urlparse,urlsplit}, but extract more leniently,
-    without raising errors."""
+    """Extract the netloc of a URL-like string.
 
+    Similar to the netloc attribute returned by
+    urllib.parse.{urlparse,urlsplit}, but extract more leniently, without
+    raising errors.
+    """
     return (
-        SCHEME_RE.sub("", url)
+        _schemeless_url(url)
         .partition("/")[0]
         .partition("?")[0]
         .partition("#")[0]
-        .split("@")[-1]
+        .rpartition("@")[-1]
         .partition(":")[0]
         .strip()
-        .rstrip(".")
+        .rstrip(".\u3002\uff0e\uff61")
     )
 
 
+def _schemeless_url(url: str) -> str:
+    double_slashes_start = url.find("//")
+    if double_slashes_start == 0:
+        return url[2:]
+    if (
+        double_slashes_start < 2
+        or not url[double_slashes_start - 1] == ":"
+        or set(url[: double_slashes_start - 1]) - scheme_chars_set
+    ):
+        return url
+    return url[double_slashes_start + 2 :]
+
+
 def looks_like_ip(maybe_ip: str) -> bool:
-    """Does the given str look like an IP address?"""
+    """Check whether the given str looks like an IP address."""
     if not maybe_ip[0].isdigit():
         return False
 

tldextract/suffix_list.py

@@ -1,4 +1,4 @@
-"tldextract helpers for testing and fetching remote resources."
+"""tldextract helpers for testing and fetching remote resources."""
 
 import logging
 import pkgutil
@@ -17,8 +17,11 @@ PUBLIC_PRIVATE_SUFFIX_SEPARATOR = "// ===BEGIN PRIVATE DOMAINS==="
 
 
 class SuffixListNotFound(LookupError):
-    """A recoverable error while looking up a suffix list. Recoverable because
-    you can specify backups, or use this library's bundled snapshot."""
+    """A recoverable error while looking up a suffix list.
+
+    Recoverable because you can specify backups, or use this library's bundled
+    snapshot.
+    """
 
 
 def find_first_response(
@@ -26,9 +29,7 @@ def find_first_response(
     urls: Sequence[str],
     cache_fetch_timeout: Union[float, int, None] = None,
 ) -> str:
-    """Decode the first successfully fetched URL, from UTF-8 encoding to
-    Python unicode.
-    """
+    """Decode the first successfully fetched URL, from UTF-8 encoding to Python unicode."""
     with requests.Session() as session:
         session.mount("file://", FileAdapter())
 
@@ -46,8 +47,7 @@ def find_first_response(
 
 
 def extract_tlds_from_suffix_list(suffix_list_text: str) -> Tuple[List[str], List[str]]:
-    """Parse the raw suffix list text for its different designations of
-    suffixes."""
+    """Parse the raw suffix list text for its different designations of suffixes."""
     public_text, _, private_text = suffix_list_text.partition(
         PUBLIC_PRIVATE_SUFFIX_SEPARATOR
     )

tldextract/tldextract.py

@@ -1,5 +1,6 @@
-""" `tldextract` accurately separates a URL's subdomain, domain, and public suffix,
-using the Public Suffix List (PSL).
+"""`tldextract` accurately separates a URL's subdomain, domain, and public suffix.
+
+It does this via the Public Suffix List (PSL).
 
     >>> import tldextract
 
@@ -48,12 +49,22 @@ or suffix were found:
     '127.0.0.1'
 """
 
+from __future__ import annotations
+
 import logging
 import os
-import re
-from functools import wraps
-from typing import FrozenSet, List, NamedTuple, Optional, Sequence, Union
 import urllib.parse
+from functools import wraps
+from typing import (
+    Collection,
+    Dict,
+    FrozenSet,
+    List,
+    NamedTuple,
+    Optional,
+    Sequence,
+    Union,
+)
 
 import idna
 
@@ -71,8 +82,6 @@ PUBLIC_SUFFIX_LIST_URLS = (
     "https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat",
 )
 
-_UNICODE_DOTS_RE = re.compile("[\u002e\u3002\uff0e\uff61]")
-
 
 class ExtractResult(NamedTuple):
     """namedtuple of a URL's subdomain, domain, and suffix."""
@@ -91,8 +100,8 @@ class ExtractResult(NamedTuple):
         >>> extract('http://localhost:8080').registered_domain
         ''
         """
-        if self.domain and self.suffix:
-            return self.domain + "." + self.suffix
+        if self.suffix and self.domain:
+            return f"{self.domain}.{self.suffix}"
         return ""
 
     @property
@@ -105,7 +114,7 @@ class ExtractResult(NamedTuple):
         >>> extract('http://localhost:8080').fqdn
         ''
         """
-        if self.domain and self.suffix:
+        if self.suffix and self.domain:
             # Disable bogus lint error (https://github.com/PyCQA/pylint/issues/2568)
             # pylint: disable-next=not-an-iterable
             return ".".join(i for i in self if i)
@@ -114,7 +123,7 @@ class ExtractResult(NamedTuple):
     @property
     def ipv4(self) -> str:
         """
-        Returns the ipv4 if that is what the presented domain/url is
+        Returns the ipv4 if that is what the presented domain/url is.
 
         >>> extract('http://127.0.0.1/path/to/file').ipv4
         '127.0.0.1'
@@ -129,8 +138,7 @@ class ExtractResult(NamedTuple):
 
 
 class TLDExtract:
-    """A callable for extracting, subdomain, domain, and suffix components from
-    a URL."""
+    """A callable for extracting, subdomain, domain, and suffix components from a URL."""
 
     # TODO: Agreed with Pylint: too-many-arguments
     def __init__(  # pylint: disable=too-many-arguments
@@ -142,9 +150,7 @@ class TLDExtract:
         extra_suffixes: Sequence[str] = (),
         cache_fetch_timeout: Union[str, float, None] = CACHE_TIMEOUT,
     ) -> None:
-        """
-        Constructs a callable for extracting subdomain, domain, and suffix
-        components from a URL.
+        """Construct a callable for extracting subdomain, domain, and suffix components from a URL.
 
         Upon calling it, it first checks for a JSON in `cache_dir`. By default,
         the `cache_dir` will live in the tldextract directory. You can disable
@@ -207,17 +213,17 @@ class TLDExtract:
         self._cache = DiskCache(cache_dir)
 
     def __call__(
-        self, url: str, include_psl_private_domains: Optional[bool] = None
+        self, url: str, include_psl_private_domains: bool | None = None
     ) -> ExtractResult:
         """Alias for `extract_str`."""
         return self.extract_str(url, include_psl_private_domains)
 
     def extract_str(
-        self, url: str, include_psl_private_domains: Optional[bool] = None
+        self, url: str, include_psl_private_domains: bool | None = None
     ) -> ExtractResult:
-        """
-        Takes a string URL and splits it into its subdomain, domain, and
-        suffix (effective TLD, gTLD, ccTLD, etc.) components.
+        """Take a string URL and splits it into its subdomain, domain, and suffix components.
+
+        I.e. its effective TLD, gTLD, ccTLD, etc. components.
 
         >>> extractor = TLDExtract()
         >>> extractor.extract_str('http://forums.news.cnn.com/')
@@ -232,10 +238,10 @@ class TLDExtract:
         url: Union[urllib.parse.ParseResult, urllib.parse.SplitResult],
         include_psl_private_domains: Optional[bool] = None,
     ) -> ExtractResult:
-        """
-        Takes the output of urllib.parse URL parsing methods and further splits
-        the parsed URL into its subdomain, domain, and suffix (effective TLD,
-        gTLD, ccTLD, etc.) components.
+        """Take the output of urllib.parse URL parsing methods and further splits the parsed URL.
+
+        Splits the parsed URL into its subdomain, domain, and suffix
+        components, i.e. its effective TLD, gTLD, ccTLD, etc. components.
 
         This method is like `extract_str` but faster, as the string's domain
         name has already been parsed.
@@ -251,18 +257,22 @@ class TLDExtract:
     def _extract_netloc(
         self, netloc: str, include_psl_private_domains: Optional[bool]
     ) -> ExtractResult:
-        labels = _UNICODE_DOTS_RE.split(netloc)
+        labels = (
+            netloc.replace("\u3002", "\u002e")
+            .replace("\uff0e", "\u002e")
+            .replace("\uff61", "\u002e")
+            .split(".")
+        )
 
-        translations = [_decode_punycode(label) for label in labels]
         suffix_index = self._get_tld_extractor().suffix_index(
-            translations, include_psl_private_domains=include_psl_private_domains
+            labels, include_psl_private_domains=include_psl_private_domains
         )
 
-        suffix = ".".join(labels[suffix_index:])
-        if not suffix and netloc and looks_like_ip(netloc):
+        if suffix_index == len(labels) and netloc and looks_like_ip(netloc):
             return ExtractResult("", netloc, "")
 
-        subdomain = ".".join(labels[: suffix_index - 1]) if suffix_index else ""
+        suffix = ".".join(labels[suffix_index:]) if suffix_index != len(labels) else ""
+        subdomain = ".".join(labels[: suffix_index - 1]) if suffix_index >= 2 else ""
         domain = labels[suffix_index - 1] if suffix_index else ""
         return ExtractResult(subdomain, domain, suffix)
 
@@ -276,22 +286,23 @@ class TLDExtract:
     @property
     def tlds(self) -> List[str]:
         """
-        Returns the list of tld's used by default
+        Returns the list of tld's used by default.
 
         This will vary based on `include_psl_private_domains` and `extra_suffixes`
         """
         return list(self._get_tld_extractor().tlds())
 
-    def _get_tld_extractor(self) -> "_PublicSuffixListTLDExtractor":
-        """Get or compute this object's TLDExtractor. Looks up the TLDExtractor
-        in roughly the following order, based on the settings passed to
-        __init__:
+    def _get_tld_extractor(self) -> _PublicSuffixListTLDExtractor:
+        """Get or compute this object's TLDExtractor.
+
+        Looks up the TLDExtractor in roughly the following order, based on the
+        settings passed to __init__:
 
         1. Memoized on `self`
         2. Local system _cache file
         3. Remote PSL, over HTTP
-        4. Bundled PSL snapshot file"""
-
+        4. Bundled PSL snapshot file
+        """
         if self._extractor:
             return self._extractor
 
@@ -317,6 +328,37 @@ class TLDExtract:
 TLD_EXTRACTOR = TLDExtract()
 
 
+class Trie:
+    """Trie for storing eTLDs with their labels in reverse-order."""
+
+    def __init__(self, matches: Optional[Dict] = None, end: bool = False) -> None:
+        self.matches = matches if matches else {}
+        self.end = end
+
+    @staticmethod
+    def create(suffixes: Collection[str]) -> Trie:
+        """Create a Trie from a list of suffixes and return its root node."""
+        root_node = Trie()
+
+        for suffix in suffixes:
+            suffix_labels = suffix.split(".")
+            suffix_labels.reverse()
+            root_node.add_suffix(suffix_labels)
+
+        return root_node
+
+    def add_suffix(self, labels: List[str]) -> None:
+        """Append a suffix's labels to this Trie node."""
+        node = self
+
+        for label in labels:
+            if label not in node.matches:
+                node.matches[label] = Trie()
+            node = node.matches[label]
+
+        node.end = True
+
+
 @wraps(TLD_EXTRACTOR.__call__)
 def extract(  # pylint: disable=missing-function-docstring
     url: str, include_psl_private_domains: Optional[bool] = False
@@ -331,9 +373,7 @@ def update(*args, **kwargs):  # type: ignore[no-untyped-def]
 
 
 class _PublicSuffixListTLDExtractor:
-    """Wrapper around this project's main algo for PSL
-    lookups.
-    """
+    """Wrapper around this project's main algo for PSL lookups."""
 
     def __init__(
         self,
@@ -348,6 +388,8 @@ class _PublicSuffixListTLDExtractor:
         self.private_tlds = private_tlds
         self.tlds_incl_private = frozenset(public_tlds + private_tlds + extra_tlds)
         self.tlds_excl_private = frozenset(public_tlds + extra_tlds)
+        self.tlds_incl_private_trie = Trie.create(self.tlds_incl_private)
+        self.tlds_excl_private_trie = Trie.create(self.tlds_excl_private)
 
     def tlds(
         self, include_psl_private_domains: Optional[bool] = None
@@ -363,27 +405,38 @@ class _PublicSuffixListTLDExtractor:
         )
 
     def suffix_index(
-        self, lower_spl: List[str], include_psl_private_domains: Optional[bool] = None
+        self, spl: List[str], include_psl_private_domains: Optional[bool] = None
     ) -> int:
-        """Returns the index of the first suffix label.
-        Returns len(spl) if no suffix is found
+        """Return the index of the first suffix label.
+
+        Returns len(spl) if no suffix is found.
         """
-        tlds = self.tlds(include_psl_private_domains)
-        length = len(lower_spl)
-        for i in range(length):
-            maybe_tld = ".".join(lower_spl[i:])
-            exception_tld = "!" + maybe_tld
-            if exception_tld in tlds:
-                return i + 1
-
-            if maybe_tld in tlds:
-                return i
-
-            wildcard_tld = "*." + ".".join(lower_spl[i + 1 :])
-            if wildcard_tld in tlds:
-                return i
-
-        return length
+        node = (
+            self.tlds_incl_private_trie
+            if include_psl_private_domains
+            else self.tlds_excl_private_trie
+        )
+        i = len(spl)
+        j = i
+        for label in reversed(spl):
+            decoded_label = _decode_punycode(label)
+            if decoded_label in node.matches:
+                j -= 1
+                if node.matches[decoded_label].end:
+                    i = j
+                node = node.matches[decoded_label]
+                continue
+
+            is_wildcard = "*" in node.matches
+            if is_wildcard:
+                is_wildcard_exception = "!" + decoded_label in node.matches
+                if is_wildcard_exception:
+                    return j
+                return j - 1
+
+            break
+
+        return i
 
 
 def _decode_punycode(label: str) -> str:

{tldextract-3.4.1.dist-info → tldextract-3.4.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tldextract
-Version: 3.4.1
+Version: 3.4.3
 Summary: Accurately separates a URL's subdomain, domain, and public suffix, using the Public Suffix List (PSL). By default, this includes the public ICANN TLDs and their exceptions. You can optionally support the Public Suffix List's private domains as well.
 Home-page: https://github.com/john-kurkowski/tldextract
 Author: John Kurkowski
@@ -24,8 +24,9 @@ Requires-Dist: requests (>=2.1.0)
 Requires-Dist: requests-file (>=1.4)
 Requires-Dist: filelock (>=3.0.8)
 
- `tldextract` accurately separates a URL's subdomain, domain, and public suffix,
-using the Public Suffix List (PSL).
+`tldextract` accurately separates a URL's subdomain, domain, and public suffix.
+
+It does this via the Public Suffix List (PSL).
 
     >>> import tldextract
     >>> tldextract.extract('http://forums.news.cnn.com/')

tldextract-3.4.3.dist-info/RECORD

@@ -0,0 +1,16 @@
+tldextract/.tld_set_snapshot,sha256=TVya0bCcmRKl_16oPKPIlNmWS09rXrjOKGgYjhvAGLE,238022
+tldextract/__init__.py,sha256=rZg3DKzS9CTARuF4Tuq50ViILwUktDED89Av8nStNuM,216
+tldextract/__main__.py,sha256=FxfCNOozXSaJP2GTjgWLAn03oNMd_EUUOWkfT1_YRgM,90
+tldextract/_version.py,sha256=48Xmhq75m8Lwk0l2LFZcUFkfMiV7Qm7TtkGOfSedCXM,160
+tldextract/cache.py,sha256=kPC5WcYRE-PbhW1CQRDdPOB17u8tAq6iaJDNVcUgybc,8725
+tldextract/cli.py,sha256=F9FZ4Hx_E9Gct1d-dA40QZFrZVlzTAp_o6Fbka1k4tw,2416
+tldextract/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tldextract/remote.py,sha256=jbliDpMe6UXxVC8GPA6GowtyOtulc7vVT9SU9MZoNus,1541
+tldextract/suffix_list.py,sha256=f0a4judZhYpbYhljtHNtD-eI7kYV7Ja0LNb1R2DWvbA,3400
+tldextract/tldextract.py,sha256=-JTu0p_q_BFBWNEjY5E5ijKlGtinAfgJasvIp87R5G4,15971
+tldextract-3.4.3.dist-info/LICENSE,sha256=oqlDTqZaKpeJ6jYsQYqTkmV8gGGg-o7cO_OnH79KjsE,1522
+tldextract-3.4.3.dist-info/METADATA,sha256=HSwKKghsTmin-2BNyQ1OLnw6P38sBd6jQGpO-D8B4dM,2313
+tldextract-3.4.3.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+tldextract-3.4.3.dist-info/entry_points.txt,sha256=EStkXC80BetCMp1UDhU3kWuXBo3qDpgKltZTJ1x4x1U,51
+tldextract-3.4.3.dist-info/top_level.txt,sha256=DWZIjV49WP30tyC1KOEP7t-EaS4IRCXQzc0KXAOn_bk,11
+tldextract-3.4.3.dist-info/RECORD,,

tldextract-3.4.1.dist-info/RECORD

@@ -1,16 +0,0 @@
-tldextract/.tld_set_snapshot,sha256=TVya0bCcmRKl_16oPKPIlNmWS09rXrjOKGgYjhvAGLE,238022
-tldextract/__init__.py,sha256=rZg3DKzS9CTARuF4Tuq50ViILwUktDED89Av8nStNuM,216
-tldextract/__main__.py,sha256=FxfCNOozXSaJP2GTjgWLAn03oNMd_EUUOWkfT1_YRgM,90
-tldextract/_version.py,sha256=WP-_SGCtdmJ9gIzEvNNB9286hJcL0KHHSpnjHpuhoiw,160
-tldextract/cache.py,sha256=cML-WAB6D99AGepWgnuuZEaIBMan7ZUsvAfFvkrtVOs,8717
-tldextract/cli.py,sha256=O9f_m60wvdtnzLHTlYnzdjHvaJU06H-vV40Pd7xKwOg,2415
-tldextract/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tldextract/remote.py,sha256=oh0JQ6Z3wx-8g3BaF9EMLSiqu4NoaMgU1G09LUYvbYI,1182
-tldextract/suffix_list.py,sha256=LLen6BeFWyKTJJIiey3uz-vlnqsP5ApJX6tXNkHmW-s,3399
-tldextract/tldextract.py,sha256=kSteeJQ11EPao3lUOCupTxCZm8jkqbth5PwyOpkBtQ8,14443
-tldextract-3.4.1.dist-info/LICENSE,sha256=oqlDTqZaKpeJ6jYsQYqTkmV8gGGg-o7cO_OnH79KjsE,1522
-tldextract-3.4.1.dist-info/METADATA,sha256=p_mFOdnkjtNCv4sE-ASdp2nZG7Yz6P5CBlDc7CDWwHM,2302
-tldextract-3.4.1.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-tldextract-3.4.1.dist-info/entry_points.txt,sha256=EStkXC80BetCMp1UDhU3kWuXBo3qDpgKltZTJ1x4x1U,51
-tldextract-3.4.1.dist-info/top_level.txt,sha256=DWZIjV49WP30tyC1KOEP7t-EaS4IRCXQzc0KXAOn_bk,11
-tldextract-3.4.1.dist-info/RECORD,,