scrapling 0.2.99__py3-none-any.whl → 0.3.1__py3-none-any.whl

scrapling/core/custom_types.py

@@ -1,159 +1,198 @@
-import re
-import typing
 from collections.abc import Mapping
 from types import MappingProxyType
+from re import compile as re_compile, UNICODE, IGNORECASE
 
 from orjson import dumps, loads
-from w3lib.html import replace_entities as _replace_entities
 
-from scrapling.core._types import (Dict, Iterable, List, Literal, Optional,
-                                   Pattern, SupportsIndex, TypeVar, Union)
-from scrapling.core.utils import _is_iterable, flatten
+from scrapling.core._types import (
+    cast,
+    Dict,
+    List,
+    Union,
+    overload,
+    TypeVar,
+    Literal,
+    Pattern,
+    Iterable,
+    Optional,
+    Generator,
+    SupportsIndex,
+)
+from scrapling.core.utils import _is_iterable, flatten, __CONSECUTIVE_SPACES_REGEX__
+from scrapling.core._html_utils import _replace_entities
 
 # Define type variable for AttributeHandler value type
-_TextHandlerType = TypeVar('_TextHandlerType', bound='TextHandler')
+_TextHandlerType = TypeVar("_TextHandlerType", bound="TextHandler")
+__CLEANING_TABLE__ = str.maketrans("\t\r\n", "   ")
 
 
 class TextHandler(str):
     """Extends standard Python string by adding more functionality"""
-    __slots__ = ()
 
-    def __new__(cls, string):
-        return super().__new__(cls, str(string))
+    __slots__ = ()
 
-    def __getitem__(self, key: Union[SupportsIndex, slice]) -> "TextHandler":
+    def __getitem__(
+        self, key: SupportsIndex | slice
+    ) -> "TextHandler":  # pragma: no cover
         lst = super().__getitem__(key)
-        return typing.cast(_TextHandlerType, TextHandler(lst))
+        return cast(_TextHandlerType, TextHandler(lst))
 
-    def split(self, sep: str = None, maxsplit: SupportsIndex = -1) -> 'TextHandlers':
+    def split(
+        self, sep: str = None, maxsplit: SupportsIndex = -1
+    ) -> "TextHandlers":  # pragma: no cover
         return TextHandlers(
-            typing.cast(List[_TextHandlerType], [TextHandler(s) for s in super().split(sep, maxsplit)])
+            cast(
+                List[_TextHandlerType],
+                [TextHandler(s) for s in super().split(sep, maxsplit)],
+            )
         )
 
-    def strip(self, chars: str = None) -> Union[str, 'TextHandler']:
+    def strip(self, chars: str = None) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().strip(chars))
 
-    def lstrip(self, chars: str = None) -> Union[str, 'TextHandler']:
+    def lstrip(
+        self, chars: str = None
+    ) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().lstrip(chars))
 
-    def rstrip(self, chars: str = None) -> Union[str, 'TextHandler']:
+    def rstrip(
+        self, chars: str = None
+    ) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().rstrip(chars))
 
-    def capitalize(self) -> Union[str, 'TextHandler']:
+    def capitalize(self) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().capitalize())
 
-    def casefold(self) -> Union[str, 'TextHandler']:
+    def casefold(self) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().casefold())
 
-    def center(self, width: SupportsIndex, fillchar: str = ' ') -> Union[str, 'TextHandler']:
+    def center(
+        self, width: SupportsIndex, fillchar: str = " "
+    ) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().center(width, fillchar))
 
-    def expandtabs(self, tabsize: SupportsIndex = 8) -> Union[str, 'TextHandler']:
+    def expandtabs(
+        self, tabsize: SupportsIndex = 8
+    ) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().expandtabs(tabsize))
 
-    def format(self, *args: str, **kwargs: str) -> Union[str, 'TextHandler']:
+    def format(
+        self, *args: str, **kwargs: str
+    ) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().format(*args, **kwargs))
 
-    def format_map(self, mapping) -> Union[str, 'TextHandler']:
+    def format_map(self, mapping) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().format_map(mapping))
 
-    def join(self, iterable: Iterable[str]) -> Union[str, 'TextHandler']:
+    def join(
+        self, iterable: Iterable[str]
+    ) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().join(iterable))
 
-    def ljust(self, width: SupportsIndex, fillchar: str = ' ') -> Union[str, 'TextHandler']:
+    def ljust(
+        self, width: SupportsIndex, fillchar: str = " "
+    ) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().ljust(width, fillchar))
 
-    def rjust(self, width: SupportsIndex, fillchar: str = ' ') -> Union[str, 'TextHandler']:
+    def rjust(
+        self, width: SupportsIndex, fillchar: str = " "
+    ) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().rjust(width, fillchar))
 
-    def swapcase(self) -> Union[str, 'TextHandler']:
+    def swapcase(self) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().swapcase())
 
-    def title(self) -> Union[str, 'TextHandler']:
+    def title(self) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().title())
 
-    def translate(self, table) -> Union[str, 'TextHandler']:
+    def translate(self, table) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().translate(table))
 
-    def zfill(self, width: SupportsIndex) -> Union[str, 'TextHandler']:
+    def zfill(
+        self, width: SupportsIndex
+    ) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().zfill(width))
 
-    def replace(self, old: str, new: str, count: SupportsIndex = -1) -> Union[str, 'TextHandler']:
+    def replace(
+        self, old: str, new: str, count: SupportsIndex = -1
+    ) -> Union[str, "TextHandler"]:
         return TextHandler(super().replace(old, new, count))
 
-    def upper(self) -> Union[str, 'TextHandler']:
+    def upper(self) -> Union[str, "TextHandler"]:
         return TextHandler(super().upper())
 
-    def lower(self) -> Union[str, 'TextHandler']:
+    def lower(self) -> Union[str, "TextHandler"]:
         return TextHandler(super().lower())
+
     ##############
 
-    def sort(self, reverse: bool = False) -> Union[str, 'TextHandler']:
+    def sort(self, reverse: bool = False) -> Union[str, "TextHandler"]:
         """Return a sorted version of the string"""
         return self.__class__("".join(sorted(self, reverse=reverse)))
 
-    def clean(self) -> Union[str, 'TextHandler']:
+    def clean(self) -> Union[str, "TextHandler"]:
         """Return a new version of the string after removing all white spaces and consecutive spaces"""
-        data = re.sub(r'[\t|\r|\n]', '', self)
-        data = re.sub(' +', ' ', data)
-        return self.__class__(data.strip())
+        data = self.translate(__CLEANING_TABLE__)
+        return self.__class__(__CONSECUTIVE_SPACES_REGEX__.sub(" ", data).strip())
 
     # For easy copy-paste from Scrapy/parsel code when needed :)
-    def get(self, default=None):
+    def get(self, default=None):  # pragma: no cover
         return self
 
-    def get_all(self):
+    def get_all(self):  # pragma: no cover
         return self
 
     extract = get_all
     extract_first = get
 
     def json(self) -> Dict:
-        """Return json response if the response is jsonable otherwise throw error"""
-        # Using str function as a workaround for orjson issue with subclasses of str
+        """Return JSON response if the response is jsonable otherwise throw error"""
+        # Using str function as a workaround for orjson issue with subclasses of str.
         # Check this out: https://github.com/ijl/orjson/issues/445
         return loads(str(self))
 
-    @typing.overload
+    @overload
     def re(
         self,
-        regex: Union[str, Pattern[str]],
+        regex: str | Pattern,
         check_match: Literal[True],
         replace_entities: bool = True,
         clean_match: bool = False,
         case_sensitive: bool = True,
-    ) -> bool:
-        ...
+    ) -> bool: ...
 
-    @typing.overload
+    @overload
     def re(
         self,
-        regex: Union[str, Pattern[str]],
+        regex: str | Pattern,
         replace_entities: bool = True,
         clean_match: bool = False,
         case_sensitive: bool = True,
         check_match: Literal[False] = False,
-    ) -> "TextHandlers[TextHandler]":
-        ...
+    ) -> "TextHandlers[TextHandler]": ...
 
     def re(
-            self, regex: Union[str, Pattern[str]], replace_entities: bool = True, clean_match: bool = False,
-            case_sensitive: bool = True, check_match: bool = False
-    ) -> Union["TextHandlers[TextHandler]", bool]:
+        self,
+        regex: str | Pattern,
+        replace_entities: bool = True,
+        clean_match: bool = False,
+        case_sensitive: bool = True,
+        check_match: bool = False,
+    ) -> Union["TextHandlers", bool]:
         """Apply the given regex to the current text and return a list of strings with the matches.
 
         :param regex: Can be either a compiled regular expression or a string.
-        :param replace_entities: if enabled character entity references are replaced by their corresponding character
-        :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
-        :param case_sensitive: if disabled, function will set the regex to ignore letters case while compiling it
-        :param check_match: used to quickly check if this regex matches or not without any operations on the results
+        :param replace_entities: If enabled character entity references are replaced by their corresponding character
+        :param clean_match: If enabled, this will ignore all whitespaces and consecutive spaces while matching
+        :param case_sensitive: If disabled, function will set the regex to ignore the letters-case while compiling it
+        :param check_match: Used to quickly check if this regex matches or not without any operations on the results
 
         """
         if isinstance(regex, str):
             if case_sensitive:
-                regex = re.compile(regex, re.UNICODE)
+                regex = re_compile(regex, UNICODE)
             else:
-                regex = re.compile(regex, flags=re.UNICODE | re.IGNORECASE)
+                regex = re_compile(regex, flags=UNICODE | IGNORECASE)
 
         input_text = self.clean() if clean_match else self
         results = regex.findall(input_text)
@@ -164,22 +203,42 @@ class TextHandler(str):
             results = flatten(results)
 
         if not replace_entities:
-            return TextHandlers(typing.cast(List[_TextHandlerType], [TextHandler(string) for string in results]))
+            return TextHandlers(
+                cast(
+                    List[_TextHandlerType], [TextHandler(string) for string in results]
+                )
+            )
 
-        return TextHandlers(typing.cast(List[_TextHandlerType], [TextHandler(_replace_entities(s)) for s in results]))
+        return TextHandlers(
+            cast(
+                List[_TextHandlerType],
+                [TextHandler(_replace_entities(s)) for s in results],
+            )
+        )
 
-    def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True,
-                 clean_match: bool = False, case_sensitive: bool = True) -> "TextHandler":
+    def re_first(
+        self,
+        regex: str | Pattern,
+        default=None,
+        replace_entities: bool = True,
+        clean_match: bool = False,
+        case_sensitive: bool = True,
+    ) -> "TextHandler":
         """Apply the given regex to text and return the first match if found, otherwise return the default value.
 
         :param regex: Can be either a compiled regular expression or a string.
         :param default: The default value to be returned if there is no match
-        :param replace_entities: if enabled character entity references are replaced by their corresponding character
-        :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
-        :param case_sensitive: if disabled, function will set the regex to ignore letters case while compiling it
+        :param replace_entities: If enabled character entity references are replaced by their corresponding character
+        :param clean_match: If enabled, this will ignore all whitespaces and consecutive spaces while matching
+        :param case_sensitive: If disabled, function will set the regex to ignore the letters-case while compiling it
 
         """
-        result = self.re(regex, replace_entities, clean_match=clean_match, case_sensitive=case_sensitive)
+        result = self.re(
+            regex,
+            replace_entities,
+            clean_match=clean_match,
+            case_sensitive=case_sensitive,
+        )
         return result[0] if result else default
 
 
@@ -187,48 +246,61 @@ class TextHandlers(List[TextHandler]):
     """
     The :class:`TextHandlers` class is a subclass of the builtin ``List`` class, which provides a few additional methods.
     """
+
     __slots__ = ()
 
-    @typing.overload
-    def __getitem__(self, pos: SupportsIndex) -> TextHandler:
+    @overload
+    def __getitem__(self, pos: SupportsIndex) -> TextHandler:  # pragma: no cover
         pass
 
-    @typing.overload
-    def __getitem__(self, pos: slice) -> "TextHandlers":
+    @overload
+    def __getitem__(self, pos: slice) -> "TextHandlers":  # pragma: no cover
         pass
 
-    def __getitem__(self, pos: Union[SupportsIndex, slice]) -> Union[TextHandler, "TextHandlers"]:
+    def __getitem__(
+        self, pos: SupportsIndex | slice
+    ) -> Union[TextHandler, "TextHandlers"]:
         lst = super().__getitem__(pos)
         if isinstance(pos, slice):
-            lst = [TextHandler(s) for s in lst]
-            return TextHandlers(typing.cast(List[_TextHandlerType], lst))
-        return typing.cast(_TextHandlerType, TextHandler(lst))
+            return TextHandlers(cast(List[_TextHandlerType], lst))
+        return cast(_TextHandlerType, TextHandler(lst))
 
-    def re(self, regex: Union[str, Pattern[str]], replace_entities: bool = True, clean_match: bool = False,
-            case_sensitive: bool = True) -> 'TextHandlers[TextHandler]':
+    def re(
+        self,
+        regex: str | Pattern,
+        replace_entities: bool = True,
+        clean_match: bool = False,
+        case_sensitive: bool = True,
+    ) -> "TextHandlers[TextHandler]":
         """Call the ``.re()`` method for each element in this list and return
         their results flattened as TextHandlers.
 
         :param regex: Can be either a compiled regular expression or a string.
-        :param replace_entities: if enabled character entity references are replaced by their corresponding character
+        :param replace_entities: If enabled character entity references are replaced by their corresponding character
         :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
-        :param case_sensitive: if disabled, function will set the regex to ignore letters case while compiling it
+        :param case_sensitive: if disabled, the function will set the regex to ignore the letters-case while compiling it
         """
         results = [
             n.re(regex, replace_entities, clean_match, case_sensitive) for n in self
         ]
         return TextHandlers(flatten(results))
 
-    def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True,
-                 clean_match: bool = False, case_sensitive: bool = True) -> TextHandler:
+    def re_first(
+        self,
+        regex: str | Pattern,
+        default=None,
+        replace_entities: bool = True,
+        clean_match: bool = False,
+        case_sensitive: bool = True,
+    ) -> TextHandler:  # pragma: no cover
         """Call the ``.re_first()`` method for each element in this list and return
         the first result or the default value otherwise.
 
         :param regex: Can be either a compiled regular expression or a string.
         :param default: The default value to be returned if there is no match
-        :param replace_entities: if enabled character entity references are replaced by their corresponding character
-        :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
-        :param case_sensitive: if disabled, function will set the regex to ignore letters case while compiling it
+        :param replace_entities: If enabled character entity references are replaced by their corresponding character
+        :param clean_match: If enabled, this will ignore all whitespaces and consecutive spaces while matching
+        :param case_sensitive: If disabled, function will set the regex to ignore the letters-case while compiling it
         """
         for n in self:
             for result in n.re(regex, replace_entities, clean_match, case_sensitive):
@@ -250,33 +322,44 @@ class TextHandlers(List[TextHandler]):
 
 
 class AttributesHandler(Mapping[str, _TextHandlerType]):
-    """A read-only mapping to use instead of the standard dictionary for the speed boost but at the same time I use it to add more functionalities.
-        If standard dictionary is needed, just convert this class to dictionary with `dict` function
+    """A read-only mapping to use instead of the standard dictionary for the speed boost, but at the same time I use it to add more functionalities.
+    If the standard dictionary is needed, convert this class to a dictionary with the `dict` function
     """
-    __slots__ = ('_data',)
+
+    __slots__ = ("_data",)
 
     def __init__(self, mapping=None, **kwargs):
-        mapping = {
-            key: TextHandler(value) if type(value) is str else value
-            for key, value in mapping.items()
-        } if mapping is not None else {}
+        mapping = (
+            {
+                key: TextHandler(value) if isinstance(value, str) else value
+                for key, value in mapping.items()
+            }
+            if mapping is not None
+            else {}
+        )
 
         if kwargs:
-            mapping.update({
-                key: TextHandler(value) if type(value) is str else value
-                for key, value in kwargs.items()
-            })
+            mapping.update(
+                {
+                    key: TextHandler(value) if isinstance(value, str) else value
+                    for key, value in kwargs.items()
+                }
+            )
 
         # Fastest read-only mapping type
         self._data = MappingProxyType(mapping)
 
-    def get(self, key: str, default: Optional[str] = None) -> Union[_TextHandlerType, None]:
-        """Acts like standard dictionary `.get()` method"""
+    def get(
+        self, key: str, default: Optional[str] = None
+    ) -> Optional[_TextHandlerType]:
+        """Acts like the standard dictionary `.get()` method"""
         return self._data.get(key, default)
 
-    def search_values(self, keyword, partial=False):
-        """Search current attributes by values and return dictionary of each matching item
-        :param keyword: The keyword to search for in the attributes values
+    def search_values(
+        self, keyword: str, partial: bool = False
+    ) -> Generator["AttributesHandler", None, None]:
+        """Search current attributes by values and return a dictionary of each matching item
+        :param keyword: The keyword to search for in the attribute values
         :param partial: If True, the function will search if keyword in each value instead of perfect match
         """
         for key, value in self._data.items():

scrapling/core/mixins.py

@@ -1,32 +1,37 @@
-
 class SelectorsGeneration:
-    """Selectors generation functions
+    """
+    Functions for generating selectors
     Trying to generate selectors like Firefox or maybe cleaner ones!? Ehm
-    Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591"""
+    Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591
+    """
 
-    def __general_selection(self, selection: str = 'css', full_path=False) -> str:
+    def __general_selection(
+        self, selection: str = "css", full_path: bool = False
+    ) -> str:
         """Generate a selector for the current element.
         :return: A string of the generated selector.
         """
         selectorPath = []
         target = self
-        css = selection.lower() == 'css'
+        css = selection.lower() == "css"
         while target is not None:
             if target.parent:
-                if target.attrib.get('id'):
+                if target.attrib.get("id"):
                     # id is enough
                     part = (
-                        f'#{target.attrib["id"]}' if css
+                        f"#{target.attrib['id']}"
+                        if css
                         else f"[@id='{target.attrib['id']}']"
                     )
                     selectorPath.append(part)
                     if not full_path:
                         return (
-                            " > ".join(reversed(selectorPath)) if css
-                            else '//*' + "/".join(reversed(selectorPath))
+                            " > ".join(reversed(selectorPath))
+                            if css
+                            else "//*" + "/".join(reversed(selectorPath))
                         )
                 else:
-                    part = f'{target.tag}'
+                    part = f"{target.tag}"
                     # We won't use classes anymore because I some websites share exact classes between elements
                     # classes = target.attrib.get('class', '').split()
                     # if classes and css:
@@ -41,23 +46,26 @@ class SelectorsGeneration:
 
                     if counter[target.tag] > 1:
                         part += (
-                            f":nth-of-type({counter[target.tag]})" if css
+                            f":nth-of-type({counter[target.tag]})"
+                            if css
                             else f"[{counter[target.tag]}]"
                         )
 
                 selectorPath.append(part)
                 target = target.parent
-                if target is None or target.tag == 'html':
+                if target is None or target.tag == "html":
                     return (
-                        " > ".join(reversed(selectorPath)) if css
-                        else '//' + "/".join(reversed(selectorPath))
+                        " > ".join(reversed(selectorPath))
+                        if css
+                        else "//" + "/".join(reversed(selectorPath))
                     )
             else:
                 break
 
         return (
-            " > ".join(reversed(selectorPath)) if css
-            else '//' + "/".join(reversed(selectorPath))
+            " > ".join(reversed(selectorPath))
+            if css
+            else "//" + "/".join(reversed(selectorPath))
         )
 
     @property
@@ -76,14 +84,14 @@ class SelectorsGeneration:
 
     @property
     def generate_xpath_selector(self) -> str:
-        """Generate a XPath selector for the current element
+        """Generate an XPath selector for the current element
         :return: A string of the generated selector.
         """
-        return self.__general_selection('xpath')
+        return self.__general_selection("xpath")
 
     @property
     def generate_full_xpath_selector(self) -> str:
         """Generate a complete XPath selector for the current element
         :return: A string of the generated selector.
         """
-        return self.__general_selection('xpath', full_path=True)
+        return self.__general_selection("xpath", full_path=True)