scrapling 0.3.1__py3-none-any.whl → 0.3.2__py3-none-any.whl

scrapling/__init__.py

@@ -1,5 +1,5 @@
 __author__ = "Karim Shoair (karim.shoair@pm.me)"
-__version__ = "0.3.1"
+__version__ = "0.3.2"
 __copyright__ = "Copyright (c) 2024 Karim Shoair"
 
 

scrapling/cli.py

@@ -2,14 +2,18 @@ from pathlib import Path
 from subprocess import check_output
 from sys import executable as python_executable
 
-from scrapling.core.utils import log
-from scrapling.engines.toolbelt import Response
+from scrapling.engines.toolbelt.custom import Response
+from scrapling.core.utils import log, _CookieParser, _ParseHeaders
 from scrapling.core._types import List, Optional, Dict, Tuple, Any, Callable
-from scrapling.fetchers import Fetcher, DynamicFetcher, StealthyFetcher
-from scrapling.core.shell import Convertor, _CookieParser, _ParseHeaders
 
 from orjson import loads as json_loads, JSONDecodeError
-from click import command, option, Choice, group, argument
+
+try:
+    from click import command, option, Choice, group, argument
+except (ImportError, ModuleNotFoundError) as e:
+    raise ModuleNotFoundError(
+        "You need to install scrapling with any of the extras to enable Shell commands. See: https://scrapling.readthedocs.io/en/latest/#installation"
+    ) from e
 
 __OUTPUT_FILE_HELP__ = "The output file path can be an HTML file, a Markdown file of the HTML content, or the text content itself. Use file extensions (`.html`/`.md`/`.txt`) respectively."
 __PACKAGE_DIR__ = Path(__file__).parent
@@ -40,6 +44,8 @@ def __Request_and_Save(
     **kwargs,
 ) -> None:
     """Make a request using the specified fetcher function and save the result"""
+    from scrapling.core.shell import Convertor
+
     # Handle relative paths - convert to an absolute path based on the current working directory
     output_path = Path(output_file)
     if not output_path.is_absolute():
@@ -72,14 +78,10 @@ def __ParseExtractArguments(
     return parsed_headers, parsed_cookies, parsed_params, parsed_json
 
 
-def __BuildRequest(
-    headers: List[str], cookies: str, params: str, json: Optional[str] = None, **kwargs
-) -> Dict:
+def __BuildRequest(headers: List[str], cookies: str, params: str, json: Optional[str] = None, **kwargs) -> Dict:
     """Build a request object using the specified arguments"""
     # Parse parameters
-    parsed_headers, parsed_cookies, parsed_params, parsed_json = (
-        __ParseExtractArguments(headers, cookies, params, json)
-    )
+    parsed_headers, parsed_cookies, parsed_params, parsed_json = __ParseExtractArguments(headers, cookies, params, json)
     # Build request arguments
     request_kwargs = {
         "headers": parsed_headers if parsed_headers else None,
@@ -106,10 +108,7 @@ def __BuildRequest(
     help="Force Scrapling to reinstall all Fetchers dependencies",
 )
 def install(force):  # pragma: no cover
-    if (
-        force
-        or not __PACKAGE_DIR__.joinpath(".scrapling_dependencies_installed").exists()
-    ):
+    if force or not __PACKAGE_DIR__.joinpath(".scrapling_dependencies_installed").exists():
         __Execute(
             [python_executable, "-m", "playwright", "install", "chromium"],
             "Playwright browsers",
@@ -158,9 +157,7 @@ def mcp():
     "level",
     is_flag=False,
     default="debug",
-    type=Choice(
-        ["debug", "info", "warning", "error", "critical", "fatal"], case_sensitive=False
-    ),
+    type=Choice(["debug", "info", "warning", "error", "critical", "fatal"], case_sensitive=False),
     help="Log level (default: DEBUG)",
 )
 def shell(code, level):
@@ -178,9 +175,7 @@ def extract():
     pass
 
 
-@extract.command(
-    help=f"Perform a GET request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}"
-)
+@extract.command(help=f"Perform a GET request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}")
 @argument("url", required=True)
 @argument("output_file", required=True)
 @option(
@@ -190,9 +185,7 @@ def extract():
     help='HTTP headers in format "Key: Value" (can be used multiple times)',
 )
 @option("--cookies", help='Cookies string in format "name1=value1; name2=value2"')
-@option(
-    "--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)"
-)
+@option("--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)")
 @option("--proxy", help='Proxy URL in format "http://username:password@host:port"')
 @option(
     "--css-selector",
@@ -264,12 +257,12 @@ def get(
         impersonate=impersonate,
         proxy=proxy,
     )
+    from scrapling.fetchers import Fetcher
+
     __Request_and_Save(Fetcher.get, url, output_file, css_selector, **kwargs)
 
 
-@extract.command(
-    help=f"Perform a POST request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}"
-)
+@extract.command(help=f"Perform a POST request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}")
 @argument("url", required=True)
 @argument("output_file", required=True)
 @option(
@@ -285,9 +278,7 @@ def get(
     help='HTTP headers in format "Key: Value" (can be used multiple times)',
 )
 @option("--cookies", help='Cookies string in format "name1=value1; name2=value2"')
-@option(
-    "--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)"
-)
+@option("--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)")
 @option("--proxy", help='Proxy URL in format "http://username:password@host:port"')
 @option(
     "--css-selector",
@@ -364,12 +355,12 @@ def post(
         proxy=proxy,
         data=data,
     )
+    from scrapling.fetchers import Fetcher
+
     __Request_and_Save(Fetcher.post, url, output_file, css_selector, **kwargs)
 
 
-@extract.command(
-    help=f"Perform a PUT request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}"
-)
+@extract.command(help=f"Perform a PUT request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}")
 @argument("url", required=True)
 @argument("output_file", required=True)
 @option("--data", "-d", help="Form data to include in the request body")
@@ -381,9 +372,7 @@ def post(
     help='HTTP headers in format "Key: Value" (can be used multiple times)',
 )
 @option("--cookies", help='Cookies string in format "name1=value1; name2=value2"')
-@option(
-    "--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)"
-)
+@option("--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)")
 @option("--proxy", help='Proxy URL in format "http://username:password@host:port"')
 @option(
     "--css-selector",
@@ -460,12 +449,12 @@ def put(
         proxy=proxy,
         data=data,
     )
+    from scrapling.fetchers import Fetcher
+
     __Request_and_Save(Fetcher.put, url, output_file, css_selector, **kwargs)
 
 
-@extract.command(
-    help=f"Perform a DELETE request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}"
-)
+@extract.command(help=f"Perform a DELETE request and save the content to a file.\n\n{__OUTPUT_FILE_HELP__}")
 @argument("url", required=True)
 @argument("output_file", required=True)
 @option(
@@ -475,9 +464,7 @@ def put(
     help='HTTP headers in format "Key: Value" (can be used multiple times)',
 )
 @option("--cookies", help='Cookies string in format "name1=value1; name2=value2"')
-@option(
-    "--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)"
-)
+@option("--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)")
 @option("--proxy", help='Proxy URL in format "http://username:password@host:port"')
 @option(
     "--css-selector",
@@ -549,12 +536,12 @@ def delete(
         impersonate=impersonate,
         proxy=proxy,
     )
+    from scrapling.fetchers import Fetcher
+
     __Request_and_Save(Fetcher.delete, url, output_file, css_selector, **kwargs)
 
 
-@extract.command(
-    help=f"Use DynamicFetcher to fetch content with browser automation.\n\n{__OUTPUT_FILE_HELP__}"
-)
+@extract.command(help=f"Use DynamicFetcher to fetch content with browser automation.\n\n{__OUTPUT_FILE_HELP__}")
 @argument("url", required=True)
 @argument("output_file", required=True)
 @option(
@@ -591,9 +578,7 @@ def delete(
 )
 @option("--wait-selector", help="CSS selector to wait for before proceeding")
 @option("--locale", default="en-US", help="Browser locale (default: en-US)")
-@option(
-    "--stealth/--no-stealth", default=False, help="Enable stealth mode (default: False)"
-)
+@option("--stealth/--no-stealth", default=False, help="Enable stealth mode (default: False)")
 @option(
     "--hide-canvas/--show-canvas",
     default=False,
@@ -672,12 +657,12 @@ def fetch(
     if parsed_headers:
         kwargs["extra_headers"] = parsed_headers
 
+    from scrapling.fetchers import DynamicFetcher
+
     __Request_and_Save(DynamicFetcher.fetch, url, output_file, css_selector, **kwargs)
 
 
-@extract.command(
-    help=f"Use StealthyFetcher to fetch content with advanced stealth features.\n\n{__OUTPUT_FILE_HELP__}"
-)
+@extract.command(help=f"Use StealthyFetcher to fetch content with advanced stealth features.\n\n{__OUTPUT_FILE_HELP__}")
 @argument("url", required=True)
 @argument("output_file", required=True)
 @option(
@@ -821,6 +806,8 @@ def stealthy_fetch(
     if parsed_headers:
         kwargs["extra_headers"] = parsed_headers
 
+    from scrapling.fetchers import StealthyFetcher
+
     __Request_and_Save(StealthyFetcher.fetch, url, output_file, css_selector, **kwargs)
 
 

scrapling/core/_html_utils.py

@@ -269,17 +269,13 @@ name2codepoint = {
 }
 
 
-def to_unicode(
-    text: StrOrBytes, encoding: Optional[str] = None, errors: str = "strict"
-) -> str:
+def to_unicode(text: StrOrBytes, encoding: Optional[str] = None, errors: str = "strict") -> str:
     """Return the Unicode representation of a bytes object `text`. If `text`
     is already a Unicode object, return it as-is."""
     if isinstance(text, str):
         return text
     if not isinstance(text, (bytes, str)):
-        raise TypeError(
-            f"to_unicode must receive bytes or str, got {type(text).__name__}"
-        )
+        raise TypeError(f"to_unicode must receive bytes or str, got {type(text).__name__}")
     if encoding is None:
         encoding = "utf-8"
     return text.decode(encoding, errors)
@@ -328,9 +324,7 @@ def _replace_entities(
             entity_name = groups["named"]
             if entity_name.lower() in keep:
                 return m.group(0)
-            number = name2codepoint.get(entity_name) or name2codepoint.get(
-                entity_name.lower()
-            )
+            number = name2codepoint.get(entity_name) or name2codepoint.get(entity_name.lower())
         if number is not None:
             # Browsers typically
             # interpret numeric character references in the 80-9F range as representing the characters mapped

scrapling/core/ai.py

@@ -4,7 +4,7 @@ from mcp.server.fastmcp import FastMCP
 from pydantic import BaseModel, Field
 
 from scrapling.core.shell import Convertor
-from scrapling.engines.toolbelt import Response as _ScraplingResponse
+from scrapling.engines.toolbelt.custom import Response as _ScraplingResponse
 from scrapling.fetchers import (
     Fetcher,
     FetcherSession,
@@ -32,21 +32,13 @@ class ResponseModel(BaseModel):
     """Request's response information structure."""
 
     status: int = Field(description="The status code returned by the website.")
-    content: list[str] = Field(
-        description="The content as Markdown/HTML or the text content of the page."
-    )
-    url: str = Field(
-        description="The URL given by the user that resulted in this response."
-    )
+    content: list[str] = Field(description="The content as Markdown/HTML or the text content of the page.")
+    url: str = Field(description="The URL given by the user that resulted in this response.")
 
 
-def _ContentTranslator(
-    content: Generator[str, None, None], page: _ScraplingResponse
-) -> ResponseModel:
+def _ContentTranslator(content: Generator[str, None, None], page: _ScraplingResponse) -> ResponseModel:
     """Convert a content generator to a list of ResponseModel objects."""
-    return ResponseModel(
-        status=page.status, content=[result for result in content], url=page.url
-    )
+    return ResponseModel(status=page.status, content=[result for result in content], url=page.url)
 
 
 class ScraplingMCPServer:

scrapling/core/custom_types.py

@@ -31,15 +31,11 @@ class TextHandler(str):
 
     __slots__ = ()
 
-    def __getitem__(
-        self, key: SupportsIndex | slice
-    ) -> "TextHandler":  # pragma: no cover
+    def __getitem__(self, key: SupportsIndex | slice) -> "TextHandler":  # pragma: no cover
         lst = super().__getitem__(key)
         return cast(_TextHandlerType, TextHandler(lst))
 
-    def split(
-        self, sep: str = None, maxsplit: SupportsIndex = -1
-    ) -> "TextHandlers":  # pragma: no cover
+    def split(self, sep: str = None, maxsplit: SupportsIndex = -1) -> "TextHandlers":  # pragma: no cover
         return TextHandlers(
             cast(
                 List[_TextHandlerType],
@@ -50,14 +46,10 @@ class TextHandler(str):
     def strip(self, chars: str = None) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().strip(chars))
 
-    def lstrip(
-        self, chars: str = None
-    ) -> Union[str, "TextHandler"]:  # pragma: no cover
+    def lstrip(self, chars: str = None) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().lstrip(chars))
 
-    def rstrip(
-        self, chars: str = None
-    ) -> Union[str, "TextHandler"]:  # pragma: no cover
+    def rstrip(self, chars: str = None) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().rstrip(chars))
 
     def capitalize(self) -> Union[str, "TextHandler"]:  # pragma: no cover
@@ -66,37 +58,25 @@ class TextHandler(str):
     def casefold(self) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().casefold())
 
-    def center(
-        self, width: SupportsIndex, fillchar: str = " "
-    ) -> Union[str, "TextHandler"]:  # pragma: no cover
+    def center(self, width: SupportsIndex, fillchar: str = " ") -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().center(width, fillchar))
 
-    def expandtabs(
-        self, tabsize: SupportsIndex = 8
-    ) -> Union[str, "TextHandler"]:  # pragma: no cover
+    def expandtabs(self, tabsize: SupportsIndex = 8) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().expandtabs(tabsize))
 
-    def format(
-        self, *args: str, **kwargs: str
-    ) -> Union[str, "TextHandler"]:  # pragma: no cover
+    def format(self, *args: str, **kwargs: str) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().format(*args, **kwargs))
 
     def format_map(self, mapping) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().format_map(mapping))
 
-    def join(
-        self, iterable: Iterable[str]
-    ) -> Union[str, "TextHandler"]:  # pragma: no cover
+    def join(self, iterable: Iterable[str]) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().join(iterable))
 
-    def ljust(
-        self, width: SupportsIndex, fillchar: str = " "
-    ) -> Union[str, "TextHandler"]:  # pragma: no cover
+    def ljust(self, width: SupportsIndex, fillchar: str = " ") -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().ljust(width, fillchar))
 
-    def rjust(
-        self, width: SupportsIndex, fillchar: str = " "
-    ) -> Union[str, "TextHandler"]:  # pragma: no cover
+    def rjust(self, width: SupportsIndex, fillchar: str = " ") -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().rjust(width, fillchar))
 
     def swapcase(self) -> Union[str, "TextHandler"]:  # pragma: no cover
@@ -108,14 +88,10 @@ class TextHandler(str):
     def translate(self, table) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().translate(table))
 
-    def zfill(
-        self, width: SupportsIndex
-    ) -> Union[str, "TextHandler"]:  # pragma: no cover
+    def zfill(self, width: SupportsIndex) -> Union[str, "TextHandler"]:  # pragma: no cover
         return TextHandler(super().zfill(width))
 
-    def replace(
-        self, old: str, new: str, count: SupportsIndex = -1
-    ) -> Union[str, "TextHandler"]:
+    def replace(self, old: str, new: str, count: SupportsIndex = -1) -> Union[str, "TextHandler"]:
         return TextHandler(super().replace(old, new, count))
 
     def upper(self) -> Union[str, "TextHandler"]:
@@ -203,11 +179,7 @@ class TextHandler(str):
             results = flatten(results)
 
         if not replace_entities:
-            return TextHandlers(
-                cast(
-                    List[_TextHandlerType], [TextHandler(string) for string in results]
-                )
-            )
+            return TextHandlers(cast(List[_TextHandlerType], [TextHandler(string) for string in results]))
 
         return TextHandlers(
             cast(
@@ -257,9 +229,7 @@ class TextHandlers(List[TextHandler]):
     def __getitem__(self, pos: slice) -> "TextHandlers":  # pragma: no cover
         pass
 
-    def __getitem__(
-        self, pos: SupportsIndex | slice
-    ) -> Union[TextHandler, "TextHandlers"]:
+    def __getitem__(self, pos: SupportsIndex | slice) -> Union[TextHandler, "TextHandlers"]:
         lst = super().__getitem__(pos)
         if isinstance(pos, slice):
             return TextHandlers(cast(List[_TextHandlerType], lst))
@@ -280,9 +250,7 @@ class TextHandlers(List[TextHandler]):
         :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
         :param case_sensitive: if disabled, the function will set the regex to ignore the letters-case while compiling it
         """
-        results = [
-            n.re(regex, replace_entities, clean_match, case_sensitive) for n in self
-        ]
+        results = [n.re(regex, replace_entities, clean_match, case_sensitive) for n in self]
         return TextHandlers(flatten(results))
 
     def re_first(
@@ -330,34 +298,24 @@ class AttributesHandler(Mapping[str, _TextHandlerType]):
 
     def __init__(self, mapping=None, **kwargs):
         mapping = (
-            {
-                key: TextHandler(value) if isinstance(value, str) else value
-                for key, value in mapping.items()
-            }
+            {key: TextHandler(value) if isinstance(value, str) else value for key, value in mapping.items()}
             if mapping is not None
             else {}
         )
 
         if kwargs:
             mapping.update(
-                {
-                    key: TextHandler(value) if isinstance(value, str) else value
-                    for key, value in kwargs.items()
-                }
+                {key: TextHandler(value) if isinstance(value, str) else value for key, value in kwargs.items()}
             )
 
         # Fastest read-only mapping type
         self._data = MappingProxyType(mapping)
 
-    def get(
-        self, key: str, default: Optional[str] = None
-    ) -> Optional[_TextHandlerType]:
+    def get(self, key: str, default: Optional[str] = None) -> Optional[_TextHandlerType]:
         """Acts like the standard dictionary `.get()` method"""
         return self._data.get(key, default)
 
-    def search_values(
-        self, keyword: str, partial: bool = False
-    ) -> Generator["AttributesHandler", None, None]:
+    def search_values(self, keyword: str, partial: bool = False) -> Generator["AttributesHandler", None, None]:
         """Search current attributes by values and return a dictionary of each matching item
         :param keyword: The keyword to search for in the attribute values
         :param partial: If True, the function will search if keyword in each value instead of perfect match

scrapling/core/mixins.py

@@ -5,9 +5,7 @@ class SelectorsGeneration:
     Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591
     """
 
-    def __general_selection(
-        self, selection: str = "css", full_path: bool = False
-    ) -> str:
+    def __general_selection(self, selection: str = "css", full_path: bool = False) -> str:
         """Generate a selector for the current element.
         :return: A string of the generated selector.
         """
@@ -18,18 +16,10 @@ class SelectorsGeneration:
             if target.parent:
                 if target.attrib.get("id"):
                     # id is enough
-                    part = (
-                        f"#{target.attrib['id']}"
-                        if css
-                        else f"[@id='{target.attrib['id']}']"
-                    )
+                    part = f"#{target.attrib['id']}" if css else f"[@id='{target.attrib['id']}']"
                     selectorPath.append(part)
                     if not full_path:
-                        return (
-                            " > ".join(reversed(selectorPath))
-                            if css
-                            else "//*" + "/".join(reversed(selectorPath))
-                        )
+                        return " > ".join(reversed(selectorPath)) if css else "//*" + "/".join(reversed(selectorPath))
                 else:
                     part = f"{target.tag}"
                     # We won't use classes anymore because I some websites share exact classes between elements
@@ -45,28 +35,16 @@ class SelectorsGeneration:
                             break
 
                     if counter[target.tag] > 1:
-                        part += (
-                            f":nth-of-type({counter[target.tag]})"
-                            if css
-                            else f"[{counter[target.tag]}]"
-                        )
+                        part += f":nth-of-type({counter[target.tag]})" if css else f"[{counter[target.tag]}]"
 
                 selectorPath.append(part)
                 target = target.parent
                 if target is None or target.tag == "html":
-                    return (
-                        " > ".join(reversed(selectorPath))
-                        if css
-                        else "//" + "/".join(reversed(selectorPath))
-                    )
+                    return " > ".join(reversed(selectorPath)) if css else "//" + "/".join(reversed(selectorPath))
             else:
                 break
 
-        return (
-            " > ".join(reversed(selectorPath))
-            if css
-            else "//" + "/".join(reversed(selectorPath))
-        )
+        return " > ".join(reversed(selectorPath)) if css else "//" + "/".join(reversed(selectorPath))
 
     @property
     def generate_css_selector(self) -> str: