scrapling 0.2.97__py3-none-any.whl → 0.2.98__py3-none-any.whl

scrapling/__init__.py

@@ -1,12 +1,41 @@
-# Declare top-level shortcuts
-from scrapling.core.custom_types import AttributesHandler, TextHandler
-from scrapling.fetchers import (AsyncFetcher, CustomFetcher, Fetcher,
-                                PlayWrightFetcher, StealthyFetcher)
-from scrapling.parser import Adaptor, Adaptors
 
 __author__ = "Karim Shoair (karim.shoair@pm.me)"
-__version__ = "0.2.97"
+__version__ = "0.2.98"
 __copyright__ = "Copyright (c) 2024 Karim Shoair"
 
 
+# A lightweight approach to create lazy loader for each import for backward compatibility
+# This will reduces initial memory footprint significantly (only loads what's used)
+def __getattr__(name):
+    if name == 'Fetcher':
+        from scrapling.fetchers import Fetcher as cls
+        return cls
+    elif name == 'Adaptor':
+        from scrapling.parser import Adaptor as cls
+        return cls
+    elif name == 'Adaptors':
+        from scrapling.parser import Adaptors as cls
+        return cls
+    elif name == 'AttributesHandler':
+        from scrapling.core.custom_types import AttributesHandler as cls
+        return cls
+    elif name == 'TextHandler':
+        from scrapling.core.custom_types import TextHandler as cls
+        return cls
+    elif name == 'AsyncFetcher':
+        from scrapling.fetchers import AsyncFetcher as cls
+        return cls
+    elif name == 'StealthyFetcher':
+        from scrapling.fetchers import StealthyFetcher as cls
+        return cls
+    elif name == 'PlayWrightFetcher':
+        from scrapling.fetchers import PlayWrightFetcher as cls
+        return cls
+    elif name == 'CustomFetcher':
+        from scrapling.fetchers import CustomFetcher as cls
+        return cls
+    else:
+        raise AttributeError(f"module 'scrapling' has no attribute '{name}'")
+
+
 __all__ = ['Adaptor', 'Fetcher', 'AsyncFetcher', 'StealthyFetcher', 'PlayWrightFetcher']

scrapling/core/storage_adaptors.py

@@ -19,7 +19,7 @@ class StorageSystemMixin(ABC):
         """
         self.url = url
 
-    @lru_cache(126, typed=True)
+    @lru_cache(64, typed=True)
     def _get_base_url(self, default_value: str = 'default') -> str:
         if not self.url or type(self.url) is not str:
             return default_value
@@ -51,7 +51,7 @@ class StorageSystemMixin(ABC):
         raise NotImplementedError('Storage system must implement `save` method')
 
     @staticmethod
-    @lru_cache(256, typed=True)
+    @lru_cache(128, typed=True)
     def _get_hash(identifier: str) -> str:
         """If you want to hash identifier in your storage system, use this safer"""
         identifier = identifier.lower().strip()
@@ -63,7 +63,7 @@ class StorageSystemMixin(ABC):
         return f"{hash_value}_{len(identifier)}"  # Length to reduce collision chance
 
 
-@lru_cache(10, typed=True)
+@lru_cache(1, typed=True)
 class SQLiteStorageSystem(StorageSystemMixin):
     """The recommended system to use, it's race condition safe and thread safe.
     Mainly built so the library can run in threaded frameworks like scrapy or threaded tools

scrapling/core/translator.py

@@ -142,3 +142,6 @@ class HTMLTranslator(TranslatorMixin, OriginalHTMLTranslator):
     @lru_cache(maxsize=256)
     def css_to_xpath(self, css: str, prefix: str = "descendant-or-self::") -> str:
         return super().css_to_xpath(css, prefix)
+
+
+translator_instance = HTMLTranslator()

scrapling/core/utils.py

@@ -115,7 +115,7 @@ class _StorageTools:
 #     return _impl
 
 
-@lru_cache(256, typed=True)
+@lru_cache(128, typed=True)
 def clean_spaces(string):
     string = string.replace('\t', ' ')
     string = re.sub('[\n|\r]', '', string)

scrapling/defaults.py

@@ -1,10 +1,19 @@
-from .fetchers import AsyncFetcher as _AsyncFetcher
-from .fetchers import Fetcher as _Fetcher
-from .fetchers import PlayWrightFetcher as _PlayWrightFetcher
-from .fetchers import StealthyFetcher as _StealthyFetcher
-
 # If you are going to use Fetchers with the default settings, import them from this file instead for a cleaner looking code
-Fetcher = _Fetcher()
-AsyncFetcher = _AsyncFetcher()
-StealthyFetcher = _StealthyFetcher()
-PlayWrightFetcher = _PlayWrightFetcher()
+
+# A lightweight approach to create lazy loader for each import for backward compatibility
+# This will reduces initial memory footprint significantly (only loads what's used)
+def __getattr__(name):
+    if name == 'Fetcher':
+        from scrapling.fetchers import Fetcher as cls
+        return cls()
+    elif name == 'AsyncFetcher':
+        from scrapling.fetchers import AsyncFetcher as cls
+        return cls()
+    elif name == 'StealthyFetcher':
+        from scrapling.fetchers import StealthyFetcher as cls
+        return cls()
+    elif name == 'PlayWrightFetcher':
+        from scrapling.fetchers import PlayWrightFetcher as cls
+        return cls()
+    else:
+        raise AttributeError(f"module 'scrapling' has no attribute '{name}'")

scrapling/engines/pw.py

@@ -126,7 +126,7 @@ class PlaywrightEngine:
 
         return cdp_url
 
-    @lru_cache(126, typed=True)
+    @lru_cache(32, typed=True)
     def __set_flags(self):
         """Returns the flags that will be used while launching the browser if stealth mode is enabled"""
         flags = DEFAULT_STEALTH_FLAGS
@@ -169,7 +169,7 @@ class PlaywrightEngine:
 
         return context_kwargs
 
-    @lru_cache(10)
+    @lru_cache(1)
     def __stealth_scripts(self):
         # Basic bypasses nothing fancy as I'm still working on it
         # But with adding these bypasses to the above config, it bypasses many online tests like

scrapling/engines/static.py

@@ -7,7 +7,7 @@ from scrapling.core.utils import log, lru_cache
 from .toolbelt import Response, generate_convincing_referer, generate_headers
 
 
-@lru_cache(5, typed=True)  # Singleton easily
+@lru_cache(2, typed=True)  # Singleton easily
 class StaticEngine:
     def __init__(
             self, url: str, proxy: Optional[str] = None, stealthy_headers: bool = True, follow_redirects: bool = True,

scrapling/engines/toolbelt/custom.py

@@ -16,7 +16,7 @@ class ResponseEncoding:
     __ISO_8859_1_CONTENT_TYPES = {"text/plain", "text/html", "text/css", "text/javascript"}
 
     @classmethod
-    @lru_cache(maxsize=256)
+    @lru_cache(maxsize=128)
     def __parse_content_type(cls, header_value: str) -> Tuple[str, Dict[str, str]]:
         """Parse content type and parameters from a content-type header value.
 
@@ -38,7 +38,7 @@ class ResponseEncoding:
         return content_type, params
 
     @classmethod
-    @lru_cache(maxsize=256)
+    @lru_cache(maxsize=128)
     def get_value(cls, content_type: Optional[str], text: Optional[str] = 'test') -> str:
         """Determine the appropriate character encoding from a content-type header.
 

scrapling/engines/toolbelt/fingerprints.py

@@ -12,7 +12,7 @@ from scrapling.core._types import Dict, Union
 from scrapling.core.utils import lru_cache
 
 
-@lru_cache(128, typed=True)
+@lru_cache(10, typed=True)
 def generate_convincing_referer(url: str) -> str:
     """Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website
 
@@ -26,7 +26,7 @@ def generate_convincing_referer(url: str) -> str:
     return f'https://www.google.com/search?q={website_name}'
 
 
-@lru_cache(128, typed=True)
+@lru_cache(1, typed=True)
 def get_os_name() -> Union[str, None]:
     """Get the current OS name in the same format needed for browserforge
 

scrapling/engines/toolbelt/navigation.py

@@ -110,7 +110,7 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
         raise ValueError(f"Invalid CDP URL: {str(e)}")
 
 
-@lru_cache(126, typed=True)
+@lru_cache(10, typed=True)
 def js_bypass_path(filename: str) -> str:
     """Takes the base filename of JS file inside the `bypasses` folder then return the full path of it
 

scrapling/parser.py

@@ -17,7 +17,7 @@ from scrapling.core.custom_types import (AttributesHandler, TextHandler,
 from scrapling.core.mixins import SelectorsGeneration
 from scrapling.core.storage_adaptors import (SQLiteStorageSystem,
                                              StorageSystemMixin, _StorageTools)
-from scrapling.core.translator import HTMLTranslator
+from scrapling.core.translator import translator_instance
 from scrapling.core.utils import (clean_spaces, flatten, html_forbidden,
                                   is_jsonable, log)
 
@@ -26,7 +26,7 @@ class Adaptor(SelectorsGeneration):
     __slots__ = (
         'url', 'encoding', '__auto_match_enabled', '_root', '_storage',
         '__keep_comments', '__huge_tree_enabled', '__attributes', '__text', '__tag',
-        '__keep_cdata', '__raw_body'
+        '__keep_cdata'
     )
 
     def __init__(
@@ -72,20 +72,17 @@ class Adaptor(SelectorsGeneration):
             raise ValueError("Adaptor class needs text, body, or root arguments to work")
 
         self.__text = ''
-        self.__raw_body = ''
         if root is None:
             if text is None:
                 if not body or not isinstance(body, bytes):
                     raise TypeError(f"body argument must be valid and of type bytes, got {body.__class__}")
 
                 body = body.replace(b"\x00", b"").strip()
-                self.__raw_body = body.replace(b"\x00", b"").strip().decode()
             else:
                 if not isinstance(text, str):
                     raise TypeError(f"text argument must be of type str, got {text.__class__}")
 
                 body = text.strip().replace("\x00", "").encode(encoding) or b"<html/>"
-                self.__raw_body = text.strip()
 
             # https://lxml.de/api/lxml.etree.HTMLParser-class.html
             parser = html.HTMLParser(
@@ -250,10 +247,7 @@ class Adaptor(SelectorsGeneration):
         """Return the inner html code of the element"""
         return TextHandler(etree.tostring(self._root, encoding='unicode', method='html', with_tail=False))
 
-    @property
-    def body(self) -> TextHandler:
-        """Return raw HTML code of the element/page without any processing when possible or return `Adaptor.html_content`"""
-        return TextHandler(self.__raw_body) or self.html_content
+    body = html_content
 
     def prettify(self) -> TextHandler:
         """Return a prettified version of the element's inner html-code"""
@@ -476,7 +470,7 @@ class Adaptor(SelectorsGeneration):
         try:
             if not self.__auto_match_enabled or ',' not in selector:
                 # No need to split selectors in this case, let's save some CPU cycles :)
-                xpath_selector = HTMLTranslator().css_to_xpath(selector)
+                xpath_selector = translator_instance.css_to_xpath(selector)
                 return self.xpath(xpath_selector, identifier or selector, auto_match, auto_save, percentage)
 
             results = []
@@ -484,7 +478,7 @@ class Adaptor(SelectorsGeneration):
                 for single_selector in split_selectors(selector):
                     # I'm doing this only so the `save` function save data correctly for combined selectors
                     # Like using the ',' to combine two different selectors that point to different elements.
-                    xpath_selector = HTMLTranslator().css_to_xpath(single_selector.canonical())
+                    xpath_selector = translator_instance.css_to_xpath(single_selector.canonical())
                     results += self.xpath(
                         xpath_selector, identifier or single_selector.canonical(), auto_match, auto_save, percentage
                     )

{scrapling-0.2.97.dist-info → scrapling-0.2.98.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: scrapling
-Version: 0.2.97
+Version: 0.2.98
 Summary: Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy again! In an internet filled with complications,
 Home-page: https://github.com/D4Vinci/Scrapling
 Author: Karim Shoair
@@ -173,7 +173,7 @@ Deep SerpApi is a dedicated search engine designed for large language models (LL
 ## Getting Started
 
 ```python
-from scrapling import Fetcher
+from scrapling.fetchers import Fetcher
 
 fetcher = Fetcher(auto_match=False)
 
@@ -255,7 +255,7 @@ Fetchers are interfaces built on top of other libraries with added features that
 ### Features
 You might be slightly confused by now so let me clear things up. All fetcher-type classes are imported in the same way
 ```python
-from scrapling import Fetcher, StealthyFetcher, PlayWrightFetcher
+from scrapling.fetchers import Fetcher, StealthyFetcher, PlayWrightFetcher
 ```
 All of them can take these initialization arguments: `auto_match`, `huge_tree`, `keep_comments`, `keep_cdata`, `storage`, and `storage_args`, which are the same ones you give to the `Adaptor` class.
 
@@ -287,7 +287,7 @@ You can route all traffic (HTTP and HTTPS) to a proxy for any of these methods i
 ```
 For Async requests, you will just replace the import like below:
 ```python
->> from scrapling import AsyncFetcher
+>> from scrapling.fetchers import AsyncFetcher
 >> page = await AsyncFetcher().get('https://httpbin.org/get', stealthy_headers=True, follow_redirects=True)
 >> page = await AsyncFetcher().post('https://httpbin.org/post', data={'key': 'value'}, proxy='http://username:password@localhost:8030')
 >> page = await AsyncFetcher().put('https://httpbin.org/put', data={'key': 'value'})
@@ -541,7 +541,7 @@ When website owners implement structural changes like
 The selector will no longer function and your code needs maintenance. That's where Scrapling's auto-matching feature comes into play.
 
 ```python
-from scrapling import Adaptor
+from scrapling.parser import Adaptor
 # Before the change
 page = Adaptor(page_source, url='example.com')
 element = page.css('#p1' auto_save=True)
@@ -559,7 +559,7 @@ To solve this issue, I will use [The Web Archive](https://archive.org/)'s [Wayba
 If I want to extract the Questions button from the old design I can use a selector like this `#hmenus > div:nth-child(1) > ul > li:nth-child(1) > a` This selector is too specific because it was generated by Google Chrome.
 Now let's test the same selector in both versions
 ```python
->> from scrapling import Fetcher
+>> from scrapling.fetchers import Fetcher
 >> selector = '#hmenus > div:nth-child(1) > ul > li:nth-child(1) > a'
 >> old_url = "https://web.archive.org/web/20100102003420/http://stackoverflow.com/"
 >> new_url = "https://stackoverflow.com/"
@@ -620,7 +620,7 @@ Note: The filtering process always starts from the first filter it finds in the
 Examples to clear any confusion :)
 
 ```python
->> from scrapling import Fetcher
+>> from scrapling.fetchers import Fetcher
 >> page = Fetcher().get('https://quotes.toscrape.com/')
 # Find all elements with tag name `div`.
 >> page.find_all('div')

{scrapling-0.2.97.dist-info → scrapling-0.2.98.dist-info}/RECORD

@@ -1,25 +1,25 @@
-scrapling/__init__.py,sha256=5yeUml2K0xHe2NAALM2x2hGSl_ORcEttIZL17b1cWtg,500
+scrapling/__init__.py,sha256=S-SWj9O2r0Tu8Z-mPxDJ-z3h5k-bBfhFOETaCY4A9dc,1510
 scrapling/cli.py,sha256=7yTsMhVAqqS8Z27T5dFKrR9_X8vuFjBlwYgAF22W7T8,1292
-scrapling/defaults.py,sha256=sdXeZjXEX7PmCtaa0weK0nRrAUzqZukNNqipZ_sltYE,469
+scrapling/defaults.py,sha256=MAn2MMLBFvoe4i3u_qlp6YEvGUiCjNPPDux1cFCdpsU,866
 scrapling/fetchers.py,sha256=xwVCjAg0VCXwhB2igSLQvb0D0bOPGfg5WNtxgE7m-W0,34987
-scrapling/parser.py,sha256=U6qFV23qeeX1pYl6mw0TZEL4FlaQw6puaoDTldUpi-M,54328
+scrapling/parser.py,sha256=1xS1UjCm1GVnKcVAtup9rSE1xuYPxXOgJe-8LJE5gUk,53956
 scrapling/py.typed,sha256=frcCV1k9oG9oKj3dpUqdJg1PxRT2RSN_XKdLCPjaYaY,2
 scrapling/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 scrapling/core/_types.py,sha256=dKVi_dUxdxNtTr7sj7ySkHXDfrsmjFTfpCQeO5tGuBY,670
 scrapling/core/custom_types.py,sha256=EWGx5t5scHEB1SMsitzc8duskq-5f-Qaj40IWkNTRzM,12947
 scrapling/core/mixins.py,sha256=sozbpaGL1_O_x3U-ABM5aYWpnxpCLfdbcA9SG3P7weY,3532
-scrapling/core/storage_adaptors.py,sha256=EkSE8LlOS9SggFblBNzgyEp0fLxl8dqYU3-MAuXUitY,6216
-scrapling/core/translator.py,sha256=ojDmNi5pFZE6Ke-AiSsTilXiPRdR8yhX3o-uVGMkap8,5236
-scrapling/core/utils.py,sha256=0e3jD029CXj3gfA_MIKcBC0Mai9fXW2scIuoKtHy1e8,3704
+scrapling/core/storage_adaptors.py,sha256=gZbUpHtLOL7o_oZbES_o40r39zShxTeTM8YK6dXA5Zo,6214
+scrapling/core/translator.py,sha256=3a2VX9KR-q-GzwT1OgGDv1UlzIkvBggkQXUdiMyL-4c,5277
+scrapling/core/utils.py,sha256=KX88B3tV1-SgCAr69TUN3LfmsTDcLnEhYJiPuWd31yA,3704
 scrapling/engines/__init__.py,sha256=zA7tzqcDXP0hllwmjVewNHWipIA4JSU9mRG4J-cud0c,267
 scrapling/engines/camo.py,sha256=oYKA0l3EpOcQW2APRj5FEmslqtp9A8i_ZljqlKvIDeI,16129
 scrapling/engines/constants.py,sha256=Gb_nXFoBB4ujJkd05SKkenMe1UDiRYQA3dkmA3DunLg,3723
-scrapling/engines/pw.py,sha256=_fy8mhkVrOnb_Qho8zKCjFyd1Y_kr2mkdo0PHrBks4M,21371
-scrapling/engines/static.py,sha256=okrEIFfYaxqVuIXPanxQDxQpN8i88AgWODo7Dnex2EI,9306
+scrapling/engines/pw.py,sha256=cZraIBWd9ulEGEdhETIGmpevi62CN9JGcUU1OIDdxkA,21369
+scrapling/engines/static.py,sha256=EjdaR0beqWfEKKavT7vlBnozoayQaVpqeVtaOuzd384,9306
 scrapling/engines/toolbelt/__init__.py,sha256=VQDdYm1zY9Apno6d8UrULk29vUjllZrQqD8mXL1E2Fc,402
-scrapling/engines/toolbelt/custom.py,sha256=dwpuEHNOd9uJbMf7sx8sXsYZhozSXStrwqfpooce1Wk,12811
-scrapling/engines/toolbelt/fingerprints.py,sha256=spJMij0qBGvbSlVjv9xJWCF8KFDf6xnNz5fWtXWhrzY,2927
-scrapling/engines/toolbelt/navigation.py,sha256=KyFQ4vHS4jR7z378VRGtUeXQHWr5NMy5nNp2-c_Evk8,4566
+scrapling/engines/toolbelt/custom.py,sha256=_-baGB8oOOHogbaddtGsq_K_01ccOjOkGA6tOKk28hM,12811
+scrapling/engines/toolbelt/fingerprints.py,sha256=Zzoqq3p6X_8D7eTxACz3z96cBZWWK61iKOGo2sZUtlg,2924
+scrapling/engines/toolbelt/navigation.py,sha256=fMjDgicqy2MoZZll2h5EvrrxkL6yNrC09v8isTpwAt0,4565
 scrapling/engines/toolbelt/bypasses/navigator_plugins.js,sha256=tbnnk3nCXB6QEQnOhDlu3n-s7lnUTAkrUsjP6FDQIQg,2104
 scrapling/engines/toolbelt/bypasses/notification_permission.js,sha256=poPM3o5WYgEX-EdiUfDCllpWfc3Umvw4jr2u6O6elus,237
 scrapling/engines/toolbelt/bypasses/pdf_viewer.js,sha256=mKjjSuP1-BOGC_2WhRYHJo_LP7lTBi2KXmP_zsHO_tI,173
@@ -41,9 +41,9 @@ tests/fetchers/sync/test_playwright.py,sha256=MEyDRaMyxDIWupG7f_xz0f0jd9Cpbd5rXC
 tests/parser/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/parser/test_automatch.py,sha256=SxsNdExE8zz8AcPRQFBUjZ3Q_1-tPOd9dzVvMSZpOYQ,4908
 tests/parser/test_general.py,sha256=dyfOsc8lleoY4AxcfDUBUaD1i95xecfYuTUhKBsYjwo,12100
-scrapling-0.2.97.dist-info/LICENSE,sha256=XHgu8DRuT7_g3Hb9Q18YGg8eShp6axPBacbnQxT_WWQ,1499
-scrapling-0.2.97.dist-info/METADATA,sha256=VnP3UEy6RcQytld-8ZYSF0Cpdd4fb-tKoX01jajFneo,69666
-scrapling-0.2.97.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
-scrapling-0.2.97.dist-info/entry_points.txt,sha256=DHyt2Blxy0P5OE2HRcP95Wz9_xo2ERCDcNqrJjYS3o8,49
-scrapling-0.2.97.dist-info/top_level.txt,sha256=ub7FkOEXeYmmYTUxd4pCrwXfBfAMIpZ1sCGmXCc14tI,16
-scrapling-0.2.97.dist-info/RECORD,,
+scrapling-0.2.98.dist-info/LICENSE,sha256=XHgu8DRuT7_g3Hb9Q18YGg8eShp6axPBacbnQxT_WWQ,1499
+scrapling-0.2.98.dist-info/METADATA,sha256=Un_ROxrGIvk_8w-ECQbwKAcJzYyx3MTWS1DHt9FRqdI,69718
+scrapling-0.2.98.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
+scrapling-0.2.98.dist-info/entry_points.txt,sha256=DHyt2Blxy0P5OE2HRcP95Wz9_xo2ERCDcNqrJjYS3o8,49
+scrapling-0.2.98.dist-info/top_level.txt,sha256=ub7FkOEXeYmmYTUxd4pCrwXfBfAMIpZ1sCGmXCc14tI,16
+scrapling-0.2.98.dist-info/RECORD,,