scrapling 0.2.94__py3-none-any.whl → 0.2.95__py3-none-any.whl

scrapling/__init__.py

@@ -5,7 +5,7 @@ from scrapling.fetchers import (AsyncFetcher, CustomFetcher, Fetcher,
 from scrapling.parser import Adaptor, Adaptors
 
 __author__ = "Karim Shoair (karim.shoair@pm.me)"
-__version__ = "0.2.94"
+__version__ = "0.2.95"
 __copyright__ = "Copyright (c) 2024 Karim Shoair"
 
 

scrapling/engines/static.py

@@ -42,16 +42,19 @@ class StaticEngine:
         :return: A dictionary of the new headers.
         """
         headers = headers or {}
-
-        # Validate headers
-        if not headers.get('user-agent') and not headers.get('User-Agent'):
-            headers['User-Agent'] = generate_headers(browser_mode=False).get('User-Agent')
-            log.debug(f"Can't find useragent in headers so '{headers['User-Agent']}' was used.")
+        headers_keys = set(map(str.lower, headers.keys()))
 
         if self.stealth:
             extra_headers = generate_headers(browser_mode=False)
+            # Don't overwrite user supplied headers
+            extra_headers = {key: value for key, value in extra_headers.items() if key.lower() not in headers_keys}
             headers.update(extra_headers)
-            headers.update({'referer': generate_convincing_referer(self.url)})
+            if 'referer' not in headers_keys:
+                headers.update({'referer': generate_convincing_referer(self.url)})
+
+        elif 'user-agent' not in headers_keys:
+            headers['User-Agent'] = generate_headers(browser_mode=False).get('User-Agent')
+            log.debug(f"Can't find useragent in headers so '{headers['User-Agent']}' was used.")
 
         return headers
 

{scrapling-0.2.94.dist-info → scrapling-0.2.95.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.2
 Name: scrapling
-Version: 0.2.94
-Summary: Scrapling is a powerful, flexible, and high-performance web scraping library for Python. It
+Version: 0.2.95
+Summary: Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy again! In an internet filled with complications,
 Home-page: https://github.com/D4Vinci/Scrapling
 Author: Karim Shoair
 Author-email: karim.shoair@pm.me
@@ -275,6 +275,8 @@ This class is built on top of [httpx](https://www.python-httpx.org/) with additi
 
 For all methods, you have `stealthy_headers` which makes `Fetcher` create and use real browser's headers then create a referer header as if this request came from Google's search of this URL's domain. It's enabled by default. You can also set the number of retries with the argument `retries` for all methods and this will make httpx retry requests if it failed for any reason. The default number of retries for all `Fetcher` methods is 3.
 
+> Hence: All headers generated by `stealthy_headers` argument can be overwritten by you through the `headers` argument
+
 You can route all traffic (HTTP and HTTPS) to a proxy for any of these methods in this format `http://username:password@localhost:8030`
 ```python
 >> page = Fetcher().get('https://httpbin.org/get', stealthy_headers=True, follow_redirects=True)

{scrapling-0.2.94.dist-info → scrapling-0.2.95.dist-info}/RECORD

@@ -1,4 +1,4 @@
-scrapling/__init__.py,sha256=pOwvxTBwxLovt0OJNZz2A5FkbfjQC0wKrDmONqoNsL0,500
+scrapling/__init__.py,sha256=7gthgq0LYOWqlly_w1GnesFh1WzPmkXwXJyjXB3JvVY,500
 scrapling/cli.py,sha256=njPdJKmbLFHeWjtSiGEm9ALBdSyfUp0IaJvxQL5C31Q,1125
 scrapling/defaults.py,sha256=sdXeZjXEX7PmCtaa0weK0nRrAUzqZukNNqipZ_sltYE,469
 scrapling/fetchers.py,sha256=qmiJ6S-bnPWvP48Z6rKxBnSuR-tdwHlJwlIsYxGxFM0,35405
@@ -15,7 +15,7 @@ scrapling/engines/__init__.py,sha256=zA7tzqcDXP0hllwmjVewNHWipIA4JSU9mRG4J-cud0c
 scrapling/engines/camo.py,sha256=SHMRnIrN6599upo5-G3fZQ10455xyB-bB_EsLMjBStA,16072
 scrapling/engines/constants.py,sha256=Gb_nXFoBB4ujJkd05SKkenMe1UDiRYQA3dkmA3DunLg,3723
 scrapling/engines/pw.py,sha256=LvS1jvTf3s7mfdeQo7_OyQ5zpiOzvBu5g88hOLlQBCQ,20856
-scrapling/engines/static.py,sha256=_bqVKcsTkm8ok6NIH6PDDaXtyQ6B2ZoGWccjZJKwvBo,10414
+scrapling/engines/static.py,sha256=8v6RmdsSP6fAtWNXaJG24evHPsZ2oDiBl7yfkLrdARU,10635
 scrapling/engines/toolbelt/__init__.py,sha256=VQDdYm1zY9Apno6d8UrULk29vUjllZrQqD8mXL1E2Fc,402
 scrapling/engines/toolbelt/custom.py,sha256=qgONLwpxUoEIAIQBF1RcakYu8cqAAmX8qdyaol5hfjA,12813
 scrapling/engines/toolbelt/fingerprints.py,sha256=ajEHdXHr7W4hw9KcNS7XlyxNBZu37p1bRj18TiICLzU,2929
@@ -41,9 +41,9 @@ tests/fetchers/sync/test_playwright.py,sha256=MEyDRaMyxDIWupG7f_xz0f0jd9Cpbd5rXC
 tests/parser/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/parser/test_automatch.py,sha256=SxsNdExE8zz8AcPRQFBUjZ3Q_1-tPOd9dzVvMSZpOYQ,4908
 tests/parser/test_general.py,sha256=dyfOsc8lleoY4AxcfDUBUaD1i95xecfYuTUhKBsYjwo,12100
-scrapling-0.2.94.dist-info/LICENSE,sha256=XHgu8DRuT7_g3Hb9Q18YGg8eShp6axPBacbnQxT_WWQ,1499
-scrapling-0.2.94.dist-info/METADATA,sha256=nF08IkBzVob418wgav0uHzbNdVXH1-FrTYZAxrTfg24,68878
-scrapling-0.2.94.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-scrapling-0.2.94.dist-info/entry_points.txt,sha256=DHyt2Blxy0P5OE2HRcP95Wz9_xo2ERCDcNqrJjYS3o8,49
-scrapling-0.2.94.dist-info/top_level.txt,sha256=ub7FkOEXeYmmYTUxd4pCrwXfBfAMIpZ1sCGmXCc14tI,16
-scrapling-0.2.94.dist-info/RECORD,,
+scrapling-0.2.95.dist-info/LICENSE,sha256=XHgu8DRuT7_g3Hb9Q18YGg8eShp6axPBacbnQxT_WWQ,1499
+scrapling-0.2.95.dist-info/METADATA,sha256=PTTxxxijblkcumiCbowId3Xy5I64lF9DvH3nAMPhEHQ,69066
+scrapling-0.2.95.dist-info/WHEEL,sha256=nn6H5-ilmfVryoAQl3ZQ2l8SH5imPWFpm1A5FgEuFV4,91
+scrapling-0.2.95.dist-info/entry_points.txt,sha256=DHyt2Blxy0P5OE2HRcP95Wz9_xo2ERCDcNqrJjYS3o8,49
+scrapling-0.2.95.dist-info/top_level.txt,sha256=ub7FkOEXeYmmYTUxd4pCrwXfBfAMIpZ1sCGmXCc14tI,16
+scrapling-0.2.95.dist-info/RECORD,,

{scrapling-0.2.94.dist-info → scrapling-0.2.95.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: setuptools (75.8.1)
 Root-Is-Purelib: true
 Tag: py3-none-any