yad2-scraper 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

yad2_scraper/constants.py

@@ -19,7 +19,7 @@ ALLOW_REQUEST_REDIRECTS = True
 VERIFY_REQUEST_SSL = True
 
 ANTIBOT_CONTENT_IDENTIFIER = b"Are you for real"  # robot-captcha
-YAD2_CONTENT_IDENTIFIER = b"https://www.yad2.co.il/"
+PAGE_CONTENT_IDENTIFIER = b"https://www.yad2.co.il/"
 
 FIRST_PAGE_NUMBER = 1
 NOT_MENTIONED_PRICE_RANGE = 0, 0

yad2_scraper/exceptions.py

@@ -1,10 +1,12 @@
 import httpx
-from typing import List
+from typing import List, Union
 
 
-class ResponseError(httpx.HTTPStatusError):
-    # This adds the request/response objects to the error
-    pass
+class ResponseError(Exception):
+    def __init__(self, msg: str, request: httpx.Request, response: httpx.Response):
+        super().__init__(msg)
+        self.request = request
+        self.response = response
 
 
 class AntiBotDetectedError(ResponseError):
@@ -15,17 +17,16 @@ class UnexpectedContentError(ResponseError):
     pass
 
 
-class MaxRetriesExceededError(Exception):
-    def __init__(self, msg: str, errors: List[Exception] = None):
+class MaxAttemptsExceededError(Exception):
+    def __init__(self, msg: str, max_attempts: int, errors: List[BaseException] = None):
         super().__init__(msg)
+        self.max_attempts = max_attempts
         self.errors = errors
 
 
-class MaxRequestRetriesExceededError(MaxRetriesExceededError):
-    def __init__(self, method: str, url: str, max_retries: int, errors: List[Exception] = None):
+class MaxRequestAttemptsExceededError(MaxAttemptsExceededError):
+    def __init__(self, method: str, url: str, max_attempts: int, errors: List[Union[httpx.HTTPError, ResponseError]]):
+        msg = f"All {max_attempts} attempts for {method} request to '{url}' have failed"
+        super().__init__(msg, max_attempts, errors)
         self.method = method
         self.url = url
-        self.max_retries = max_retries
-
-        msg = f"All {self.max_retries} retry attempts for {self.method} request to '{self.url}' have failed"
-        super().__init__(msg, errors)

yad2_scraper/scraper.py

@@ -1,25 +1,25 @@
 import logging
 import httpx
 import time
-import random
-from typing import Optional, Dict, Any, Tuple, Union, Type, TypeVar
+from fake_useragent import FakeUserAgent
+from typing import Optional, Dict, Any, Callable, Union, Type, TypeVar
 
 from yad2_scraper.category import Yad2Category
 from yad2_scraper.query import QueryFilters
-from yad2_scraper.utils import get_random_user_agent
-from yad2_scraper.exceptions import AntiBotDetectedError, UnexpectedContentError, MaxRequestRetriesExceededError
+from yad2_scraper.exceptions import AntiBotDetectedError, UnexpectedContentError, MaxRequestAttemptsExceededError
 from yad2_scraper.constants import (
     DEFAULT_REQUEST_HEADERS,
     ALLOW_REQUEST_REDIRECTS,
     VERIFY_REQUEST_SSL,
     ANTIBOT_CONTENT_IDENTIFIER,
-    YAD2_CONTENT_IDENTIFIER
+    PAGE_CONTENT_IDENTIFIER
 )
 
 Category = TypeVar("Category", bound=Yad2Category)
-DelayRange = Tuple[float, float]
+WaitStrategy = Callable[[int], Optional[float]]
 QueryParamTypes = Union[QueryFilters, Dict[str, Any]]
 
+fua = FakeUserAgent()
 logger = logging.getLogger(__name__)
 
 
@@ -28,9 +28,9 @@ class Yad2Scraper:
             self,
             client: Optional[httpx.Client] = None,
             request_defaults: Optional[Dict[str, Any]] = None,
-            randomize_user_agent: bool = False,
-            random_delay_range: Optional[DelayRange] = None,
-            max_retries: int = 0
+            randomize_user_agent: bool = True,
+            wait_strategy: Optional[WaitStrategy] = None,
+            max_request_attempts: int = 1
     ):
         self.client = client or httpx.Client(
             headers=DEFAULT_REQUEST_HEADERS,
@@ -39,16 +39,25 @@ class Yad2Scraper:
         )
         self.request_defaults = request_defaults or {}
         self.randomize_user_agent = randomize_user_agent
-        self.random_delay_range = random_delay_range
-        self.max_retries = max_retries
+        self.wait_strategy = wait_strategy
+        self.max_request_attempts = max_request_attempts
 
         logger.debug(f"Scraper initialized with client: {self.client}")
 
+    def set_user_agent(self, user_agent: str) -> None:
+        self.client.headers["User-Agent"] = user_agent
+        logger.debug(f"User-Agent client header set to: '{user_agent}'")
+
+    def set_no_script(self, no_script: bool) -> None:
+        value = "1" if no_script else "0"
+        self.client.cookies.set("noscript", value)
+        logger.debug(f"NoScript (noscript) client cookie set to: '{value}'")
+
     def fetch_category(
             self,
             url: str,
-            params: Optional[QueryParamTypes] = None,
-            category_type: Type[Category] = Yad2Category
+            category_type: Type[Category] = Yad2Category,
+            params: Optional[QueryParamTypes] = None
     ) -> Category:
         logger.debug(f"Fetching category from URL: '{url}'")
         response = self.get(url, params)
@@ -59,72 +68,48 @@ class Yad2Scraper:
         return self.request("GET", url, params=params)
 
     def request(self, method: str, url: str, params: Optional[QueryParamTypes] = None) -> httpx.Response:
+        if not isinstance(self.max_request_attempts, int):
+            raise TypeError(f"max_request_attempts must be of type 'int', but got {type(self.max_request_attempts)}")
+
+        if self.max_request_attempts <= 0:
+            raise ValueError(f"max_request_attempts must be a positive integer, but got {self.max_request_attempts}")
+
         request_options = self._prepare_request_options(params=params)
+        error_list = []
 
-        try:
-            return self._send_request(method, url, request_options)
-        except Exception as error:
-            return self._handle_request_error(method, url, request_options, error)
+        for attempt in range(1, self.max_request_attempts + 1):
+            try:
+                return self._send_request(method, url, request_options, attempt)
+            except Exception as error:
+                logger.error(f"{method} request to '{url}' failed {self._format_attempt_info(attempt)}: {error}")
+                error_list.append(error)
 
-    def set_user_agent(self, user_agent: str) -> None:
-        self.client.headers["User-Agent"] = user_agent
-        logger.debug(f"User-Agent client header set to: '{user_agent}'")
+        if self.max_request_attempts == 1:
+            raise error_list[0]  # only one error exists, raise it
 
-    def set_no_script(self, no_script: bool) -> None:
-        value = "1" if no_script else "0"
-        self.client.cookies.set("noscript", value)
-        logger.debug(f"noscript client cookie set to: '{value}'")
+        max_attempts_error = MaxRequestAttemptsExceededError(method, url, self.max_request_attempts, error_list)
+        logger.error(str(max_attempts_error))
+        raise max_attempts_error from error_list[-1]  # multiple errors exist, raise from the last one
 
     def close(self) -> None:
         logger.debug("Closing scraper client")
         self.client.close()
         logger.info("Scraper client closed")
 
-    def _send_request(self, method: str, url: str, request_options: Dict[str, Any]) -> httpx.Response:
+    def _send_request(self, method: str, url: str, request_options: Dict[str, Any], attempt: int) -> httpx.Response:
         if self.randomize_user_agent:
             self._set_random_user_agent(request_options)
 
-        if self.random_delay_range:
-            self._apply_request_delay()
+        if self.wait_strategy:
+            self._apply_wait_strategy(attempt)
 
-        logger.info(f"Making {method} request to URL: '{url}'")
+        logger.info(f"Sending {method} request to URL: '{url}' {self._format_attempt_info(attempt)}")
         response = self.client.request(method, url, **request_options)
-        logger.debug(f"Received response with status code: {response.status_code}")
+        logger.debug(f"Received response {response.status_code} from '{url}' {self._format_attempt_info(attempt)}")
         self._validate_response(response)
 
         return response
 
-    def _handle_request_error(
-            self,
-            method: str,
-            url: str,
-            request_options: Dict[str, Any],
-            error: Exception
-    ) -> httpx.Response:
-        logger.error(f"{method} request to '{url}' failed: {error}")
-
-        if self.max_retries == 0:
-            raise error
-
-        return self._retry_request(method, url, request_options)
-
-    def _retry_request(self, method: str, url: str, request_options: Dict[str, Any]) -> httpx.Response:
-        logger.info(f"Retrying {method} request to '{url}' (max retries: {self.max_retries})")
-
-        errors = []
-
-        for retry_attempt in range(1, self.max_retries + 1):
-            try:
-                logger.debug(f"Retry attempt {retry_attempt}/{self.max_retries}")
-                return self._send_request(method, url, request_options)
-            except Exception as error:
-                logger.warning(f"Retry attempt {retry_attempt} failed: {error}")
-                errors.append(error)
-
-        error_to_raise = MaxRequestRetriesExceededError(method, url, self.max_retries, errors)
-        logger.error(str(error_to_raise))
-        raise error_to_raise from errors[-1]
-
     def _prepare_request_options(self, params: Optional[QueryParamTypes] = None) -> Dict[str, Any]:
         logger.debug("Preparing request options from defaults")
         request_options = self.request_defaults.copy()
@@ -135,17 +120,20 @@ class Yad2Scraper:
 
         return request_options
 
-    def _apply_request_delay(self):
-        delay = random.uniform(*self.random_delay_range)
-        logger.debug(f"Applying request delay of {delay:.2f} seconds")
-        time.sleep(delay)
-
     @staticmethod
     def _set_random_user_agent(request_options: Dict[str, str]):
-        user_agent = get_random_user_agent()
+        user_agent = fua.random
         request_options.setdefault("headers", {})["User-Agent"] = user_agent
         logger.debug(f"Updated request options with random User-Agent header: '{user_agent}'")
 
+    def _apply_wait_strategy(self, attempt: int):
+        wait_time = self.wait_strategy(attempt)
+        if not wait_time:
+            return
+
+        logger.debug(f"Waiting {wait_time:.2f} seconds before request {self._format_attempt_info(attempt)}")
+        time.sleep(wait_time)
+
     @staticmethod
     def _validate_response(response: httpx.Response):
         response.raise_for_status()
@@ -156,15 +144,18 @@ class Yad2Scraper:
                 request=response.request,
                 response=response
             )
-        if YAD2_CONTENT_IDENTIFIER not in response.content:
+        if response.request.method == "GET" and PAGE_CONTENT_IDENTIFIER not in response.content:
             raise UnexpectedContentError(
-                "The response does not contain yad2 content",
+                "The GET response does not contain yad2 related content",
                 request=response.request,
                 response=response
             )
 
         logger.debug("Response validation succeeded")
 
+    def _format_attempt_info(self, attempt: int) -> str:
+        return f"(attempt {attempt}/{self.max_request_attempts})"
+
     def __enter__(self):
         logger.debug("Entering scraper context")
         return self

yad2_scraper/utils.py

@@ -1,13 +1,6 @@
-from fake_useragent import FakeUserAgent
 from bs4 import BeautifulSoup, Tag
 from typing import Union, List
 
-fua = FakeUserAgent()
-
-
-def get_random_user_agent() -> str:
-    return fua.random
-
 
 def join_url(url: str, path: str) -> str:
     return url.rstrip("/") + "/" + path.lstrip("/")

{yad2_scraper-0.3.0.dist-info → yad2_scraper-0.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: yad2-scraper
-Version: 0.3.0
+Version: 0.4.0
 Summary: Scrape Yad2 in Python.
 License: LICENSE
 Author: dav ost

yad2_scraper-0.4.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+yad2_scraper/__init__.py,sha256=UUiIk6TAHTAP4IY86bIR4TcY3VVMTCyEF0Sq1MSneMM,141
+yad2_scraper/category.py,sha256=KXLyjMOlPzu3xj08-uRmffAMD83DbqFVm-y1-T83Djw,910
+yad2_scraper/constants.py,sha256=8zXJ31fRqkDIOJp96BRK1PJofGXX8SG64YcfmJnVW8Q,910
+yad2_scraper/exceptions.py,sha256=_IcuDdJPKAznSUp_c3fLEuTnDdKf0NLJqpRPs0IzdXw,979
+yad2_scraper/next_data.py,sha256=-vqvXJqugk-895_kOnwb7J8kUjugg28Aqrh4Z_ct11M,512
+yad2_scraper/query.py,sha256=WaOWUlyNye9MNXv3hkiUaBFDeV9lbkvHiaDHWYKzgtY,1194
+yad2_scraper/scraper.py,sha256=sgDpfnKlBSDIWEb2enpQ5O9E5fJvXz3cDOnGXHGCJL4,6653
+yad2_scraper/utils.py,sha256=e6tqaN5Gw9BXunOQ1V919NkLrZREN7TdMsDuOvZgrcY,713
+yad2_scraper-0.4.0.dist-info/LICENSE,sha256=JCpnDxMx2kE40e0UQ1svSmifrLWg2Gni5VTkJR68thY,1065
+yad2_scraper-0.4.0.dist-info/METADATA,sha256=TDO8cS7t4aGZ-B-XikL4hSneizcb7TzLtBsV7dQGl1k,925
+yad2_scraper-0.4.0.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+yad2_scraper-0.4.0.dist-info/RECORD,,

yad2_scraper-0.3.0.dist-info/RECORD

@@ -1,12 +0,0 @@
-yad2_scraper/__init__.py,sha256=UUiIk6TAHTAP4IY86bIR4TcY3VVMTCyEF0Sq1MSneMM,141
-yad2_scraper/category.py,sha256=KXLyjMOlPzu3xj08-uRmffAMD83DbqFVm-y1-T83Djw,910
-yad2_scraper/constants.py,sha256=RAikaxRILQyiNeZG-_MPAwPi83abK5sscHdzDOrFge8,910
-yad2_scraper/exceptions.py,sha256=Vewa3CmVEdH6Wok3YP2686RoIrA7myKnjDQTNEZAn7w,830
-yad2_scraper/next_data.py,sha256=-vqvXJqugk-895_kOnwb7J8kUjugg28Aqrh4Z_ct11M,512
-yad2_scraper/query.py,sha256=WaOWUlyNye9MNXv3hkiUaBFDeV9lbkvHiaDHWYKzgtY,1194
-yad2_scraper/scraper.py,sha256=QeLNFxwTQSN9Dq3zotBFnnTU5XHQrnEoWOJD3qfj2w8,6564
-yad2_scraper/utils.py,sha256=48flvJPUje3nDHL3F_C3pPw3pf3ycke0f1WoXq2cSeE,837
-yad2_scraper-0.3.0.dist-info/LICENSE,sha256=JCpnDxMx2kE40e0UQ1svSmifrLWg2Gni5VTkJR68thY,1065
-yad2_scraper-0.3.0.dist-info/METADATA,sha256=YOj8J10dvwS7fYdC6nEzu0Ea-D-CpndhzN9o5LujvNk,925
-yad2_scraper-0.3.0.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-yad2_scraper-0.3.0.dist-info/RECORD,,