yad2-scraper 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

yad2_scraper/constants.py

@@ -18,9 +18,10 @@ DEFAULT_REQUEST_HEADERS = {
 ALLOW_REQUEST_REDIRECTS = True
 VERIFY_REQUEST_SSL = True
 
-ANTIBOT_CONTENT = b"Are you for real"  # robot-captcha
+ANTIBOT_CONTENT_IDENTIFIER = b"Are you for real"  # robot-captcha
+YAD2_CONTENT_IDENTIFIER = b"https://www.yad2.co.il/"
 
-FIRST_PAGE = 1
+FIRST_PAGE_NUMBER = 1
 NOT_MENTIONED_PRICE_RANGE = 0, 0
 
 NEXT_DATA_SCRIPT_ID = "__NEXT_DATA__"

yad2_scraper/exceptions.py

@@ -1,2 +1,31 @@
-class AntiBotDetectedError(Exception):
+import httpx
+from typing import List
+
+
+class ResponseError(httpx.HTTPStatusError):
+    # This adds the request/response objects to the error
     pass
+
+
+class AntiBotDetectedError(ResponseError):
+    pass
+
+
+class UnexpectedContentError(ResponseError):
+    pass
+
+
+class MaxRetriesExceededError(Exception):
+    def __init__(self, msg: str, errors: List[Exception] = None):
+        super().__init__(msg)
+        self.errors = errors
+
+
+class MaxRequestRetriesExceededError(MaxRetriesExceededError):
+    def __init__(self, method: str, url: str, max_retries: int, errors: List[Exception] = None):
+        self.method = method
+        self.url = url
+        self.max_retries = max_retries
+
+        msg = f"All {self.max_retries} retry attempts for {self.method} request to '{self.url}' have failed"
+        super().__init__(msg, errors)

yad2_scraper/scraper.py

@@ -6,16 +6,19 @@ from typing import Optional, Dict, Any, Tuple, Union, Type, TypeVar
 
 from yad2_scraper.category import Yad2Category
 from yad2_scraper.query import QueryFilters
-from yad2_scraper.utils import get_random_user_agent, validate_http_response
+from yad2_scraper.utils import get_random_user_agent
+from yad2_scraper.exceptions import AntiBotDetectedError, UnexpectedContentError, MaxRequestRetriesExceededError
 from yad2_scraper.constants import (
     DEFAULT_REQUEST_HEADERS,
     ALLOW_REQUEST_REDIRECTS,
-    VERIFY_REQUEST_SSL
+    VERIFY_REQUEST_SSL,
+    ANTIBOT_CONTENT_IDENTIFIER,
+    YAD2_CONTENT_IDENTIFIER
 )
 
 Category = TypeVar("Category", bound=Yad2Category)
 DelayRange = Tuple[float, float]
-QueryParams = Union[QueryFilters, Dict[str, Any]]
+QueryParamTypes = Union[QueryFilters, Dict[str, Any]]
 
 logger = logging.getLogger(__name__)
 
@@ -23,85 +26,149 @@ logger = logging.getLogger(__name__)
 class Yad2Scraper:
     def __init__(
             self,
-            session: Optional[httpx.Client] = None,
-            request_kwargs: Dict[str, Any] = None,
+            client: Optional[httpx.Client] = None,
+            request_defaults: Optional[Dict[str, Any]] = None,
             randomize_user_agent: bool = False,
-            requests_delay_range: Optional[DelayRange] = None,
+            random_delay_range: Optional[DelayRange] = None,
+            max_retries: int = 0
     ):
-        self.session = session or httpx.Client(
+        self.client = client or httpx.Client(
             headers=DEFAULT_REQUEST_HEADERS,
             follow_redirects=ALLOW_REQUEST_REDIRECTS,
             verify=VERIFY_REQUEST_SSL
         )
-        self.request_kwargs = request_kwargs or {}
+        self.request_defaults = request_defaults or {}
         self.randomize_user_agent = randomize_user_agent
-        self.requests_delay_range = requests_delay_range
+        self.random_delay_range = random_delay_range
+        self.max_retries = max_retries
 
-        logger.debug(f"Initialized with session {self.session} and request kwargs: {self.request_kwargs}")
-
-    def set_user_agent(self, user_agent: str):
-        self.session.headers["User-Agent"] = user_agent
-        logger.debug(f"User-Agent session header set to: '{user_agent}'")
-
-    def set_no_script(self, no_script: bool):
-        value = "1" if no_script else "0"  # str(int(no_script))
-        self.session.cookies.set("noscript", value)
-        logger.debug(f"NoScript session cookie set to: '{value}'")
+        logger.debug(f"Scraper initialized with client: {self.client}")
 
     def fetch_category(
             self,
             url: str,
-            query_params: Optional[QueryParams] = None,
+            params: Optional[QueryParamTypes] = None,
             category_type: Type[Category] = Yad2Category
     ) -> Category:
         logger.debug(f"Fetching category from URL: '{url}'")
-        response = self.get(url, query_params)
+        response = self.get(url, params)
         logger.debug(f"Category fetched successfully from URL: '{url}'")
         return category_type.from_html_io(response)
 
-    def get(self, url: str, query_params: Optional[QueryParams] = None) -> httpx.Response:
-        return self.request("GET", url, query_params=query_params)
+    def get(self, url: str, params: Optional[QueryParamTypes] = None) -> httpx.Response:
+        return self.request("GET", url, params=params)
+
+    def request(self, method: str, url: str, params: Optional[QueryParamTypes] = None) -> httpx.Response:
+        request_options = self._prepare_request_options(params=params)
+
+        try:
+            return self._send_request(method, url, request_options)
+        except Exception as error:
+            return self._handle_request_error(method, url, request_options, error)
 
-    def request(self, method: str, url: str, query_params: Optional[QueryParams] = None) -> httpx.Response:
-        request_kwargs = self._prepare_request_kwargs(query_params=query_params)
+    def set_user_agent(self, user_agent: str) -> None:
+        self.client.headers["User-Agent"] = user_agent
+        logger.debug(f"User-Agent client header set to: '{user_agent}'")
 
-        if self.requests_delay_range:
+    def set_no_script(self, no_script: bool) -> None:
+        value = "1" if no_script else "0"
+        self.client.cookies.set("noscript", value)
+        logger.debug(f"noscript client cookie set to: '{value}'")
+
+    def close(self) -> None:
+        logger.debug("Closing scraper client")
+        self.client.close()
+        logger.info("Scraper client closed")
+
+    def _send_request(self, method: str, url: str, request_options: Dict[str, Any]) -> httpx.Response:
+        if self.randomize_user_agent:
+            self._set_random_user_agent(request_options)
+
+        if self.random_delay_range:
             self._apply_request_delay()
 
-        try:
-            logger.info(f"Making {method} request to URL: '{url}'")  # request kwargs not logged - may be sensitive
-            response = self.session.request(method, url, **request_kwargs)
-            logger.debug(f"Received response with status code: {response.status_code}")
+        logger.info(f"Making {method} request to URL: '{url}'")
+        response = self.client.request(method, url, **request_options)
+        logger.debug(f"Received response with status code: {response.status_code}")
+        self._validate_response(response)
 
-            validate_http_response(response)
-            logger.debug("Response validation succeeded")
-        except Exception as error:
-            logger.error(f"Request to '{url}' failed: {error}")
+        return response
+
+    def _handle_request_error(
+            self,
+            method: str,
+            url: str,
+            request_options: Dict[str, Any],
+            error: Exception
+    ) -> httpx.Response:
+        logger.error(f"{method} request to '{url}' failed: {error}")
+
+        if self.max_retries == 0:
             raise error
 
-        return response
+        return self._retry_request(method, url, request_options)
 
-    def _prepare_request_kwargs(self, query_params: Optional[QueryParams] = None) -> Dict[str, Any]:
-        logger.debug("Preparing request kwargs from defaults")
-        request_kwargs = self.request_kwargs.copy()
+    def _retry_request(self, method: str, url: str, request_options: Dict[str, Any]) -> httpx.Response:
+        logger.info(f"Retrying {method} request to '{url}' (max retries: {self.max_retries})")
 
-        if query_params:
-            request_kwargs.setdefault("params", {}).update(query_params)
-            logger.debug(f"Updated request kwargs with query params: {query_params}")
+        errors = []
 
-        if self.randomize_user_agent:
-            random_user_agent = get_random_user_agent()
-            request_kwargs.setdefault("headers", {})["User-Agent"] = random_user_agent
-            logger.debug(f"Updated request kwargs with random 'User-Agent' header: '{random_user_agent}'")
+        for retry_attempt in range(1, self.max_retries + 1):
+            try:
+                logger.debug(f"Retry attempt {retry_attempt}/{self.max_retries}")
+                return self._send_request(method, url, request_options)
+            except Exception as error:
+                logger.warning(f"Retry attempt {retry_attempt} failed: {error}")
+                errors.append(error)
+
+        error_to_raise = MaxRequestRetriesExceededError(method, url, self.max_retries, errors)
+        logger.error(str(error_to_raise))
+        raise error_to_raise from errors[-1]
+
+    def _prepare_request_options(self, params: Optional[QueryParamTypes] = None) -> Dict[str, Any]:
+        logger.debug("Preparing request options from defaults")
+        request_options = self.request_defaults.copy()
+
+        if params:
+            request_options.setdefault("params", {}).update(params)
+            logger.debug(f"Updated request options with query params: {params}")
 
-        return request_kwargs
+        return request_options
 
     def _apply_request_delay(self):
-        delay = random.uniform(*self.requests_delay_range)
+        delay = random.uniform(*self.random_delay_range)
         logger.debug(f"Applying request delay of {delay:.2f} seconds")
         time.sleep(delay)
 
-    def close(self):
-        logger.debug("Closing scraper session")
-        self.session.close()
-        logger.info("Scraper session closed")
+    @staticmethod
+    def _set_random_user_agent(request_options: Dict[str, str]):
+        user_agent = get_random_user_agent()
+        request_options.setdefault("headers", {})["User-Agent"] = user_agent
+        logger.debug(f"Updated request options with random User-Agent header: '{user_agent}'")
+
+    @staticmethod
+    def _validate_response(response: httpx.Response):
+        response.raise_for_status()
+
+        if ANTIBOT_CONTENT_IDENTIFIER in response.content:
+            raise AntiBotDetectedError(
+                f"The response contains Anti-Bot content",
+                request=response.request,
+                response=response
+            )
+        if YAD2_CONTENT_IDENTIFIER not in response.content:
+            raise UnexpectedContentError(
+                "The response does not contain yad2 content",
+                request=response.request,
+                response=response
+            )
+
+        logger.debug("Response validation succeeded")
+
+    def __enter__(self):
+        logger.debug("Entering scraper context")
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        logger.debug("Exiting scraper context")
+        self.close()

yad2_scraper/utils.py

@@ -1,11 +1,7 @@
-import httpx
 from fake_useragent import FakeUserAgent
 from bs4 import BeautifulSoup, Tag
 from typing import Union, List
 
-from yad2_scraper.exceptions import AntiBotDetectedError
-from yad2_scraper.constants import ANTIBOT_CONTENT
-
 fua = FakeUserAgent()
 
 
@@ -24,13 +20,6 @@ def get_parent_url(url: str) -> str:
     return url.rstrip("/").rsplit("/", 1)[0]
 
 
-def validate_http_response(response: httpx.Response):
-    response.raise_for_status()
-
-    if ANTIBOT_CONTENT in response.content:
-        raise AntiBotDetectedError(f"The response contains Anti-Bot content")
-
-
 def find_html_tag_by_class_substring(e: Union[BeautifulSoup, Tag], tag_name: str, substring: str) -> Tag:
     return e.find(tag_name, class_=lambda class_name: class_name and substring in class_name)
 

{yad2_scraper-0.2.0.dist-info → yad2_scraper-0.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: yad2-scraper
-Version: 0.2.0
+Version: 0.3.0
 Summary: Scrape Yad2 in Python.
 License: LICENSE
 Author: dav ost

yad2_scraper-0.3.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+yad2_scraper/__init__.py,sha256=UUiIk6TAHTAP4IY86bIR4TcY3VVMTCyEF0Sq1MSneMM,141
+yad2_scraper/category.py,sha256=KXLyjMOlPzu3xj08-uRmffAMD83DbqFVm-y1-T83Djw,910
+yad2_scraper/constants.py,sha256=RAikaxRILQyiNeZG-_MPAwPi83abK5sscHdzDOrFge8,910
+yad2_scraper/exceptions.py,sha256=Vewa3CmVEdH6Wok3YP2686RoIrA7myKnjDQTNEZAn7w,830
+yad2_scraper/next_data.py,sha256=-vqvXJqugk-895_kOnwb7J8kUjugg28Aqrh4Z_ct11M,512
+yad2_scraper/query.py,sha256=WaOWUlyNye9MNXv3hkiUaBFDeV9lbkvHiaDHWYKzgtY,1194
+yad2_scraper/scraper.py,sha256=QeLNFxwTQSN9Dq3zotBFnnTU5XHQrnEoWOJD3qfj2w8,6564
+yad2_scraper/utils.py,sha256=48flvJPUje3nDHL3F_C3pPw3pf3ycke0f1WoXq2cSeE,837
+yad2_scraper-0.3.0.dist-info/LICENSE,sha256=JCpnDxMx2kE40e0UQ1svSmifrLWg2Gni5VTkJR68thY,1065
+yad2_scraper-0.3.0.dist-info/METADATA,sha256=YOj8J10dvwS7fYdC6nEzu0Ea-D-CpndhzN9o5LujvNk,925
+yad2_scraper-0.3.0.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+yad2_scraper-0.3.0.dist-info/RECORD,,

yad2_scraper-0.2.0.dist-info/RECORD

@@ -1,12 +0,0 @@
-yad2_scraper/__init__.py,sha256=UUiIk6TAHTAP4IY86bIR4TcY3VVMTCyEF0Sq1MSneMM,141
-yad2_scraper/category.py,sha256=KXLyjMOlPzu3xj08-uRmffAMD83DbqFVm-y1-T83Djw,910
-yad2_scraper/constants.py,sha256=HtZh0okD7FhbeAw3VladHsXIBlbxvZDoXaWm8aaVYvk,839
-yad2_scraper/exceptions.py,sha256=Ym3k3aQHe4-SQtfDch5DdnNN3RK6dbXLCZej1chSEGQ,48
-yad2_scraper/next_data.py,sha256=-vqvXJqugk-895_kOnwb7J8kUjugg28Aqrh4Z_ct11M,512
-yad2_scraper/query.py,sha256=WaOWUlyNye9MNXv3hkiUaBFDeV9lbkvHiaDHWYKzgtY,1194
-yad2_scraper/scraper.py,sha256=bGAjuu-ZhNYo4NBmrZU096NC2uYHvnxusAFZs1mjnRU,4202
-yad2_scraper/utils.py,sha256=9R-XJgWcmt_HTR_JQbFIFHcCg-II4ClFqlLQQZOU74g,1170
-yad2_scraper-0.2.0.dist-info/LICENSE,sha256=JCpnDxMx2kE40e0UQ1svSmifrLWg2Gni5VTkJR68thY,1065
-yad2_scraper-0.2.0.dist-info/METADATA,sha256=fznOi8yGp07DU7AMDSAg3LKuYJ5Gu6X6G8qS-SFfJt0,925
-yad2_scraper-0.2.0.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-yad2_scraper-0.2.0.dist-info/RECORD,,