yad2-scraper 0.1.0__tar.gz → 0.3.0__tar.gz

{yad2_scraper-0.1.0 → yad2_scraper-0.3.0}/LICENSE

@@ -1,21 +1,21 @@
-MIT License
-
-Copyright (c) 2025 DavOstx7
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+MIT License
+
+Copyright (c) 2025 DavOstx7
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

{yad2_scraper-0.1.0 → yad2_scraper-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: yad2-scraper
-Version: 0.1.0
+Version: 0.3.0
 Summary: Scrape Yad2 in Python.
 License: LICENSE
 Author: dav ost
@@ -13,9 +13,13 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: beautifulsoup4 (>=4.11.1,<5.0.0)
 Requires-Dist: fake-useragent (>=0.1.11,<0.2.0)
+Requires-Dist: httpcore (>=0.15.0)
 Requires-Dist: httpx (>=0.24.0,<0.25.0)
+Requires-Dist: pydantic (>=1.10.0,<2.0.0)
 Description-Content-Type: text/markdown
 
 # yad2-scraper

{yad2_scraper-0.1.0 → yad2_scraper-0.3.0}/README.md

@@ -1,2 +1,2 @@
-# yad2-scraper
-Scrape Yad2 in Python.
+# yad2-scraper
+Scrape Yad2 in Python.

{yad2_scraper-0.1.0 → yad2_scraper-0.3.0}/pyproject.toml

@@ -1,21 +1,23 @@
-[tool.poetry]
-name = "yad2-scraper"
-version = "0.1.0"
-description = "Scrape Yad2 in Python."
-authors = ["dav ost <davidost2003@gmail.com>"]
-license = "LICENSE"
-readme = "README.md"
-
-[tool.poetry.dependencies]
-python = ">=3.7"
-httpx = "^0.24.0"
-fake-useragent = "^0.1.11"
-beautifulsoup4 = "^4.11.1"
-
-[tool.poetry.dev-dependencies]
-pytest = "^6.2.2"
-respx = "^0.15.0"
-
-[build-system]
-requires = ["poetry-core"]
+[tool.poetry]
+name = "yad2-scraper"
+version = "0.3.0"
+description = "Scrape Yad2 in Python."
+authors = ["dav ost <davidost2003@gmail.com>"]
+license = "LICENSE"
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = ">=3.7"
+httpx = "^0.24.0"
+httpcore = ">=0.15.0"
+fake-useragent = "^0.1.11"
+beautifulsoup4 = "^4.11.1"
+pydantic = "^1.10.0"
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^6.2.2"
+respx = "^0.20.0"
+
+[build-system]
+requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"

{yad2_scraper-0.1.0 → yad2_scraper-0.3.0}/yad2_scraper/__init__.py

@@ -1,4 +1,4 @@
-from .scraper import Yad2Scraper
-from .query import QueryFilters, OrderBy
-from .category import Yad2Category
-from .next_data import NextData
+from .scraper import Yad2Scraper
+from .query import QueryFilters, OrderBy
+from .category import Yad2Category
+from .next_data import NextData

{yad2_scraper-0.1.0 → yad2_scraper-0.3.0}/yad2_scraper/category.py

@@ -1,25 +1,25 @@
-import json
-from bs4 import BeautifulSoup, Tag
-from typing import Optional, List, Union, TextIO, BinaryIO
-
-from yad2_scraper.next_data import NextData
-from yad2_scraper.utils import find_all_html_tags_by_class_substring
-from yad2_scraper.constants import NEXT_DATA_SCRIPT_ID
-
-
-class Yad2Category:
-    def __init__(self, soup: BeautifulSoup):
-        self.soup = soup
-
-    @classmethod
-    def from_html_io(cls, html_io: Union[TextIO, BinaryIO]):
-        html = html_io.read()
-        soup = BeautifulSoup(html, "html.parser")
-        return cls(soup)
-
-    def load_next_data(self) -> Optional[NextData]:
-        tag = self.soup.find("script", id=NEXT_DATA_SCRIPT_ID)
-        return NextData(json.loads(tag.string)) if tag else None
-
-    def find_all_tags_by_class_substring(self, tag_name: str, substring: str) -> List[Tag]:
-        return find_all_html_tags_by_class_substring(self.soup, tag_name, substring)
+import json
+from bs4 import BeautifulSoup, Tag
+from typing import Optional, List, Union, TextIO, BinaryIO
+
+from yad2_scraper.next_data import NextData
+from yad2_scraper.utils import find_all_html_tags_by_class_substring
+from yad2_scraper.constants import NEXT_DATA_SCRIPT_ID
+
+
+class Yad2Category:
+    def __init__(self, soup: BeautifulSoup):
+        self.soup = soup
+
+    @classmethod
+    def from_html_io(cls, html_io: Union[TextIO, BinaryIO]):
+        html = html_io.read()
+        soup = BeautifulSoup(html, "html.parser")
+        return cls(soup)
+
+    def load_next_data(self) -> Optional[NextData]:
+        tag = self.soup.find("script", id=NEXT_DATA_SCRIPT_ID)
+        return NextData(json.loads(tag.string)) if tag else None
+
+    def find_all_tags_by_class_substring(self, tag_name: str, substring: str) -> List[Tag]:
+        return find_all_html_tags_by_class_substring(self.soup, tag_name, substring)

{yad2_scraper-0.1.0 → yad2_scraper-0.3.0}/yad2_scraper/constants.py

@@ -1,26 +1,27 @@
-BASE_URL = "https://www.yad2.co.il"
-
-DEFAULT_REQUEST_HEADERS = {
-    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Safari/605.1.15",
-    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
-    "Accept-Encoding": "gzip, deflate, br",
-    "Accept-Language": "en-US,en;q=0.9",
-    "Connection": "keep-alive",
-    "Upgrade-Insecure-Requests": "1",
-    "DNT": "1",
-    "Cache-Control": "max-age=0",
-    "Sec-Fetch-Site": "none",
-    "Sec-Fetch-Mode": "navigate",
-    "Sec-Fetch-User": "?1",
-    "Sec-Fetch-Dest": "document",
-}
-
-ALLOW_REQUEST_REDIRECTS = True
-VERIFY_REQUEST_SSL = True
-
-ANTIBOT_CONTENT = b"Are you for real"  # robot-captcha
-
-FIRST_PAGE = 1
-NOT_MENTIONED_PRICE_RANGE = 0, 0
-
-NEXT_DATA_SCRIPT_ID = "__NEXT_DATA__"
+BASE_URL = "https://www.yad2.co.il"
+
+DEFAULT_REQUEST_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Safari/605.1.15",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
+    "Accept-Encoding": "gzip, deflate, br",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Connection": "keep-alive",
+    "Upgrade-Insecure-Requests": "1",
+    "DNT": "1",
+    "Cache-Control": "max-age=0",
+    "Sec-Fetch-Site": "none",
+    "Sec-Fetch-Mode": "navigate",
+    "Sec-Fetch-User": "?1",
+    "Sec-Fetch-Dest": "document",
+}
+
+ALLOW_REQUEST_REDIRECTS = True
+VERIFY_REQUEST_SSL = True
+
+ANTIBOT_CONTENT_IDENTIFIER = b"Are you for real"  # robot-captcha
+YAD2_CONTENT_IDENTIFIER = b"https://www.yad2.co.il/"
+
+FIRST_PAGE_NUMBER = 1
+NOT_MENTIONED_PRICE_RANGE = 0, 0
+
+NEXT_DATA_SCRIPT_ID = "__NEXT_DATA__"

yad2_scraper-0.3.0/yad2_scraper/exceptions.py

@@ -0,0 +1,31 @@
+import httpx
+from typing import List
+
+
+class ResponseError(httpx.HTTPStatusError):
+    # This adds the request/response objects to the error
+    pass
+
+
+class AntiBotDetectedError(ResponseError):
+    pass
+
+
+class UnexpectedContentError(ResponseError):
+    pass
+
+
+class MaxRetriesExceededError(Exception):
+    def __init__(self, msg: str, errors: List[Exception] = None):
+        super().__init__(msg)
+        self.errors = errors
+
+
+class MaxRequestRetriesExceededError(MaxRetriesExceededError):
+    def __init__(self, method: str, url: str, max_retries: int, errors: List[Exception] = None):
+        self.method = method
+        self.url = url
+        self.max_retries = max_retries
+
+        msg = f"All {self.max_retries} retry attempts for {self.method} request to '{self.url}' have failed"
+        super().__init__(msg, errors)

{yad2_scraper-0.1.0 → yad2_scraper-0.3.0}/yad2_scraper/next_data.py

@@ -1,27 +1,27 @@
-from enum import Enum
-from typing import List, Union
-
-
-class Field(str, Enum):
-    ID = "id"
-    TEXT = "text"
-    ENGLISH_TEXT = "textEng"
-
-
-FieldTypes = Union[str, int]
-
-
-class NextData:
-    def __init__(self, data: dict):
-        self.data = data
-
-    @property
-    def json(self) -> dict:
-        return self.data
-
-    @property
-    def queries(self) -> List[dict]:
-        return self.data["props"]["pageProps"]["dehydratedState"]["queries"]
-
-    def __getitem__(self, item):
-        return self.data[item]
+from enum import Enum
+from typing import List, Union
+
+
+class Field(str, Enum):
+    ID = "id"
+    TEXT = "text"
+    ENGLISH_TEXT = "textEng"
+
+
+FieldTypes = Union[str, int]
+
+
+class NextData:
+    def __init__(self, data: dict):
+        self.data = data
+
+    @property
+    def json(self) -> dict:
+        return self.data
+
+    @property
+    def queries(self) -> List[dict]:
+        return self.data["props"]["pageProps"]["dehydratedState"]["queries"]
+
+    def __getitem__(self, item):
+        return self.data[item]

{yad2_scraper-0.1.0 → yad2_scraper-0.3.0}/yad2_scraper/query.py

@@ -1,45 +1,46 @@
-from pydantic import BaseModel
-from enum import Enum
-from typing import Optional, Tuple
-
-PriceRange = Tuple[int, int]
-
-
-class OrderBy(int, Enum):
-    DATE = 1
-    PRICE_LOWEST_TO_HIGHEST = 3
-    PRICE_HIGHEST_TO_LOWEST = 4
-    ...
-
-
-def format_number_range(number_range: Optional[Tuple[int, int]]) -> Optional[str]:
-    if number_range is None:
-        return None
-
-    try:
-        min_value, max_value = min(*number_range), max(*number_range)
-    except TypeError:
-        raise ValueError("Number range is incomplete, both values must be set")
-
-    return f"{min_value}-{max_value}"
-
-
-class QueryFilters(BaseModel):
-    page: Optional[int] = None
-    order_by: Optional[OrderBy] = None
-    price_range: Optional[PriceRange] = None
-    ...
-
-    def to_dict_raw(self) -> dict:
-        return {
-            "page": self.page,
-            "Order": self.order_by,
-            "price": format_number_range(self.price_range)
-        }
-
-    # TODO: add helper methods for managing the attribute values
-
-    def __iter__(self):
-        for key, value in self.to_dict_raw().items():
-            if value is not None:
-                yield key, value
+from pydantic import BaseModel
+from enum import Enum
+from typing import Optional, Tuple
+
+PriceRange = Tuple[int, int]
+
+
+class OrderBy(int, Enum):
+    DATE = 1
+    PRICE_LOWEST_TO_HIGHEST = 3
+    PRICE_HIGHEST_TO_LOWEST = 4
+    ...
+
+
+def format_number_range(number_range: Optional[Tuple[int, int]]) -> Optional[str]:
+    if number_range is None:
+        return None
+
+    try:
+        min_value, max_value = min(*number_range), max(*number_range)
+    except TypeError:
+        raise ValueError("Number range is incomplete, both values must be set")
+
+    return f"{min_value}-{max_value}"
+
+
+class QueryFilters(BaseModel):
+    page: Optional[int] = None
+    order_by: Optional[OrderBy] = None
+    price_range: Optional[PriceRange] = None
+    ...
+
+    def to_params(self) -> dict:
+        return {
+            "page": self.page,
+            "Order": self.order_by,
+            "price": format_number_range(self.price_range)
+        }
+
+    def to_clean_params(self):
+        return {key: value for key, value in self.to_params().items() if value is not None}
+
+    # TODO: add helper methods for managing the attribute values
+
+    def __iter__(self):
+        yield from self.to_clean_params().items()

yad2_scraper-0.3.0/yad2_scraper/scraper.py

@@ -0,0 +1,174 @@
+import logging
+import httpx
+import time
+import random
+from typing import Optional, Dict, Any, Tuple, Union, Type, TypeVar
+
+from yad2_scraper.category import Yad2Category
+from yad2_scraper.query import QueryFilters
+from yad2_scraper.utils import get_random_user_agent
+from yad2_scraper.exceptions import AntiBotDetectedError, UnexpectedContentError, MaxRequestRetriesExceededError
+from yad2_scraper.constants import (
+    DEFAULT_REQUEST_HEADERS,
+    ALLOW_REQUEST_REDIRECTS,
+    VERIFY_REQUEST_SSL,
+    ANTIBOT_CONTENT_IDENTIFIER,
+    YAD2_CONTENT_IDENTIFIER
+)
+
+Category = TypeVar("Category", bound=Yad2Category)
+DelayRange = Tuple[float, float]
+QueryParamTypes = Union[QueryFilters, Dict[str, Any]]
+
+logger = logging.getLogger(__name__)
+
+
+class Yad2Scraper:
+    def __init__(
+            self,
+            client: Optional[httpx.Client] = None,
+            request_defaults: Optional[Dict[str, Any]] = None,
+            randomize_user_agent: bool = False,
+            random_delay_range: Optional[DelayRange] = None,
+            max_retries: int = 0
+    ):
+        self.client = client or httpx.Client(
+            headers=DEFAULT_REQUEST_HEADERS,
+            follow_redirects=ALLOW_REQUEST_REDIRECTS,
+            verify=VERIFY_REQUEST_SSL
+        )
+        self.request_defaults = request_defaults or {}
+        self.randomize_user_agent = randomize_user_agent
+        self.random_delay_range = random_delay_range
+        self.max_retries = max_retries
+
+        logger.debug(f"Scraper initialized with client: {self.client}")
+
+    def fetch_category(
+            self,
+            url: str,
+            params: Optional[QueryParamTypes] = None,
+            category_type: Type[Category] = Yad2Category
+    ) -> Category:
+        logger.debug(f"Fetching category from URL: '{url}'")
+        response = self.get(url, params)
+        logger.debug(f"Category fetched successfully from URL: '{url}'")
+        return category_type.from_html_io(response)
+
+    def get(self, url: str, params: Optional[QueryParamTypes] = None) -> httpx.Response:
+        return self.request("GET", url, params=params)
+
+    def request(self, method: str, url: str, params: Optional[QueryParamTypes] = None) -> httpx.Response:
+        request_options = self._prepare_request_options(params=params)
+
+        try:
+            return self._send_request(method, url, request_options)
+        except Exception as error:
+            return self._handle_request_error(method, url, request_options, error)
+
+    def set_user_agent(self, user_agent: str) -> None:
+        self.client.headers["User-Agent"] = user_agent
+        logger.debug(f"User-Agent client header set to: '{user_agent}'")
+
+    def set_no_script(self, no_script: bool) -> None:
+        value = "1" if no_script else "0"
+        self.client.cookies.set("noscript", value)
+        logger.debug(f"noscript client cookie set to: '{value}'")
+
+    def close(self) -> None:
+        logger.debug("Closing scraper client")
+        self.client.close()
+        logger.info("Scraper client closed")
+
+    def _send_request(self, method: str, url: str, request_options: Dict[str, Any]) -> httpx.Response:
+        if self.randomize_user_agent:
+            self._set_random_user_agent(request_options)
+
+        if self.random_delay_range:
+            self._apply_request_delay()
+
+        logger.info(f"Making {method} request to URL: '{url}'")
+        response = self.client.request(method, url, **request_options)
+        logger.debug(f"Received response with status code: {response.status_code}")
+        self._validate_response(response)
+
+        return response
+
+    def _handle_request_error(
+            self,
+            method: str,
+            url: str,
+            request_options: Dict[str, Any],
+            error: Exception
+    ) -> httpx.Response:
+        logger.error(f"{method} request to '{url}' failed: {error}")
+
+        if self.max_retries == 0:
+            raise error
+
+        return self._retry_request(method, url, request_options)
+
+    def _retry_request(self, method: str, url: str, request_options: Dict[str, Any]) -> httpx.Response:
+        logger.info(f"Retrying {method} request to '{url}' (max retries: {self.max_retries})")
+
+        errors = []
+
+        for retry_attempt in range(1, self.max_retries + 1):
+            try:
+                logger.debug(f"Retry attempt {retry_attempt}/{self.max_retries}")
+                return self._send_request(method, url, request_options)
+            except Exception as error:
+                logger.warning(f"Retry attempt {retry_attempt} failed: {error}")
+                errors.append(error)
+
+        error_to_raise = MaxRequestRetriesExceededError(method, url, self.max_retries, errors)
+        logger.error(str(error_to_raise))
+        raise error_to_raise from errors[-1]
+
+    def _prepare_request_options(self, params: Optional[QueryParamTypes] = None) -> Dict[str, Any]:
+        logger.debug("Preparing request options from defaults")
+        request_options = self.request_defaults.copy()
+
+        if params:
+            request_options.setdefault("params", {}).update(params)
+            logger.debug(f"Updated request options with query params: {params}")
+
+        return request_options
+
+    def _apply_request_delay(self):
+        delay = random.uniform(*self.random_delay_range)
+        logger.debug(f"Applying request delay of {delay:.2f} seconds")
+        time.sleep(delay)
+
+    @staticmethod
+    def _set_random_user_agent(request_options: Dict[str, str]):
+        user_agent = get_random_user_agent()
+        request_options.setdefault("headers", {})["User-Agent"] = user_agent
+        logger.debug(f"Updated request options with random User-Agent header: '{user_agent}'")
+
+    @staticmethod
+    def _validate_response(response: httpx.Response):
+        response.raise_for_status()
+
+        if ANTIBOT_CONTENT_IDENTIFIER in response.content:
+            raise AntiBotDetectedError(
+                f"The response contains Anti-Bot content",
+                request=response.request,
+                response=response
+            )
+        if YAD2_CONTENT_IDENTIFIER not in response.content:
+            raise UnexpectedContentError(
+                "The response does not contain yad2 content",
+                request=response.request,
+                response=response
+            )
+
+        logger.debug("Response validation succeeded")
+
+    def __enter__(self):
+        logger.debug("Entering scraper context")
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        logger.debug("Exiting scraper context")
+        self.close()

{yad2_scraper-0.1.0 → yad2_scraper-0.3.0}/yad2_scraper/utils.py

@@ -1,39 +1,28 @@
-import httpx
-from fake_useragent import FakeUserAgent
-from bs4 import BeautifulSoup, Tag
-from typing import Union, List
-
-from yad2_scraper.exceptions import AntiBotDetectedError
-from yad2_scraper.constants import ANTIBOT_CONTENT
-
-fua = FakeUserAgent()
-
-
-def get_random_user_agent() -> str:
-    return fua.random
-
-
-def join_url(url: str, path: str) -> str:
-    return url.rstrip("/") + "/" + path.lstrip("/")
-
-
-def get_parent_url(url: str) -> str:
-    if url.count("/") <= 2:
-        return url
-
-    return url.rstrip("/").rsplit("/", 1)[0]
-
-
-def validate_http_response(response: httpx.Response):
-    response.raise_for_status()
-
-    if ANTIBOT_CONTENT in response.content:
-        raise AntiBotDetectedError(f"The response contains Anti-Bot content")
-
-
-def find_html_tag_by_class_substring(e: Union[BeautifulSoup, Tag], tag_name: str, substring: str) -> Tag:
-    return e.find(tag_name, class_=lambda class_name: class_name and substring in class_name)
-
-
-def find_all_html_tags_by_class_substring(e: Union[BeautifulSoup, Tag], tag_name: str, substring: str) -> List[Tag]:
-    return e.find_all(tag_name, class_=lambda class_name: class_name and substring in class_name)
+from fake_useragent import FakeUserAgent
+from bs4 import BeautifulSoup, Tag
+from typing import Union, List
+
+fua = FakeUserAgent()
+
+
+def get_random_user_agent() -> str:
+    return fua.random
+
+
+def join_url(url: str, path: str) -> str:
+    return url.rstrip("/") + "/" + path.lstrip("/")
+
+
+def get_parent_url(url: str) -> str:
+    if url.count("/") <= 2:
+        return url
+
+    return url.rstrip("/").rsplit("/", 1)[0]
+
+
+def find_html_tag_by_class_substring(e: Union[BeautifulSoup, Tag], tag_name: str, substring: str) -> Tag:
+    return e.find(tag_name, class_=lambda class_name: class_name and substring in class_name)
+
+
+def find_all_html_tags_by_class_substring(e: Union[BeautifulSoup, Tag], tag_name: str, substring: str) -> List[Tag]:
+    return e.find_all(tag_name, class_=lambda class_name: class_name and substring in class_name)

yad2_scraper-0.1.0/yad2_scraper/exceptions.py

@@ -1,2 +0,0 @@
-class AntiBotDetectedError(Exception):
-    pass

yad2_scraper-0.1.0/yad2_scraper/scraper.py

@@ -1,107 +0,0 @@
-import logging
-import httpx
-import time
-import random
-from typing import Optional, Dict, Any, Tuple, Union, Type, TypeVar
-
-from yad2_scraper.category import Yad2Category
-from yad2_scraper.query import QueryFilters
-from yad2_scraper.utils import get_random_user_agent, validate_http_response
-from yad2_scraper.constants import (
-    DEFAULT_REQUEST_HEADERS,
-    ALLOW_REQUEST_REDIRECTS,
-    VERIFY_REQUEST_SSL
-)
-
-Category = TypeVar("Category", bound=Yad2Category)
-DelayRange = Tuple[float, float]
-QueryParams = Union[QueryFilters, Dict[str, Any]]
-
-logger = logging.getLogger(__name__)
-
-
-class Yad2Scraper:
-    def __init__(
-            self,
-            session: Optional[httpx.Client] = None,
-            request_kwargs: Dict[str, Any] = None,
-            randomize_user_agent: bool = False,
-            requests_delay_range: Optional[DelayRange] = None,
-    ):
-        self.session = session or httpx.Client(
-            headers=DEFAULT_REQUEST_HEADERS,
-            follow_redirects=ALLOW_REQUEST_REDIRECTS,
-            verify=VERIFY_REQUEST_SSL
-        )
-        self.request_kwargs = request_kwargs or {}
-        self.randomize_user_agent = randomize_user_agent
-        self.requests_delay_range = requests_delay_range
-
-        logger.debug(f"Initialized with session {self.session} and request kwargs: {self.request_kwargs}")
-
-    def set_user_agent(self, user_agent: str):
-        self.session.headers["User-Agent"] = user_agent
-        logger.debug(f"User-Agent session header set to: '{user_agent}'")
-
-    def set_no_script(self, no_script: bool):
-        value = '1' if no_script else '0'  # str(int(no_script))
-        self.session.cookies.set("noscript", value)
-        logger.debug(f"NoScript session cookie set to: '{value}'")
-
-    def fetch_category(
-            self,
-            url: str,
-            query_params: Optional[QueryParams] = None,
-            category_type: Type[Category] = Yad2Category
-    ) -> Category:
-        logger.debug(f"Fetching category from URL: '{url}'")
-        response = self.get(url, query_params)
-        logger.debug(f"Category fetched successfully from URL: '{url}'")
-        return category_type.from_html_io(response)
-
-    def get(self, url: str, query_params: Optional[QueryParams] = None) -> httpx.Response:
-        return self.request("GET", url, query_params=query_params)
-
-    def request(self, method: str, url: str, query_params: Optional[QueryParams] = None) -> httpx.Response:
-        request_kwargs = self._prepare_request_kwargs(query_params=query_params)
-
-        if self.requests_delay_range:
-            self._apply_request_delay()
-
-        try:
-            logger.info(f"Making {method} request to URL: '{url}'")  # request kwargs not logged - may be sensitive
-            response = self.session.request(method, url, **request_kwargs)
-            logger.debug(f"Received response with status code: {response.status_code}")
-
-            validate_http_response(response)
-            logger.debug("Response validation succeeded")
-        except Exception as error:
-            logger.error(f"Request to '{url}' failed: {error}")
-            raise error
-
-        return response
-
-    def _prepare_request_kwargs(self, query_params: Optional[QueryParams] = None) -> Dict[str, Any]:
-        logger.debug("Preparing request kwargs from defaults")
-        request_kwargs = self.request_kwargs.copy()
-
-        if query_params:
-            request_kwargs.setdefault("params", {}).update(query_params)
-            logger.debug(f"Updated request kwargs with query params: {query_params}")
-
-        if self.randomize_user_agent:
-            random_user_agent = get_random_user_agent()
-            request_kwargs.setdefault("headers", {})["User-Agent"] = random_user_agent
-            logger.debug(f"Updated request kwargs with random 'User-Agent' header: '{random_user_agent}'")
-
-        return request_kwargs
-
-    def _apply_request_delay(self):
-        delay = random.uniform(*self.requests_delay_range)
-        logger.debug(f"Applying request delay of {delay:.2f} seconds")
-        time.sleep(delay)
-
-    def close(self):
-        logger.debug("Closing scraper session")
-        self.session.close()
-        logger.info("Scraper session closed")