yad2-scraper 0.1.0__py3-none-any.whl

yad2_scraper/__init__.py

@@ -0,0 +1,4 @@
+from .scraper import Yad2Scraper
+from .query import QueryFilters, OrderBy
+from .category import Yad2Category
+from .next_data import NextData

yad2_scraper/category.py

@@ -0,0 +1,25 @@
+import json
+from bs4 import BeautifulSoup, Tag
+from typing import Optional, List, Union, TextIO, BinaryIO
+
+from yad2_scraper.next_data import NextData
+from yad2_scraper.utils import find_all_html_tags_by_class_substring
+from yad2_scraper.constants import NEXT_DATA_SCRIPT_ID
+
+
+class Yad2Category:
+    def __init__(self, soup: BeautifulSoup):
+        self.soup = soup
+
+    @classmethod
+    def from_html_io(cls, html_io: Union[TextIO, BinaryIO]):
+        html = html_io.read()
+        soup = BeautifulSoup(html, "html.parser")
+        return cls(soup)
+
+    def load_next_data(self) -> Optional[NextData]:
+        tag = self.soup.find("script", id=NEXT_DATA_SCRIPT_ID)
+        return NextData(json.loads(tag.string)) if tag else None
+
+    def find_all_tags_by_class_substring(self, tag_name: str, substring: str) -> List[Tag]:
+        return find_all_html_tags_by_class_substring(self.soup, tag_name, substring)

yad2_scraper/constants.py

@@ -0,0 +1,26 @@
+BASE_URL = "https://www.yad2.co.il"
+
+DEFAULT_REQUEST_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Safari/605.1.15",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
+    "Accept-Encoding": "gzip, deflate, br",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Connection": "keep-alive",
+    "Upgrade-Insecure-Requests": "1",
+    "DNT": "1",
+    "Cache-Control": "max-age=0",
+    "Sec-Fetch-Site": "none",
+    "Sec-Fetch-Mode": "navigate",
+    "Sec-Fetch-User": "?1",
+    "Sec-Fetch-Dest": "document",
+}
+
+ALLOW_REQUEST_REDIRECTS = True
+VERIFY_REQUEST_SSL = True
+
+ANTIBOT_CONTENT = b"Are you for real"  # robot-captcha
+
+FIRST_PAGE = 1
+NOT_MENTIONED_PRICE_RANGE = 0, 0
+
+NEXT_DATA_SCRIPT_ID = "__NEXT_DATA__"

yad2_scraper/exceptions.py

@@ -0,0 +1,2 @@
+class AntiBotDetectedError(Exception):
+    pass

yad2_scraper/next_data.py

@@ -0,0 +1,27 @@
+from enum import Enum
+from typing import List, Union
+
+
+class Field(str, Enum):
+    ID = "id"
+    TEXT = "text"
+    ENGLISH_TEXT = "textEng"
+
+
+FieldTypes = Union[str, int]
+
+
+class NextData:
+    def __init__(self, data: dict):
+        self.data = data
+
+    @property
+    def json(self) -> dict:
+        return self.data
+
+    @property
+    def queries(self) -> List[dict]:
+        return self.data["props"]["pageProps"]["dehydratedState"]["queries"]
+
+    def __getitem__(self, item):
+        return self.data[item]

yad2_scraper/query.py

@@ -0,0 +1,45 @@
+from pydantic import BaseModel
+from enum import Enum
+from typing import Optional, Tuple
+
+PriceRange = Tuple[int, int]
+
+
+class OrderBy(int, Enum):
+    DATE = 1
+    PRICE_LOWEST_TO_HIGHEST = 3
+    PRICE_HIGHEST_TO_LOWEST = 4
+    ...
+
+
+def format_number_range(number_range: Optional[Tuple[int, int]]) -> Optional[str]:
+    if number_range is None:
+        return None
+
+    try:
+        min_value, max_value = min(*number_range), max(*number_range)
+    except TypeError:
+        raise ValueError("Number range is incomplete, both values must be set")
+
+    return f"{min_value}-{max_value}"
+
+
+class QueryFilters(BaseModel):
+    page: Optional[int] = None
+    order_by: Optional[OrderBy] = None
+    price_range: Optional[PriceRange] = None
+    ...
+
+    def to_dict_raw(self) -> dict:
+        return {
+            "page": self.page,
+            "Order": self.order_by,
+            "price": format_number_range(self.price_range)
+        }
+
+    # TODO: add helper methods for managing the attribute values
+
+    def __iter__(self):
+        for key, value in self.to_dict_raw().items():
+            if value is not None:
+                yield key, value

yad2_scraper/scraper.py

@@ -0,0 +1,107 @@
+import logging
+import httpx
+import time
+import random
+from typing import Optional, Dict, Any, Tuple, Union, Type, TypeVar
+
+from yad2_scraper.category import Yad2Category
+from yad2_scraper.query import QueryFilters
+from yad2_scraper.utils import get_random_user_agent, validate_http_response
+from yad2_scraper.constants import (
+    DEFAULT_REQUEST_HEADERS,
+    ALLOW_REQUEST_REDIRECTS,
+    VERIFY_REQUEST_SSL
+)
+
+Category = TypeVar("Category", bound=Yad2Category)
+DelayRange = Tuple[float, float]
+QueryParams = Union[QueryFilters, Dict[str, Any]]
+
+logger = logging.getLogger(__name__)
+
+
+class Yad2Scraper:
+    def __init__(
+            self,
+            session: Optional[httpx.Client] = None,
+            request_kwargs: Dict[str, Any] = None,
+            randomize_user_agent: bool = False,
+            requests_delay_range: Optional[DelayRange] = None,
+    ):
+        self.session = session or httpx.Client(
+            headers=DEFAULT_REQUEST_HEADERS,
+            follow_redirects=ALLOW_REQUEST_REDIRECTS,
+            verify=VERIFY_REQUEST_SSL
+        )
+        self.request_kwargs = request_kwargs or {}
+        self.randomize_user_agent = randomize_user_agent
+        self.requests_delay_range = requests_delay_range
+
+        logger.debug(f"Initialized with session {self.session} and request kwargs: {self.request_kwargs}")
+
+    def set_user_agent(self, user_agent: str):
+        self.session.headers["User-Agent"] = user_agent
+        logger.debug(f"User-Agent session header set to: '{user_agent}'")
+
+    def set_no_script(self, no_script: bool):
+        value = '1' if no_script else '0'  # str(int(no_script))
+        self.session.cookies.set("noscript", value)
+        logger.debug(f"NoScript session cookie set to: '{value}'")
+
+    def fetch_category(
+            self,
+            url: str,
+            query_params: Optional[QueryParams] = None,
+            category_type: Type[Category] = Yad2Category
+    ) -> Category:
+        logger.debug(f"Fetching category from URL: '{url}'")
+        response = self.get(url, query_params)
+        logger.debug(f"Category fetched successfully from URL: '{url}'")
+        return category_type.from_html_io(response)
+
+    def get(self, url: str, query_params: Optional[QueryParams] = None) -> httpx.Response:
+        return self.request("GET", url, query_params=query_params)
+
+    def request(self, method: str, url: str, query_params: Optional[QueryParams] = None) -> httpx.Response:
+        request_kwargs = self._prepare_request_kwargs(query_params=query_params)
+
+        if self.requests_delay_range:
+            self._apply_request_delay()
+
+        try:
+            logger.info(f"Making {method} request to URL: '{url}'")  # request kwargs not logged - may be sensitive
+            response = self.session.request(method, url, **request_kwargs)
+            logger.debug(f"Received response with status code: {response.status_code}")
+
+            validate_http_response(response)
+            logger.debug("Response validation succeeded")
+        except Exception as error:
+            logger.error(f"Request to '{url}' failed: {error}")
+            raise error
+
+        return response
+
+    def _prepare_request_kwargs(self, query_params: Optional[QueryParams] = None) -> Dict[str, Any]:
+        logger.debug("Preparing request kwargs from defaults")
+        request_kwargs = self.request_kwargs.copy()
+
+        if query_params:
+            request_kwargs.setdefault("params", {}).update(query_params)
+            logger.debug(f"Updated request kwargs with query params: {query_params}")
+
+        if self.randomize_user_agent:
+            random_user_agent = get_random_user_agent()
+            request_kwargs.setdefault("headers", {})["User-Agent"] = random_user_agent
+            logger.debug(f"Updated request kwargs with random 'User-Agent' header: '{random_user_agent}'")
+
+        return request_kwargs
+
+    def _apply_request_delay(self):
+        delay = random.uniform(*self.requests_delay_range)
+        logger.debug(f"Applying request delay of {delay:.2f} seconds")
+        time.sleep(delay)
+
+    def close(self):
+        logger.debug("Closing scraper session")
+        self.session.close()
+        logger.info("Scraper session closed")

yad2_scraper/utils.py

@@ -0,0 +1,39 @@
+import httpx
+from fake_useragent import FakeUserAgent
+from bs4 import BeautifulSoup, Tag
+from typing import Union, List
+
+from yad2_scraper.exceptions import AntiBotDetectedError
+from yad2_scraper.constants import ANTIBOT_CONTENT
+
+fua = FakeUserAgent()
+
+
+def get_random_user_agent() -> str:
+    return fua.random
+
+
+def join_url(url: str, path: str) -> str:
+    return url.rstrip("/") + "/" + path.lstrip("/")
+
+
+def get_parent_url(url: str) -> str:
+    if url.count("/") <= 2:
+        return url
+
+    return url.rstrip("/").rsplit("/", 1)[0]
+
+
+def validate_http_response(response: httpx.Response):
+    response.raise_for_status()
+
+    if ANTIBOT_CONTENT in response.content:
+        raise AntiBotDetectedError(f"The response contains Anti-Bot content")
+
+
+def find_html_tag_by_class_substring(e: Union[BeautifulSoup, Tag], tag_name: str, substring: str) -> Tag:
+    return e.find(tag_name, class_=lambda class_name: class_name and substring in class_name)
+
+
+def find_all_html_tags_by_class_substring(e: Union[BeautifulSoup, Tag], tag_name: str, substring: str) -> List[Tag]:
+    return e.find_all(tag_name, class_=lambda class_name: class_name and substring in class_name)

yad2_scraper-0.1.0.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 DavOstx7
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

yad2_scraper-0.1.0.dist-info/METADATA

@@ -0,0 +1,23 @@
+Metadata-Version: 2.1
+Name: yad2-scraper
+Version: 0.1.0
+Summary: Scrape Yad2 in Python.
+License: LICENSE
+Author: dav ost
+Author-email: davidost2003@gmail.com
+Requires-Python: >=3.7
+Classifier: License :: Other/Proprietary License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Requires-Dist: beautifulsoup4 (>=4.11.1,<5.0.0)
+Requires-Dist: fake-useragent (>=0.1.11,<0.2.0)
+Requires-Dist: httpx (>=0.24.0,<0.25.0)
+Description-Content-Type: text/markdown
+
+# yad2-scraper
+Scrape Yad2 in Python.
+

yad2_scraper-0.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+yad2_scraper/__init__.py,sha256=HPe6DlHwLH1P-eSDBEH7h_c2Oy_aA-kt5ZFiO7GQYHE,145
+yad2_scraper/category.py,sha256=Me8UtO0641fYJriqWSMwL702guhJeRKC185L-PtaDrk,935
+yad2_scraper/constants.py,sha256=yhDWQ4LDoHV0Rrly4isWqkSL_9H2qrM7nTcb3xGBAwU,865
+yad2_scraper/exceptions.py,sha256=1xn3ncfaSJZ6WoqGeWckJ8FT2TDU8Yd8fX_Fe8PgbMc,50
+yad2_scraper/next_data.py,sha256=eGO821iZ92ezxKOXrYubP2GomJWohF-rqFJhQQ7lqO0,539
+yad2_scraper/query.py,sha256=Sd07pPZpfKor8-HY6RAapAEc9m3H6ALiYE9AxiKnp30,1188
+yad2_scraper/scraper.py,sha256=hcFXkcOvaj-BCrA9YJNJxgCOGHhlgFYRNm1hw3RCKKI,4309
+yad2_scraper/utils.py,sha256=rmVtziIbhsogIUdDb-um3vLYA30VMZKJI52-cifJv5I,1209
+yad2_scraper-0.1.0.dist-info/LICENSE,sha256=iLHgAD6INoAKYUaoJb5XlbyhlMaKOg-41pTDZFEfaf8,1086
+yad2_scraper-0.1.0.dist-info/METADATA,sha256=RxgLidUCMruhR5o1oPDPxkOmnjqIPNuYXTYuyWEucgc,746
+yad2_scraper-0.1.0.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
+yad2_scraper-0.1.0.dist-info/RECORD,,

yad2_scraper-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 1.7.0
+Root-Is-Purelib: true
+Tag: py3-none-any