yad2-scraper 0.3.0__tar.gz → 0.5.0__tar.gz

{yad2_scraper-0.3.0 → yad2_scraper-0.5.0}/PKG-INFO

@@ -1,14 +1,13 @@
 Metadata-Version: 2.3
 Name: yad2-scraper
-Version: 0.3.0
+Version: 0.5.0
 Summary: Scrape Yad2 in Python.
 License: LICENSE
 Author: dav ost
 Author-email: davidost2003@gmail.com
-Requires-Python: >=3.7
+Requires-Python: >=3.8
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10

{yad2_scraper-0.3.0 → yad2_scraper-0.5.0}/pyproject.toml

@@ -1,13 +1,13 @@
 [tool.poetry]
 name = "yad2-scraper"
-version = "0.3.0"
+version = "0.5.0"
 description = "Scrape Yad2 in Python."
 authors = ["dav ost <davidost2003@gmail.com>"]
 license = "LICENSE"
 readme = "README.md"
 
 [tool.poetry.dependencies]
-python = ">=3.7"
+python = ">=3.8"
 httpx = "^0.24.0"
 httpcore = ">=0.15.0"
 fake-useragent = "^0.1.11"

{yad2_scraper-0.3.0 → yad2_scraper-0.5.0}/yad2_scraper/constants.py

@@ -19,7 +19,7 @@ ALLOW_REQUEST_REDIRECTS = True
 VERIFY_REQUEST_SSL = True
 
 ANTIBOT_CONTENT_IDENTIFIER = b"Are you for real"  # robot-captcha
-YAD2_CONTENT_IDENTIFIER = b"https://www.yad2.co.il/"
+PAGE_CONTENT_IDENTIFIER = b"https://www.yad2.co.il/"
 
 FIRST_PAGE_NUMBER = 1
 NOT_MENTIONED_PRICE_RANGE = 0, 0

yad2_scraper-0.5.0/yad2_scraper/exceptions.py

@@ -0,0 +1,32 @@
+import httpx
+from typing import List, Union
+
+
+class ResponseError(Exception):
+    def __init__(self, msg: str, request: httpx.Request, response: httpx.Response):
+        super().__init__(msg)
+        self.request = request
+        self.response = response
+
+
+class AntiBotDetectedError(ResponseError):
+    pass
+
+
+class UnexpectedContentError(ResponseError):
+    pass
+
+
+class MaxAttemptsExceededError(Exception):
+    def __init__(self, msg: str, max_attempts: int, errors: List[BaseException] = None):
+        super().__init__(msg)
+        self.max_attempts = max_attempts
+        self.errors = errors
+
+
+class MaxRequestAttemptsExceededError(MaxAttemptsExceededError):
+    def __init__(self, method: str, url: str, max_attempts: int, errors: List[Union[httpx.HTTPError, ResponseError]]):
+        msg = f"All {max_attempts} attempts for {method} request to '{url}' have failed"
+        super().__init__(msg, max_attempts, errors)
+        self.method = method
+        self.url = url

yad2_scraper-0.5.0/yad2_scraper/next_data.py

@@ -0,0 +1,50 @@
+from datetime import datetime
+from enum import Enum
+from typing import List, Union
+
+from yad2_scraper.utils import safe_access
+
+FieldTypes = Union[str, int]
+
+safe_access_optional_keys = safe_access(exceptions=(KeyError, TypeError), default=None)
+
+
+class SafeAccessOptionalKeysMeta(type):
+    def __new__(cls, name, bases, dictionary):
+        for attr_name, attr_value in dictionary.items():
+            if callable(attr_value):  # Wrap methods
+                dictionary[attr_name] = safe_access_optional_keys(attr_value)
+            elif isinstance(attr_value, property):  # Wrap properties
+                dictionary[attr_name] = property(
+                    safe_access_optional_keys(attr_value.fget) if attr_value.fget else None,
+                    safe_access_optional_keys(attr_value.fset) if attr_value.fset else None,
+                    safe_access_optional_keys(attr_value.fdel) if attr_value.fdel else None,
+                    attr_value.__doc__,
+                )
+        return super().__new__(cls, name, bases, dictionary)
+
+
+class Field(str, Enum):
+    ID = "id"
+    TEXT = "text"
+    ENGLISH_TEXT = "textEng"
+
+
+def convert_string_date_to_datetime(date_string: str) -> datetime:
+    return datetime.fromisoformat(date_string)
+
+
+class NextData:
+    def __init__(self, data: dict):
+        self.data = data
+
+    @property
+    def json(self) -> dict:
+        return self.data
+
+    @property
+    def queries(self) -> List[dict]:
+        return self.data["props"]["pageProps"]["dehydratedState"]["queries"]
+
+    def __getitem__(self, item):
+        return self.data[item]

{yad2_scraper-0.3.0 → yad2_scraper-0.5.0}/yad2_scraper/query.py

@@ -2,7 +2,7 @@ from pydantic import BaseModel
 from enum import Enum
 from typing import Optional, Tuple
 
-PriceRange = Tuple[int, int]
+NumberRange = Tuple[int, int]
 
 
 class OrderBy(int, Enum):
@@ -27,7 +27,7 @@ def format_number_range(number_range: Optional[Tuple[int, int]]) -> Optional[str
 class QueryFilters(BaseModel):
     page: Optional[int] = None
     order_by: Optional[OrderBy] = None
-    price_range: Optional[PriceRange] = None
+    price_range: Optional[NumberRange] = None
     ...
 
     def to_params(self) -> dict:

{yad2_scraper-0.3.0 → yad2_scraper-0.5.0}/yad2_scraper/scraper.py

@@ -1,25 +1,25 @@
 import logging
 import httpx
 import time
-import random
-from typing import Optional, Dict, Any, Tuple, Union, Type, TypeVar
+from fake_useragent import FakeUserAgent
+from typing import Optional, Dict, Any, Callable, Union, Type, TypeVar
 
 from yad2_scraper.category import Yad2Category
 from yad2_scraper.query import QueryFilters
-from yad2_scraper.utils import get_random_user_agent
-from yad2_scraper.exceptions import AntiBotDetectedError, UnexpectedContentError, MaxRequestRetriesExceededError
+from yad2_scraper.exceptions import AntiBotDetectedError, UnexpectedContentError, MaxRequestAttemptsExceededError
 from yad2_scraper.constants import (
     DEFAULT_REQUEST_HEADERS,
     ALLOW_REQUEST_REDIRECTS,
     VERIFY_REQUEST_SSL,
     ANTIBOT_CONTENT_IDENTIFIER,
-    YAD2_CONTENT_IDENTIFIER
+    PAGE_CONTENT_IDENTIFIER
 )
 
 Category = TypeVar("Category", bound=Yad2Category)
-DelayRange = Tuple[float, float]
+WaitStrategy = Callable[[int], Optional[float]]
 QueryParamTypes = Union[QueryFilters, Dict[str, Any]]
 
+fua = FakeUserAgent()
 logger = logging.getLogger(__name__)
 
 
@@ -28,9 +28,9 @@ class Yad2Scraper:
             self,
             client: Optional[httpx.Client] = None,
             request_defaults: Optional[Dict[str, Any]] = None,
-            randomize_user_agent: bool = False,
-            random_delay_range: Optional[DelayRange] = None,
-            max_retries: int = 0
+            randomize_user_agent: bool = True,
+            wait_strategy: Optional[WaitStrategy] = None,
+            max_request_attempts: int = 1
     ):
         self.client = client or httpx.Client(
             headers=DEFAULT_REQUEST_HEADERS,
@@ -39,16 +39,25 @@ class Yad2Scraper:
         )
         self.request_defaults = request_defaults or {}
         self.randomize_user_agent = randomize_user_agent
-        self.random_delay_range = random_delay_range
-        self.max_retries = max_retries
+        self.wait_strategy = wait_strategy
+        self.max_request_attempts = max_request_attempts
 
         logger.debug(f"Scraper initialized with client: {self.client}")
 
+    def set_user_agent(self, user_agent: str) -> None:
+        self.client.headers["User-Agent"] = user_agent
+        logger.debug(f"User-Agent client header set to: '{user_agent}'")
+
+    def set_no_script(self, no_script: bool) -> None:
+        value = "1" if no_script else "0"
+        self.client.cookies.set("noscript", value)
+        logger.debug(f"NoScript (noscript) client cookie set to: '{value}'")
+
     def fetch_category(
             self,
             url: str,
-            params: Optional[QueryParamTypes] = None,
-            category_type: Type[Category] = Yad2Category
+            category_type: Type[Category] = Yad2Category,
+            params: Optional[QueryParamTypes] = None
     ) -> Category:
         logger.debug(f"Fetching category from URL: '{url}'")
         response = self.get(url, params)
@@ -59,72 +68,48 @@ class Yad2Scraper:
         return self.request("GET", url, params=params)
 
     def request(self, method: str, url: str, params: Optional[QueryParamTypes] = None) -> httpx.Response:
+        if not isinstance(self.max_request_attempts, int):
+            raise TypeError(f"max_request_attempts must be of type 'int', but got {type(self.max_request_attempts)}")
+
+        if self.max_request_attempts <= 0:
+            raise ValueError(f"max_request_attempts must be a positive integer, but got {self.max_request_attempts}")
+
         request_options = self._prepare_request_options(params=params)
+        error_list = []
 
-        try:
-            return self._send_request(method, url, request_options)
-        except Exception as error:
-            return self._handle_request_error(method, url, request_options, error)
+        for attempt in range(1, self.max_request_attempts + 1):
+            try:
+                return self._send_request(method, url, request_options, attempt)
+            except Exception as error:
+                logger.error(f"{method} request to '{url}' failed {self._format_attempt_info(attempt)}: {error}")
+                error_list.append(error)
 
-    def set_user_agent(self, user_agent: str) -> None:
-        self.client.headers["User-Agent"] = user_agent
-        logger.debug(f"User-Agent client header set to: '{user_agent}'")
+        if self.max_request_attempts == 1:
+            raise error_list[0]  # only one error exists, raise it
 
-    def set_no_script(self, no_script: bool) -> None:
-        value = "1" if no_script else "0"
-        self.client.cookies.set("noscript", value)
-        logger.debug(f"noscript client cookie set to: '{value}'")
+        max_attempts_error = MaxRequestAttemptsExceededError(method, url, self.max_request_attempts, error_list)
+        logger.error(str(max_attempts_error))
+        raise max_attempts_error from error_list[-1]  # multiple errors exist, raise from the last one
 
     def close(self) -> None:
         logger.debug("Closing scraper client")
         self.client.close()
         logger.info("Scraper client closed")
 
-    def _send_request(self, method: str, url: str, request_options: Dict[str, Any]) -> httpx.Response:
+    def _send_request(self, method: str, url: str, request_options: Dict[str, Any], attempt: int) -> httpx.Response:
         if self.randomize_user_agent:
             self._set_random_user_agent(request_options)
 
-        if self.random_delay_range:
-            self._apply_request_delay()
+        if self.wait_strategy:
+            self._apply_wait_strategy(attempt)
 
-        logger.info(f"Making {method} request to URL: '{url}'")
+        logger.info(f"Sending {method} request to URL: '{url}' {self._format_attempt_info(attempt)}")
         response = self.client.request(method, url, **request_options)
-        logger.debug(f"Received response with status code: {response.status_code}")
+        logger.debug(f"Received response {response.status_code} from '{url}' {self._format_attempt_info(attempt)}")
         self._validate_response(response)
 
         return response
 
-    def _handle_request_error(
-            self,
-            method: str,
-            url: str,
-            request_options: Dict[str, Any],
-            error: Exception
-    ) -> httpx.Response:
-        logger.error(f"{method} request to '{url}' failed: {error}")
-
-        if self.max_retries == 0:
-            raise error
-
-        return self._retry_request(method, url, request_options)
-
-    def _retry_request(self, method: str, url: str, request_options: Dict[str, Any]) -> httpx.Response:
-        logger.info(f"Retrying {method} request to '{url}' (max retries: {self.max_retries})")
-
-        errors = []
-
-        for retry_attempt in range(1, self.max_retries + 1):
-            try:
-                logger.debug(f"Retry attempt {retry_attempt}/{self.max_retries}")
-                return self._send_request(method, url, request_options)
-            except Exception as error:
-                logger.warning(f"Retry attempt {retry_attempt} failed: {error}")
-                errors.append(error)
-
-        error_to_raise = MaxRequestRetriesExceededError(method, url, self.max_retries, errors)
-        logger.error(str(error_to_raise))
-        raise error_to_raise from errors[-1]
-
     def _prepare_request_options(self, params: Optional[QueryParamTypes] = None) -> Dict[str, Any]:
         logger.debug("Preparing request options from defaults")
         request_options = self.request_defaults.copy()
@@ -135,17 +120,20 @@ class Yad2Scraper:
 
         return request_options
 
-    def _apply_request_delay(self):
-        delay = random.uniform(*self.random_delay_range)
-        logger.debug(f"Applying request delay of {delay:.2f} seconds")
-        time.sleep(delay)
-
     @staticmethod
     def _set_random_user_agent(request_options: Dict[str, str]):
-        user_agent = get_random_user_agent()
+        user_agent = fua.random
         request_options.setdefault("headers", {})["User-Agent"] = user_agent
         logger.debug(f"Updated request options with random User-Agent header: '{user_agent}'")
 
+    def _apply_wait_strategy(self, attempt: int):
+        wait_time = self.wait_strategy(attempt)
+        if not wait_time:
+            return
+
+        logger.debug(f"Waiting {wait_time:.2f} seconds before request {self._format_attempt_info(attempt)}")
+        time.sleep(wait_time)
+
     @staticmethod
     def _validate_response(response: httpx.Response):
         response.raise_for_status()
@@ -156,15 +144,18 @@ class Yad2Scraper:
                 request=response.request,
                 response=response
             )
-        if YAD2_CONTENT_IDENTIFIER not in response.content:
+        if response.request.method == "GET" and PAGE_CONTENT_IDENTIFIER not in response.content:
             raise UnexpectedContentError(
-                "The response does not contain yad2 content",
+                "The GET response does not contain yad2 related content",
                 request=response.request,
                 response=response
             )
 
         logger.debug("Response validation succeeded")
 
+    def _format_attempt_info(self, attempt: int) -> str:
+        return f"(attempt {attempt}/{self.max_request_attempts})"
+
     def __enter__(self):
         logger.debug("Entering scraper context")
         return self

{yad2_scraper-0.3.0 → yad2_scraper-0.5.0}/yad2_scraper/utils.py

@@ -1,12 +1,6 @@
-from fake_useragent import FakeUserAgent
+import functools
 from bs4 import BeautifulSoup, Tag
-from typing import Union, List
-
-fua = FakeUserAgent()
-
-
-def get_random_user_agent() -> str:
-    return fua.random
+from typing import Union, List, Tuple, Any
 
 
 def join_url(url: str, path: str) -> str:
@@ -26,3 +20,17 @@ def find_html_tag_by_class_substring(e: Union[BeautifulSoup, Tag], tag_name: str
 
 def find_all_html_tags_by_class_substring(e: Union[BeautifulSoup, Tag], tag_name: str, substring: str) -> List[Tag]:
     return e.find_all(tag_name, class_=lambda class_name: class_name and substring in class_name)
+
+
+def safe_access(exceptions: Tuple = (), default: Any = None):
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            try:
+                return func(*args, **kwargs)
+            except exceptions:
+                return default
+
+        return wrapper
+
+    return decorator

yad2_scraper-0.5.0/yad2_scraper/vehicles/__init__.py

@@ -0,0 +1,5 @@
+from .urls import VEHICLES_URL, VehicleType, get_vehicle_url
+from .query import VehiclesQueryFilters, OrderVehiclesBy
+from .category import Yad2VehiclesCategory
+from .tag import VehicleTag
+from .next_data import VehiclesNextData

yad2_scraper-0.5.0/yad2_scraper/vehicles/category.py

@@ -0,0 +1,15 @@
+from typing import List, Optional
+
+from yad2_scraper.category import Yad2Category
+from yad2_scraper.vehicles.tag import VehicleTag
+from yad2_scraper.vehicles.next_data import VehiclesNextData
+
+
+class Yad2VehiclesCategory(Yad2Category):
+    def get_vehicle_tags(self) -> List[VehicleTag]:
+        tags = self.find_all_tags_by_class_substring("div", "feedItemBox")
+        return [VehicleTag(tag) for tag in tags]
+
+    def load_next_data(self) -> Optional[VehiclesNextData]:
+        next_data = super().load_next_data()
+        return VehiclesNextData(next_data) if next_data else None

yad2_scraper-0.5.0/yad2_scraper/vehicles/next_data.py

@@ -0,0 +1,318 @@
+import itertools
+from datetime import datetime
+from typing import List, Any, Iterator, Optional
+
+from yad2_scraper.next_data import (
+    SafeAccessOptionalKeysMeta,
+    NextData,
+    Field,
+    FieldTypes,
+    convert_string_date_to_datetime
+)
+from yad2_scraper.utils import join_url
+from yad2_scraper.vehicles.urls import VEHICLES_URL
+
+
+class VehicleData(metaclass=SafeAccessOptionalKeysMeta):
+    def __init__(self, data: dict):
+        self.data = data
+
+    @property
+    def token(self) -> str:
+        return self["token"]
+
+    @property
+    def page_link(self) -> str:
+        return join_url(VEHICLES_URL, f"item/{self.token}")
+
+    @property
+    def price(self) -> int:
+        return self["price"]
+
+    @property
+    def customer(self) -> dict:
+        return self["customer"]
+
+    @property
+    def customer_name(self) -> str:
+        return self.customer["name"]
+
+    @property
+    def customer_phone(self) -> str:
+        return self.customer["phone"]
+
+    @property
+    def address(self) -> dict:
+        return self["address"]
+
+    def top_area(self, field: Field = Field.TEXT) -> Optional[FieldTypes]:
+        return self["address"]["topArea"][field]
+
+    def area(self, field: Field = Field.TEXT) -> Optional[FieldTypes]:
+        return self["address"]["area"][field]
+
+    def city(self, field: Field = Field.TEXT) -> Optional[FieldTypes]:
+        return self["address"]["city"][field]
+
+    @property
+    def metadata(self) -> dict:
+        return self["metaData"]
+
+    @property
+    def video(self) -> str:
+        return self.metadata["video"]
+
+    @property
+    def cover_image(self) -> str:
+        return self.metadata["coverImage"]
+
+    @property
+    def images(self) -> str:
+        return self.metadata["images"]
+
+    @property
+    def description(self) -> str:
+        return self.metadata["description"]
+
+    @property
+    def dates(self) -> dict:
+        return self["dates"]
+
+    @property
+    def updated_at(self) -> datetime:
+        return convert_string_date_to_datetime(self.dates["updatedAt"])
+
+    @property
+    def created_at(self) -> datetime:
+        return convert_string_date_to_datetime(self.dates["createdAt"])
+
+    @property
+    def ends_at(self) -> datetime:
+        return convert_string_date_to_datetime(self.dates["endsAt"])
+
+    @property
+    def rebounced_at(self) -> datetime:
+        return convert_string_date_to_datetime(self.dates["rebouncedAt"])
+
+    def manufacturer(self, field: Field = Field.TEXT) -> Optional[FieldTypes]:
+        return self["manufacturer"][field]
+
+    def color(self, field: Field = Field.TEXT) -> Optional[FieldTypes]:
+        return self["color"][field]
+
+    @property
+    def km(self) -> Optional[int]:
+        return self["km"]
+
+    @property
+    def hand(self, field: Field = Field.ID) -> Optional[FieldTypes]:
+        return self["hand"][field]
+
+    @property
+    def engine_volume(self) -> Optional[int]:
+        return self["engineVolume"]
+
+    @property
+    def horse_power(self) -> Optional[int]:
+        return self["horsePower"]
+
+    @property
+    def previous_owner(self, field: Field = Field.TEXT) -> Optional[FieldTypes]:
+        return self["previousOwner"][field]
+
+    @property
+    def above_price(self) -> Optional[int]:
+        return self["abovePrice"]
+
+    @property
+    def tags(self) -> List[dict]:
+        return self["tags"]
+
+    @property
+    def is_contact_lead_supported(self) -> Optional[bool]:
+        return self["isContactLeadSupported"]
+
+    @property
+    def vehicle_dates(self) -> dict:
+        return self["vehicleDates"]
+
+    @property
+    def year_of_production(self) -> Optional[int]:
+        return self.vehicle_dates["yearOfProduction"]
+
+    @property
+    def month_of_production(self) -> Optional[int]:
+        return self.vehicle_dates["monthOfProduction"]["id"]
+
+    @property
+    def test_date(self) -> Optional[datetime]:
+        return convert_string_date_to_datetime(self.vehicle_dates["testDate"])
+
+    def model(self, field: Field = Field.TEXT) -> Optional[FieldTypes]:
+        return self["model"][field]
+
+    @property
+    def sub_model(self) -> Optional[str]:
+        return self["subModel"]
+
+    def gear_box(self, field: Field = Field.TEXT) -> Optional[FieldTypes]:
+        return self["gearBox"][field]
+
+    def car_family_types(self, field: Field = Field.TEXT) -> Optional[List[FieldTypes]]:
+        return [obj[field] for obj in self["carFamilyType"]]
+
+    def engine_type(self, field: Field = Field.TEXT) -> Optional[FieldTypes]:
+        return self["engineType"][field]
+
+    @property
+    def seats(self) -> Optional[int]:
+        return self["seats"]
+
+    @property
+    def number_of_doors(self) -> Optional[int]:
+        return self["numberOfDoors"]
+
+    @property
+    def owner(self) -> Optional[str]:
+        return self["owner"]["text"]
+
+    @property
+    def body_type(self) -> Optional[str]:
+        return self["bodyType"]["text"]
+
+    @property
+    def combined_fuel_consumption(self) -> Optional[float]:
+        return self["combinedFuelConsumption"]
+
+    @property
+    def power_train_architecture(self) -> Optional[str]:
+        return self["powertrainArchitecture"]
+
+    def car_tags(self, field: Field = Field.TEXT) -> Optional[List[FieldTypes]]:
+        return [obj[field] for obj in self["carTag"]]
+
+    @property
+    def specification(self) -> dict:
+        return self["specification"]
+
+    @property
+    def has_air_conditioner(self) -> Optional[bool]:
+        return self.specification["airConditioner"]
+
+    @property
+    def has_power_steering(self) -> Optional[bool]:
+        return self.specification["powerSteering"]
+
+    @property
+    def has_magnesium_wheel(self) -> Optional[bool]:
+        return self.specification["magnesiumWheel"]
+
+    @property
+    def has_tire_pressure_monitoring_system(self) -> Optional[bool]:
+        return self.specification["tirePressureMonitoringSystem"]
+
+    @property
+    def has_abs(self) -> Optional[bool]:
+        return self.specification["abs"]
+
+    @property
+    def air_bags(self) -> Optional[int]:
+        return self.specification["airBags"]
+
+    @property
+    def has_control_stability(self) -> Optional[bool]:
+        return self.specification["controlStability"]
+
+    @property
+    def has_electric_window(self) -> Optional[int]:
+        return self.specification["electricWindow"]
+
+    @property
+    def has_breaking_assist_system(self) -> Optional[bool]:
+        return self.specification["breakingAssistSystem"]
+
+    @property
+    def has_reverse_camera(self) -> Optional[bool]:
+        return self.specification["reverseCamera"]
+
+    @property
+    def has_adaptive_cruise_control(self) -> Optional[bool]:
+        return self.specification["adaptiveCruiseControl"]
+
+    @property
+    def has_high_beams_auto_control(self) -> Optional[bool]:
+        return self.specification["highBeamsAutoControl"]
+
+    @property
+    def has_blind_spot_assist(self) -> Optional[bool]:
+        return self.specification["blindSpotAssist"]
+
+    @property
+    def has_identify_pedestrians(self) -> Optional[bool]:
+        return self.specification["identifyPedestrians"]
+
+    @property
+    def has_seat_belts_sensors(self) -> Optional[bool]:
+        return self.specification["seatBeltsSensors"]
+
+    @property
+    def has_identifying_dangerous_nearing(self) -> Optional[bool]:
+        return self.specification["identifyingDangerousNearing"]
+
+    @property
+    def has_auto_lighting_in_forward(self) -> Optional[bool]:
+        return self.specification["autoLightingInForward"]
+
+    @property
+    def has_identify_traffic_signs(self) -> Optional[bool]:
+        return self.specification["identifyTrafficSigns"]
+
+    def ignition(self, field: Field = Field.TEXT) -> Optional[FieldTypes]:
+        return self.specification["ignition"][field]
+
+    @property
+    def safety_points(self) -> Optional[int]:
+        return self.specification["safetyPoints"]
+
+    @property
+    def is_handicapped_friendly(self) -> Optional[bool]:
+        return self.specification["isHandicappedFriendly"]
+
+    @property
+    def has_sun_roof(self) -> Optional[bool]:
+        return self.specification["sunRoof"]
+
+    @property
+    def is_turbo(self) -> Optional[bool]:
+        return self.specification["isTurbo"]
+
+    @property
+    def has_road_deviation_control(self) -> Optional[bool]:
+        return self.specification["roadDeviationControl"]
+
+    @property
+    def has_forward_distance_monitor(self) -> Optional[bool]:
+        return self.specification["forwardDistanceMonitor"]
+
+    @property
+    def has_box(self) -> Optional[bool]:
+        return self.specification["box"]
+
+    def __getitem__(self, key: str) -> Any:
+        return self.data[key]
+
+
+class VehiclesNextData(NextData):
+    def iterate_vehicles(self) -> Iterator[VehicleData]:
+        for query in self.queries:
+            data = query["state"].get("data")
+
+            if not data or isinstance(data, list):
+                continue
+
+            for vehicle_data in itertools.chain.from_iterable(data.values()):
+                if isinstance(vehicle_data, dict):
+                    yield VehicleData(vehicle_data)
+
+    def __getitem__(self, item):
+        return self.data[item]

yad2_scraper-0.5.0/yad2_scraper/vehicles/query.py

@@ -0,0 +1,25 @@
+from enum import Enum
+from typing import Optional
+
+from yad2_scraper.query import QueryFilters, OrderBy, NumberRange, format_number_range
+
+
+class OrderVehiclesBy(int, Enum):
+    DATE = OrderBy.DATE
+    PRICE_LOWEST_TO_HIGHEST = OrderBy.PRICE_LOWEST_TO_HIGHEST
+    PRICE_HIGHEST_TO_LOWEST = OrderBy.PRICE_HIGHEST_TO_LOWEST
+    DISTANCE_LOWEST_TO_HIGHEST = 5
+    YEAR_HIGHEST_TO_LOWEST = 6
+
+
+class VehiclesQueryFilters(QueryFilters):
+    year_range: Optional[NumberRange] = None
+    ...
+
+    def to_params(self) -> dict:
+        return {
+            **super().to_params(),
+            "year": format_number_range(self.year_range)
+        }
+
+# TODO: add QueryParams class for each vehicle type (some share the same attributes - sometimes with different enums)

yad2_scraper-0.5.0/yad2_scraper/vehicles/tag.py

@@ -0,0 +1,63 @@
+from functools import cached_property
+from bs4 import Tag
+from typing import Optional
+
+from yad2_scraper.utils import join_url, find_html_tag_by_class_substring
+from yad2_scraper.vehicles.urls import VEHICLES_URL
+
+YEAR_AND_HAND_TAG_SEPARATOR = " • "
+
+
+class VehicleTag:
+    def __init__(self, tag: Tag):
+        self.tag = tag
+
+    @cached_property
+    def relative_link(self) -> str:
+        return self.find_tag_by_class_substring("a", "itemLink")["href"]
+
+    @property
+    def page_link(self) -> str:
+        return join_url(VEHICLES_URL, self.relative_link)
+
+    @cached_property
+    def image_url(self) -> str:
+        return self.find_tag_by_class_substring("img", "image")["src"]
+
+    @cached_property
+    def model(self) -> str:
+        return self.find_tag_by_class_substring("span", "heading").text.strip()
+
+    @cached_property
+    def marketing_text(self) -> str:
+        return self.find_tag_by_class_substring("span", "marketingText").text.strip()
+
+    @cached_property
+    def year_and_hand_string(self) -> str:
+        return self.find_tag_by_class_substring("span", "yearAndHand").text.strip()
+
+    @property
+    def year(self) -> int:
+        year, _ = self.year_and_hand_string.split(YEAR_AND_HAND_TAG_SEPARATOR)
+        return int(year)
+
+    @property
+    def hand(self) -> int:
+        _, hand_string = self.year_and_hand_string.split(YEAR_AND_HAND_TAG_SEPARATOR)
+        _, hand = hand_string.split()
+        return int(hand)
+
+    @cached_property
+    def price_string(self) -> str:
+        return self.find_tag_by_class_substring("span", "price").text.strip()
+
+    @property
+    def price(self) -> Optional[int]:
+        try:
+            price, _ = self.price_string.split()
+            return int(price.replace(",", ""))
+        except ValueError:
+            return None
+
+    def find_tag_by_class_substring(self, tag_name: str, substring: str) -> Tag:
+        return find_html_tag_by_class_substring(self.tag, tag_name, substring)

yad2_scraper-0.5.0/yad2_scraper/vehicles/urls.py

@@ -0,0 +1,16 @@
+from typing import Literal, get_args
+
+from yad2_scraper.utils import join_url
+from yad2_scraper.constants import BASE_URL
+
+VEHICLES_URL = join_url(BASE_URL, "vehicles")
+
+VehicleType = Literal["cars", "motorcycles", "scooters", "trucks", "watercraft", "others"]
+
+_VALID_VEHICLE_TYPES = get_args(VehicleType)
+
+
+def get_vehicle_url(vehicle_type: VehicleType) -> str:
+    if vehicle_type not in _VALID_VEHICLE_TYPES:
+        raise ValueError(f"Invalid vehicle type: {repr(vehicle_type)}. Expected one of {_VALID_VEHICLE_TYPES}")
+    return join_url(VEHICLES_URL, vehicle_type)

yad2_scraper-0.3.0/yad2_scraper/exceptions.py

@@ -1,31 +0,0 @@
-import httpx
-from typing import List
-
-
-class ResponseError(httpx.HTTPStatusError):
-    # This adds the request/response objects to the error
-    pass
-
-
-class AntiBotDetectedError(ResponseError):
-    pass
-
-
-class UnexpectedContentError(ResponseError):
-    pass
-
-
-class MaxRetriesExceededError(Exception):
-    def __init__(self, msg: str, errors: List[Exception] = None):
-        super().__init__(msg)
-        self.errors = errors
-
-
-class MaxRequestRetriesExceededError(MaxRetriesExceededError):
-    def __init__(self, method: str, url: str, max_retries: int, errors: List[Exception] = None):
-        self.method = method
-        self.url = url
-        self.max_retries = max_retries
-
-        msg = f"All {self.max_retries} retry attempts for {self.method} request to '{self.url}' have failed"
-        super().__init__(msg, errors)

yad2_scraper-0.3.0/yad2_scraper/next_data.py

@@ -1,27 +0,0 @@
-from enum import Enum
-from typing import List, Union
-
-
-class Field(str, Enum):
-    ID = "id"
-    TEXT = "text"
-    ENGLISH_TEXT = "textEng"
-
-
-FieldTypes = Union[str, int]
-
-
-class NextData:
-    def __init__(self, data: dict):
-        self.data = data
-
-    @property
-    def json(self) -> dict:
-        return self.data
-
-    @property
-    def queries(self) -> List[dict]:
-        return self.data["props"]["pageProps"]["dehydratedState"]["queries"]
-
-    def __getitem__(self, item):
-        return self.data[item]