yad2-scraper 0.4.0__tar.gz → 0.5.0__tar.gz

{yad2_scraper-0.4.0 → yad2_scraper-0.5.0}/PKG-INFO

@@ -1,14 +1,13 @@
 Metadata-Version: 2.3
 Name: yad2-scraper
-Version: 0.4.0
+Version: 0.5.0
 Summary: Scrape Yad2 in Python.
 License: LICENSE
 Author: dav ost
 Author-email: davidost2003@gmail.com
-Requires-Python: >=3.7
+Requires-Python: >=3.8
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10

{yad2_scraper-0.4.0 → yad2_scraper-0.5.0}/pyproject.toml

@@ -1,13 +1,13 @@
 [tool.poetry]
 name = "yad2-scraper"
-version = "0.4.0"
+version = "0.5.0"
 description = "Scrape Yad2 in Python."
 authors = ["dav ost <davidost2003@gmail.com>"]
 license = "LICENSE"
 readme = "README.md"
 
 [tool.poetry.dependencies]
-python = ">=3.7"
+python = ">=3.8"
 httpx = "^0.24.0"
 httpcore = ">=0.15.0"
 fake-useragent = "^0.1.11"

yad2_scraper-0.5.0/yad2_scraper/next_data.py

@@ -0,0 +1,50 @@
+from datetime import datetime
+from enum import Enum
+from typing import List, Union
+
+from yad2_scraper.utils import safe_access
+
+FieldTypes = Union[str, int]
+
+safe_access_optional_keys = safe_access(exceptions=(KeyError, TypeError), default=None)
+
+
+class SafeAccessOptionalKeysMeta(type):
+    def __new__(cls, name, bases, dictionary):
+        for attr_name, attr_value in dictionary.items():
+            if callable(attr_value):  # Wrap methods
+                dictionary[attr_name] = safe_access_optional_keys(attr_value)
+            elif isinstance(attr_value, property):  # Wrap properties
+                dictionary[attr_name] = property(
+                    safe_access_optional_keys(attr_value.fget) if attr_value.fget else None,
+                    safe_access_optional_keys(attr_value.fset) if attr_value.fset else None,
+                    safe_access_optional_keys(attr_value.fdel) if attr_value.fdel else None,
+                    attr_value.__doc__,
+                )
+        return super().__new__(cls, name, bases, dictionary)
+
+
+class Field(str, Enum):
+    ID = "id"
+    TEXT = "text"
+    ENGLISH_TEXT = "textEng"
+
+
+def convert_string_date_to_datetime(date_string: str) -> datetime:
+    return datetime.fromisoformat(date_string)
+
+
+class NextData:
+    def __init__(self, data: dict):
+        self.data = data
+
+    @property
+    def json(self) -> dict:
+        return self.data
+
+    @property
+    def queries(self) -> List[dict]:
+        return self.data["props"]["pageProps"]["dehydratedState"]["queries"]
+
+    def __getitem__(self, item):
+        return self.data[item]

{yad2_scraper-0.4.0 → yad2_scraper-0.5.0}/yad2_scraper/query.py

@@ -2,7 +2,7 @@ from pydantic import BaseModel
 from enum import Enum
 from typing import Optional, Tuple
 
-PriceRange = Tuple[int, int]
+NumberRange = Tuple[int, int]
 
 
 class OrderBy(int, Enum):
@@ -27,7 +27,7 @@ def format_number_range(number_range: Optional[Tuple[int, int]]) -> Optional[str
 class QueryFilters(BaseModel):
     page: Optional[int] = None
     order_by: Optional[OrderBy] = None
-    price_range: Optional[PriceRange] = None
+    price_range: Optional[NumberRange] = None
     ...
 
     def to_params(self) -> dict:

{yad2_scraper-0.4.0 → yad2_scraper-0.5.0}/yad2_scraper/utils.py

@@ -1,5 +1,6 @@
+import functools
 from bs4 import BeautifulSoup, Tag
-from typing import Union, List
+from typing import Union, List, Tuple, Any
 
 
 def join_url(url: str, path: str) -> str:
@@ -19,3 +20,17 @@ def find_html_tag_by_class_substring(e: Union[BeautifulSoup, Tag], tag_name: str
 
 def find_all_html_tags_by_class_substring(e: Union[BeautifulSoup, Tag], tag_name: str, substring: str) -> List[Tag]:
     return e.find_all(tag_name, class_=lambda class_name: class_name and substring in class_name)
+
+
+def safe_access(exceptions: Tuple = (), default: Any = None):
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            try:
+                return func(*args, **kwargs)
+            except exceptions:
+                return default
+
+        return wrapper
+
+    return decorator

yad2_scraper-0.5.0/yad2_scraper/vehicles/__init__.py

@@ -0,0 +1,5 @@
+from .urls import VEHICLES_URL, VehicleType, get_vehicle_url
+from .query import VehiclesQueryFilters, OrderVehiclesBy
+from .category import Yad2VehiclesCategory
+from .tag import VehicleTag
+from .next_data import VehiclesNextData

yad2_scraper-0.5.0/yad2_scraper/vehicles/category.py

@@ -0,0 +1,15 @@
+from typing import List, Optional
+
+from yad2_scraper.category import Yad2Category
+from yad2_scraper.vehicles.tag import VehicleTag
+from yad2_scraper.vehicles.next_data import VehiclesNextData
+
+
+class Yad2VehiclesCategory(Yad2Category):
+    def get_vehicle_tags(self) -> List[VehicleTag]:
+        tags = self.find_all_tags_by_class_substring("div", "feedItemBox")
+        return [VehicleTag(tag) for tag in tags]
+
+    def load_next_data(self) -> Optional[VehiclesNextData]:
+        next_data = super().load_next_data()
+        return VehiclesNextData(next_data) if next_data else None

yad2_scraper-0.5.0/yad2_scraper/vehicles/next_data.py

@@ -0,0 +1,318 @@
+import itertools
+from datetime import datetime
+from typing import List, Any, Iterator, Optional
+
+from yad2_scraper.next_data import (
+    SafeAccessOptionalKeysMeta,
+    NextData,
+    Field,
+    FieldTypes,
+    convert_string_date_to_datetime
+)
+from yad2_scraper.utils import join_url
+from yad2_scraper.vehicles.urls import VEHICLES_URL
+
+
+class VehicleData(metaclass=SafeAccessOptionalKeysMeta):
+    def __init__(self, data: dict):
+        self.data = data
+
+    @property
+    def token(self) -> str:
+        return self["token"]
+
+    @property
+    def page_link(self) -> str:
+        return join_url(VEHICLES_URL, f"item/{self.token}")
+
+    @property
+    def price(self) -> int:
+        return self["price"]
+
+    @property
+    def customer(self) -> dict:
+        return self["customer"]
+
+    @property
+    def customer_name(self) -> str:
+        return self.customer["name"]
+
+    @property
+    def customer_phone(self) -> str:
+        return self.customer["phone"]
+
+    @property
+    def address(self) -> dict:
+        return self["address"]
+
+    def top_area(self, field: Field = Field.TEXT) -> Optional[FieldTypes]:
+        return self["address"]["topArea"][field]
+
+    def area(self, field: Field = Field.TEXT) -> Optional[FieldTypes]:
+        return self["address"]["area"][field]
+
+    def city(self, field: Field = Field.TEXT) -> Optional[FieldTypes]:
+        return self["address"]["city"][field]
+
+    @property
+    def metadata(self) -> dict:
+        return self["metaData"]
+
+    @property
+    def video(self) -> str:
+        return self.metadata["video"]
+
+    @property
+    def cover_image(self) -> str:
+        return self.metadata["coverImage"]
+
+    @property
+    def images(self) -> str:
+        return self.metadata["images"]
+
+    @property
+    def description(self) -> str:
+        return self.metadata["description"]
+
+    @property
+    def dates(self) -> dict:
+        return self["dates"]
+
+    @property
+    def updated_at(self) -> datetime:
+        return convert_string_date_to_datetime(self.dates["updatedAt"])
+
+    @property
+    def created_at(self) -> datetime:
+        return convert_string_date_to_datetime(self.dates["createdAt"])
+
+    @property
+    def ends_at(self) -> datetime:
+        return convert_string_date_to_datetime(self.dates["endsAt"])
+
+    @property
+    def rebounced_at(self) -> datetime:
+        return convert_string_date_to_datetime(self.dates["rebouncedAt"])
+
+    def manufacturer(self, field: Field = Field.TEXT) -> Optional[FieldTypes]:
+        return self["manufacturer"][field]
+
+    def color(self, field: Field = Field.TEXT) -> Optional[FieldTypes]:
+        return self["color"][field]
+
+    @property
+    def km(self) -> Optional[int]:
+        return self["km"]
+
+    @property
+    def hand(self, field: Field = Field.ID) -> Optional[FieldTypes]:
+        return self["hand"][field]
+
+    @property
+    def engine_volume(self) -> Optional[int]:
+        return self["engineVolume"]
+
+    @property
+    def horse_power(self) -> Optional[int]:
+        return self["horsePower"]
+
+    @property
+    def previous_owner(self, field: Field = Field.TEXT) -> Optional[FieldTypes]:
+        return self["previousOwner"][field]
+
+    @property
+    def above_price(self) -> Optional[int]:
+        return self["abovePrice"]
+
+    @property
+    def tags(self) -> List[dict]:
+        return self["tags"]
+
+    @property
+    def is_contact_lead_supported(self) -> Optional[bool]:
+        return self["isContactLeadSupported"]
+
+    @property
+    def vehicle_dates(self) -> dict:
+        return self["vehicleDates"]
+
+    @property
+    def year_of_production(self) -> Optional[int]:
+        return self.vehicle_dates["yearOfProduction"]
+
+    @property
+    def month_of_production(self) -> Optional[int]:
+        return self.vehicle_dates["monthOfProduction"]["id"]
+
+    @property
+    def test_date(self) -> Optional[datetime]:
+        return convert_string_date_to_datetime(self.vehicle_dates["testDate"])
+
+    def model(self, field: Field = Field.TEXT) -> Optional[FieldTypes]:
+        return self["model"][field]
+
+    @property
+    def sub_model(self) -> Optional[str]:
+        return self["subModel"]
+
+    def gear_box(self, field: Field = Field.TEXT) -> Optional[FieldTypes]:
+        return self["gearBox"][field]
+
+    def car_family_types(self, field: Field = Field.TEXT) -> Optional[List[FieldTypes]]:
+        return [obj[field] for obj in self["carFamilyType"]]
+
+    def engine_type(self, field: Field = Field.TEXT) -> Optional[FieldTypes]:
+        return self["engineType"][field]
+
+    @property
+    def seats(self) -> Optional[int]:
+        return self["seats"]
+
+    @property
+    def number_of_doors(self) -> Optional[int]:
+        return self["numberOfDoors"]
+
+    @property
+    def owner(self) -> Optional[str]:
+        return self["owner"]["text"]
+
+    @property
+    def body_type(self) -> Optional[str]:
+        return self["bodyType"]["text"]
+
+    @property
+    def combined_fuel_consumption(self) -> Optional[float]:
+        return self["combinedFuelConsumption"]
+
+    @property
+    def power_train_architecture(self) -> Optional[str]:
+        return self["powertrainArchitecture"]
+
+    def car_tags(self, field: Field = Field.TEXT) -> Optional[List[FieldTypes]]:
+        return [obj[field] for obj in self["carTag"]]
+
+    @property
+    def specification(self) -> dict:
+        return self["specification"]
+
+    @property
+    def has_air_conditioner(self) -> Optional[bool]:
+        return self.specification["airConditioner"]
+
+    @property
+    def has_power_steering(self) -> Optional[bool]:
+        return self.specification["powerSteering"]
+
+    @property
+    def has_magnesium_wheel(self) -> Optional[bool]:
+        return self.specification["magnesiumWheel"]
+
+    @property
+    def has_tire_pressure_monitoring_system(self) -> Optional[bool]:
+        return self.specification["tirePressureMonitoringSystem"]
+
+    @property
+    def has_abs(self) -> Optional[bool]:
+        return self.specification["abs"]
+
+    @property
+    def air_bags(self) -> Optional[int]:
+        return self.specification["airBags"]
+
+    @property
+    def has_control_stability(self) -> Optional[bool]:
+        return self.specification["controlStability"]
+
+    @property
+    def has_electric_window(self) -> Optional[int]:
+        return self.specification["electricWindow"]
+
+    @property
+    def has_breaking_assist_system(self) -> Optional[bool]:
+        return self.specification["breakingAssistSystem"]
+
+    @property
+    def has_reverse_camera(self) -> Optional[bool]:
+        return self.specification["reverseCamera"]
+
+    @property
+    def has_adaptive_cruise_control(self) -> Optional[bool]:
+        return self.specification["adaptiveCruiseControl"]
+
+    @property
+    def has_high_beams_auto_control(self) -> Optional[bool]:
+        return self.specification["highBeamsAutoControl"]
+
+    @property
+    def has_blind_spot_assist(self) -> Optional[bool]:
+        return self.specification["blindSpotAssist"]
+
+    @property
+    def has_identify_pedestrians(self) -> Optional[bool]:
+        return self.specification["identifyPedestrians"]
+
+    @property
+    def has_seat_belts_sensors(self) -> Optional[bool]:
+        return self.specification["seatBeltsSensors"]
+
+    @property
+    def has_identifying_dangerous_nearing(self) -> Optional[bool]:
+        return self.specification["identifyingDangerousNearing"]
+
+    @property
+    def has_auto_lighting_in_forward(self) -> Optional[bool]:
+        return self.specification["autoLightingInForward"]
+
+    @property
+    def has_identify_traffic_signs(self) -> Optional[bool]:
+        return self.specification["identifyTrafficSigns"]
+
+    def ignition(self, field: Field = Field.TEXT) -> Optional[FieldTypes]:
+        return self.specification["ignition"][field]
+
+    @property
+    def safety_points(self) -> Optional[int]:
+        return self.specification["safetyPoints"]
+
+    @property
+    def is_handicapped_friendly(self) -> Optional[bool]:
+        return self.specification["isHandicappedFriendly"]
+
+    @property
+    def has_sun_roof(self) -> Optional[bool]:
+        return self.specification["sunRoof"]
+
+    @property
+    def is_turbo(self) -> Optional[bool]:
+        return self.specification["isTurbo"]
+
+    @property
+    def has_road_deviation_control(self) -> Optional[bool]:
+        return self.specification["roadDeviationControl"]
+
+    @property
+    def has_forward_distance_monitor(self) -> Optional[bool]:
+        return self.specification["forwardDistanceMonitor"]
+
+    @property
+    def has_box(self) -> Optional[bool]:
+        return self.specification["box"]
+
+    def __getitem__(self, key: str) -> Any:
+        return self.data[key]
+
+
+class VehiclesNextData(NextData):
+    def iterate_vehicles(self) -> Iterator[VehicleData]:
+        for query in self.queries:
+            data = query["state"].get("data")
+
+            if not data or isinstance(data, list):
+                continue
+
+            for vehicle_data in itertools.chain.from_iterable(data.values()):
+                if isinstance(vehicle_data, dict):
+                    yield VehicleData(vehicle_data)
+
+    def __getitem__(self, item):
+        return self.data[item]

yad2_scraper-0.5.0/yad2_scraper/vehicles/query.py

@@ -0,0 +1,25 @@
+from enum import Enum
+from typing import Optional
+
+from yad2_scraper.query import QueryFilters, OrderBy, NumberRange, format_number_range
+
+
+class OrderVehiclesBy(int, Enum):
+    DATE = OrderBy.DATE
+    PRICE_LOWEST_TO_HIGHEST = OrderBy.PRICE_LOWEST_TO_HIGHEST
+    PRICE_HIGHEST_TO_LOWEST = OrderBy.PRICE_HIGHEST_TO_LOWEST
+    DISTANCE_LOWEST_TO_HIGHEST = 5
+    YEAR_HIGHEST_TO_LOWEST = 6
+
+
+class VehiclesQueryFilters(QueryFilters):
+    year_range: Optional[NumberRange] = None
+    ...
+
+    def to_params(self) -> dict:
+        return {
+            **super().to_params(),
+            "year": format_number_range(self.year_range)
+        }
+
+# TODO: add QueryParams class for each vehicle type (some share the same attributes - sometimes with different enums)

yad2_scraper-0.5.0/yad2_scraper/vehicles/tag.py

@@ -0,0 +1,63 @@
+from functools import cached_property
+from bs4 import Tag
+from typing import Optional
+
+from yad2_scraper.utils import join_url, find_html_tag_by_class_substring
+from yad2_scraper.vehicles.urls import VEHICLES_URL
+
+YEAR_AND_HAND_TAG_SEPARATOR = " • "
+
+
+class VehicleTag:
+    def __init__(self, tag: Tag):
+        self.tag = tag
+
+    @cached_property
+    def relative_link(self) -> str:
+        return self.find_tag_by_class_substring("a", "itemLink")["href"]
+
+    @property
+    def page_link(self) -> str:
+        return join_url(VEHICLES_URL, self.relative_link)
+
+    @cached_property
+    def image_url(self) -> str:
+        return self.find_tag_by_class_substring("img", "image")["src"]
+
+    @cached_property
+    def model(self) -> str:
+        return self.find_tag_by_class_substring("span", "heading").text.strip()
+
+    @cached_property
+    def marketing_text(self) -> str:
+        return self.find_tag_by_class_substring("span", "marketingText").text.strip()
+
+    @cached_property
+    def year_and_hand_string(self) -> str:
+        return self.find_tag_by_class_substring("span", "yearAndHand").text.strip()
+
+    @property
+    def year(self) -> int:
+        year, _ = self.year_and_hand_string.split(YEAR_AND_HAND_TAG_SEPARATOR)
+        return int(year)
+
+    @property
+    def hand(self) -> int:
+        _, hand_string = self.year_and_hand_string.split(YEAR_AND_HAND_TAG_SEPARATOR)
+        _, hand = hand_string.split()
+        return int(hand)
+
+    @cached_property
+    def price_string(self) -> str:
+        return self.find_tag_by_class_substring("span", "price").text.strip()
+
+    @property
+    def price(self) -> Optional[int]:
+        try:
+            price, _ = self.price_string.split()
+            return int(price.replace(",", ""))
+        except ValueError:
+            return None
+
+    def find_tag_by_class_substring(self, tag_name: str, substring: str) -> Tag:
+        return find_html_tag_by_class_substring(self.tag, tag_name, substring)

yad2_scraper-0.5.0/yad2_scraper/vehicles/urls.py

@@ -0,0 +1,16 @@
+from typing import Literal, get_args
+
+from yad2_scraper.utils import join_url
+from yad2_scraper.constants import BASE_URL
+
+VEHICLES_URL = join_url(BASE_URL, "vehicles")
+
+VehicleType = Literal["cars", "motorcycles", "scooters", "trucks", "watercraft", "others"]
+
+_VALID_VEHICLE_TYPES = get_args(VehicleType)
+
+
+def get_vehicle_url(vehicle_type: VehicleType) -> str:
+    if vehicle_type not in _VALID_VEHICLE_TYPES:
+        raise ValueError(f"Invalid vehicle type: {repr(vehicle_type)}. Expected one of {_VALID_VEHICLE_TYPES}")
+    return join_url(VEHICLES_URL, vehicle_type)

yad2_scraper-0.4.0/yad2_scraper/next_data.py

@@ -1,27 +0,0 @@
-from enum import Enum
-from typing import List, Union
-
-
-class Field(str, Enum):
-    ID = "id"
-    TEXT = "text"
-    ENGLISH_TEXT = "textEng"
-
-
-FieldTypes = Union[str, int]
-
-
-class NextData:
-    def __init__(self, data: dict):
-        self.data = data
-
-    @property
-    def json(self) -> dict:
-        return self.data
-
-    @property
-    def queries(self) -> List[dict]:
-        return self.data["props"]["pageProps"]["dehydratedState"]["queries"]
-
-    def __getitem__(self, item):
-        return self.data[item]