yad2-scraper 0.2.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
yad2_scraper/constants.py CHANGED
@@ -18,9 +18,10 @@ DEFAULT_REQUEST_HEADERS = {
18
18
  ALLOW_REQUEST_REDIRECTS = True
19
19
  VERIFY_REQUEST_SSL = True
20
20
 
21
- ANTIBOT_CONTENT = b"Are you for real" # robot-captcha
21
+ ANTIBOT_CONTENT_IDENTIFIER = b"Are you for real" # robot-captcha
22
+ PAGE_CONTENT_IDENTIFIER = b"https://www.yad2.co.il/"
22
23
 
23
- FIRST_PAGE = 1
24
+ FIRST_PAGE_NUMBER = 1
24
25
  NOT_MENTIONED_PRICE_RANGE = 0, 0
25
26
 
26
27
  NEXT_DATA_SCRIPT_ID = "__NEXT_DATA__"
@@ -1,2 +1,32 @@
1
- class AntiBotDetectedError(Exception):
1
+ import httpx
2
+ from typing import List, Union
3
+
4
+
5
+ class ResponseError(Exception):
6
+ def __init__(self, msg: str, request: httpx.Request, response: httpx.Response):
7
+ super().__init__(msg)
8
+ self.request = request
9
+ self.response = response
10
+
11
+
12
+ class AntiBotDetectedError(ResponseError):
2
13
  pass
14
+
15
+
16
+ class UnexpectedContentError(ResponseError):
17
+ pass
18
+
19
+
20
+ class MaxAttemptsExceededError(Exception):
21
+ def __init__(self, msg: str, max_attempts: int, errors: List[BaseException] = None):
22
+ super().__init__(msg)
23
+ self.max_attempts = max_attempts
24
+ self.errors = errors
25
+
26
+
27
+ class MaxRequestAttemptsExceededError(MaxAttemptsExceededError):
28
+ def __init__(self, method: str, url: str, max_attempts: int, errors: List[Union[httpx.HTTPError, ResponseError]]):
29
+ msg = f"All {max_attempts} attempts for {method} request to '{url}' have failed"
30
+ super().__init__(msg, max_attempts, errors)
31
+ self.method = method
32
+ self.url = url
yad2_scraper/scraper.py CHANGED
@@ -1,107 +1,165 @@
1
1
  import logging
2
2
  import httpx
3
3
  import time
4
- import random
5
- from typing import Optional, Dict, Any, Tuple, Union, Type, TypeVar
4
+ from fake_useragent import FakeUserAgent
5
+ from typing import Optional, Dict, Any, Callable, Union, Type, TypeVar
6
6
 
7
7
  from yad2_scraper.category import Yad2Category
8
8
  from yad2_scraper.query import QueryFilters
9
- from yad2_scraper.utils import get_random_user_agent, validate_http_response
9
+ from yad2_scraper.exceptions import AntiBotDetectedError, UnexpectedContentError, MaxRequestAttemptsExceededError
10
10
  from yad2_scraper.constants import (
11
11
  DEFAULT_REQUEST_HEADERS,
12
12
  ALLOW_REQUEST_REDIRECTS,
13
- VERIFY_REQUEST_SSL
13
+ VERIFY_REQUEST_SSL,
14
+ ANTIBOT_CONTENT_IDENTIFIER,
15
+ PAGE_CONTENT_IDENTIFIER
14
16
  )
15
17
 
16
18
  Category = TypeVar("Category", bound=Yad2Category)
17
- DelayRange = Tuple[float, float]
18
- QueryParams = Union[QueryFilters, Dict[str, Any]]
19
+ WaitStrategy = Callable[[int], Optional[float]]
20
+ QueryParamTypes = Union[QueryFilters, Dict[str, Any]]
19
21
 
22
+ fua = FakeUserAgent()
20
23
  logger = logging.getLogger(__name__)
21
24
 
22
25
 
23
26
  class Yad2Scraper:
24
27
  def __init__(
25
28
  self,
26
- session: Optional[httpx.Client] = None,
27
- request_kwargs: Dict[str, Any] = None,
28
- randomize_user_agent: bool = False,
29
- requests_delay_range: Optional[DelayRange] = None,
29
+ client: Optional[httpx.Client] = None,
30
+ request_defaults: Optional[Dict[str, Any]] = None,
31
+ randomize_user_agent: bool = True,
32
+ wait_strategy: Optional[WaitStrategy] = None,
33
+ max_request_attempts: int = 1
30
34
  ):
31
- self.session = session or httpx.Client(
35
+ self.client = client or httpx.Client(
32
36
  headers=DEFAULT_REQUEST_HEADERS,
33
37
  follow_redirects=ALLOW_REQUEST_REDIRECTS,
34
38
  verify=VERIFY_REQUEST_SSL
35
39
  )
36
- self.request_kwargs = request_kwargs or {}
40
+ self.request_defaults = request_defaults or {}
37
41
  self.randomize_user_agent = randomize_user_agent
38
- self.requests_delay_range = requests_delay_range
42
+ self.wait_strategy = wait_strategy
43
+ self.max_request_attempts = max_request_attempts
39
44
 
40
- logger.debug(f"Initialized with session {self.session} and request kwargs: {self.request_kwargs}")
45
+ logger.debug(f"Scraper initialized with client: {self.client}")
41
46
 
42
- def set_user_agent(self, user_agent: str):
43
- self.session.headers["User-Agent"] = user_agent
44
- logger.debug(f"User-Agent session header set to: '{user_agent}'")
47
+ def set_user_agent(self, user_agent: str) -> None:
48
+ self.client.headers["User-Agent"] = user_agent
49
+ logger.debug(f"User-Agent client header set to: '{user_agent}'")
45
50
 
46
- def set_no_script(self, no_script: bool):
47
- value = "1" if no_script else "0" # str(int(no_script))
48
- self.session.cookies.set("noscript", value)
49
- logger.debug(f"NoScript session cookie set to: '{value}'")
51
+ def set_no_script(self, no_script: bool) -> None:
52
+ value = "1" if no_script else "0"
53
+ self.client.cookies.set("noscript", value)
54
+ logger.debug(f"NoScript (noscript) client cookie set to: '{value}'")
50
55
 
51
56
  def fetch_category(
52
57
  self,
53
58
  url: str,
54
- query_params: Optional[QueryParams] = None,
55
- category_type: Type[Category] = Yad2Category
59
+ category_type: Type[Category] = Yad2Category,
60
+ params: Optional[QueryParamTypes] = None
56
61
  ) -> Category:
57
62
  logger.debug(f"Fetching category from URL: '{url}'")
58
- response = self.get(url, query_params)
63
+ response = self.get(url, params)
59
64
  logger.debug(f"Category fetched successfully from URL: '{url}'")
60
65
  return category_type.from_html_io(response)
61
66
 
62
- def get(self, url: str, query_params: Optional[QueryParams] = None) -> httpx.Response:
63
- return self.request("GET", url, query_params=query_params)
67
+ def get(self, url: str, params: Optional[QueryParamTypes] = None) -> httpx.Response:
68
+ return self.request("GET", url, params=params)
64
69
 
65
- def request(self, method: str, url: str, query_params: Optional[QueryParams] = None) -> httpx.Response:
66
- request_kwargs = self._prepare_request_kwargs(query_params=query_params)
70
+ def request(self, method: str, url: str, params: Optional[QueryParamTypes] = None) -> httpx.Response:
71
+ if not isinstance(self.max_request_attempts, int):
72
+ raise TypeError(f"max_request_attempts must be of type 'int', but got {type(self.max_request_attempts)}")
67
73
 
68
- if self.requests_delay_range:
69
- self._apply_request_delay()
74
+ if self.max_request_attempts <= 0:
75
+ raise ValueError(f"max_request_attempts must be a positive integer, but got {self.max_request_attempts}")
70
76
 
71
- try:
72
- logger.info(f"Making {method} request to URL: '{url}'") # request kwargs not logged - may be sensitive
73
- response = self.session.request(method, url, **request_kwargs)
74
- logger.debug(f"Received response with status code: {response.status_code}")
77
+ request_options = self._prepare_request_options(params=params)
78
+ error_list = []
75
79
 
76
- validate_http_response(response)
77
- logger.debug("Response validation succeeded")
78
- except Exception as error:
79
- logger.error(f"Request to '{url}' failed: {error}")
80
- raise error
80
+ for attempt in range(1, self.max_request_attempts + 1):
81
+ try:
82
+ return self._send_request(method, url, request_options, attempt)
83
+ except Exception as error:
84
+ logger.error(f"{method} request to '{url}' failed {self._format_attempt_info(attempt)}: {error}")
85
+ error_list.append(error)
81
86
 
82
- return response
87
+ if self.max_request_attempts == 1:
88
+ raise error_list[0] # only one error exists, raise it
83
89
 
84
- def _prepare_request_kwargs(self, query_params: Optional[QueryParams] = None) -> Dict[str, Any]:
85
- logger.debug("Preparing request kwargs from defaults")
86
- request_kwargs = self.request_kwargs.copy()
90
+ max_attempts_error = MaxRequestAttemptsExceededError(method, url, self.max_request_attempts, error_list)
91
+ logger.error(str(max_attempts_error))
92
+ raise max_attempts_error from error_list[-1] # multiple errors exist, raise from the last one
87
93
 
88
- if query_params:
89
- request_kwargs.setdefault("params", {}).update(query_params)
90
- logger.debug(f"Updated request kwargs with query params: {query_params}")
94
+ def close(self) -> None:
95
+ logger.debug("Closing scraper client")
96
+ self.client.close()
97
+ logger.info("Scraper client closed")
91
98
 
99
+ def _send_request(self, method: str, url: str, request_options: Dict[str, Any], attempt: int) -> httpx.Response:
92
100
  if self.randomize_user_agent:
93
- random_user_agent = get_random_user_agent()
94
- request_kwargs.setdefault("headers", {})["User-Agent"] = random_user_agent
95
- logger.debug(f"Updated request kwargs with random 'User-Agent' header: '{random_user_agent}'")
101
+ self._set_random_user_agent(request_options)
102
+
103
+ if self.wait_strategy:
104
+ self._apply_wait_strategy(attempt)
96
105
 
97
- return request_kwargs
106
+ logger.info(f"Sending {method} request to URL: '{url}' {self._format_attempt_info(attempt)}")
107
+ response = self.client.request(method, url, **request_options)
108
+ logger.debug(f"Received response {response.status_code} from '{url}' {self._format_attempt_info(attempt)}")
109
+ self._validate_response(response)
98
110
 
99
- def _apply_request_delay(self):
100
- delay = random.uniform(*self.requests_delay_range)
101
- logger.debug(f"Applying request delay of {delay:.2f} seconds")
102
- time.sleep(delay)
111
+ return response
103
112
 
104
- def close(self):
105
- logger.debug("Closing scraper session")
106
- self.session.close()
107
- logger.info("Scraper session closed")
113
+ def _prepare_request_options(self, params: Optional[QueryParamTypes] = None) -> Dict[str, Any]:
114
+ logger.debug("Preparing request options from defaults")
115
+ request_options = self.request_defaults.copy()
116
+
117
+ if params:
118
+ request_options.setdefault("params", {}).update(params)
119
+ logger.debug(f"Updated request options with query params: {params}")
120
+
121
+ return request_options
122
+
123
+ @staticmethod
124
+ def _set_random_user_agent(request_options: Dict[str, str]):
125
+ user_agent = fua.random
126
+ request_options.setdefault("headers", {})["User-Agent"] = user_agent
127
+ logger.debug(f"Updated request options with random User-Agent header: '{user_agent}'")
128
+
129
+ def _apply_wait_strategy(self, attempt: int):
130
+ wait_time = self.wait_strategy(attempt)
131
+ if not wait_time:
132
+ return
133
+
134
+ logger.debug(f"Waiting {wait_time:.2f} seconds before request {self._format_attempt_info(attempt)}")
135
+ time.sleep(wait_time)
136
+
137
+ @staticmethod
138
+ def _validate_response(response: httpx.Response):
139
+ response.raise_for_status()
140
+
141
+ if ANTIBOT_CONTENT_IDENTIFIER in response.content:
142
+ raise AntiBotDetectedError(
143
+ f"The response contains Anti-Bot content",
144
+ request=response.request,
145
+ response=response
146
+ )
147
+ if response.request.method == "GET" and PAGE_CONTENT_IDENTIFIER not in response.content:
148
+ raise UnexpectedContentError(
149
+ "The GET response does not contain yad2 related content",
150
+ request=response.request,
151
+ response=response
152
+ )
153
+
154
+ logger.debug("Response validation succeeded")
155
+
156
+ def _format_attempt_info(self, attempt: int) -> str:
157
+ return f"(attempt {attempt}/{self.max_request_attempts})"
158
+
159
+ def __enter__(self):
160
+ logger.debug("Entering scraper context")
161
+ return self
162
+
163
+ def __exit__(self, exc_type, exc_val, exc_tb):
164
+ logger.debug("Exiting scraper context")
165
+ self.close()
yad2_scraper/utils.py CHANGED
@@ -1,17 +1,6 @@
1
- import httpx
2
- from fake_useragent import FakeUserAgent
3
1
  from bs4 import BeautifulSoup, Tag
4
2
  from typing import Union, List
5
3
 
6
- from yad2_scraper.exceptions import AntiBotDetectedError
7
- from yad2_scraper.constants import ANTIBOT_CONTENT
8
-
9
- fua = FakeUserAgent()
10
-
11
-
12
- def get_random_user_agent() -> str:
13
- return fua.random
14
-
15
4
 
16
5
  def join_url(url: str, path: str) -> str:
17
6
  return url.rstrip("/") + "/" + path.lstrip("/")
@@ -24,13 +13,6 @@ def get_parent_url(url: str) -> str:
24
13
  return url.rstrip("/").rsplit("/", 1)[0]
25
14
 
26
15
 
27
- def validate_http_response(response: httpx.Response):
28
- response.raise_for_status()
29
-
30
- if ANTIBOT_CONTENT in response.content:
31
- raise AntiBotDetectedError(f"The response contains Anti-Bot content")
32
-
33
-
34
16
  def find_html_tag_by_class_substring(e: Union[BeautifulSoup, Tag], tag_name: str, substring: str) -> Tag:
35
17
  return e.find(tag_name, class_=lambda class_name: class_name and substring in class_name)
36
18
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: yad2-scraper
3
- Version: 0.2.0
3
+ Version: 0.4.0
4
4
  Summary: Scrape Yad2 in Python.
5
5
  License: LICENSE
6
6
  Author: dav ost
@@ -0,0 +1,12 @@
1
+ yad2_scraper/__init__.py,sha256=UUiIk6TAHTAP4IY86bIR4TcY3VVMTCyEF0Sq1MSneMM,141
2
+ yad2_scraper/category.py,sha256=KXLyjMOlPzu3xj08-uRmffAMD83DbqFVm-y1-T83Djw,910
3
+ yad2_scraper/constants.py,sha256=8zXJ31fRqkDIOJp96BRK1PJofGXX8SG64YcfmJnVW8Q,910
4
+ yad2_scraper/exceptions.py,sha256=_IcuDdJPKAznSUp_c3fLEuTnDdKf0NLJqpRPs0IzdXw,979
5
+ yad2_scraper/next_data.py,sha256=-vqvXJqugk-895_kOnwb7J8kUjugg28Aqrh4Z_ct11M,512
6
+ yad2_scraper/query.py,sha256=WaOWUlyNye9MNXv3hkiUaBFDeV9lbkvHiaDHWYKzgtY,1194
7
+ yad2_scraper/scraper.py,sha256=sgDpfnKlBSDIWEb2enpQ5O9E5fJvXz3cDOnGXHGCJL4,6653
8
+ yad2_scraper/utils.py,sha256=e6tqaN5Gw9BXunOQ1V919NkLrZREN7TdMsDuOvZgrcY,713
9
+ yad2_scraper-0.4.0.dist-info/LICENSE,sha256=JCpnDxMx2kE40e0UQ1svSmifrLWg2Gni5VTkJR68thY,1065
10
+ yad2_scraper-0.4.0.dist-info/METADATA,sha256=TDO8cS7t4aGZ-B-XikL4hSneizcb7TzLtBsV7dQGl1k,925
11
+ yad2_scraper-0.4.0.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
12
+ yad2_scraper-0.4.0.dist-info/RECORD,,
@@ -1,12 +0,0 @@
1
- yad2_scraper/__init__.py,sha256=UUiIk6TAHTAP4IY86bIR4TcY3VVMTCyEF0Sq1MSneMM,141
2
- yad2_scraper/category.py,sha256=KXLyjMOlPzu3xj08-uRmffAMD83DbqFVm-y1-T83Djw,910
3
- yad2_scraper/constants.py,sha256=HtZh0okD7FhbeAw3VladHsXIBlbxvZDoXaWm8aaVYvk,839
4
- yad2_scraper/exceptions.py,sha256=Ym3k3aQHe4-SQtfDch5DdnNN3RK6dbXLCZej1chSEGQ,48
5
- yad2_scraper/next_data.py,sha256=-vqvXJqugk-895_kOnwb7J8kUjugg28Aqrh4Z_ct11M,512
6
- yad2_scraper/query.py,sha256=WaOWUlyNye9MNXv3hkiUaBFDeV9lbkvHiaDHWYKzgtY,1194
7
- yad2_scraper/scraper.py,sha256=bGAjuu-ZhNYo4NBmrZU096NC2uYHvnxusAFZs1mjnRU,4202
8
- yad2_scraper/utils.py,sha256=9R-XJgWcmt_HTR_JQbFIFHcCg-II4ClFqlLQQZOU74g,1170
9
- yad2_scraper-0.2.0.dist-info/LICENSE,sha256=JCpnDxMx2kE40e0UQ1svSmifrLWg2Gni5VTkJR68thY,1065
10
- yad2_scraper-0.2.0.dist-info/METADATA,sha256=fznOi8yGp07DU7AMDSAg3LKuYJ5Gu6X6G8qS-SFfJt0,925
11
- yad2_scraper-0.2.0.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
12
- yad2_scraper-0.2.0.dist-info/RECORD,,