notify-tls-client 0.1.6__py3-none-any.whl → 0.1.9__py3-none-any.whl

notify_tls_client/__init__.py

@@ -1,2 +1,2 @@
-from .core.notifytlsclient import NotifyTLSClient
-from .tls_client import Session
+from .core.notifytlsclient import NotifyTLSClient
+from .tls_client import Session

notify_tls_client/core/client/__init__.py

@@ -1,6 +1 @@
-from .decorators import (
-    increment_requests_decorator,
-    handle_forbidden_requests_decorator,
-    print_request_infos_decorator,
-    change_state_decorator,
-    handle_exception_requests_decorator)
+from .decorators import (request_guard_decorator)

notify_tls_client/core/client/decorators.py

@@ -1,90 +1,239 @@
-import time
-import traceback
-from datetime import datetime
-
-
-
-def change_state_decorator(callback):
-    def wrapper(*args, **kwargs):
-        self = args[0]
-        self.free = False
-        response = callback(*args, **kwargs)
-        self.free = True
-        return response
-
-    return wrapper
-
-
-def print_request_infos_decorator(callback):
-    def wrapper(*args, **kwargs):
-        start = time.time()
-        response = callback(*args, **kwargs)
-        response.elapsed = round((time.time() - start) * 1000, 2)
-        self = args[0]
-
-        print("Date: ", datetime.now().strftime("%d-%m-%Y %H:%M:%S.%f")[:-3])
-        print(f"Request URL: {response.url}")
-        print(f"Response Status Code: {response.status_code}")
-        print(f"Response Headers: {response.headers}")
-        print(f"Response Elapsed Time: {response.elapsed} ms")
-
-        if self.client.proxies:
-            print(f"Proxy: {self.client.proxies['http']}")
-
-        print(f"-")
-
-        return response
-
-    return wrapper
-
-
-def handle_exception_requests_decorator(callback):
-    def wrapper(*args, **kwargs):
-        response = None
-        try:
-            response = callback(*args, **kwargs)
-        except Exception as e:
-            print(traceback.format_exc())
-            self = args[0]
-            self.get_tls().close()
-            self._create_new_client(self.client_identifier,self.random_tls_extension_order)
-            self.change_proxy()
-
-        return response
-
-    return wrapper
-
-
-def handle_forbidden_requests_decorator(callback):
-    def wrapper(*args, **kwargs):
-        response = callback(*args, **kwargs)
-        if response.status_code == 403 or response.status_code == 429:
-            self = args[0]
-            self.get_tls().close()
-            self._create_new_client(self.client_identifier,self.random_tls_extension_order)
-
-            if self.proxies_manager:
-                self.change_proxy()
-
-        return response
-
-    return wrapper
-
-
-def increment_requests_decorator(callback):
-    def wrapper(*args, **kwargs):
-
-        self = args[0]
-        response = callback(*args, **kwargs)
-        self.requests_amount += 1
-
-        if self.requests_amount >= self.requests_limit_same_proxy:
-            self.get_tls().close()
-            self._create_new_client(self.client_identifier,self.random_tls_extension_order)
-            self.change_proxy()
-
-        return response
-
-    return wrapper
-
-
+import logging
+from datetime import datetime
+from typing import Optional
+from urllib.parse import urlparse
+
+from notify_tls_client.tls_client.response import Response
+
+logger = logging.getLogger(__name__)
+
+
+def __is_valid_url(url: str) -> bool:
+    if not isinstance(url, str):
+        return False
+
+    parsed = urlparse(url)
+    return all([parsed.scheme, parsed.netloc])
+
+
+def request_guard_decorator(callback):
+    def wrapper(*args, **kwargs):
+        self = args[0]
+        self.free = False
+        request_url = None
+        request_headers = {}
+        res = None
+
+        for arg in args:
+            if isinstance(arg, str) and __is_valid_url(arg):
+                request_url = arg
+                break
+
+        for key, value in kwargs.items():
+            if key == 'url':
+                request_url = value
+                break
+
+            if key == 'headers':
+                request_headers = value
+
+        try:
+            self.requests_amount += 1
+            res = callback(*args, **kwargs)
+            _log_request_info(self, res, request_url, request_headers)
+            _requests_handler(self, res)
+            return res
+
+        except Exception as e:
+            logging.exception("Exception occurred during TLS request")
+            _log_request_info(self, res, request_url, request_headers)
+            _recover_on_exception(self)
+            _requests_handler(self, res)
+
+        finally:
+            self.free = True
+
+            # _forbidden_request_handler(self, res)
+
+        return None
+
+    return wrapper
+
+
+def _recover_on_exception(self):
+    if self.instantiate_new_client_on_exception:
+        logging.info("Instantiating new TLS client due to exception...")
+        self.get_tls().close()
+        self._create_new_client()
+
+    if self.proxies_manager:
+        logging.info("Changing proxy due to exception...")
+        self.change_proxy()
+
+
+def _log_request_info(self,
+                      response: Optional[Response],
+                      request_url: Optional[str],
+                      request_headers: dict):
+    if response:
+        logger.debug(f"""Request finished
+                                        client_identifier={self.get_current_client_identifier()}
+                                        request_url={request_url}
+                                        request_headers={request_headers}
+                                        response_url={response.url}
+                                        status_code={response.status_code}
+                                        response_time={response.elapsed}ms
+                                        response_headers={dict(response.headers)}
+                                        proxy={self.client.proxies['http'] if self.client.proxies else None}
+                             """,
+                     extra={
+                         "date": datetime.now().strftime("%d-%m-%Y %H:%M:%S.%f")[:-3],
+                         "request_url": request_url,
+                         "request_headers": request_headers,
+                         "response_url": response.url,
+                         "status_code": response.status_code,
+                         "response_headers": dict(response.headers),
+                         "response_elapsed_ms": response.elapsed,
+                         "proxy": self.client.proxies['http'] if self.client.proxies else None,
+                         "client_identifier": self.get_current_client_identifier()
+
+                     })
+
+    if not response:
+        logger.debug(f"""Request failed before getting a response
+                                        client_identifier={self.get_current_client_identifier()}
+                                        request_url={request_url}
+                                        request_headers={request_headers}
+                                        response_url=None
+                                        status_code=None
+                                        response_headers=None
+                                        response_time=0
+                                        proxy={self.client.proxies['http'] if self.client.proxies else None}
+                             """,
+                     extra={
+                         "date": datetime.now().strftime("%d-%m-%Y %H:%M:%S.%f")[:-3],
+                         "request_url": request_url,
+                         "request_headers": request_headers,
+                         "response_url": None,
+                         "status_code": None,
+                         "response_headers": None,
+                         "response_elapsed_ms": 0,
+                         "proxy": self.client.proxies['http'] if self.client.proxies else None,
+                         "client_identifier": self.get_current_client_identifier()
+
+                     })
+
+
+def _forbidden_request_handler(self, response: Optional[Response]):
+    if not response:
+        return
+
+    if response.status_code in self.status_codes_to_forbidden_response_handle:
+        if self.instantiate_new_client_on_forbidden_response:
+            logging.info("Instantiating new TLS client due to forbidden response...")
+            self.get_tls().close()
+            self._create_new_client()
+
+        if self.proxies_manager:
+            logging.info("Changing proxy due to forbidden response...")
+            self.change_proxy()
+
+
+# tem casos que está retornando 403 como erro da API, tenho que estudar o que fazer nesses casos
+def _requests_handler(self, response: Optional[Response]):
+    self.requests_amount_with_current_client_identifier += 1
+
+    if self.proxies_manager:
+        self.requests_amount_with_current_proxy += 1
+
+    if not response:
+        return
+
+    client_identifier_changed_by_limit = self.is_client_identifier_changeable_by_requests_limit_reached()
+    is_same_proxy_limit = self.is_same_proxy_request_limit_reached()
+    is_forbidden_request_status = self.is_forbidden_request_status(response.status_code)
+    create_new_client_on_forbidden = self.instantiate_new_client_on_forbidden_response
+    change_client_identifier_on_forbidden = self.change_client_identifier_on_forbidden_response
+    has_proxies_manager = self.has_proxies_manager()
+
+    if client_identifier_changed_by_limit:
+        logging.info("Requests limit reached for current client identifier")
+        if is_forbidden_request_status or is_same_proxy_limit:
+            self._create_new_client(self.client_identifiers_manager.get_next(),
+                                    proxy=None if not has_proxies_manager else self.proxies_manager.get_next())
+
+        else:
+            self._create_new_client(self.client_identifiers_manager.get_next(),
+                                    proxy=None if not has_proxies_manager else self.proxies_manager.get_current_proxy())
+
+        return
+
+    if is_forbidden_request_status:
+
+        if not create_new_client_on_forbidden:
+            if has_proxies_manager:
+                logging.info("Changing proxy due to forbidden response...")
+                self.change_proxy()
+
+            return
+
+        if create_new_client_on_forbidden:
+                if change_client_identifier_on_forbidden:
+                    logging.info("Instantiating new TLS client due to forbidden response and changing client identifier...")
+                    self._create_new_client(self.client_identifiers_manager.get_next(),
+                                            proxy=None if not has_proxies_manager else self.proxies_manager.get_next())
+
+                else:
+                    logging.info("Instantiating new TLS client due to forbidden response...")
+                    item = self.client_identifiers_manager.get_current_item()
+                    self._create_new_client(str(item),
+                                            proxy=None if not has_proxies_manager else self.proxies_manager.get_next())
+
+
+    if is_same_proxy_limit:
+        if has_proxies_manager:
+            logging.info("Changing proxy due to requests limit reached...")
+            self.change_proxy()
+
+    # if client_identifier_changed_by_limit:
+    #     logging.info("Requests limit reached for current client identifier")
+    #     logging.info("Changing client identifier due to requests limit reached...")
+    #     # self._create_new_client(self.client_identifiers_manager.get_next(),
+    #     #                         proxy=self.proxies_manager.get_current_proxy() if self.proxies_manager else None)
+    #
+    #
+    #
+    #
+    # if self.requests_limit_with_same_proxy and self.requests_amount_with_current_proxy >= self.requests_limit_with_same_proxy:
+    #     logging.info("Requests limit reached for current proxy")
+    #     if self.proxies_manager:
+    #         proxy_changed_by_limit = True
+    #         logging.info("Changing proxy due to requests limit reached...")
+    #
+    #         if response.status_code in self.status_codes_to_forbidden_response_handle and self.instantiate_new_client_on_forbidden_response:
+    #             logging.info("Instantiating new TLS client due to forbidden response before changing proxy...")
+    #             self.get_tls().close()
+    #             if self.change_client_identifier_on_forbidden_response:
+    #                 self._create_new_client(self.client_identifiers_manager.get_next())
+    #             else:
+    #                 self._create_new_client(self.client_identifiers_manager.get_current())
+    #
+    #         self.change_proxy()
+    #
+    # # Metodo vai ter que ser usado para não trocar o proxy duas vezes, uma pelo status 403 e outra pelo limite de requests
+    # if not proxy_changed_by_limit:
+    #     return
+    #
+    # if response.status_code in self.status_codes_to_forbidden_response_handle:
+    #     if self.instantiate_new_client_on_forbidden_response:
+    #         logging.info("Instantiating new TLS client due to forbidden response...")
+    #         self.get_tls().close()
+    #
+    #         if self.change_client_identifier_on_forbidden_response:
+    #             self._create_new_client(self.client_identifiers_manager.get_next())
+    #         else:
+    #             self._create_new_client()
+    #
+    #     if self.proxies_manager:
+    #         logging.info("Changing proxy due to forbidden response...")
+    #         self.change_proxy()

notify_tls_client/core/client_identifiers_manager.py

@@ -0,0 +1,45 @@
+from dataclasses import dataclass, field
+from typing import Optional
+
+from dataclasses_json import dataclass_json
+
+from notify_tls_client.tls_client.settings import ClientIdentifiers
+
+
+@dataclass_json
+@dataclass
+class ClientIdentifiersManager:
+    _items: list[ClientIdentifiers] = field(default_factory=list)
+    _current_index: int = 0
+    _current_item: Optional[ClientIdentifiers] = None
+
+    def get_next(self) -> ClientIdentifiers:
+        if not self._items:
+            raise Exception('No items available')
+
+        if self._current_index < len(self._items):
+            _item = self._items[self._current_index]
+            self._current_index += 1
+        else:
+            self._current_index = 0
+            _item = self._items[self._current_index]
+
+        return _item
+
+    def get_current_item(self) -> Optional[ClientIdentifiers]:
+        if not self._items:
+            return None
+
+        return self._items[0] if self._current_index == 0 else self._items[self._current_index - 1]
+
+
+    def set_items(self, items: list[ClientIdentifiers]):
+        self._items = items
+        self._current_proxy_index = 0
+        self._current_proxy = None
+
+    def get_item(self) -> list[ClientIdentifiers]:
+        return self._items
+
+    def get_total_items(self) -> int:
+        return len(self._items)