notify-tls-client 0.1.6__tar.gz → 0.1.9__tar.gz

{notify_tls_client-0.1.6 → notify_tls_client-0.1.9}/MANIFEST.in

@@ -1,3 +1,3 @@
-exclude proxies.txt
-exclude notify_tls_client/main.py
+exclude proxies.txt
+exclude notify_tls_client/main.py
 include notify_tls_client/tls_client/dependencies/*

{notify_tls_client-0.1.6 → notify_tls_client-0.1.9}/PKG-INFO

@@ -1,16 +1,14 @@
-Metadata-Version: 2.4
-Name: notify_tls_client
-Version: 0.1.6
-Summary: Sem descrição
-Author: Jeferson Albara
-Author-email: Naruto Uzumaki <naruto_uzumaki@gmail.com>
-License: MIT
-Project-URL: Homepage, https://github.com/jefersonAlbara/notify-tls-client
-Requires-Python: >=3.12
-Description-Content-Type: text/markdown
-Requires-Dist: dataclasses_json
-Requires-Dist: typing_extensions
-Dynamic: author
-
-# MY TLS Client
-Pacote python privado com modificações na classe 'tls client'
+Metadata-Version: 2.4
+Name: notify_tls_client
+Version: 0.1.9
+Summary: Sem descrição
+Author-email: Naruto Uzumaki <naruto_uzumaki@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/jefersonAlbara/notify-tls-client
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: dataclasses_json
+Requires-Dist: typing_extensions
+
+# MY TLS Client
+Pacote python privado com modificações na classe 'tls client'

notify_tls_client-0.1.9/README.md

@@ -0,0 +1,2 @@
+# MY TLS Client
+Pacote python privado com modificações na classe 'tls client'

{notify_tls_client-0.1.6 → notify_tls_client-0.1.9}/notify_tls_client/__init__.py

@@ -1,2 +1,2 @@
-from .core.notifytlsclient import NotifyTLSClient
-from .tls_client import Session
+from .core.notifytlsclient import NotifyTLSClient
+from .tls_client import Session

notify_tls_client-0.1.9/notify_tls_client/core/client/__init__.py

@@ -0,0 +1 @@
+from .decorators import (request_guard_decorator)

notify_tls_client-0.1.9/notify_tls_client/core/client/decorators.py

@@ -0,0 +1,239 @@
+import logging
+from datetime import datetime
+from typing import Optional
+from urllib.parse import urlparse
+
+from notify_tls_client.tls_client.response import Response
+
+logger = logging.getLogger(__name__)
+
+
+def __is_valid_url(url: str) -> bool:
+    if not isinstance(url, str):
+        return False
+
+    parsed = urlparse(url)
+    return all([parsed.scheme, parsed.netloc])
+
+
+def request_guard_decorator(callback):
+    def wrapper(*args, **kwargs):
+        self = args[0]
+        self.free = False
+        request_url = None
+        request_headers = {}
+        res = None
+
+        for arg in args:
+            if isinstance(arg, str) and __is_valid_url(arg):
+                request_url = arg
+                break
+
+        for key, value in kwargs.items():
+            if key == 'url':
+                request_url = value
+                break
+
+            if key == 'headers':
+                request_headers = value
+
+        try:
+            self.requests_amount += 1
+            res = callback(*args, **kwargs)
+            _log_request_info(self, res, request_url, request_headers)
+            _requests_handler(self, res)
+            return res
+
+        except Exception as e:
+            logging.exception("Exception occurred during TLS request")
+            _log_request_info(self, res, request_url, request_headers)
+            _recover_on_exception(self)
+            _requests_handler(self, res)
+
+        finally:
+            self.free = True
+
+            # _forbidden_request_handler(self, res)
+
+        return None
+
+    return wrapper
+
+
+def _recover_on_exception(self):
+    if self.instantiate_new_client_on_exception:
+        logging.info("Instantiating new TLS client due to exception...")
+        self.get_tls().close()
+        self._create_new_client()
+
+    if self.proxies_manager:
+        logging.info("Changing proxy due to exception...")
+        self.change_proxy()
+
+
+def _log_request_info(self,
+                      response: Optional[Response],
+                      request_url: Optional[str],
+                      request_headers: dict):
+    if response:
+        logger.debug(f"""Request finished
+                                        client_identifier={self.get_current_client_identifier()}
+                                        request_url={request_url}
+                                        request_headers={request_headers}
+                                        response_url={response.url}
+                                        status_code={response.status_code}
+                                        response_time={response.elapsed}ms
+                                        response_headers={dict(response.headers)}
+                                        proxy={self.client.proxies['http'] if self.client.proxies else None}
+                             """,
+                     extra={
+                         "date": datetime.now().strftime("%d-%m-%Y %H:%M:%S.%f")[:-3],
+                         "request_url": request_url,
+                         "request_headers": request_headers,
+                         "response_url": response.url,
+                         "status_code": response.status_code,
+                         "response_headers": dict(response.headers),
+                         "response_elapsed_ms": response.elapsed,
+                         "proxy": self.client.proxies['http'] if self.client.proxies else None,
+                         "client_identifier": self.get_current_client_identifier()
+
+                     })
+
+    if not response:
+        logger.debug(f"""Request failed before getting a response
+                                        client_identifier={self.get_current_client_identifier()}
+                                        request_url={request_url}
+                                        request_headers={request_headers}
+                                        response_url=None
+                                        status_code=None
+                                        response_headers=None
+                                        response_time=0
+                                        proxy={self.client.proxies['http'] if self.client.proxies else None}
+                             """,
+                     extra={
+                         "date": datetime.now().strftime("%d-%m-%Y %H:%M:%S.%f")[:-3],
+                         "request_url": request_url,
+                         "request_headers": request_headers,
+                         "response_url": None,
+                         "status_code": None,
+                         "response_headers": None,
+                         "response_elapsed_ms": 0,
+                         "proxy": self.client.proxies['http'] if self.client.proxies else None,
+                         "client_identifier": self.get_current_client_identifier()
+
+                     })
+
+
+def _forbidden_request_handler(self, response: Optional[Response]):
+    if not response:
+        return
+
+    if response.status_code in self.status_codes_to_forbidden_response_handle:
+        if self.instantiate_new_client_on_forbidden_response:
+            logging.info("Instantiating new TLS client due to forbidden response...")
+            self.get_tls().close()
+            self._create_new_client()
+
+        if self.proxies_manager:
+            logging.info("Changing proxy due to forbidden response...")
+            self.change_proxy()
+
+
+# tem casos que está retornando 403 como erro da API, tenho que estudar o que fazer nesses casos
+def _requests_handler(self, response: Optional[Response]):
+    self.requests_amount_with_current_client_identifier += 1
+
+    if self.proxies_manager:
+        self.requests_amount_with_current_proxy += 1
+
+    if not response:
+        return
+
+    client_identifier_changed_by_limit = self.is_client_identifier_changeable_by_requests_limit_reached()
+    is_same_proxy_limit = self.is_same_proxy_request_limit_reached()
+    is_forbidden_request_status = self.is_forbidden_request_status(response.status_code)
+    create_new_client_on_forbidden = self.instantiate_new_client_on_forbidden_response
+    change_client_identifier_on_forbidden = self.change_client_identifier_on_forbidden_response
+    has_proxies_manager = self.has_proxies_manager()
+
+    if client_identifier_changed_by_limit:
+        logging.info("Requests limit reached for current client identifier")
+        if is_forbidden_request_status or is_same_proxy_limit:
+            self._create_new_client(self.client_identifiers_manager.get_next(),
+                                    proxy=None if not has_proxies_manager else self.proxies_manager.get_next())
+
+        else:
+            self._create_new_client(self.client_identifiers_manager.get_next(),
+                                    proxy=None if not has_proxies_manager else self.proxies_manager.get_current_proxy())
+
+        return
+
+    if is_forbidden_request_status:
+
+        if not create_new_client_on_forbidden:
+            if has_proxies_manager:
+                logging.info("Changing proxy due to forbidden response...")
+                self.change_proxy()
+
+            return
+
+        if create_new_client_on_forbidden:
+                if change_client_identifier_on_forbidden:
+                    logging.info("Instantiating new TLS client due to forbidden response and changing client identifier...")
+                    self._create_new_client(self.client_identifiers_manager.get_next(),
+                                            proxy=None if not has_proxies_manager else self.proxies_manager.get_next())
+
+                else:
+                    logging.info("Instantiating new TLS client due to forbidden response...")
+                    item = self.client_identifiers_manager.get_current_item()
+                    self._create_new_client(str(item),
+                                            proxy=None if not has_proxies_manager else self.proxies_manager.get_next())
+
+
+    if is_same_proxy_limit:
+        if has_proxies_manager:
+            logging.info("Changing proxy due to requests limit reached...")
+            self.change_proxy()
+
+    # if client_identifier_changed_by_limit:
+    #     logging.info("Requests limit reached for current client identifier")
+    #     logging.info("Changing client identifier due to requests limit reached...")
+    #     # self._create_new_client(self.client_identifiers_manager.get_next(),
+    #     #                         proxy=self.proxies_manager.get_current_proxy() if self.proxies_manager else None)
+    #
+    #
+    #
+    #
+    # if self.requests_limit_with_same_proxy and self.requests_amount_with_current_proxy >= self.requests_limit_with_same_proxy:
+    #     logging.info("Requests limit reached for current proxy")
+    #     if self.proxies_manager:
+    #         proxy_changed_by_limit = True
+    #         logging.info("Changing proxy due to requests limit reached...")
+    #
+    #         if response.status_code in self.status_codes_to_forbidden_response_handle and self.instantiate_new_client_on_forbidden_response:
+    #             logging.info("Instantiating new TLS client due to forbidden response before changing proxy...")
+    #             self.get_tls().close()
+    #             if self.change_client_identifier_on_forbidden_response:
+    #                 self._create_new_client(self.client_identifiers_manager.get_next())
+    #             else:
+    #                 self._create_new_client(self.client_identifiers_manager.get_current())
+    #
+    #         self.change_proxy()
+    #
+    # # Metodo vai ter que ser usado para não trocar o proxy duas vezes, uma pelo status 403 e outra pelo limite de requests
+    # if not proxy_changed_by_limit:
+    #     return
+    #
+    # if response.status_code in self.status_codes_to_forbidden_response_handle:
+    #     if self.instantiate_new_client_on_forbidden_response:
+    #         logging.info("Instantiating new TLS client due to forbidden response...")
+    #         self.get_tls().close()
+    #
+    #         if self.change_client_identifier_on_forbidden_response:
+    #             self._create_new_client(self.client_identifiers_manager.get_next())
+    #         else:
+    #             self._create_new_client()
+    #
+    #     if self.proxies_manager:
+    #         logging.info("Changing proxy due to forbidden response...")
+    #         self.change_proxy()

notify_tls_client-0.1.9/notify_tls_client/core/client_identifiers_manager.py

@@ -0,0 +1,45 @@
+from dataclasses import dataclass, field
+from typing import Optional
+
+from dataclasses_json import dataclass_json
+
+from notify_tls_client.tls_client.settings import ClientIdentifiers
+
+
+@dataclass_json
+@dataclass
+class ClientIdentifiersManager:
+    _items: list[ClientIdentifiers] = field(default_factory=list)
+    _current_index: int = 0
+    _current_item: Optional[ClientIdentifiers] = None
+
+    def get_next(self) -> ClientIdentifiers:
+        if not self._items:
+            raise Exception('No items available')
+
+        if self._current_index < len(self._items):
+            _item = self._items[self._current_index]
+            self._current_index += 1
+        else:
+            self._current_index = 0
+            _item = self._items[self._current_index]
+
+        return _item
+
+    def get_current_item(self) -> Optional[ClientIdentifiers]:
+        if not self._items:
+            return None
+
+        return self._items[0] if self._current_index == 0 else self._items[self._current_index - 1]
+
+
+    def set_items(self, items: list[ClientIdentifiers]):
+        self._items = items
+        self._current_proxy_index = 0
+        self._current_proxy = None
+
+    def get_item(self) -> list[ClientIdentifiers]:
+        return self._items
+
+    def get_total_items(self) -> int:
+        return len(self._items)

notify_tls_client-0.1.9/notify_tls_client/core/notifytlsclient.py

@@ -0,0 +1,196 @@
+import logging
+from typing import Optional
+
+from typing_extensions import TypeAlias, Literal
+
+from notify_tls_client.core.client import *
+from notify_tls_client.core.client_identifiers_manager import ClientIdentifiersManager
+from notify_tls_client.core.proxiesmanager import ProxiesManager, Proxy
+from notify_tls_client.tls_client.response import Response
+from notify_tls_client.tls_client.sessions import Session
+from notify_tls_client.tls_client.settings import ClientIdentifiers
+from notify_tls_client.tls_client.structures import CaseInsensitiveDict
+
+HttpMethods: TypeAlias = Literal["GET", "POST", "PUT", "DELETE", "PATCH"]
+logger = logging.Logger(__name__)
+
+
+class NotifyTLSClient:
+    def __init__(self,
+                 proxies_manager: Optional[ProxiesManager] = None,
+                 client_identifiers: list[ClientIdentifiers] = None,
+                 requests_limit_same_proxy: int = 1000,
+                 random_tls_extension_order: bool = True,
+                 requests_limit_with_same_client_identifier: int = -1,
+                 instantiate_new_client_on_forbidden_response: bool = False,
+                 instantiate_new_client_on_exception: bool = False,
+                 debug_mode: bool = False,
+                 status_codes_to_forbidden_response_handle: list[int] = None,
+                 change_client_identifier_on_forbidden_response: bool = False,
+                 default_headers: dict = None,
+                 disable_http3: bool = False
+                 ):
+
+        self.client: Optional[Session] = None
+        self.change_client_identifier_on_forbidden_response = change_client_identifier_on_forbidden_response
+        self.client_identifiers_manager = ClientIdentifiersManager(client_identifiers or ["chrome_133"])
+        self.status_codes_to_forbidden_response_handle = status_codes_to_forbidden_response_handle or [403]
+        self.free = True
+        self.requests_amount = 0
+        self.requests_amount_with_current_proxy = 0
+        self.requests_amount_with_current_client_identifier = 0
+        self.last_request_status = 0
+        self.headers = default_headers or CaseInsensitiveDict({
+            "User-Agent": f"tls-client/1.0.0",
+            "Accept-Encoding": "gzip, deflate, br",
+            "Accept": "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8",
+            "Connection": "keep-alive",
+        })
+
+        self.disable_http3 = disable_http3
+        self.proxies_manager = proxies_manager
+        self.requests_limit_with_same_proxy = requests_limit_same_proxy
+        self.requests_limit_with_same_client_identifier = requests_limit_with_same_client_identifier
+        self.debug_mode = debug_mode
+
+        self.instantiate_new_client_on_forbidden_response = instantiate_new_client_on_forbidden_response
+        self.instantiate_new_client_on_exception = instantiate_new_client_on_exception
+
+        self.current_proxy = None
+        self._create_new_client(self.client_identifiers_manager.get_next(),
+                                random_tls_extension_order,
+                                proxy=self.proxies_manager.get_next() if self.proxies_manager else None)
+
+
+    def _create_new_client(self,
+                           client_identifier: ClientIdentifiers = 'chrome_133',
+                           random_tls_extension_order: bool = False,
+                           proxy: Optional[Proxy] = None):
+
+
+
+        if self.client:
+            self.client.close()
+
+        old_client_identifier = self.client.client_identifier if self.client else None
+
+        self.client = Session(client_identifier=client_identifier,
+                              random_tls_extension_order=random_tls_extension_order)
+
+        if old_client_identifier != client_identifier:
+            self.requests_amount_with_current_client_identifier = 0
+
+        if proxy:
+
+            self.client.proxies = proxy.to_proxy_dict()
+
+            if self.current_proxy != proxy:
+                self.current_proxy = proxy
+                self.requests_amount_with_current_proxy = 0
+
+
+    def set_requests_limit_same_proxy(self, requests_limit_same_proxy: int):
+        self.requests_limit_with_same_proxy = requests_limit_same_proxy
+
+    def set_proxies_manager(self, proxies_manager: ProxiesManager):
+        self.proxies_manager = proxies_manager
+
+    def change_proxy(self):
+        if self.proxies_manager and self.proxies_manager.get_proxies():
+            self.current_proxy = self.proxies_manager.get_next()
+            self.client.proxies = self.current_proxy.to_proxy_dict()
+            self.requests_amount_with_current_proxy = 0
+
+    def set_proxies(self, proxies: Proxy):
+        self.client.proxies = proxies.to_proxy_dict()
+
+    def set_headers(self, headers: dict):
+        self.client.headers = CaseInsensitiveDict(headers)
+
+    def get_cookies(self):
+        return self.client.cookies
+
+    def get_cookie_by_name(self, name: str) -> Optional[str]:
+        try:
+            return self.client.cookies.get(name)
+        except Exception as e:
+            logging.exception(f"Error getting cookie '{name}': {e}")
+
+    @request_guard_decorator
+    def execute_request(self, method: HttpMethods, url: str, **kwargs) -> Optional[Response]:
+        method = method.upper()
+        if method == "GET":
+            return self.client.get(url, **kwargs)
+        elif method == "POST":
+            return self.client.post(url, **kwargs)
+        elif method == "PUT":
+            return self.client.put(url, **kwargs)
+        elif method == "DELETE":
+            return self.client.delete(url, **kwargs)
+        elif method == "PATCH":
+            return self.client.delete(url, **kwargs)
+        else:
+            raise ValueError(f"Unsupported HTTP method: {method}")
+
+    def get(self, url: str, **kwargs) -> Optional[Response]:
+        return self.execute_request("GET", url, **kwargs)
+
+    def post(self, url: str, **kwargs) -> Optional[Response]:
+        return self.execute_request("POST", url, **kwargs)
+
+    def put(self, url: str, **kwargs) -> Optional[Response]:
+        return self.execute_request("PUT", url, **kwargs)
+
+    def delete(self, url: str, **kwargs) -> Optional[Response]:
+        return self.execute_request("DELETE", url, **kwargs)
+
+    def patch(self, url: str, **kwargs) -> Optional[Response]:
+        return self.execute_request("PATCH", url, **kwargs)
+
+    def _change_to_free(self):
+        self.free = True
+
+    def _change_to_busy(self):
+        self.free = False
+
+    def get_cookie_value_by_name(self, name: str):
+        return self.client.cookies.get(name)
+
+    def set_cookie(self, name: str, value: str):
+        self.client.cookies.set(name, value)
+
+    def get_tls(self):
+        return self.client
+
+    def get_current_client_identifier(self) -> ClientIdentifiers:
+        return self.client.client_identifier
+
+    def is_same_proxy_request_limit_reached(self) -> bool:
+        if self.requests_limit_with_same_proxy == -1:
+            return False
+
+        return self.requests_amount_with_current_proxy >= self.requests_limit_with_same_proxy
+
+    def is_same_client_identifier_request_limit_reached(self) -> bool:
+        if self.requests_limit_with_same_client_identifier == -1:
+            return False
+
+        return self.requests_amount_with_current_client_identifier >= self.requests_limit_with_same_client_identifier
+
+    def is_client_identifier_changeable_by_requests_limit_reached(self) -> bool:
+        return self.is_same_client_identifier_request_limit_reached() and self.has_multiple_client_identifiers()
+
+
+    def has_multiple_client_identifiers(self) -> bool:
+        return self.client_identifiers_manager.get_total_items() > 1
+
+
+    def is_forbidden_request_status(self, status_code: int) -> bool:
+        return status_code in self.status_codes_to_forbidden_response_handle
+
+    def has_proxies_manager(self) -> bool:
+        return self.proxies_manager is not None and self.proxies_manager.get_proxies()
+
+    def set_default_headers(self, headers: dict):
+        self.headers = headers
+        self.client.headers = self.headers

notify_tls_client-0.1.9/notify_tls_client/core/proxiesmanager/__init__.py

@@ -0,0 +1,2 @@
+from .proxiesmanager import ProxiesManager, Proxy
+from .proxiesmanagerloader import ProxiesManagerLoader

{notify_tls_client-0.1.6 → notify_tls_client-0.1.9}/notify_tls_client/core/proxiesmanager/proxiesmanager.py

@@ -1,67 +1,67 @@
-from dataclasses import dataclass, field
-from typing import Optional
-
-from dataclasses_json import dataclass_json
-
-
-@dataclass_json
-@dataclass
-class Proxy:
-    host: str
-    port: int
-    username: Optional[str] = None
-    password: Optional[str] = None
-
-    def to_proxy_dict(self):
-        if self.username is None:
-            return {
-                'http': f'http://{self.host}:{self.port}',
-                'https': f'https://{self.host}:{self.port}'
-            }
-
-        return {
-            'http': f'http://{self.username}:{self.password}@{self.host}:{self.port}',
-            'https': f'https://{self.username}:{self.password}@{self.host}:{self.port}'
-        }
-
-
-@dataclass_json
-@dataclass
-class ProxiesManager:
-    _proxies: list[Proxy] = field(default_factory=list)
-    _current_proxy_index: int = 0
-    _current_proxy: Optional[Proxy] = None
-
-    def get_next(self) -> Proxy:
-
-        if not self._proxies:
-            raise Exception('No proxies available')
-
-        if self._current_proxy_index < len(self._proxies):
-            proxy = self._proxies[self._current_proxy_index]
-            self._current_proxy_index += 1
-        else:
-            self._current_proxy_index = 0
-            proxy = self._proxies[self._current_proxy_index]
-
-        return proxy
-
-    def get_current_proxy(self) -> Optional[Proxy]:
-        if not self._current_proxy:
-            return None
-
-        return self._current_proxy
-
-    def set_proxies(self, proxies: list[Proxy]):
-        self._proxies = proxies
-        self._current_proxy_index = 0
-        self._current_proxy = None
-
-    def add_proxy(self, proxy: Proxy):
-        self._proxies.append(proxy)
-        self._current_proxy_index = 0
-    def get_proxies(self) -> list[Proxy]:
-        return self._proxies
-
-
-
+from dataclasses import dataclass, field
+from typing import Optional
+
+from dataclasses_json import dataclass_json
+
+
+@dataclass_json
+@dataclass
+class Proxy:
+    host: str
+    port: int
+    username: Optional[str] = None
+    password: Optional[str] = None
+
+    def to_proxy_dict(self):
+        if self.username is None:
+            return {
+                'http': f'http://{self.host}:{self.port}',
+                'https': f'https://{self.host}:{self.port}'
+            }
+
+        return {
+            'http': f'http://{self.username}:{self.password}@{self.host}:{self.port}',
+            'https': f'https://{self.username}:{self.password}@{self.host}:{self.port}'
+        }
+
+
+@dataclass_json
+@dataclass
+class ProxiesManager:
+    _proxies: list[Proxy] = field(default_factory=list)
+    _current_proxy_index: int = 0
+    _current_proxy: Optional[Proxy] = None
+
+    def get_next(self) -> Proxy:
+
+        if not self._proxies:
+            raise Exception('No proxies available')
+
+        if self._current_proxy_index < len(self._proxies):
+            proxy = self._proxies[self._current_proxy_index]
+            self._current_proxy_index += 1
+        else:
+            self._current_proxy_index = 0
+            proxy = self._proxies[self._current_proxy_index]
+
+        return proxy
+
+    def get_current_proxy(self) -> Optional[Proxy]:
+        if not self._current_proxy:
+            return None
+
+        return self._current_proxy
+
+    def set_proxies(self, proxies: list[Proxy]):
+        self._proxies = proxies
+        self._current_proxy_index = 0
+        self._current_proxy = None
+
+    def add_proxy(self, proxy: Proxy):
+        self._proxies.append(proxy)
+        self._current_proxy_index = 0
+    def get_proxies(self) -> list[Proxy]:
+        return self._proxies
+
+
+