notify-tls-client 0.1.9__py3-none-any.whl → 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- notify_tls_client/config/__init__.py +18 -0
- notify_tls_client/config/client_config.py +76 -0
- notify_tls_client/config/client_configuration.py +237 -0
- notify_tls_client/config/recovery_config.py +66 -0
- notify_tls_client/config/rotation_config.py +56 -0
- notify_tls_client/core/client/decorators.py +9 -48
- notify_tls_client/core/client_identifiers_manager.py +20 -21
- notify_tls_client/core/notifytlsclient.py +125 -50
- notify_tls_client/core/proxiesmanager/proxiesmanager.py +32 -20
- notify_tls_client/tls_client/cffi.py +2 -0
- notify_tls_client/tls_client/response.py +2 -1
- notify_tls_client/tls_client/sessions.py +6 -4
- notify_tls_client-2.0.0.dist-info/METADATA +38 -0
- {notify_tls_client-0.1.9.dist-info → notify_tls_client-2.0.0.dist-info}/RECORD +16 -11
- {notify_tls_client-0.1.9.dist-info → notify_tls_client-2.0.0.dist-info}/WHEEL +1 -1
- notify_tls_client-0.1.9.dist-info/METADATA +0 -14
- {notify_tls_client-0.1.9.dist-info → notify_tls_client-2.0.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Módulo de configuração para NotifyTLSClient.
|
|
3
|
+
|
|
4
|
+
Fornece classes de configuração especializadas para organizar
|
|
5
|
+
os parâmetros do cliente TLS de forma mais limpa e reutilizável.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from .rotation_config import RotationConfig
|
|
9
|
+
from .recovery_config import RecoveryConfig
|
|
10
|
+
from .client_config import ClientConfig
|
|
11
|
+
from .client_configuration import ClientConfiguration
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
'RotationConfig',
|
|
15
|
+
'RecoveryConfig',
|
|
16
|
+
'ClientConfig',
|
|
17
|
+
'ClientConfiguration',
|
|
18
|
+
]
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Configurações do cliente TLS (identifiers, headers, protocolos).
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from typing import Optional
|
|
7
|
+
|
|
8
|
+
from notify_tls_client.tls_client.settings import ClientIdentifiers
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass
|
|
12
|
+
class ClientConfig:
|
|
13
|
+
"""
|
|
14
|
+
Configurações relacionadas ao cliente TLS e fingerprinting.
|
|
15
|
+
|
|
16
|
+
Attributes:
|
|
17
|
+
client_identifiers: Lista de client identifiers para rotação.
|
|
18
|
+
Identifiers disponíveis incluem chrome_133, firefox_120, safari_17_0, etc.
|
|
19
|
+
Se apenas um identifier for fornecido, não haverá rotação.
|
|
20
|
+
Default: ["chrome_133"]
|
|
21
|
+
|
|
22
|
+
default_headers: Headers padrão a serem enviados em todas as requisições.
|
|
23
|
+
Pode ser sobrescrito por requisição individual.
|
|
24
|
+
Default: None (usa headers padrão do tls-client)
|
|
25
|
+
|
|
26
|
+
disable_http3: Se True, força o uso de HTTP/2 ou HTTP/1.1, desabilitando HTTP/3.
|
|
27
|
+
Útil para ambientes que não suportam QUIC.
|
|
28
|
+
Default: False
|
|
29
|
+
|
|
30
|
+
debug_mode: Se True, habilita logs de debug detalhados.
|
|
31
|
+
IMPORTANTE: Pode vazar informações sensíveis nos logs.
|
|
32
|
+
Default: False
|
|
33
|
+
|
|
34
|
+
Examples:
|
|
35
|
+
>>> # Configuração padrão
|
|
36
|
+
>>> config = ClientConfig()
|
|
37
|
+
|
|
38
|
+
>>> # Múltiplos identifiers para stealth
|
|
39
|
+
>>> config = ClientConfig(
|
|
40
|
+
... client_identifiers=["chrome_133", "firefox_120", "safari_17_0"]
|
|
41
|
+
... )
|
|
42
|
+
|
|
43
|
+
>>> # Headers customizados
|
|
44
|
+
>>> config = ClientConfig(
|
|
45
|
+
... default_headers={
|
|
46
|
+
... "User-Agent": "Mozilla/5.0...",
|
|
47
|
+
... "Accept-Language": "pt-BR,pt;q=0.9"
|
|
48
|
+
... }
|
|
49
|
+
... )
|
|
50
|
+
|
|
51
|
+
>>> # Forçar HTTP/2
|
|
52
|
+
>>> config = ClientConfig(disable_http3=True)
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
client_identifiers: list[ClientIdentifiers] = field(default_factory=lambda: ["chrome_133"])
|
|
56
|
+
default_headers: Optional[dict] = None
|
|
57
|
+
disable_http3: bool = False
|
|
58
|
+
debug_mode: bool = False
|
|
59
|
+
|
|
60
|
+
def __post_init__(self):
|
|
61
|
+
"""Valida os parâmetros de configuração."""
|
|
62
|
+
if not self.client_identifiers:
|
|
63
|
+
raise ValueError("client_identifiers não pode ser vazio")
|
|
64
|
+
|
|
65
|
+
if len(self.client_identifiers) != len(set(self.client_identifiers)):
|
|
66
|
+
raise ValueError("client_identifiers contém duplicatas")
|
|
67
|
+
|
|
68
|
+
# Valida default_headers se fornecido
|
|
69
|
+
if self.default_headers is not None:
|
|
70
|
+
if not isinstance(self.default_headers, dict):
|
|
71
|
+
raise ValueError("default_headers deve ser um dicionário")
|
|
72
|
+
|
|
73
|
+
# Valida que keys são strings
|
|
74
|
+
for key in self.default_headers.keys():
|
|
75
|
+
if not isinstance(key, str):
|
|
76
|
+
raise ValueError(f"Header key inválido: {key}. Deve ser string")
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Configuração completa do NotifyTLSClient com factory methods para casos de uso comuns.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from typing import Optional
|
|
7
|
+
|
|
8
|
+
from notify_tls_client.core.proxiesmanager import ProxiesManager
|
|
9
|
+
from .rotation_config import RotationConfig
|
|
10
|
+
from .recovery_config import RecoveryConfig
|
|
11
|
+
from .client_config import ClientConfig
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass
|
|
15
|
+
class ClientConfiguration:
|
|
16
|
+
"""
|
|
17
|
+
Configuração completa para NotifyTLSClient.
|
|
18
|
+
|
|
19
|
+
Agrupa todas as configurações específicas em um único objeto para
|
|
20
|
+
simplificar a criação e reutilização de configurações.
|
|
21
|
+
|
|
22
|
+
Attributes:
|
|
23
|
+
proxies_manager: Gerenciador de proxies para rotação automática.
|
|
24
|
+
Default: None (sem proxies)
|
|
25
|
+
|
|
26
|
+
rotation: Configurações de rotação de proxies e client identifiers.
|
|
27
|
+
Default: RotationConfig com valores padrão
|
|
28
|
+
|
|
29
|
+
recovery: Configurações de recuperação automática.
|
|
30
|
+
Default: RecoveryConfig com valores padrão
|
|
31
|
+
|
|
32
|
+
client: Configurações do cliente TLS.
|
|
33
|
+
Default: ClientConfig com valores padrão
|
|
34
|
+
|
|
35
|
+
Examples:
|
|
36
|
+
>>> # Configuração padrão
|
|
37
|
+
>>> config = ClientConfiguration()
|
|
38
|
+
|
|
39
|
+
>>> # Configuração customizada
|
|
40
|
+
>>> config = ClientConfiguration(
|
|
41
|
+
... proxies_manager=ProxiesManager([...]),
|
|
42
|
+
... rotation=RotationConfig(requests_limit_same_proxy=10),
|
|
43
|
+
... recovery=RecoveryConfig(
|
|
44
|
+
... instantiate_new_client_on_forbidden_response=True,
|
|
45
|
+
... status_codes_to_forbidden_response_handle=[403, 429]
|
|
46
|
+
... ),
|
|
47
|
+
... client=ClientConfig(
|
|
48
|
+
... client_identifiers=["chrome_133", "firefox_120"]
|
|
49
|
+
... )
|
|
50
|
+
... )
|
|
51
|
+
|
|
52
|
+
>>> # Usando factory method
|
|
53
|
+
>>> config = ClientConfiguration.aggressive(proxies_manager)
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
proxies_manager: Optional[ProxiesManager] = None
|
|
57
|
+
rotation: RotationConfig = field(default_factory=RotationConfig)
|
|
58
|
+
recovery: RecoveryConfig = field(default_factory=RecoveryConfig)
|
|
59
|
+
client: ClientConfig = field(default_factory=ClientConfig)
|
|
60
|
+
|
|
61
|
+
@classmethod
|
|
62
|
+
def aggressive(cls, proxies_manager: Optional[ProxiesManager] = None) -> 'ClientConfiguration':
|
|
63
|
+
"""
|
|
64
|
+
Configuração agressiva: rotação rápida e recuperação automática completa.
|
|
65
|
+
|
|
66
|
+
Ideal para cenários de scraping intensivo onde detectar e contornar
|
|
67
|
+
bloqueios rapidamente é crítico.
|
|
68
|
+
|
|
69
|
+
Características:
|
|
70
|
+
- Troca proxy a cada 10 requisições
|
|
71
|
+
- Troca client identifier a cada 50 requisições
|
|
72
|
+
- Recuperação automática em erros e respostas proibidas
|
|
73
|
+
- Troca client identifier em respostas proibidas
|
|
74
|
+
- Monitora status codes 403, 429 e 503
|
|
75
|
+
|
|
76
|
+
Args:
|
|
77
|
+
proxies_manager: Gerenciador de proxies (obrigatório para rotação)
|
|
78
|
+
|
|
79
|
+
Returns:
|
|
80
|
+
ClientConfiguration configurado para modo agressivo
|
|
81
|
+
|
|
82
|
+
Example:
|
|
83
|
+
>>> tls = NotifyTLSClient(ClientConfiguration.aggressive(proxies))
|
|
84
|
+
"""
|
|
85
|
+
return cls(
|
|
86
|
+
proxies_manager=proxies_manager,
|
|
87
|
+
rotation=RotationConfig(
|
|
88
|
+
requests_limit_same_proxy=10,
|
|
89
|
+
requests_limit_same_client_identifier=50,
|
|
90
|
+
random_tls_extension_order=True
|
|
91
|
+
),
|
|
92
|
+
recovery=RecoveryConfig(
|
|
93
|
+
instantiate_new_client_on_forbidden_response=True,
|
|
94
|
+
instantiate_new_client_on_exception=True,
|
|
95
|
+
change_client_identifier_on_forbidden_response=True,
|
|
96
|
+
status_codes_to_forbidden_response_handle=[403, 429, 503]
|
|
97
|
+
),
|
|
98
|
+
client=ClientConfig(
|
|
99
|
+
client_identifiers=["chrome_133", "firefox_120", "safari_17_0"]
|
|
100
|
+
)
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
@classmethod
|
|
104
|
+
def stealth(cls, proxies_manager: Optional[ProxiesManager] = None) -> 'ClientConfiguration':
|
|
105
|
+
"""
|
|
106
|
+
Configuração stealth: foco em evitar detecção através de diversidade.
|
|
107
|
+
|
|
108
|
+
Ideal para cenários onde ser discreto é mais importante que velocidade,
|
|
109
|
+
utilizando múltiplos client identifiers e randomização.
|
|
110
|
+
|
|
111
|
+
Características:
|
|
112
|
+
- Múltiplos client identifiers (Chrome, Firefox, Safari, Edge)
|
|
113
|
+
- Ordem de extensões TLS randomizada
|
|
114
|
+
- Rotação moderada de proxies (100 requisições)
|
|
115
|
+
- Rotação de client identifier a cada 30 requisições
|
|
116
|
+
- Recuperação básica apenas para erros críticos
|
|
117
|
+
|
|
118
|
+
Args:
|
|
119
|
+
proxies_manager: Gerenciador de proxies (opcional mas recomendado)
|
|
120
|
+
|
|
121
|
+
Returns:
|
|
122
|
+
ClientConfiguration configurado para modo stealth
|
|
123
|
+
|
|
124
|
+
Example:
|
|
125
|
+
>>> tls = NotifyTLSClient(ClientConfiguration.stealth(proxies))
|
|
126
|
+
"""
|
|
127
|
+
return cls(
|
|
128
|
+
proxies_manager=proxies_manager,
|
|
129
|
+
rotation=RotationConfig(
|
|
130
|
+
requests_limit_same_proxy=100,
|
|
131
|
+
requests_limit_same_client_identifier=30,
|
|
132
|
+
random_tls_extension_order=True
|
|
133
|
+
),
|
|
134
|
+
recovery=RecoveryConfig(
|
|
135
|
+
instantiate_new_client_on_forbidden_response=False,
|
|
136
|
+
instantiate_new_client_on_exception=True,
|
|
137
|
+
change_client_identifier_on_forbidden_response=False,
|
|
138
|
+
status_codes_to_forbidden_response_handle=[403]
|
|
139
|
+
),
|
|
140
|
+
client=ClientConfig(
|
|
141
|
+
client_identifiers=[
|
|
142
|
+
"chrome_133",
|
|
143
|
+
"firefox_120",
|
|
144
|
+
"safari_17_0",
|
|
145
|
+
"safari_ios_16_0"
|
|
146
|
+
]
|
|
147
|
+
)
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
@classmethod
|
|
151
|
+
def simple(cls, proxies_manager: Optional[ProxiesManager] = None) -> 'ClientConfiguration':
|
|
152
|
+
"""
|
|
153
|
+
Configuração simples: apenas rotação básica de proxies.
|
|
154
|
+
|
|
155
|
+
Ideal para uso geral onde não há necessidade de evasão agressiva,
|
|
156
|
+
mantendo a simplicidade e confiabilidade.
|
|
157
|
+
|
|
158
|
+
Características:
|
|
159
|
+
- Client identifier único (Chrome 133)
|
|
160
|
+
- Rotação de proxy padrão (1000 requisições)
|
|
161
|
+
- Sem rotação de client identifier
|
|
162
|
+
- Sem recuperação automática
|
|
163
|
+
- Monitora apenas status 403
|
|
164
|
+
|
|
165
|
+
Args:
|
|
166
|
+
proxies_manager: Gerenciador de proxies (opcional)
|
|
167
|
+
|
|
168
|
+
Returns:
|
|
169
|
+
ClientConfiguration configurado para modo simples
|
|
170
|
+
|
|
171
|
+
Example:
|
|
172
|
+
>>> tls = NotifyTLSClient(ClientConfiguration.simple())
|
|
173
|
+
"""
|
|
174
|
+
return cls(
|
|
175
|
+
proxies_manager=proxies_manager,
|
|
176
|
+
rotation=RotationConfig(
|
|
177
|
+
requests_limit_same_proxy=1000,
|
|
178
|
+
requests_limit_same_client_identifier=-1,
|
|
179
|
+
random_tls_extension_order=False
|
|
180
|
+
),
|
|
181
|
+
recovery=RecoveryConfig(
|
|
182
|
+
instantiate_new_client_on_forbidden_response=False,
|
|
183
|
+
instantiate_new_client_on_exception=False,
|
|
184
|
+
change_client_identifier_on_forbidden_response=False,
|
|
185
|
+
status_codes_to_forbidden_response_handle=[403]
|
|
186
|
+
),
|
|
187
|
+
client=ClientConfig(
|
|
188
|
+
client_identifiers=["chrome_133"]
|
|
189
|
+
)
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
@classmethod
|
|
193
|
+
def mobile(cls, proxies_manager: Optional[ProxiesManager] = None, platform: str = "android") -> 'ClientConfiguration':
|
|
194
|
+
"""
|
|
195
|
+
Configuração mobile: simula dispositivos móveis.
|
|
196
|
+
|
|
197
|
+
Ideal para APIs ou sites que têm comportamento diferente para mobile.
|
|
198
|
+
|
|
199
|
+
Características:
|
|
200
|
+
- Client identifiers mobile (Android ou iOS)
|
|
201
|
+
- Rotação moderada
|
|
202
|
+
- Recuperação automática em erros
|
|
203
|
+
|
|
204
|
+
Args:
|
|
205
|
+
proxies_manager: Gerenciador de proxies (opcional)
|
|
206
|
+
platform: "android" ou "ios"
|
|
207
|
+
|
|
208
|
+
Returns:
|
|
209
|
+
ClientConfiguration configurado para modo mobile
|
|
210
|
+
|
|
211
|
+
Example:
|
|
212
|
+
>>> tls = NotifyTLSClient(ClientConfiguration.mobile(platform="ios"))
|
|
213
|
+
"""
|
|
214
|
+
if platform.lower() == "android":
|
|
215
|
+
identifiers = ["okhttp4_android_13", "okhttp4_android_12"]
|
|
216
|
+
elif platform.lower() == "ios":
|
|
217
|
+
identifiers = ["safari_ios_16_0", "safari_ios_15_6"]
|
|
218
|
+
else:
|
|
219
|
+
raise ValueError(f"Platform inválida: {platform}. Use 'android' ou 'ios'")
|
|
220
|
+
|
|
221
|
+
return cls(
|
|
222
|
+
proxies_manager=proxies_manager,
|
|
223
|
+
rotation=RotationConfig(
|
|
224
|
+
requests_limit_same_proxy=50,
|
|
225
|
+
requests_limit_same_client_identifier=100,
|
|
226
|
+
random_tls_extension_order=True
|
|
227
|
+
),
|
|
228
|
+
recovery=RecoveryConfig(
|
|
229
|
+
instantiate_new_client_on_forbidden_response=True,
|
|
230
|
+
instantiate_new_client_on_exception=True,
|
|
231
|
+
change_client_identifier_on_forbidden_response=False,
|
|
232
|
+
status_codes_to_forbidden_response_handle=[403, 429]
|
|
233
|
+
),
|
|
234
|
+
client=ClientConfig(
|
|
235
|
+
client_identifiers=identifiers
|
|
236
|
+
)
|
|
237
|
+
)
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Configurações de recuperação automática em caso de erros e respostas proibidas.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass
|
|
9
|
+
class RecoveryConfig:
|
|
10
|
+
"""
|
|
11
|
+
Configurações para controlar o comportamento de recuperação automática.
|
|
12
|
+
|
|
13
|
+
Attributes:
|
|
14
|
+
instantiate_new_client_on_forbidden_response: Se True, cria uma nova sessão TLS
|
|
15
|
+
quando receber um status code configurado como "proibido".
|
|
16
|
+
Default: False
|
|
17
|
+
|
|
18
|
+
instantiate_new_client_on_exception: Se True, cria uma nova sessão TLS
|
|
19
|
+
quando ocorrer uma exceção durante a requisição.
|
|
20
|
+
Default: False
|
|
21
|
+
|
|
22
|
+
change_client_identifier_on_forbidden_response: Se True, além de criar uma nova
|
|
23
|
+
sessão, também troca o client identifier quando receber resposta proibida.
|
|
24
|
+
Requer instantiate_new_client_on_forbidden_response=True para ter efeito.
|
|
25
|
+
Default: False
|
|
26
|
+
|
|
27
|
+
status_codes_to_forbidden_response_handle: Lista de status codes HTTP que devem
|
|
28
|
+
acionar os mecanismos de recuperação automática.
|
|
29
|
+
Default: [403]
|
|
30
|
+
|
|
31
|
+
Examples:
|
|
32
|
+
>>> # Recuperação agressiva
|
|
33
|
+
>>> config = RecoveryConfig(
|
|
34
|
+
... instantiate_new_client_on_forbidden_response=True,
|
|
35
|
+
... instantiate_new_client_on_exception=True,
|
|
36
|
+
... change_client_identifier_on_forbidden_response=True,
|
|
37
|
+
... status_codes_to_forbidden_response_handle=[403, 429, 503]
|
|
38
|
+
... )
|
|
39
|
+
|
|
40
|
+
>>> # Sem recuperação automática
|
|
41
|
+
>>> config = RecoveryConfig()
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
instantiate_new_client_on_forbidden_response: bool = False
|
|
45
|
+
instantiate_new_client_on_exception: bool = False
|
|
46
|
+
change_client_identifier_on_forbidden_response: bool = False
|
|
47
|
+
status_codes_to_forbidden_response_handle: list[int] = field(default_factory=lambda: [403])
|
|
48
|
+
|
|
49
|
+
def __post_init__(self):
|
|
50
|
+
"""Valida os parâmetros de configuração."""
|
|
51
|
+
if not self.status_codes_to_forbidden_response_handle:
|
|
52
|
+
raise ValueError("status_codes_to_forbidden_response_handle não pode ser vazio")
|
|
53
|
+
|
|
54
|
+
for code in self.status_codes_to_forbidden_response_handle:
|
|
55
|
+
if not isinstance(code, int) or code < 100 or code > 599:
|
|
56
|
+
raise ValueError(f"Status code inválido: {code}. Deve estar entre 100 e 599")
|
|
57
|
+
|
|
58
|
+
# Aviso sobre configuração inconsistente
|
|
59
|
+
if self.change_client_identifier_on_forbidden_response and not self.instantiate_new_client_on_forbidden_response:
|
|
60
|
+
import warnings
|
|
61
|
+
warnings.warn(
|
|
62
|
+
"change_client_identifier_on_forbidden_response=True não terá efeito "
|
|
63
|
+
"porque instantiate_new_client_on_forbidden_response=False. "
|
|
64
|
+
"Considere habilitar ambos.",
|
|
65
|
+
UserWarning
|
|
66
|
+
)
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Configurações de rotação de proxies e client identifiers.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass
|
|
9
|
+
class RotationConfig:
|
|
10
|
+
"""
|
|
11
|
+
Configurações para controlar a rotação automática de proxies e client identifiers.
|
|
12
|
+
|
|
13
|
+
Attributes:
|
|
14
|
+
requests_limit_same_proxy: Número máximo de requisições com o mesmo proxy.
|
|
15
|
+
Use -1 para desabilitar rotação automática por limite.
|
|
16
|
+
Default: 1000
|
|
17
|
+
|
|
18
|
+
requests_limit_same_client_identifier: Número máximo de requisições com o mesmo
|
|
19
|
+
client identifier. Use -1 para desabilitar rotação automática por limite.
|
|
20
|
+
Default: -1 (desabilitado)
|
|
21
|
+
|
|
22
|
+
random_tls_extension_order: Se True, randomiza a ordem das extensões TLS
|
|
23
|
+
para dificultar fingerprinting.
|
|
24
|
+
Default: True
|
|
25
|
+
|
|
26
|
+
Examples:
|
|
27
|
+
>>> # Rotação agressiva
|
|
28
|
+
>>> config = RotationConfig(
|
|
29
|
+
... requests_limit_same_proxy=10,
|
|
30
|
+
... requests_limit_same_client_identifier=50
|
|
31
|
+
... )
|
|
32
|
+
|
|
33
|
+
>>> # Sem rotação automática
|
|
34
|
+
>>> config = RotationConfig(
|
|
35
|
+
... requests_limit_same_proxy=-1,
|
|
36
|
+
... requests_limit_same_client_identifier=-1
|
|
37
|
+
... )
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
requests_limit_same_proxy: int = 1000
|
|
41
|
+
requests_limit_same_client_identifier: int = -1
|
|
42
|
+
random_tls_extension_order: bool = True
|
|
43
|
+
|
|
44
|
+
def __post_init__(self):
|
|
45
|
+
"""Valida os parâmetros de configuração."""
|
|
46
|
+
if self.requests_limit_same_proxy < -1:
|
|
47
|
+
raise ValueError("requests_limit_same_proxy deve ser >= -1")
|
|
48
|
+
|
|
49
|
+
if self.requests_limit_same_client_identifier < -1:
|
|
50
|
+
raise ValueError("requests_limit_same_client_identifier deve ser >= -1")
|
|
51
|
+
|
|
52
|
+
if self.requests_limit_same_proxy == 0:
|
|
53
|
+
raise ValueError("requests_limit_same_proxy não pode ser 0 (use -1 para desabilitar)")
|
|
54
|
+
|
|
55
|
+
if self.requests_limit_same_client_identifier == 0:
|
|
56
|
+
raise ValueError("requests_limit_same_client_identifier não pode ser 0 (use -1 para desabilitar)")
|
|
@@ -38,8 +38,8 @@ def request_guard_decorator(callback):
|
|
|
38
38
|
request_headers = value
|
|
39
39
|
|
|
40
40
|
try:
|
|
41
|
-
self.requests_amount += 1
|
|
42
41
|
res = callback(*args, **kwargs)
|
|
42
|
+
self.requests_amount += 1
|
|
43
43
|
_log_request_info(self, res, request_url, request_headers)
|
|
44
44
|
_requests_handler(self, res)
|
|
45
45
|
return res
|
|
@@ -63,10 +63,14 @@ def request_guard_decorator(callback):
|
|
|
63
63
|
def _recover_on_exception(self):
|
|
64
64
|
if self.instantiate_new_client_on_exception:
|
|
65
65
|
logging.info("Instantiating new TLS client due to exception...")
|
|
66
|
-
self.
|
|
67
|
-
self.
|
|
68
|
-
|
|
69
|
-
|
|
66
|
+
current_identifier = self.client_identifiers_manager.get_current_item()
|
|
67
|
+
next_proxy = self.proxies_manager.get_next() if self.proxies_manager else None
|
|
68
|
+
self._create_new_client(
|
|
69
|
+
client_identifier=current_identifier or 'chrome_133',
|
|
70
|
+
random_tls_extension_order=self.random_tls_extension_order,
|
|
71
|
+
proxy=next_proxy
|
|
72
|
+
)
|
|
73
|
+
elif self.proxies_manager:
|
|
70
74
|
logging.info("Changing proxy due to exception...")
|
|
71
75
|
self.change_proxy()
|
|
72
76
|
|
|
@@ -194,46 +198,3 @@ def _requests_handler(self, response: Optional[Response]):
|
|
|
194
198
|
if has_proxies_manager:
|
|
195
199
|
logging.info("Changing proxy due to requests limit reached...")
|
|
196
200
|
self.change_proxy()
|
|
197
|
-
|
|
198
|
-
# if client_identifier_changed_by_limit:
|
|
199
|
-
# logging.info("Requests limit reached for current client identifier")
|
|
200
|
-
# logging.info("Changing client identifier due to requests limit reached...")
|
|
201
|
-
# # self._create_new_client(self.client_identifiers_manager.get_next(),
|
|
202
|
-
# # proxy=self.proxies_manager.get_current_proxy() if self.proxies_manager else None)
|
|
203
|
-
#
|
|
204
|
-
#
|
|
205
|
-
#
|
|
206
|
-
#
|
|
207
|
-
# if self.requests_limit_with_same_proxy and self.requests_amount_with_current_proxy >= self.requests_limit_with_same_proxy:
|
|
208
|
-
# logging.info("Requests limit reached for current proxy")
|
|
209
|
-
# if self.proxies_manager:
|
|
210
|
-
# proxy_changed_by_limit = True
|
|
211
|
-
# logging.info("Changing proxy due to requests limit reached...")
|
|
212
|
-
#
|
|
213
|
-
# if response.status_code in self.status_codes_to_forbidden_response_handle and self.instantiate_new_client_on_forbidden_response:
|
|
214
|
-
# logging.info("Instantiating new TLS client due to forbidden response before changing proxy...")
|
|
215
|
-
# self.get_tls().close()
|
|
216
|
-
# if self.change_client_identifier_on_forbidden_response:
|
|
217
|
-
# self._create_new_client(self.client_identifiers_manager.get_next())
|
|
218
|
-
# else:
|
|
219
|
-
# self._create_new_client(self.client_identifiers_manager.get_current())
|
|
220
|
-
#
|
|
221
|
-
# self.change_proxy()
|
|
222
|
-
#
|
|
223
|
-
# # Metodo vai ter que ser usado para não trocar o proxy duas vezes, uma pelo status 403 e outra pelo limite de requests
|
|
224
|
-
# if not proxy_changed_by_limit:
|
|
225
|
-
# return
|
|
226
|
-
#
|
|
227
|
-
# if response.status_code in self.status_codes_to_forbidden_response_handle:
|
|
228
|
-
# if self.instantiate_new_client_on_forbidden_response:
|
|
229
|
-
# logging.info("Instantiating new TLS client due to forbidden response...")
|
|
230
|
-
# self.get_tls().close()
|
|
231
|
-
#
|
|
232
|
-
# if self.change_client_identifier_on_forbidden_response:
|
|
233
|
-
# self._create_new_client(self.client_identifiers_manager.get_next())
|
|
234
|
-
# else:
|
|
235
|
-
# self._create_new_client()
|
|
236
|
-
#
|
|
237
|
-
# if self.proxies_manager:
|
|
238
|
-
# logging.info("Changing proxy due to forbidden response...")
|
|
239
|
-
# self.change_proxy()
|
|
@@ -1,45 +1,44 @@
|
|
|
1
|
+
import threading
|
|
1
2
|
from dataclasses import dataclass, field
|
|
2
3
|
from typing import Optional
|
|
3
4
|
|
|
4
|
-
from dataclasses_json import dataclass_json
|
|
5
|
-
|
|
6
5
|
from notify_tls_client.tls_client.settings import ClientIdentifiers
|
|
7
6
|
|
|
8
7
|
|
|
9
|
-
@dataclass_json
|
|
10
8
|
@dataclass
|
|
11
9
|
class ClientIdentifiersManager:
|
|
12
10
|
_items: list[ClientIdentifiers] = field(default_factory=list)
|
|
13
11
|
_current_index: int = 0
|
|
14
12
|
_current_item: Optional[ClientIdentifiers] = None
|
|
15
13
|
|
|
14
|
+
def __post_init__(self):
|
|
15
|
+
self._lock = threading.Lock()
|
|
16
|
+
|
|
16
17
|
def get_next(self) -> ClientIdentifiers:
|
|
17
|
-
|
|
18
|
-
|
|
18
|
+
with self._lock:
|
|
19
|
+
if not self._items:
|
|
20
|
+
raise Exception('No items available')
|
|
19
21
|
|
|
20
|
-
if self._current_index < len(self._items):
|
|
21
|
-
_item = self._items[self._current_index]
|
|
22
|
-
self._current_index += 1
|
|
23
|
-
else:
|
|
24
|
-
self._current_index = 0
|
|
25
22
|
_item = self._items[self._current_index]
|
|
23
|
+
self._current_item = _item
|
|
24
|
+
self._current_index = (self._current_index + 1) % len(self._items)
|
|
26
25
|
|
|
27
|
-
|
|
26
|
+
return _item
|
|
28
27
|
|
|
29
28
|
def get_current_item(self) -> Optional[ClientIdentifiers]:
|
|
30
|
-
|
|
31
|
-
return
|
|
32
|
-
|
|
33
|
-
return self._items[0] if self._current_index == 0 else self._items[self._current_index - 1]
|
|
34
|
-
|
|
29
|
+
with self._lock:
|
|
30
|
+
return self._current_item
|
|
35
31
|
|
|
36
32
|
def set_items(self, items: list[ClientIdentifiers]):
|
|
37
|
-
self.
|
|
38
|
-
|
|
39
|
-
|
|
33
|
+
with self._lock:
|
|
34
|
+
self._items = items
|
|
35
|
+
self._current_index = 0
|
|
36
|
+
self._current_item = None
|
|
40
37
|
|
|
41
38
|
def get_item(self) -> list[ClientIdentifiers]:
|
|
42
|
-
|
|
39
|
+
with self._lock:
|
|
40
|
+
return self._items.copy()
|
|
43
41
|
|
|
44
42
|
def get_total_items(self) -> int:
|
|
45
|
-
|
|
43
|
+
with self._lock:
|
|
44
|
+
return len(self._items)
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import logging
|
|
2
|
+
import threading
|
|
3
|
+
import warnings
|
|
2
4
|
from typing import Optional
|
|
3
5
|
|
|
4
6
|
from typing_extensions import TypeAlias, Literal
|
|
@@ -6,60 +8,134 @@ from typing_extensions import TypeAlias, Literal
|
|
|
6
8
|
from notify_tls_client.core.client import *
|
|
7
9
|
from notify_tls_client.core.client_identifiers_manager import ClientIdentifiersManager
|
|
8
10
|
from notify_tls_client.core.proxiesmanager import ProxiesManager, Proxy
|
|
11
|
+
from notify_tls_client.config import ClientConfiguration, RotationConfig, RecoveryConfig, ClientConfig
|
|
9
12
|
from notify_tls_client.tls_client.response import Response
|
|
10
13
|
from notify_tls_client.tls_client.sessions import Session
|
|
11
14
|
from notify_tls_client.tls_client.settings import ClientIdentifiers
|
|
12
15
|
from notify_tls_client.tls_client.structures import CaseInsensitiveDict
|
|
13
16
|
|
|
14
17
|
HttpMethods: TypeAlias = Literal["GET", "POST", "PUT", "DELETE", "PATCH"]
|
|
15
|
-
logger = logging.
|
|
18
|
+
logger = logging.getLogger(__name__)
|
|
16
19
|
|
|
17
20
|
|
|
18
21
|
class NotifyTLSClient:
|
|
19
22
|
def __init__(self,
|
|
23
|
+
config: Optional[ClientConfiguration] = None,
|
|
24
|
+
# Parâmetros legados - mantidos para compatibilidade mas deprecados
|
|
20
25
|
proxies_manager: Optional[ProxiesManager] = None,
|
|
21
26
|
client_identifiers: list[ClientIdentifiers] = None,
|
|
22
|
-
requests_limit_same_proxy: int =
|
|
23
|
-
random_tls_extension_order: bool =
|
|
24
|
-
requests_limit_with_same_client_identifier: int =
|
|
25
|
-
instantiate_new_client_on_forbidden_response: bool =
|
|
26
|
-
instantiate_new_client_on_exception: bool =
|
|
27
|
-
debug_mode: bool =
|
|
27
|
+
requests_limit_same_proxy: int = None,
|
|
28
|
+
random_tls_extension_order: bool = None,
|
|
29
|
+
requests_limit_with_same_client_identifier: int = None,
|
|
30
|
+
instantiate_new_client_on_forbidden_response: bool = None,
|
|
31
|
+
instantiate_new_client_on_exception: bool = None,
|
|
32
|
+
debug_mode: bool = None,
|
|
28
33
|
status_codes_to_forbidden_response_handle: list[int] = None,
|
|
29
|
-
change_client_identifier_on_forbidden_response: bool =
|
|
34
|
+
change_client_identifier_on_forbidden_response: bool = None,
|
|
30
35
|
default_headers: dict = None,
|
|
31
|
-
disable_http3: bool =
|
|
36
|
+
disable_http3: bool = None
|
|
32
37
|
):
|
|
33
|
-
|
|
38
|
+
"""
|
|
39
|
+
Inicializa um NotifyTLSClient.
|
|
40
|
+
|
|
41
|
+
Modo Recomendado - Usando ClientConfiguration:
|
|
42
|
+
>>> config = ClientConfiguration.aggressive(proxies_manager)
|
|
43
|
+
>>> client = NotifyTLSClient(config)
|
|
44
|
+
|
|
45
|
+
Modo Legado - Parâmetros individuais (DEPRECADO):
|
|
46
|
+
>>> client = NotifyTLSClient(
|
|
47
|
+
... proxies_manager=proxies,
|
|
48
|
+
... client_identifiers=["chrome_133"],
|
|
49
|
+
... requests_limit_same_proxy=15
|
|
50
|
+
... )
|
|
51
|
+
|
|
52
|
+
Args:
|
|
53
|
+
config: Objeto ClientConfiguration com todas as configurações.
|
|
54
|
+
Se fornecido, sobrescreve todos os outros parâmetros.
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
# Se config não fornecido, criar a partir de parâmetros legados
|
|
58
|
+
if config is None:
|
|
59
|
+
# Emitir warning se usar API legada
|
|
60
|
+
if any(param is not None for param in [
|
|
61
|
+
proxies_manager, client_identifiers, requests_limit_same_proxy,
|
|
62
|
+
random_tls_extension_order, requests_limit_with_same_client_identifier,
|
|
63
|
+
instantiate_new_client_on_forbidden_response, instantiate_new_client_on_exception,
|
|
64
|
+
debug_mode, status_codes_to_forbidden_response_handle,
|
|
65
|
+
change_client_identifier_on_forbidden_response, default_headers, disable_http3
|
|
66
|
+
]):
|
|
67
|
+
warnings.warn(
|
|
68
|
+
"API com parâmetros individuais está DEPRECADA. "
|
|
69
|
+
"Use ClientConfiguration ao invés. Exemplo: "
|
|
70
|
+
"NotifyTLSClient(ClientConfiguration.aggressive(proxies))",
|
|
71
|
+
DeprecationWarning,
|
|
72
|
+
stacklevel=2
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
# Construir config a partir de parâmetros legados
|
|
76
|
+
config = ClientConfiguration(
|
|
77
|
+
proxies_manager=proxies_manager,
|
|
78
|
+
rotation=RotationConfig(
|
|
79
|
+
requests_limit_same_proxy=requests_limit_same_proxy if requests_limit_same_proxy is not None else 1000,
|
|
80
|
+
requests_limit_same_client_identifier=requests_limit_with_same_client_identifier if requests_limit_with_same_client_identifier is not None else -1,
|
|
81
|
+
random_tls_extension_order=random_tls_extension_order if random_tls_extension_order is not None else True
|
|
82
|
+
),
|
|
83
|
+
recovery=RecoveryConfig(
|
|
84
|
+
instantiate_new_client_on_forbidden_response=instantiate_new_client_on_forbidden_response if instantiate_new_client_on_forbidden_response is not None else False,
|
|
85
|
+
instantiate_new_client_on_exception=instantiate_new_client_on_exception if instantiate_new_client_on_exception is not None else False,
|
|
86
|
+
change_client_identifier_on_forbidden_response=change_client_identifier_on_forbidden_response if change_client_identifier_on_forbidden_response is not None else False,
|
|
87
|
+
status_codes_to_forbidden_response_handle=status_codes_to_forbidden_response_handle or [403]
|
|
88
|
+
),
|
|
89
|
+
client=ClientConfig(
|
|
90
|
+
client_identifiers=client_identifiers or ["chrome_133"],
|
|
91
|
+
default_headers=default_headers,
|
|
92
|
+
disable_http3=disable_http3 if disable_http3 is not None else False,
|
|
93
|
+
debug_mode=debug_mode if debug_mode is not None else False
|
|
94
|
+
)
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
# Armazenar config
|
|
98
|
+
self.config = config
|
|
99
|
+
|
|
100
|
+
# Inicializar estado
|
|
34
101
|
self.client: Optional[Session] = None
|
|
35
|
-
self.change_client_identifier_on_forbidden_response = change_client_identifier_on_forbidden_response
|
|
36
|
-
self.client_identifiers_manager = ClientIdentifiersManager(client_identifiers or ["chrome_133"])
|
|
37
|
-
self.status_codes_to_forbidden_response_handle = status_codes_to_forbidden_response_handle or [403]
|
|
38
102
|
self.free = True
|
|
39
103
|
self.requests_amount = 0
|
|
40
104
|
self.requests_amount_with_current_proxy = 0
|
|
41
105
|
self.requests_amount_with_current_client_identifier = 0
|
|
42
106
|
self.last_request_status = 0
|
|
43
|
-
self.
|
|
107
|
+
self.current_proxy = None
|
|
108
|
+
self._lock = threading.Lock()
|
|
109
|
+
|
|
110
|
+
# Configurar a partir de config
|
|
111
|
+
self.client_identifiers_manager = ClientIdentifiersManager(config.client.client_identifiers)
|
|
112
|
+
self.proxies_manager = config.proxies_manager
|
|
113
|
+
|
|
114
|
+
# Aliases para compatibilidade interna
|
|
115
|
+
self.requests_limit_with_same_proxy = config.rotation.requests_limit_same_proxy
|
|
116
|
+
self.requests_limit_with_same_client_identifier = config.rotation.requests_limit_same_client_identifier
|
|
117
|
+
self.random_tls_extension_order = config.rotation.random_tls_extension_order
|
|
118
|
+
|
|
119
|
+
self.instantiate_new_client_on_forbidden_response = config.recovery.instantiate_new_client_on_forbidden_response
|
|
120
|
+
self.instantiate_new_client_on_exception = config.recovery.instantiate_new_client_on_exception
|
|
121
|
+
self.change_client_identifier_on_forbidden_response = config.recovery.change_client_identifier_on_forbidden_response
|
|
122
|
+
self.status_codes_to_forbidden_response_handle = config.recovery.status_codes_to_forbidden_response_handle
|
|
123
|
+
|
|
124
|
+
self.disable_http3 = config.client.disable_http3
|
|
125
|
+
self.debug_mode = config.client.debug_mode
|
|
126
|
+
self.headers = config.client.default_headers or CaseInsensitiveDict({
|
|
44
127
|
"User-Agent": f"tls-client/1.0.0",
|
|
45
128
|
"Accept-Encoding": "gzip, deflate, br",
|
|
46
129
|
"Accept": "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8",
|
|
47
130
|
"Connection": "keep-alive",
|
|
48
131
|
})
|
|
49
132
|
|
|
50
|
-
|
|
51
|
-
self.
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
self.instantiate_new_client_on_forbidden_response = instantiate_new_client_on_forbidden_response
|
|
57
|
-
self.instantiate_new_client_on_exception = instantiate_new_client_on_exception
|
|
58
|
-
|
|
59
|
-
self.current_proxy = None
|
|
60
|
-
self._create_new_client(self.client_identifiers_manager.get_next(),
|
|
61
|
-
random_tls_extension_order,
|
|
62
|
-
proxy=self.proxies_manager.get_next() if self.proxies_manager else None)
|
|
133
|
+
# Criar primeiro cliente
|
|
134
|
+
self._create_new_client(
|
|
135
|
+
self.client_identifiers_manager.get_next(),
|
|
136
|
+
self.random_tls_extension_order,
|
|
137
|
+
proxy=self.proxies_manager.get_next() if self.proxies_manager else None
|
|
138
|
+
)
|
|
63
139
|
|
|
64
140
|
|
|
65
141
|
def _create_new_client(self,
|
|
@@ -67,26 +143,34 @@ class NotifyTLSClient:
|
|
|
67
143
|
random_tls_extension_order: bool = False,
|
|
68
144
|
proxy: Optional[Proxy] = None):
|
|
69
145
|
|
|
146
|
+
with self._lock:
|
|
147
|
+
old_client_identifier = None
|
|
148
|
+
old_client = None
|
|
70
149
|
|
|
150
|
+
if self.client:
|
|
151
|
+
old_client_identifier = self.client.client_identifier
|
|
152
|
+
old_client = self.client
|
|
71
153
|
|
|
72
|
-
|
|
73
|
-
|
|
154
|
+
self.client = Session(client_identifier=client_identifier,
|
|
155
|
+
random_tls_extension_order=random_tls_extension_order,
|
|
156
|
+
disable_http3=self.disable_http3)
|
|
74
157
|
|
|
75
|
-
|
|
158
|
+
if old_client:
|
|
159
|
+
old_client.close()
|
|
76
160
|
|
|
77
|
-
|
|
78
|
-
|
|
161
|
+
if old_client_identifier != client_identifier:
|
|
162
|
+
self.requests_amount_with_current_client_identifier = 0
|
|
79
163
|
|
|
80
|
-
|
|
81
|
-
|
|
164
|
+
if proxy:
|
|
165
|
+
self.client.proxies = proxy.to_proxy_dict()
|
|
82
166
|
|
|
83
|
-
|
|
167
|
+
if self.current_proxy != proxy:
|
|
168
|
+
self.current_proxy = proxy
|
|
169
|
+
self.requests_amount_with_current_proxy = 0
|
|
84
170
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
self.current_proxy = proxy
|
|
89
|
-
self.requests_amount_with_current_proxy = 0
|
|
171
|
+
# Aplicar headers padrão
|
|
172
|
+
if self.headers:
|
|
173
|
+
self.client.headers = CaseInsensitiveDict(self.headers)
|
|
90
174
|
|
|
91
175
|
|
|
92
176
|
def set_requests_limit_same_proxy(self, requests_limit_same_proxy: int):
|
|
@@ -128,7 +212,7 @@ class NotifyTLSClient:
|
|
|
128
212
|
elif method == "DELETE":
|
|
129
213
|
return self.client.delete(url, **kwargs)
|
|
130
214
|
elif method == "PATCH":
|
|
131
|
-
return self.client.
|
|
215
|
+
return self.client.patch(url, **kwargs)
|
|
132
216
|
else:
|
|
133
217
|
raise ValueError(f"Unsupported HTTP method: {method}")
|
|
134
218
|
|
|
@@ -147,15 +231,6 @@ class NotifyTLSClient:
|
|
|
147
231
|
def patch(self, url: str, **kwargs) -> Optional[Response]:
|
|
148
232
|
return self.execute_request("PATCH", url, **kwargs)
|
|
149
233
|
|
|
150
|
-
def _change_to_free(self):
|
|
151
|
-
self.free = True
|
|
152
|
-
|
|
153
|
-
def _change_to_busy(self):
|
|
154
|
-
self.free = False
|
|
155
|
-
|
|
156
|
-
def get_cookie_value_by_name(self, name: str):
|
|
157
|
-
return self.client.cookies.get(name)
|
|
158
|
-
|
|
159
234
|
def set_cookie(self, name: str, value: str):
|
|
160
235
|
self.client.cookies.set(name, value)
|
|
161
236
|
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import threading
|
|
1
2
|
from dataclasses import dataclass, field
|
|
2
3
|
from typing import Optional
|
|
3
4
|
|
|
@@ -24,44 +25,55 @@ class Proxy:
|
|
|
24
25
|
'https': f'https://{self.username}:{self.password}@{self.host}:{self.port}'
|
|
25
26
|
}
|
|
26
27
|
|
|
28
|
+
def __eq__(self, other):
|
|
29
|
+
if not isinstance(other, Proxy):
|
|
30
|
+
return False
|
|
31
|
+
return (self.host == other.host and
|
|
32
|
+
self.port == other.port and
|
|
33
|
+
self.username == other.username and
|
|
34
|
+
self.password == other.password)
|
|
35
|
+
|
|
36
|
+
def __hash__(self):
|
|
37
|
+
return hash((self.host, self.port, self.username, self.password))
|
|
38
|
+
|
|
27
39
|
|
|
28
|
-
@dataclass_json
|
|
29
40
|
@dataclass
|
|
30
41
|
class ProxiesManager:
|
|
31
42
|
_proxies: list[Proxy] = field(default_factory=list)
|
|
32
43
|
_current_proxy_index: int = 0
|
|
33
44
|
_current_proxy: Optional[Proxy] = None
|
|
34
45
|
|
|
35
|
-
def
|
|
46
|
+
def __post_init__(self):
|
|
47
|
+
self._lock = threading.Lock()
|
|
36
48
|
|
|
37
|
-
|
|
38
|
-
|
|
49
|
+
def get_next(self) -> Proxy:
|
|
50
|
+
with self._lock:
|
|
51
|
+
if not self._proxies:
|
|
52
|
+
raise Exception('No proxies available')
|
|
39
53
|
|
|
40
|
-
if self._current_proxy_index < len(self._proxies):
|
|
41
|
-
proxy = self._proxies[self._current_proxy_index]
|
|
42
|
-
self._current_proxy_index += 1
|
|
43
|
-
else:
|
|
44
|
-
self._current_proxy_index = 0
|
|
45
54
|
proxy = self._proxies[self._current_proxy_index]
|
|
55
|
+
self._current_proxy = proxy
|
|
56
|
+
self._current_proxy_index = (self._current_proxy_index + 1) % len(self._proxies)
|
|
46
57
|
|
|
47
|
-
|
|
58
|
+
return proxy
|
|
48
59
|
|
|
49
60
|
def get_current_proxy(self) -> Optional[Proxy]:
|
|
50
|
-
|
|
51
|
-
return
|
|
52
|
-
|
|
53
|
-
return self._current_proxy
|
|
61
|
+
with self._lock:
|
|
62
|
+
return self._current_proxy
|
|
54
63
|
|
|
55
64
|
def set_proxies(self, proxies: list[Proxy]):
|
|
56
|
-
self.
|
|
57
|
-
|
|
58
|
-
|
|
65
|
+
with self._lock:
|
|
66
|
+
self._proxies = proxies
|
|
67
|
+
self._current_proxy_index = 0
|
|
68
|
+
self._current_proxy = None
|
|
59
69
|
|
|
60
70
|
def add_proxy(self, proxy: Proxy):
|
|
61
|
-
self.
|
|
62
|
-
|
|
71
|
+
with self._lock:
|
|
72
|
+
self._proxies.append(proxy)
|
|
73
|
+
|
|
63
74
|
def get_proxies(self) -> list[Proxy]:
|
|
64
|
-
|
|
75
|
+
with self._lock:
|
|
76
|
+
return self._proxies.copy()
|
|
65
77
|
|
|
66
78
|
|
|
67
79
|
|
|
@@ -18,6 +18,8 @@ else:
|
|
|
18
18
|
file_ext = '-amd64-*.so'
|
|
19
19
|
|
|
20
20
|
root_dir = os.path.abspath(os.path.dirname(__file__))
|
|
21
|
+
lib_dir = os.path.join(root_dir, 'dependencies')
|
|
22
|
+
lib_path = None
|
|
21
23
|
matches = glob(f'{root_dir}/dependencies/tls-client{file_ext}')
|
|
22
24
|
if not matches:
|
|
23
25
|
raise FileNotFoundError(f'No tls-client library found for the current platform: {platform} {machine()}')
|
|
@@ -328,6 +328,9 @@ class Session:
|
|
|
328
328
|
# --- Request Body ---------------------------------------------------------------------------------------------
|
|
329
329
|
# Prepare request body - build request body
|
|
330
330
|
# Data has priority. JSON is only used if data is None.
|
|
331
|
+
request_body = None
|
|
332
|
+
content_type = None
|
|
333
|
+
|
|
331
334
|
if data is None and json is not None:
|
|
332
335
|
if type(json) in [dict, list]:
|
|
333
336
|
json = dumps(json)
|
|
@@ -336,9 +339,8 @@ class Session:
|
|
|
336
339
|
elif data is not None and type(data) not in [str, bytes]:
|
|
337
340
|
request_body = urllib.parse.urlencode(data, doseq=True)
|
|
338
341
|
content_type = "application/x-www-form-urlencoded"
|
|
339
|
-
|
|
342
|
+
elif data is not None:
|
|
340
343
|
request_body = data
|
|
341
|
-
content_type = None
|
|
342
344
|
# set content type if it isn't set
|
|
343
345
|
# if content_type is not None and "content-type" not in self.headers:
|
|
344
346
|
# self.headers["Content-Type"] = content_type
|
|
@@ -391,7 +393,7 @@ class Session:
|
|
|
391
393
|
certificate_pinning = self.certificate_pinning
|
|
392
394
|
|
|
393
395
|
# --- Request --------------------------------------------------------------------------------------------------
|
|
394
|
-
is_byte_request = isinstance(request_body, (bytes, bytearray))
|
|
396
|
+
is_byte_request = isinstance(request_body, (bytes, bytearray)) if request_body is not None else False
|
|
395
397
|
request_payload = {
|
|
396
398
|
"sessionId": self._session_id,
|
|
397
399
|
"followRedirects": allow_redirects,
|
|
@@ -406,7 +408,7 @@ class Session:
|
|
|
406
408
|
"proxyUrl": proxy,
|
|
407
409
|
"requestUrl": url,
|
|
408
410
|
"requestMethod": method,
|
|
409
|
-
"requestBody": base64.b64encode(request_body).decode() if is_byte_request else request_body,
|
|
411
|
+
"requestBody": base64.b64encode(request_body).decode() if is_byte_request else (request_body or ""),
|
|
410
412
|
"requestCookies": request_cookies,
|
|
411
413
|
"timeoutSeconds": timeout_seconds,
|
|
412
414
|
"disableHttp3": self.disable_http3,
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: notify-tls-client
|
|
3
|
+
Version: 2.0.0
|
|
4
|
+
Summary: Cliente HTTP avançado com TLS fingerprinting, rotação de proxies e recuperação automática para web scraping resiliente
|
|
5
|
+
Author-email: Jeferson Albara <jeferson.albara@example.com>
|
|
6
|
+
Maintainer-email: Jeferson Albara <jeferson.albara@example.com>
|
|
7
|
+
License: MIT
|
|
8
|
+
Project-URL: Homepage, https://github.com/jefersonAlbara/notify-tls-client
|
|
9
|
+
Project-URL: Documentation, https://github.com/jefersonAlbara/notify-tls-client#readme
|
|
10
|
+
Project-URL: Repository, https://github.com/jefersonAlbara/notify-tls-client
|
|
11
|
+
Project-URL: Issues, https://github.com/jefersonAlbara/notify-tls-client/issues
|
|
12
|
+
Project-URL: Changelog, https://github.com/jefersonAlbara/notify-tls-client/blob/main/CHANGELOG.md
|
|
13
|
+
Keywords: tls-client,http-client,web-scraping,proxy-rotation,tls-fingerprinting,browser-emulation,http2,http3,requests,automation
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
21
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
22
|
+
Classifier: Topic :: Internet :: Proxy Servers
|
|
23
|
+
Classifier: Operating System :: OS Independent
|
|
24
|
+
Classifier: Typing :: Typed
|
|
25
|
+
Requires-Python: >=3.12
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
Requires-Dist: dataclasses-json>=0.6.0
|
|
28
|
+
Requires-Dist: typing-extensions>=4.8.0
|
|
29
|
+
Requires-Dist: orjson>=3.9.0
|
|
30
|
+
Provides-Extra: dev
|
|
31
|
+
Requires-Dist: pytest>=7.4.0; extra == "dev"
|
|
32
|
+
Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
|
|
33
|
+
Requires-Dist: black>=23.0.0; extra == "dev"
|
|
34
|
+
Requires-Dist: mypy>=1.5.0; extra == "dev"
|
|
35
|
+
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
36
|
+
|
|
37
|
+
# MY TLS Client
|
|
38
|
+
Pacote python privado com modificações na classe 'tls client'
|
|
@@ -1,19 +1,24 @@
|
|
|
1
1
|
notify_tls_client/__init__.py,sha256=APA8wxvkGYZiVMMbTWGo7prB3iKE-OzUajd0bfc8PwE,84
|
|
2
|
+
notify_tls_client/config/__init__.py,sha256=2xsw7M2E8-ncpIDySYcgH500xAnYyO3qy3PRXkP2inQ,495
|
|
3
|
+
notify_tls_client/config/client_config.py,sha256=Fwg6LS1zP8XFRPbTxNj2svCqeWh2GJcqrq7hnpQtYRQ,2837
|
|
4
|
+
notify_tls_client/config/client_configuration.py,sha256=Zl71lxZSKU7oVJwnHPSYV9uufk2IgFtHW3u9deqo5HU,9103
|
|
5
|
+
notify_tls_client/config/recovery_config.py,sha256=L5a_jMUfdUMdFBXeLPz9RAU0oXz-Ma4ZuzG7B-h7hDs,2876
|
|
6
|
+
notify_tls_client/config/rotation_config.py,sha256=Av_K6BzgPknFYZXxe9WmDra8VGI9JEpDJSm0dDs8QqM,2089
|
|
2
7
|
notify_tls_client/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
-
notify_tls_client/core/client_identifiers_manager.py,sha256
|
|
4
|
-
notify_tls_client/core/notifytlsclient.py,sha256=
|
|
8
|
+
notify_tls_client/core/client_identifiers_manager.py,sha256=ko4Y5IVMxNTRgk4e6jiJF_kcMjzej9rRt7BJtX-OMyg,1346
|
|
9
|
+
notify_tls_client/core/notifytlsclient.py,sha256=Mj6myXtzc_M52oOj_-cj4NcSx2WOuDIfg396CtOLmDo,12598
|
|
5
10
|
notify_tls_client/core/client/__init__.py,sha256=VKjpq02hC-r9ER6mbfkwG6W2ybv_jND9d9bidte0CjU,51
|
|
6
|
-
notify_tls_client/core/client/decorators.py,sha256=
|
|
11
|
+
notify_tls_client/core/client/decorators.py,sha256=GjGUl5eE5KZGB7Bc6XRFgmJ1Xjz6gNBXxOH8jldCAl8,8554
|
|
7
12
|
notify_tls_client/core/proxiesmanager/__init__.py,sha256=emF4vnZvSfb9zlHkt9dDdTcGwkfs1DADi7XVw_DxsWs,105
|
|
8
|
-
notify_tls_client/core/proxiesmanager/proxiesmanager.py,sha256=
|
|
13
|
+
notify_tls_client/core/proxiesmanager/proxiesmanager.py,sha256=V-X5dWskGHUjrX-hpl9E4L6pv53LBg1M0giEbWmG6JE,2286
|
|
9
14
|
notify_tls_client/core/proxiesmanager/proxiesmanagerloader.py,sha256=7xr3SVdRnr95KWOdk15iehOCXG2huA-rY1j9VIe30YQ,1179
|
|
10
15
|
notify_tls_client/tls_client/__init__.py,sha256=sThiIAzA650rfBlqZ_xalTjgTysUsjKua5ODnqyvhUE,669
|
|
11
16
|
notify_tls_client/tls_client/__version__.py,sha256=32ZZ-ufGC9Yo6oPk5a1xig8YKJ2ZkRXnXoVqiqO0ptg,395
|
|
12
|
-
notify_tls_client/tls_client/cffi.py,sha256=
|
|
17
|
+
notify_tls_client/tls_client/cffi.py,sha256=pedwBcQOwJvI66yp5GpyNU6zoqrQhTv3ocM1-1PtUm0,1291
|
|
13
18
|
notify_tls_client/tls_client/cookies.py,sha256=1fIOnFDMWMeXuAjQybSrUJXnyjhP-_jJ68AxBUZYgYU,15609
|
|
14
19
|
notify_tls_client/tls_client/exceptions.py,sha256=xIoFb5H3Suk7XssQ-yw3I1DBkPLqnDXsiAe2MMp1qNQ,80
|
|
15
|
-
notify_tls_client/tls_client/response.py,sha256=
|
|
16
|
-
notify_tls_client/tls_client/sessions.py,sha256=
|
|
20
|
+
notify_tls_client/tls_client/response.py,sha256=7j4iJa59Hcdw5ZWPvaxgLTvC_AWrch9hwMbrK4HNJjo,2618
|
|
21
|
+
notify_tls_client/tls_client/sessions.py,sha256=1xeUDR9H42wS900mCzkpu4At7Lx1O07QGoDuThbLG0M,19233
|
|
17
22
|
notify_tls_client/tls_client/settings.py,sha256=x_2Vrph-QebbCjkxmnX8UPd5oYYU4ClqPOpfoqelno4,1485
|
|
18
23
|
notify_tls_client/tls_client/structures.py,sha256=md-tJmo8X5bad0KrMUTVN8jxUIvui7NiBxaf10bLULU,2517
|
|
19
24
|
notify_tls_client/tls_client/dependencies/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -21,7 +26,7 @@ notify_tls_client/tls_client/dependencies/tls-client-darwin-amd64-1.12.0.dylib,s
|
|
|
21
26
|
notify_tls_client/tls_client/dependencies/tls-client-linux-arm64-1.12.0.so,sha256=fQvdtCiRRS228WrFUE_ucq4OPC4Z7QU4_KI4B3Gf97Y,14404088
|
|
22
27
|
notify_tls_client/tls_client/dependencies/tls-client-linux-ubuntu-amd64-1.12.0.so,sha256=UTvtZa93fWLWaSBINC_Cu8mNoLwsVdcQbQZsdQnZrJM,15210112
|
|
23
28
|
notify_tls_client/tls_client/dependencies/tls-client-windows-64-1.12.0.dll,sha256=uk80UEGW8WepYMglh1Yo6VSrBSNDwon-OyFqE_1bWmM,24849278
|
|
24
|
-
notify_tls_client-0.
|
|
25
|
-
notify_tls_client-0.
|
|
26
|
-
notify_tls_client-0.
|
|
27
|
-
notify_tls_client-0.
|
|
29
|
+
notify_tls_client-2.0.0.dist-info/METADATA,sha256=O_m_lTWjraTbcqLAHr1GxwtNwOzHWnb6T1InhT5qiUA,1941
|
|
30
|
+
notify_tls_client-2.0.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
|
|
31
|
+
notify_tls_client-2.0.0.dist-info/top_level.txt,sha256=fq9YA0cFdpCuUO7cdMFN7oxm1zDfZm_m1KPXehUqA5o,18
|
|
32
|
+
notify_tls_client-2.0.0.dist-info/RECORD,,
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: notify_tls_client
|
|
3
|
-
Version: 0.1.9
|
|
4
|
-
Summary: Sem descrição
|
|
5
|
-
Author-email: Naruto Uzumaki <naruto_uzumaki@gmail.com>
|
|
6
|
-
License-Expression: MIT
|
|
7
|
-
Project-URL: Homepage, https://github.com/jefersonAlbara/notify-tls-client
|
|
8
|
-
Requires-Python: >=3.12
|
|
9
|
-
Description-Content-Type: text/markdown
|
|
10
|
-
Requires-Dist: dataclasses_json
|
|
11
|
-
Requires-Dist: typing_extensions
|
|
12
|
-
|
|
13
|
-
# MY TLS Client
|
|
14
|
-
Pacote python privado com modificações na classe 'tls client'
|
|
File without changes
|