webscout 7.4__py3-none-any.whl → 7.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of webscout might be problematic. Click here for more details.
- webscout/AIauto.py +5 -53
- webscout/AIutel.py +8 -318
- webscout/DWEBS.py +460 -489
- webscout/Extra/YTToolkit/YTdownloader.py +14 -53
- webscout/Extra/YTToolkit/transcriber.py +12 -13
- webscout/Extra/YTToolkit/ytapi/video.py +0 -1
- webscout/Extra/__init__.py +0 -1
- webscout/Extra/autocoder/autocoder_utiles.py +0 -4
- webscout/Extra/autocoder/rawdog.py +13 -41
- webscout/Extra/gguf.py +652 -428
- webscout/Extra/weather.py +178 -156
- webscout/Extra/weather_ascii.py +70 -17
- webscout/Litlogger/core/logger.py +1 -2
- webscout/Litlogger/handlers/file.py +1 -1
- webscout/Litlogger/styles/formats.py +0 -2
- webscout/Litlogger/utils/detectors.py +0 -1
- webscout/Provider/AISEARCH/DeepFind.py +0 -1
- webscout/Provider/AISEARCH/ISou.py +1 -1
- webscout/Provider/AISEARCH/felo_search.py +0 -1
- webscout/Provider/AllenAI.py +24 -9
- webscout/Provider/C4ai.py +432 -0
- webscout/Provider/ChatGPTGratis.py +24 -56
- webscout/Provider/Cloudflare.py +18 -21
- webscout/Provider/DeepSeek.py +27 -48
- webscout/Provider/Deepinfra.py +129 -53
- webscout/Provider/Gemini.py +1 -1
- webscout/Provider/GithubChat.py +362 -0
- webscout/Provider/Glider.py +25 -8
- webscout/Provider/HF_space/qwen_qwen2.py +2 -2
- webscout/Provider/HeckAI.py +38 -5
- webscout/Provider/HuggingFaceChat.py +462 -0
- webscout/Provider/Jadve.py +20 -5
- webscout/Provider/Marcus.py +7 -50
- webscout/Provider/Netwrck.py +43 -67
- webscout/Provider/PI.py +4 -2
- webscout/Provider/Perplexitylabs.py +26 -6
- webscout/Provider/Phind.py +29 -3
- webscout/Provider/PizzaGPT.py +10 -51
- webscout/Provider/TTI/AiForce/async_aiforce.py +4 -37
- webscout/Provider/TTI/AiForce/sync_aiforce.py +41 -38
- webscout/Provider/TTI/FreeAIPlayground/__init__.py +9 -9
- webscout/Provider/TTI/FreeAIPlayground/async_freeaiplayground.py +206 -206
- webscout/Provider/TTI/FreeAIPlayground/sync_freeaiplayground.py +192 -192
- webscout/Provider/TTI/MagicStudio/__init__.py +2 -0
- webscout/Provider/TTI/MagicStudio/async_magicstudio.py +111 -0
- webscout/Provider/TTI/MagicStudio/sync_magicstudio.py +109 -0
- webscout/Provider/TTI/PollinationsAI/async_pollinations.py +5 -24
- webscout/Provider/TTI/PollinationsAI/sync_pollinations.py +2 -22
- webscout/Provider/TTI/__init__.py +2 -3
- webscout/Provider/TTI/aiarta/__init__.py +2 -0
- webscout/Provider/TTI/aiarta/async_aiarta.py +482 -0
- webscout/Provider/TTI/aiarta/sync_aiarta.py +440 -0
- webscout/Provider/TTI/fastflux/__init__.py +22 -0
- webscout/Provider/TTI/fastflux/async_fastflux.py +257 -0
- webscout/Provider/TTI/fastflux/sync_fastflux.py +247 -0
- webscout/Provider/TTS/__init__.py +2 -2
- webscout/Provider/TTS/deepgram.py +12 -39
- webscout/Provider/TTS/elevenlabs.py +14 -40
- webscout/Provider/TTS/gesserit.py +11 -35
- webscout/Provider/TTS/murfai.py +13 -39
- webscout/Provider/TTS/parler.py +17 -40
- webscout/Provider/TTS/speechma.py +180 -0
- webscout/Provider/TTS/streamElements.py +17 -44
- webscout/Provider/TextPollinationsAI.py +39 -59
- webscout/Provider/Venice.py +217 -200
- webscout/Provider/WiseCat.py +27 -5
- webscout/Provider/Youchat.py +63 -36
- webscout/Provider/__init__.py +13 -8
- webscout/Provider/akashgpt.py +28 -10
- webscout/Provider/copilot.py +416 -0
- webscout/Provider/flowith.py +196 -0
- webscout/Provider/freeaichat.py +32 -45
- webscout/Provider/granite.py +17 -53
- webscout/Provider/koala.py +20 -5
- webscout/Provider/llamatutor.py +7 -47
- webscout/Provider/llmchat.py +36 -53
- webscout/Provider/multichat.py +92 -98
- webscout/Provider/talkai.py +1 -0
- webscout/Provider/turboseek.py +3 -0
- webscout/Provider/tutorai.py +2 -0
- webscout/Provider/typegpt.py +154 -64
- webscout/Provider/x0gpt.py +3 -1
- webscout/Provider/yep.py +102 -20
- webscout/__init__.py +3 -0
- webscout/cli.py +4 -40
- webscout/conversation.py +1 -10
- webscout/exceptions.py +19 -9
- webscout/litagent/__init__.py +2 -2
- webscout/litagent/agent.py +351 -20
- webscout/litagent/constants.py +34 -5
- webscout/litprinter/__init__.py +0 -3
- webscout/models.py +181 -0
- webscout/optimizers.py +1 -1
- webscout/prompt_manager.py +2 -8
- webscout/scout/core/scout.py +1 -4
- webscout/scout/core/search_result.py +1 -1
- webscout/scout/core/text_utils.py +1 -1
- webscout/scout/core.py +2 -5
- webscout/scout/element.py +1 -1
- webscout/scout/parsers/html_parser.py +1 -1
- webscout/scout/utils.py +0 -1
- webscout/swiftcli/__init__.py +1 -3
- webscout/tempid.py +1 -1
- webscout/update_checker.py +55 -95
- webscout/version.py +1 -1
- webscout/webscout_search_async.py +1 -2
- webscout/yep_search.py +297 -297
- webscout-7.6.dist-info/LICENSE.md +146 -0
- {webscout-7.4.dist-info → webscout-7.6.dist-info}/METADATA +104 -514
- {webscout-7.4.dist-info → webscout-7.6.dist-info}/RECORD +113 -120
- webscout/Extra/autollama.py +0 -231
- webscout/Local/__init__.py +0 -10
- webscout/Local/_version.py +0 -3
- webscout/Local/formats.py +0 -747
- webscout/Local/model.py +0 -1368
- webscout/Local/samplers.py +0 -125
- webscout/Local/thread.py +0 -539
- webscout/Local/ui.py +0 -401
- webscout/Local/utils.py +0 -388
- webscout/Provider/Amigo.py +0 -274
- webscout/Provider/Bing.py +0 -243
- webscout/Provider/DiscordRocks.py +0 -253
- webscout/Provider/TTI/blackbox/__init__.py +0 -4
- webscout/Provider/TTI/blackbox/async_blackbox.py +0 -212
- webscout/Provider/TTI/blackbox/sync_blackbox.py +0 -199
- webscout/Provider/TTI/deepinfra/__init__.py +0 -4
- webscout/Provider/TTI/deepinfra/async_deepinfra.py +0 -227
- webscout/Provider/TTI/deepinfra/sync_deepinfra.py +0 -199
- webscout/Provider/TTI/imgninza/__init__.py +0 -4
- webscout/Provider/TTI/imgninza/async_ninza.py +0 -214
- webscout/Provider/TTI/imgninza/sync_ninza.py +0 -209
- webscout/Provider/TTS/voicepod.py +0 -117
- webscout/Provider/dgaf.py +0 -214
- webscout-7.4.dist-info/LICENSE.md +0 -211
- {webscout-7.4.dist-info → webscout-7.6.dist-info}/WHEEL +0 -0
- {webscout-7.4.dist-info → webscout-7.6.dist-info}/entry_points.txt +0 -0
- {webscout-7.4.dist-info → webscout-7.6.dist-info}/top_level.txt +0 -0
webscout/litagent/agent.py
CHANGED
|
@@ -1,21 +1,31 @@
|
|
|
1
1
|
"""Main LitAgent implementation."""
|
|
2
2
|
|
|
3
3
|
import random
|
|
4
|
-
|
|
5
|
-
from
|
|
6
|
-
|
|
4
|
+
import threading
|
|
5
|
+
from typing import List, Dict, Any, Optional
|
|
6
|
+
|
|
7
|
+
from webscout.litagent.constants import BROWSERS, OS_VERSIONS, DEVICES, FINGERPRINTS
|
|
7
8
|
|
|
8
|
-
logger = Logger(
|
|
9
|
-
name="LitAgent",
|
|
10
|
-
format=LogFormat.MODERN_EMOJI,
|
|
11
|
-
)
|
|
12
9
|
|
|
13
10
|
class LitAgent:
|
|
14
11
|
"""A lit user agent generator that keeps it fresh! 🌟"""
|
|
15
12
|
|
|
16
|
-
def __init__(self):
|
|
17
|
-
"""Initialize LitAgent with style! 💫
|
|
13
|
+
def __init__(self, thread_safe: bool = False):
|
|
14
|
+
"""Initialize LitAgent with style! 💫
|
|
15
|
+
|
|
16
|
+
Args:
|
|
17
|
+
thread_safe (bool): Enable thread-safety for multi-threaded applications
|
|
18
|
+
"""
|
|
18
19
|
self.agents = self._generate_agents(100) # Keep 100 agents in memory
|
|
20
|
+
self.thread_safe = thread_safe
|
|
21
|
+
self.lock = threading.RLock() if thread_safe else None
|
|
22
|
+
self._refresh_timer = None
|
|
23
|
+
self._stats = {
|
|
24
|
+
"total_generated": 100,
|
|
25
|
+
"requests_served": 0,
|
|
26
|
+
"browser_usage": {browser: 0 for browser in BROWSERS.keys()},
|
|
27
|
+
"device_usage": {device: 0 for device in DEVICES.keys()}
|
|
28
|
+
}
|
|
19
29
|
|
|
20
30
|
def _generate_agents(self, count: int) -> List[str]:
|
|
21
31
|
"""Generate some lit user agents! 🛠️"""
|
|
@@ -24,7 +34,7 @@ class LitAgent:
|
|
|
24
34
|
browser = random.choice(list(BROWSERS.keys()))
|
|
25
35
|
version = random.randint(*BROWSERS[browser])
|
|
26
36
|
|
|
27
|
-
if browser in ['chrome', 'firefox', 'edge', 'opera']:
|
|
37
|
+
if browser in ['chrome', 'firefox', 'edge', 'opera', 'brave', 'vivaldi']:
|
|
28
38
|
os_type = random.choice(['windows', 'mac', 'linux'])
|
|
29
39
|
os_ver = random.choice(OS_VERSIONS[os_type])
|
|
30
40
|
|
|
@@ -44,6 +54,10 @@ class LitAgent:
|
|
|
44
54
|
agent += f"Edge/{version}.0.0.0"
|
|
45
55
|
elif browser == 'opera':
|
|
46
56
|
agent += f"OPR/{version}.0.0.0"
|
|
57
|
+
elif browser == 'brave':
|
|
58
|
+
agent += f"Chrome/{version}.0.0.0 Safari/537.36 Brave/{version}.0.0.0"
|
|
59
|
+
elif browser == 'vivaldi':
|
|
60
|
+
agent += f"Chrome/{version}.0.0.0 Safari/537.36 Vivaldi/{version}.0.{random.randint(1000, 9999)}"
|
|
47
61
|
|
|
48
62
|
elif browser == 'safari':
|
|
49
63
|
device = random.choice(['mac', 'ios'])
|
|
@@ -60,29 +74,127 @@ class LitAgent:
|
|
|
60
74
|
|
|
61
75
|
return list(set(agents)) # Remove any duplicates
|
|
62
76
|
|
|
77
|
+
def _update_stats(self, browser_type=None, device_type=None):
|
|
78
|
+
"""Update usage statistics."""
|
|
79
|
+
if self.thread_safe and self.lock:
|
|
80
|
+
with self.lock:
|
|
81
|
+
self._stats["requests_served"] += 1
|
|
82
|
+
if browser_type:
|
|
83
|
+
self._stats["browser_usage"][browser_type] = self._stats["browser_usage"].get(browser_type, 0) + 1
|
|
84
|
+
if device_type:
|
|
85
|
+
self._stats["device_usage"][device_type] = self._stats["device_usage"].get(device_type, 0) + 1
|
|
86
|
+
else:
|
|
87
|
+
self._stats["requests_served"] += 1
|
|
88
|
+
if browser_type:
|
|
89
|
+
self._stats["browser_usage"][browser_type] = self._stats["browser_usage"].get(browser_type, 0) + 1
|
|
90
|
+
if device_type:
|
|
91
|
+
self._stats["device_usage"][device_type] = self._stats["device_usage"].get(device_type, 0) + 1
|
|
92
|
+
|
|
63
93
|
def random(self) -> str:
|
|
64
94
|
"""Get a random user agent! 🎲"""
|
|
65
|
-
|
|
95
|
+
if self.thread_safe and self.lock:
|
|
96
|
+
with self.lock:
|
|
97
|
+
agent = random.choice(self.agents)
|
|
98
|
+
self._update_stats()
|
|
99
|
+
return agent
|
|
100
|
+
else:
|
|
101
|
+
agent = random.choice(self.agents)
|
|
102
|
+
self._update_stats()
|
|
103
|
+
return agent
|
|
66
104
|
|
|
67
105
|
def browser(self, name: str) -> str:
|
|
68
106
|
"""Get a browser-specific agent! 🌐"""
|
|
69
107
|
name = name.lower()
|
|
70
108
|
if name not in BROWSERS:
|
|
71
|
-
logger.warning(f"Unknown browser: {name} - Using random browser")
|
|
72
109
|
return self.random()
|
|
73
110
|
|
|
74
|
-
|
|
75
|
-
|
|
111
|
+
if self.thread_safe and self.lock:
|
|
112
|
+
with self.lock:
|
|
113
|
+
agents = [a for a in self.agents if name in a.lower()]
|
|
114
|
+
agent = random.choice(agents) if agents else self.random()
|
|
115
|
+
self._update_stats(browser_type=name)
|
|
116
|
+
return agent
|
|
117
|
+
else:
|
|
118
|
+
agents = [a for a in self.agents if name in a.lower()]
|
|
119
|
+
agent = random.choice(agents) if agents else self.random()
|
|
120
|
+
self._update_stats(browser_type=name)
|
|
121
|
+
return agent
|
|
76
122
|
|
|
77
123
|
def mobile(self) -> str:
|
|
78
124
|
"""Get a mobile device agent! 📱"""
|
|
79
|
-
|
|
80
|
-
|
|
125
|
+
if self.thread_safe and self.lock:
|
|
126
|
+
with self.lock:
|
|
127
|
+
agents = [a for a in self.agents if any(d in a for d in DEVICES['mobile'])]
|
|
128
|
+
agent = random.choice(agents) if agents else self.random()
|
|
129
|
+
self._update_stats(device_type="mobile")
|
|
130
|
+
return agent
|
|
131
|
+
else:
|
|
132
|
+
agents = [a for a in self.agents if any(d in a for d in DEVICES['mobile'])]
|
|
133
|
+
agent = random.choice(agents) if agents else self.random()
|
|
134
|
+
self._update_stats(device_type="mobile")
|
|
135
|
+
return agent
|
|
81
136
|
|
|
82
137
|
def desktop(self) -> str:
|
|
83
138
|
"""Get a desktop agent! 💻"""
|
|
84
|
-
|
|
85
|
-
|
|
139
|
+
if self.thread_safe and self.lock:
|
|
140
|
+
with self.lock:
|
|
141
|
+
agents = [a for a in self.agents if 'Windows' in a or 'Macintosh' in a or 'Linux' in a]
|
|
142
|
+
agent = random.choice(agents) if agents else self.random()
|
|
143
|
+
self._update_stats(device_type="desktop")
|
|
144
|
+
return agent
|
|
145
|
+
else:
|
|
146
|
+
agents = [a for a in self.agents if 'Windows' in a or 'Macintosh' in a or 'Linux' in a]
|
|
147
|
+
agent = random.choice(agents) if agents else self.random()
|
|
148
|
+
self._update_stats(device_type="desktop")
|
|
149
|
+
return agent
|
|
150
|
+
|
|
151
|
+
def tablet(self) -> str:
|
|
152
|
+
"""Get a tablet agent! 📱"""
|
|
153
|
+
if self.thread_safe and self.lock:
|
|
154
|
+
with self.lock:
|
|
155
|
+
# Focus on iPad and Android tablets
|
|
156
|
+
agents = [a for a in self.agents if 'iPad' in a or 'Android' in a and not 'Mobile' in a]
|
|
157
|
+
agent = random.choice(agents) if agents else self.random()
|
|
158
|
+
self._update_stats(device_type="tablet")
|
|
159
|
+
return agent
|
|
160
|
+
else:
|
|
161
|
+
agents = [a for a in self.agents if 'iPad' in a or 'Android' in a and not 'Mobile' in a]
|
|
162
|
+
agent = random.choice(agents) if agents else self.random()
|
|
163
|
+
self._update_stats(device_type="tablet")
|
|
164
|
+
return agent
|
|
165
|
+
|
|
166
|
+
def smart_tv(self) -> str:
|
|
167
|
+
"""Get a Smart TV agent! 📺"""
|
|
168
|
+
# Create a TV-specific agent since they may not be in our standard pool
|
|
169
|
+
tv_type = random.choice(DEVICES['tv'])
|
|
170
|
+
if 'Samsung' in tv_type:
|
|
171
|
+
agent = f"Mozilla/5.0 (SMART-TV; SAMSUNG; {tv_type}; Tizen 5.5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.38 Safari/537.36"
|
|
172
|
+
elif 'LG' in tv_type:
|
|
173
|
+
agent = f"Mozilla/5.0 (Web0S; {tv_type}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36"
|
|
174
|
+
elif 'Android' in tv_type:
|
|
175
|
+
agent = f"Mozilla/5.0 (Linux; Android 9; {tv_type}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36"
|
|
176
|
+
elif 'Apple' in tv_type:
|
|
177
|
+
agent = f"Mozilla/5.0 (AppleTV; CPU like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148"
|
|
178
|
+
else:
|
|
179
|
+
agent = f"Mozilla/5.0 (Linux; {tv_type}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36"
|
|
180
|
+
|
|
181
|
+
self._update_stats(device_type="tv")
|
|
182
|
+
return agent
|
|
183
|
+
|
|
184
|
+
def gaming(self) -> str:
|
|
185
|
+
"""Get a gaming console agent! 🎮"""
|
|
186
|
+
console_type = random.choice(DEVICES['console'])
|
|
187
|
+
if 'PlayStation' in console_type:
|
|
188
|
+
agent = f"Mozilla/5.0 ({console_type}/5.0) AppleWebKit/601.2 (KHTML, like Gecko)"
|
|
189
|
+
elif 'Xbox' in console_type:
|
|
190
|
+
agent = f"Mozilla/5.0 ({console_type}; Xbox; Xbox One) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.19041"
|
|
191
|
+
elif 'Nintendo' in console_type:
|
|
192
|
+
agent = f"Mozilla/5.0 (Nintendo Switch; {console_type}) AppleWebKit/601.6 (KHTML, like Gecko) NintendoBrowser/5.1.0.13343"
|
|
193
|
+
else:
|
|
194
|
+
agent = self.random()
|
|
195
|
+
|
|
196
|
+
self._update_stats(device_type="console")
|
|
197
|
+
return agent
|
|
86
198
|
|
|
87
199
|
def chrome(self) -> str:
|
|
88
200
|
"""Get a Chrome agent! 🌐"""
|
|
@@ -103,10 +215,220 @@ class LitAgent:
|
|
|
103
215
|
def opera(self) -> str:
|
|
104
216
|
"""Get an Opera agent! 🎭"""
|
|
105
217
|
return self.browser('opera')
|
|
218
|
+
|
|
219
|
+
def brave(self) -> str:
|
|
220
|
+
"""Get a Brave agent! 🦁"""
|
|
221
|
+
return self.browser('brave')
|
|
222
|
+
|
|
223
|
+
def vivaldi(self) -> str:
|
|
224
|
+
"""Get a Vivaldi agent! 🎨"""
|
|
225
|
+
return self.browser('vivaldi')
|
|
226
|
+
|
|
227
|
+
# OS-specific agents
|
|
228
|
+
def windows(self) -> str:
|
|
229
|
+
"""Get a Windows agent! 🪟"""
|
|
230
|
+
agents = [a for a in self.agents if 'Windows' in a]
|
|
231
|
+
agent = random.choice(agents) if agents else self.random()
|
|
232
|
+
self._update_stats()
|
|
233
|
+
return agent
|
|
234
|
+
|
|
235
|
+
def macos(self) -> str:
|
|
236
|
+
"""Get a macOS agent! 🍎"""
|
|
237
|
+
agents = [a for a in self.agents if 'Macintosh' in a]
|
|
238
|
+
agent = random.choice(agents) if agents else self.random()
|
|
239
|
+
self._update_stats()
|
|
240
|
+
return agent
|
|
241
|
+
|
|
242
|
+
def linux(self) -> str:
|
|
243
|
+
"""Get a Linux agent! 🐧"""
|
|
244
|
+
agents = [a for a in self.agents if 'Linux' in a and 'Android' not in a]
|
|
245
|
+
agent = random.choice(agents) if agents else self.random()
|
|
246
|
+
self._update_stats()
|
|
247
|
+
return agent
|
|
248
|
+
|
|
249
|
+
def android(self) -> str:
|
|
250
|
+
"""Get an Android agent! 🤖"""
|
|
251
|
+
agents = [a for a in self.agents if 'Android' in a]
|
|
252
|
+
agent = random.choice(agents) if agents else self.random()
|
|
253
|
+
self._update_stats()
|
|
254
|
+
return agent
|
|
255
|
+
|
|
256
|
+
def ios(self) -> str:
|
|
257
|
+
"""Get an iOS agent! 📱"""
|
|
258
|
+
agents = [a for a in self.agents if 'iPhone' in a or 'iPad' in a]
|
|
259
|
+
agent = random.choice(agents) if agents else self.random()
|
|
260
|
+
self._update_stats()
|
|
261
|
+
return agent
|
|
262
|
+
|
|
263
|
+
def custom(self, browser: str, version: Optional[str] = None,
|
|
264
|
+
os: Optional[str] = None, os_version: Optional[str] = None,
|
|
265
|
+
device_type: Optional[str] = None) -> str:
|
|
266
|
+
"""Generate a custom user agent with specified parameters! 🛠️
|
|
267
|
+
|
|
268
|
+
Args:
|
|
269
|
+
browser: Browser name (chrome, firefox, safari, edge, opera)
|
|
270
|
+
version: Browser version (optional)
|
|
271
|
+
os: Operating system (windows, mac, linux, android, ios)
|
|
272
|
+
os_version: OS version (optional)
|
|
273
|
+
device_type: Device type (desktop, mobile, tablet)
|
|
274
|
+
|
|
275
|
+
Returns:
|
|
276
|
+
Customized user agent string
|
|
277
|
+
"""
|
|
278
|
+
browser = browser.lower() if browser else 'chrome'
|
|
279
|
+
if browser not in BROWSERS:
|
|
280
|
+
browser = 'chrome'
|
|
281
|
+
|
|
282
|
+
if version:
|
|
283
|
+
try:
|
|
284
|
+
version_num = int(version.split('.')[0])
|
|
285
|
+
except (ValueError, IndexError):
|
|
286
|
+
version_num = random.randint(*BROWSERS[browser])
|
|
287
|
+
else:
|
|
288
|
+
version_num = random.randint(*BROWSERS[browser])
|
|
289
|
+
|
|
290
|
+
os = os.lower() if os else random.choice(['windows', 'mac', 'linux'])
|
|
291
|
+
if os not in OS_VERSIONS:
|
|
292
|
+
os = 'windows'
|
|
293
|
+
|
|
294
|
+
os_ver = os_version or random.choice(OS_VERSIONS[os])
|
|
295
|
+
|
|
296
|
+
device_type = device_type.lower() if device_type else 'desktop'
|
|
297
|
+
|
|
298
|
+
# Build the user agent
|
|
299
|
+
if os == 'windows':
|
|
300
|
+
platform = f"Windows NT {os_ver}"
|
|
301
|
+
elif os == 'mac':
|
|
302
|
+
platform = f"Macintosh; Intel Mac OS X {os_ver}"
|
|
303
|
+
elif os == 'linux':
|
|
304
|
+
platform = f"X11; Linux {OS_VERSIONS['linux'][0]}"
|
|
305
|
+
elif os == 'android':
|
|
306
|
+
platform = f"Linux; Android {os_ver}; {random.choice(DEVICES['mobile'])}"
|
|
307
|
+
elif os == 'ios':
|
|
308
|
+
device = 'iPhone' if device_type == 'mobile' else 'iPad'
|
|
309
|
+
platform = f"{device}; CPU OS {os_ver} like Mac OS X"
|
|
310
|
+
else:
|
|
311
|
+
platform = f"Windows NT 10.0" # Default fallback
|
|
312
|
+
|
|
313
|
+
agent = f"Mozilla/5.0 ({platform}) AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
314
|
+
|
|
315
|
+
if browser == 'chrome':
|
|
316
|
+
agent += f"Chrome/{version_num}.0.0.0 Safari/537.36"
|
|
317
|
+
elif browser == 'firefox':
|
|
318
|
+
agent += f"Firefox/{version_num}.0"
|
|
319
|
+
elif browser == 'safari':
|
|
320
|
+
safari_ver = random.randint(*BROWSERS['safari'])
|
|
321
|
+
agent += f"Version/{version_num}.0 Safari/{safari_ver}.0"
|
|
322
|
+
elif browser == 'edge':
|
|
323
|
+
agent += f"Chrome/{version_num}.0.0.0 Safari/537.36 Edg/{version_num}.0.0.0"
|
|
324
|
+
elif browser == 'opera':
|
|
325
|
+
agent += f"Chrome/{version_num}.0.0.0 Safari/537.36 OPR/{version_num}.0.0.0"
|
|
326
|
+
elif browser == 'brave':
|
|
327
|
+
agent += f"Chrome/{version_num}.0.0.0 Safari/537.36 Brave/{version_num}.1.0"
|
|
328
|
+
|
|
329
|
+
self._update_stats(browser_type=browser, device_type=device_type)
|
|
330
|
+
return agent
|
|
331
|
+
|
|
332
|
+
def generate_fingerprint(self, browser: Optional[str] = None) -> Dict[str, str]:
|
|
333
|
+
"""Generate a consistent browser fingerprint! 👆
|
|
334
|
+
|
|
335
|
+
This creates a coherent set of headers for anti-fingerprinting.
|
|
336
|
+
|
|
337
|
+
Args:
|
|
338
|
+
browser: Specific browser to generate fingerprint for
|
|
339
|
+
|
|
340
|
+
Returns:
|
|
341
|
+
Dictionary with fingerprinting headers
|
|
342
|
+
"""
|
|
343
|
+
browser = browser.lower() if browser else random.choice(list(BROWSERS.keys()))
|
|
344
|
+
if browser not in BROWSERS:
|
|
345
|
+
browser = 'chrome'
|
|
346
|
+
|
|
347
|
+
version = random.randint(*BROWSERS[browser])
|
|
348
|
+
user_agent = self.custom(browser=browser, version=str(version))
|
|
349
|
+
|
|
350
|
+
accept_language = random.choice(FINGERPRINTS["accept_language"])
|
|
351
|
+
accept = random.choice(FINGERPRINTS["accept"])
|
|
352
|
+
platform = random.choice(FINGERPRINTS["platforms"])
|
|
353
|
+
|
|
354
|
+
# Generate sec-ch-ua
|
|
355
|
+
sec_ch_ua = ""
|
|
356
|
+
if browser in FINGERPRINTS["sec_ch_ua"]:
|
|
357
|
+
sec_ch_ua = FINGERPRINTS["sec_ch_ua"][browser].format(version, version)
|
|
358
|
+
|
|
359
|
+
fingerprint = {
|
|
360
|
+
"user_agent": user_agent,
|
|
361
|
+
"accept_language": accept_language,
|
|
362
|
+
"accept": accept,
|
|
363
|
+
"sec_ch_ua": sec_ch_ua,
|
|
364
|
+
"platform": platform
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
self._update_stats(browser_type=browser)
|
|
368
|
+
return fingerprint
|
|
106
369
|
|
|
107
370
|
def refresh(self) -> None:
|
|
108
371
|
"""Refresh the agents with new ones! 🔄"""
|
|
109
|
-
self.
|
|
372
|
+
if self.thread_safe and self.lock:
|
|
373
|
+
with self.lock:
|
|
374
|
+
self.agents = self._generate_agents(100)
|
|
375
|
+
self._stats["total_generated"] += 100
|
|
376
|
+
else:
|
|
377
|
+
self.agents = self._generate_agents(100)
|
|
378
|
+
self._stats["total_generated"] += 100
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def auto_refresh(self, interval_minutes: int = 30) -> None:
|
|
382
|
+
"""Set up automatic refreshing of agents pool! ⏱️
|
|
383
|
+
|
|
384
|
+
Args:
|
|
385
|
+
interval_minutes: Minutes between refreshes
|
|
386
|
+
"""
|
|
387
|
+
if self._refresh_timer:
|
|
388
|
+
self._refresh_timer.cancel()
|
|
389
|
+
|
|
390
|
+
def _refresh_task():
|
|
391
|
+
self.refresh()
|
|
392
|
+
self._refresh_timer = threading.Timer(interval_minutes * 60, _refresh_task)
|
|
393
|
+
self._refresh_timer.daemon = True
|
|
394
|
+
self._refresh_timer.start()
|
|
395
|
+
|
|
396
|
+
self._refresh_timer = threading.Timer(interval_minutes * 60, _refresh_task)
|
|
397
|
+
self._refresh_timer.daemon = True
|
|
398
|
+
self._refresh_timer.start()
|
|
399
|
+
|
|
400
|
+
def get_stats(self) -> Dict[str, Any]:
|
|
401
|
+
"""Get statistics about agent usage! 📊
|
|
402
|
+
|
|
403
|
+
Returns:
|
|
404
|
+
Dictionary with usage statistics
|
|
405
|
+
"""
|
|
406
|
+
stats_copy = self._stats.copy()
|
|
407
|
+
# Calculate top browser
|
|
408
|
+
top_browser = max(stats_copy["browser_usage"].items(), key=lambda x: x[1])[0] if stats_copy["browser_usage"] else None
|
|
409
|
+
stats_copy["top_browser"] = top_browser
|
|
410
|
+
|
|
411
|
+
# Calculate fake detection avoidance rate (just for fun)
|
|
412
|
+
stats_copy["avoidance_rate"] = min(99.9, 90 + (stats_copy["total_generated"] / 1000))
|
|
413
|
+
|
|
414
|
+
return stats_copy
|
|
415
|
+
|
|
416
|
+
def export_stats(self, filename: str) -> bool:
|
|
417
|
+
"""Export usage statistics to a file! 💾
|
|
418
|
+
|
|
419
|
+
Args:
|
|
420
|
+
filename: Path to export the stats
|
|
421
|
+
|
|
422
|
+
Returns:
|
|
423
|
+
True if export was successful, False otherwise
|
|
424
|
+
"""
|
|
425
|
+
try:
|
|
426
|
+
import json
|
|
427
|
+
with open(filename, 'w') as f:
|
|
428
|
+
json.dump(self.get_stats(), f, indent=2)
|
|
429
|
+
return True
|
|
430
|
+
except Exception as e:
|
|
431
|
+
return False
|
|
110
432
|
|
|
111
433
|
if __name__ == "__main__":
|
|
112
434
|
# Test it out! 🧪
|
|
@@ -116,4 +438,13 @@ if __name__ == "__main__":
|
|
|
116
438
|
print("Firefox:", agent.firefox())
|
|
117
439
|
print("Safari:", agent.safari())
|
|
118
440
|
print("Mobile:", agent.mobile())
|
|
119
|
-
print("Desktop:", agent.desktop())
|
|
441
|
+
print("Desktop:", agent.desktop())
|
|
442
|
+
print("Tablet:", agent.tablet())
|
|
443
|
+
print("Smart TV:", agent.smart_tv())
|
|
444
|
+
print("Gaming:", agent.gaming())
|
|
445
|
+
|
|
446
|
+
# Test custom agent
|
|
447
|
+
print("Custom:", agent.custom(browser="chrome", os="windows", os_version="10.0"))
|
|
448
|
+
|
|
449
|
+
# Test fingerprinting
|
|
450
|
+
print("Fingerprint:", agent.generate_fingerprint("chrome"))
|
webscout/litagent/constants.py
CHANGED
|
@@ -6,7 +6,9 @@ BROWSERS = {
|
|
|
6
6
|
"firefox": (48, 121),
|
|
7
7
|
"safari": (605, 617),
|
|
8
8
|
"edge": (79, 120),
|
|
9
|
-
"opera": (48, 104)
|
|
9
|
+
"opera": (48, 104),
|
|
10
|
+
"brave": (100, 120),
|
|
11
|
+
"vivaldi": (5, 6)
|
|
10
12
|
}
|
|
11
13
|
|
|
12
14
|
# OS versions
|
|
@@ -15,7 +17,8 @@ OS_VERSIONS = {
|
|
|
15
17
|
"mac": ["10_15_7", "11_0", "12_0", "13_0", "14_0"],
|
|
16
18
|
"linux": ["x86_64", "i686"],
|
|
17
19
|
"android": ["10", "11", "12", "13", "14"],
|
|
18
|
-
"ios": ["14_0", "15_0", "16_0", "17_0"]
|
|
20
|
+
"ios": ["14_0", "15_0", "16_0", "17_0"],
|
|
21
|
+
"chrome_os": ["13.0", "14.0", "15.0"]
|
|
19
22
|
}
|
|
20
23
|
|
|
21
24
|
# Device types
|
|
@@ -25,7 +28,33 @@ DEVICES = {
|
|
|
25
28
|
"OnePlus", "Xiaomi", "Huawei", "OPPO", "Vivo"
|
|
26
29
|
],
|
|
27
30
|
"desktop": ["Windows PC", "MacBook", "iMac", "Linux Desktop"],
|
|
28
|
-
"tablet": ["iPad", "Samsung Galaxy Tab", "Microsoft Surface"],
|
|
29
|
-
"console": ["PlayStation 5", "Xbox Series X", "Nintendo Switch"],
|
|
30
|
-
"tv": ["Samsung Smart TV", "LG WebOS", "Android TV", "Apple TV"]
|
|
31
|
+
"tablet": ["iPad", "Samsung Galaxy Tab", "Microsoft Surface", "Huawei MatePad", "Lenovo Tab"],
|
|
32
|
+
"console": ["PlayStation 5", "Xbox Series X", "Nintendo Switch", "PlayStation 4", "Xbox One"],
|
|
33
|
+
"tv": ["Samsung Smart TV", "LG WebOS", "Android TV", "Apple TV", "Sony Bravia"],
|
|
34
|
+
"wearable": ["Apple Watch", "Samsung Galaxy Watch", "Fitbit", "Garmin"]
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
# Browser fingerprinting components
|
|
38
|
+
FINGERPRINTS = {
|
|
39
|
+
"accept_language": [
|
|
40
|
+
"en-US,en;q=0.9",
|
|
41
|
+
"en-GB,en;q=0.8,en-US;q=0.6",
|
|
42
|
+
"es-ES,es;q=0.9,en;q=0.8",
|
|
43
|
+
"fr-FR,fr;q=0.9,en;q=0.8",
|
|
44
|
+
"de-DE,de;q=0.9,en;q=0.8"
|
|
45
|
+
],
|
|
46
|
+
"accept": [
|
|
47
|
+
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
|
48
|
+
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
49
|
+
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"
|
|
50
|
+
],
|
|
51
|
+
"sec_ch_ua": {
|
|
52
|
+
"chrome": "\" Not A;Brand\";v=\"99\", \"Chromium\";v=\"{}\", \"Google Chrome\";v=\"{}\"",
|
|
53
|
+
"edge": "\" Not A;Brand\";v=\"99\", \"Chromium\";v=\"{}\", \"Microsoft Edge\";v=\"{}\"",
|
|
54
|
+
"firefox": "\"Firefox\";v=\"{}\", \"Not;A=Brand\";v=\"8\"",
|
|
55
|
+
"safari": "\"Safari\";v=\"{}\", \"Not;A=Brand\";v=\"99\""
|
|
56
|
+
},
|
|
57
|
+
"platforms": [
|
|
58
|
+
"Windows", "macOS", "Linux", "Android", "iOS"
|
|
59
|
+
]
|
|
31
60
|
}
|
webscout/litprinter/__init__.py
CHANGED
|
@@ -5,11 +5,8 @@ import time
|
|
|
5
5
|
import threading
|
|
6
6
|
import re
|
|
7
7
|
from typing import Any, Optional, TextIO, Union, Sequence, Dict, List
|
|
8
|
-
from datetime import datetime
|
|
9
8
|
import textwrap
|
|
10
|
-
from collections import defaultdict
|
|
11
9
|
import shutil
|
|
12
|
-
import inspect
|
|
13
10
|
from .colors import Colors
|
|
14
11
|
import ctypes
|
|
15
12
|
|
webscout/models.py
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
import importlib
|
|
2
|
+
import pkgutil
|
|
3
|
+
from typing import Dict, List, Any, Union
|
|
4
|
+
from webscout.AIbase import Provider, TTSProvider
|
|
5
|
+
|
|
6
|
+
class _LLMModels:
|
|
7
|
+
"""
|
|
8
|
+
A class for managing LLM provider models in the webscout package.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
def list(self) -> Dict[str, List[str]]:
|
|
12
|
+
"""
|
|
13
|
+
Gets all available models from each provider that has an AVAILABLE_MODELS attribute.
|
|
14
|
+
|
|
15
|
+
Returns:
|
|
16
|
+
Dictionary mapping provider names to their available models
|
|
17
|
+
"""
|
|
18
|
+
return self._get_provider_models()
|
|
19
|
+
|
|
20
|
+
def get(self, provider_name: str) -> List[str]:
|
|
21
|
+
"""
|
|
22
|
+
Gets all available models for a specific provider.
|
|
23
|
+
|
|
24
|
+
Args:
|
|
25
|
+
provider_name: The name of the provider
|
|
26
|
+
|
|
27
|
+
Returns:
|
|
28
|
+
List of available models for the provider
|
|
29
|
+
"""
|
|
30
|
+
all_models = self._get_provider_models()
|
|
31
|
+
return all_models.get(provider_name, [])
|
|
32
|
+
|
|
33
|
+
def summary(self) -> Dict[str, int]:
|
|
34
|
+
"""
|
|
35
|
+
Returns a summary of available providers and models.
|
|
36
|
+
|
|
37
|
+
Returns:
|
|
38
|
+
Dictionary with provider and model counts
|
|
39
|
+
"""
|
|
40
|
+
provider_models = self._get_provider_models()
|
|
41
|
+
total_providers = len(provider_models)
|
|
42
|
+
total_models = sum(len(models) if isinstance(models, (list, tuple, set))
|
|
43
|
+
else 1 for models in provider_models.values())
|
|
44
|
+
|
|
45
|
+
return {
|
|
46
|
+
"providers": total_providers,
|
|
47
|
+
"models": total_models,
|
|
48
|
+
"provider_model_counts": {
|
|
49
|
+
provider: len(models) if isinstance(models, (list, tuple, set)) else 1
|
|
50
|
+
for provider, models in provider_models.items()
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
def _get_provider_models(self) -> Dict[str, List[str]]:
|
|
55
|
+
"""
|
|
56
|
+
Internal method to get all available models from each provider.
|
|
57
|
+
|
|
58
|
+
Returns:
|
|
59
|
+
Dictionary mapping provider names to their available models
|
|
60
|
+
"""
|
|
61
|
+
provider_models = {}
|
|
62
|
+
provider_package = importlib.import_module("webscout.Provider")
|
|
63
|
+
|
|
64
|
+
for _, module_name, _ in pkgutil.iter_modules(provider_package.__path__):
|
|
65
|
+
try:
|
|
66
|
+
module = importlib.import_module(f"webscout.Provider.{module_name}")
|
|
67
|
+
for attr_name in dir(module):
|
|
68
|
+
attr = getattr(module, attr_name)
|
|
69
|
+
if isinstance(attr, type) and issubclass(attr, Provider) and attr != Provider:
|
|
70
|
+
if hasattr(attr, 'AVAILABLE_MODELS'):
|
|
71
|
+
# Convert any sets to lists to ensure serializability
|
|
72
|
+
models = attr.AVAILABLE_MODELS
|
|
73
|
+
if isinstance(models, set):
|
|
74
|
+
models = list(models)
|
|
75
|
+
provider_models[attr_name] = models
|
|
76
|
+
except Exception:
|
|
77
|
+
pass
|
|
78
|
+
|
|
79
|
+
return provider_models
|
|
80
|
+
|
|
81
|
+
class _TTSModels:
|
|
82
|
+
"""
|
|
83
|
+
A class for managing TTS provider voices in the webscout package.
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
def list(self) -> Dict[str, List[str]]:
|
|
87
|
+
"""
|
|
88
|
+
Gets all available voices from each TTS provider that has an all_voices attribute.
|
|
89
|
+
|
|
90
|
+
Returns:
|
|
91
|
+
Dictionary mapping TTS provider names to their available voices
|
|
92
|
+
"""
|
|
93
|
+
return self._get_tts_voices()
|
|
94
|
+
|
|
95
|
+
def get(self, provider_name: str) -> Union[List[str], Dict[str, str]]:
|
|
96
|
+
"""
|
|
97
|
+
Gets all available voices for a specific TTS provider.
|
|
98
|
+
|
|
99
|
+
Args:
|
|
100
|
+
provider_name: The name of the TTS provider
|
|
101
|
+
|
|
102
|
+
Returns:
|
|
103
|
+
List or Dictionary of available voices for the provider
|
|
104
|
+
"""
|
|
105
|
+
all_voices = self._get_tts_voices()
|
|
106
|
+
return all_voices.get(provider_name, [])
|
|
107
|
+
|
|
108
|
+
def summary(self) -> Dict[str, Any]:
|
|
109
|
+
"""
|
|
110
|
+
Returns a summary of available TTS providers and voices.
|
|
111
|
+
|
|
112
|
+
Returns:
|
|
113
|
+
Dictionary with provider and voice counts
|
|
114
|
+
"""
|
|
115
|
+
provider_voices = self._get_tts_voices()
|
|
116
|
+
total_providers = len(provider_voices)
|
|
117
|
+
|
|
118
|
+
# Count voices, handling both list and dict formats
|
|
119
|
+
total_voices = 0
|
|
120
|
+
provider_voice_counts = {}
|
|
121
|
+
|
|
122
|
+
for provider, voices in provider_voices.items():
|
|
123
|
+
if isinstance(voices, dict):
|
|
124
|
+
count = len(voices)
|
|
125
|
+
elif isinstance(voices, (list, tuple, set)):
|
|
126
|
+
count = len(voices)
|
|
127
|
+
else:
|
|
128
|
+
count = 1
|
|
129
|
+
|
|
130
|
+
total_voices += count
|
|
131
|
+
provider_voice_counts[provider] = count
|
|
132
|
+
|
|
133
|
+
return {
|
|
134
|
+
"providers": total_providers,
|
|
135
|
+
"voices": total_voices,
|
|
136
|
+
"provider_voice_counts": provider_voice_counts
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
def _get_tts_voices(self) -> Dict[str, Union[List[str], Dict[str, str]]]:
|
|
140
|
+
"""
|
|
141
|
+
Internal method to get all available voices from each TTS provider.
|
|
142
|
+
|
|
143
|
+
Returns:
|
|
144
|
+
Dictionary mapping TTS provider names to their available voices
|
|
145
|
+
"""
|
|
146
|
+
provider_voices = {}
|
|
147
|
+
|
|
148
|
+
try:
|
|
149
|
+
# Import the TTS package specifically
|
|
150
|
+
tts_package = importlib.import_module("webscout.Provider.TTS")
|
|
151
|
+
|
|
152
|
+
# Iterate through TTS modules
|
|
153
|
+
for _, module_name, _ in pkgutil.iter_modules(tts_package.__path__):
|
|
154
|
+
try:
|
|
155
|
+
module = importlib.import_module(f"webscout.Provider.TTS.{module_name}")
|
|
156
|
+
for attr_name in dir(module):
|
|
157
|
+
attr = getattr(module, attr_name)
|
|
158
|
+
if isinstance(attr, type) and issubclass(attr, TTSProvider) and attr != TTSProvider:
|
|
159
|
+
# TTS providers typically use 'all_voices' instead of 'AVAILABLE_MODELS'
|
|
160
|
+
if hasattr(attr, 'all_voices'):
|
|
161
|
+
voices = attr.all_voices
|
|
162
|
+
provider_voices[attr_name] = voices
|
|
163
|
+
except Exception as e:
|
|
164
|
+
pass
|
|
165
|
+
except Exception as e:
|
|
166
|
+
pass
|
|
167
|
+
|
|
168
|
+
return provider_voices
|
|
169
|
+
|
|
170
|
+
# Create singleton instances
|
|
171
|
+
llm = _LLMModels()
|
|
172
|
+
tts = _TTSModels()
|
|
173
|
+
|
|
174
|
+
# Container class for all model types
|
|
175
|
+
class Models:
|
|
176
|
+
def __init__(self):
|
|
177
|
+
self.llm = llm
|
|
178
|
+
self.tts = tts
|
|
179
|
+
|
|
180
|
+
# Create a singleton instance
|
|
181
|
+
model = Models()
|