webscout 8.3.2__py3-none-any.whl → 8.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of webscout might be problematic. Click here for more details.
- webscout/AIutel.py +146 -37
- webscout/Bing_search.py +1 -2
- webscout/Provider/AISEARCH/__init__.py +1 -0
- webscout/Provider/AISEARCH/stellar_search.py +132 -0
- webscout/Provider/ExaChat.py +84 -58
- webscout/Provider/HeckAI.py +85 -80
- webscout/Provider/Jadve.py +56 -50
- webscout/Provider/MiniMax.py +207 -0
- webscout/Provider/Nemotron.py +41 -13
- webscout/Provider/Netwrck.py +34 -51
- webscout/Provider/OPENAI/BLACKBOXAI.py +0 -1
- webscout/Provider/OPENAI/MiniMax.py +298 -0
- webscout/Provider/OPENAI/README.md +30 -29
- webscout/Provider/OPENAI/TogetherAI.py +4 -17
- webscout/Provider/OPENAI/__init__.py +3 -1
- webscout/Provider/OPENAI/autoproxy.py +752 -17
- webscout/Provider/OPENAI/base.py +7 -76
- webscout/Provider/OPENAI/deepinfra.py +42 -108
- webscout/Provider/OPENAI/flowith.py +179 -166
- webscout/Provider/OPENAI/friendli.py +233 -0
- webscout/Provider/OPENAI/monochat.py +329 -0
- webscout/Provider/OPENAI/pydantic_imports.py +1 -172
- webscout/Provider/OPENAI/toolbaz.py +1 -0
- webscout/Provider/OPENAI/typegpt.py +1 -1
- webscout/Provider/OPENAI/utils.py +19 -42
- webscout/Provider/OPENAI/x0gpt.py +14 -2
- webscout/Provider/OpenGPT.py +54 -32
- webscout/Provider/PI.py +58 -84
- webscout/Provider/StandardInput.py +32 -13
- webscout/Provider/TTI/README.md +9 -9
- webscout/Provider/TTI/__init__.py +2 -1
- webscout/Provider/TTI/aiarta.py +92 -78
- webscout/Provider/TTI/infip.py +212 -0
- webscout/Provider/TTI/monochat.py +220 -0
- webscout/Provider/TeachAnything.py +11 -3
- webscout/Provider/TextPollinationsAI.py +78 -70
- webscout/Provider/TogetherAI.py +32 -48
- webscout/Provider/Venice.py +37 -46
- webscout/Provider/VercelAI.py +27 -24
- webscout/Provider/WiseCat.py +35 -35
- webscout/Provider/WrDoChat.py +22 -26
- webscout/Provider/WritingMate.py +26 -22
- webscout/Provider/__init__.py +2 -2
- webscout/Provider/granite.py +48 -57
- webscout/Provider/koala.py +51 -39
- webscout/Provider/learnfastai.py +49 -64
- webscout/Provider/llmchat.py +79 -93
- webscout/Provider/llmchatco.py +63 -78
- webscout/Provider/multichat.py +51 -40
- webscout/Provider/oivscode.py +1 -1
- webscout/Provider/scira_chat.py +159 -96
- webscout/Provider/scnet.py +13 -13
- webscout/Provider/searchchat.py +13 -13
- webscout/Provider/sonus.py +12 -11
- webscout/Provider/toolbaz.py +25 -8
- webscout/Provider/turboseek.py +41 -42
- webscout/Provider/typefully.py +27 -12
- webscout/Provider/typegpt.py +41 -46
- webscout/Provider/uncovr.py +55 -90
- webscout/Provider/x0gpt.py +33 -17
- webscout/Provider/yep.py +79 -96
- webscout/auth/__init__.py +12 -1
- webscout/auth/providers.py +27 -5
- webscout/auth/routes.py +128 -104
- webscout/auth/server.py +367 -312
- webscout/client.py +121 -116
- webscout/litagent/Readme.md +68 -55
- webscout/litagent/agent.py +99 -9
- webscout/version.py +1 -1
- {webscout-8.3.2.dist-info → webscout-8.3.3.dist-info}/METADATA +102 -90
- {webscout-8.3.2.dist-info → webscout-8.3.3.dist-info}/RECORD +75 -87
- webscout/Provider/TTI/fastflux.py +0 -233
- webscout/Provider/Writecream.py +0 -246
- webscout/auth/static/favicon.svg +0 -11
- webscout/auth/swagger_ui.py +0 -203
- webscout/auth/templates/components/authentication.html +0 -237
- webscout/auth/templates/components/base.html +0 -103
- webscout/auth/templates/components/endpoints.html +0 -750
- webscout/auth/templates/components/examples.html +0 -491
- webscout/auth/templates/components/footer.html +0 -75
- webscout/auth/templates/components/header.html +0 -27
- webscout/auth/templates/components/models.html +0 -286
- webscout/auth/templates/components/navigation.html +0 -70
- webscout/auth/templates/static/api.js +0 -455
- webscout/auth/templates/static/icons.js +0 -168
- webscout/auth/templates/static/main.js +0 -784
- webscout/auth/templates/static/particles.js +0 -201
- webscout/auth/templates/static/styles.css +0 -3353
- webscout/auth/templates/static/ui.js +0 -374
- webscout/auth/templates/swagger_ui.html +0 -170
- {webscout-8.3.2.dist-info → webscout-8.3.3.dist-info}/WHEEL +0 -0
- {webscout-8.3.2.dist-info → webscout-8.3.3.dist-info}/entry_points.txt +0 -0
- {webscout-8.3.2.dist-info → webscout-8.3.3.dist-info}/licenses/LICENSE.md +0 -0
- {webscout-8.3.2.dist-info → webscout-8.3.3.dist-info}/top_level.txt +0 -0
webscout/client.py
CHANGED
|
@@ -1,116 +1,121 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Unified client import for OpenAI-compatible providers and server utilities.
|
|
3
|
-
|
|
4
|
-
This module provides a unified import interface for all OpenAI-compatible providers and exposes
|
|
5
|
-
helper functions to start the OpenAI-compatible API server programmatically.
|
|
6
|
-
|
|
7
|
-
Usage:
|
|
8
|
-
from webscout.client import FreeAIChat, AI4Chat, ExaChat, MultiChatAI, TwoAI, SciraChat, ChatSandbox, C4AI, Groq, TypeGPT, LLMChat, Cleeai, DeepInfra, BlackboxAI, Cloudflare, Netwrck, OIVSCode, Venice, Writecream, Wisecat, Yep, X0GPT, Sonus, Toolbaz, TextPollinations, StandardInput, Opkfc, Flowith, ExaAI, FreeGemini, GeminiAPI, Gemini, GithubChat, GizAI, Glider, HeckAI, HuggingFaceChat, Hunyuan, Jadve, Julius, Koala, LearnFast, LearnFastAI, NEMOTRON, MCPCore, PydanticImports, TeachAnything, UncovrAI, API, Base
|
|
9
|
-
client = FreeAIChat()
|
|
10
|
-
response = client.chat.completions.create(...)
|
|
11
|
-
|
|
12
|
-
# To start the server programmatically:
|
|
13
|
-
from webscout.client import start_server
|
|
14
|
-
start_server()
|
|
15
|
-
|
|
16
|
-
# For advanced server control:
|
|
17
|
-
from webscout.client import run_api
|
|
18
|
-
run_api(host="0.0.0.0", port=8000, debug=True)
|
|
19
|
-
|
|
20
|
-
# Instantly start the API server from the command line:
|
|
21
|
-
# python -m webscout.client
|
|
22
|
-
# or, if installed as a script:
|
|
23
|
-
# webscout-api
|
|
24
|
-
|
|
25
|
-
Exports:
|
|
26
|
-
- All OpenAI-compatible provider classes
|
|
27
|
-
- start_server (function): Start the OpenAI-compatible API server with default or custom settings.
|
|
28
|
-
- run_api (function): Advanced server startup with full control over host, port, and other options.
|
|
29
|
-
"""
|
|
30
|
-
|
|
31
|
-
from webscout.Provider.OPENAI import *
|
|
32
|
-
|
|
33
|
-
# Import server utilities from the FastAPI-compatible backend
|
|
34
|
-
try:
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
#
|
|
53
|
-
#
|
|
54
|
-
#
|
|
55
|
-
#
|
|
56
|
-
#
|
|
57
|
-
#
|
|
58
|
-
#
|
|
59
|
-
#
|
|
60
|
-
#
|
|
61
|
-
#
|
|
62
|
-
#
|
|
63
|
-
#
|
|
64
|
-
#
|
|
65
|
-
#
|
|
66
|
-
#
|
|
67
|
-
#
|
|
68
|
-
#
|
|
69
|
-
#
|
|
70
|
-
#
|
|
71
|
-
#
|
|
72
|
-
#
|
|
73
|
-
#
|
|
74
|
-
#
|
|
75
|
-
#
|
|
76
|
-
#
|
|
77
|
-
#
|
|
78
|
-
#
|
|
79
|
-
#
|
|
80
|
-
#
|
|
81
|
-
#
|
|
82
|
-
#
|
|
83
|
-
#
|
|
84
|
-
#
|
|
85
|
-
#
|
|
86
|
-
#
|
|
87
|
-
#
|
|
88
|
-
#
|
|
89
|
-
#
|
|
90
|
-
#
|
|
91
|
-
#
|
|
92
|
-
#
|
|
93
|
-
#
|
|
94
|
-
#
|
|
95
|
-
#
|
|
96
|
-
#
|
|
97
|
-
#
|
|
98
|
-
#
|
|
99
|
-
#
|
|
100
|
-
#
|
|
101
|
-
#
|
|
102
|
-
#
|
|
103
|
-
#
|
|
104
|
-
#
|
|
105
|
-
#
|
|
106
|
-
#
|
|
107
|
-
#
|
|
108
|
-
#
|
|
109
|
-
#
|
|
110
|
-
#
|
|
111
|
-
#
|
|
112
|
-
#
|
|
113
|
-
#
|
|
114
|
-
#
|
|
115
|
-
#
|
|
116
|
-
#
|
|
1
|
+
"""
|
|
2
|
+
Unified client import for OpenAI-compatible providers and server utilities.
|
|
3
|
+
|
|
4
|
+
This module provides a unified import interface for all OpenAI-compatible providers and exposes
|
|
5
|
+
helper functions to start the OpenAI-compatible API server programmatically.
|
|
6
|
+
|
|
7
|
+
Usage:
|
|
8
|
+
from webscout.client import FreeAIChat, AI4Chat, ExaChat, MultiChatAI, TwoAI, SciraChat, ChatSandbox, C4AI, Groq, TypeGPT, LLMChat, Cleeai, DeepInfra, BlackboxAI, Cloudflare, Netwrck, OIVSCode, Venice, Writecream, Wisecat, Yep, X0GPT, Sonus, Toolbaz, TextPollinations, StandardInput, Opkfc, Flowith, ExaAI, FreeGemini, GeminiAPI, Gemini, GithubChat, GizAI, Glider, HeckAI, HuggingFaceChat, Hunyuan, Jadve, Julius, Koala, LearnFast, LearnFastAI, NEMOTRON, MCPCore, PydanticImports, TeachAnything, UncovrAI, API, Base
|
|
9
|
+
client = FreeAIChat()
|
|
10
|
+
response = client.chat.completions.create(...)
|
|
11
|
+
|
|
12
|
+
# To start the server programmatically:
|
|
13
|
+
from webscout.client import start_server
|
|
14
|
+
start_server()
|
|
15
|
+
|
|
16
|
+
# For advanced server control:
|
|
17
|
+
from webscout.client import run_api
|
|
18
|
+
run_api(host="0.0.0.0", port=8000, debug=True)
|
|
19
|
+
|
|
20
|
+
# Instantly start the API server from the command line:
|
|
21
|
+
# python -m webscout.client
|
|
22
|
+
# or, if installed as a script:
|
|
23
|
+
# webscout-api
|
|
24
|
+
|
|
25
|
+
Exports:
|
|
26
|
+
- All OpenAI-compatible provider classes
|
|
27
|
+
- start_server (function): Start the OpenAI-compatible API server with default or custom settings.
|
|
28
|
+
- run_api (function): Advanced server startup with full control over host, port, and other options.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from webscout.Provider.OPENAI import *
|
|
32
|
+
|
|
33
|
+
# Import server utilities from the FastAPI-compatible backend
|
|
34
|
+
try:
|
|
35
|
+
# Use lazy import to avoid module execution issues
|
|
36
|
+
def run_api(*args, **kwargs):
|
|
37
|
+
"""Run the Webscout OpenAI-compatible API server (FastAPI backend)."""
|
|
38
|
+
from webscout.auth.server import run_api as _run_api
|
|
39
|
+
return _run_api(*args, **kwargs)
|
|
40
|
+
|
|
41
|
+
def start_server(**kwargs):
|
|
42
|
+
"""Start the Webscout OpenAI-compatible API server (FastAPI backend)."""
|
|
43
|
+
from webscout.auth.server import run_api as _run_api
|
|
44
|
+
return _run_api(**kwargs)
|
|
45
|
+
except ImportError:
|
|
46
|
+
# Fallback for environments where the backend is not available
|
|
47
|
+
def run_api(*args, **kwargs):
|
|
48
|
+
raise ImportError("webscout.auth.server.run_api is not available in this environment.")
|
|
49
|
+
def start_server(*args, **kwargs):
|
|
50
|
+
raise ImportError("webscout.auth.server.start_server is not available in this environment.")
|
|
51
|
+
|
|
52
|
+
# ---
|
|
53
|
+
# API Documentation
|
|
54
|
+
#
|
|
55
|
+
# start_server
|
|
56
|
+
# -------------
|
|
57
|
+
# def start_server(
|
|
58
|
+
# port: int = 8000,
|
|
59
|
+
# host: str = "0.0.0.0",
|
|
60
|
+
# api_key: str = None,
|
|
61
|
+
# default_provider: str = None,
|
|
62
|
+
# base_url: str = None,
|
|
63
|
+
# workers: int = 1,
|
|
64
|
+
# log_level: str = 'info',
|
|
65
|
+
# debug: bool = False,
|
|
66
|
+
# no_auth: bool = False,
|
|
67
|
+
# no_rate_limit: bool = False
|
|
68
|
+
# ):
|
|
69
|
+
# """
|
|
70
|
+
# Start the OpenAI-compatible API server with optional configuration.
|
|
71
|
+
#
|
|
72
|
+
# Parameters:
|
|
73
|
+
# port (int, optional): The port to run the server on. Defaults to 8000.
|
|
74
|
+
# host (str, optional): Host address to bind the server. Defaults to '0.0.0.0'.
|
|
75
|
+
# api_key (str, optional): API key for authentication. If None, authentication is disabled.
|
|
76
|
+
# default_provider (str, optional): The default provider to use. If None, uses the package default.
|
|
77
|
+
# base_url (str, optional): Base URL prefix for the API (e.g., '/api/v1'). If None, no prefix is used.
|
|
78
|
+
# workers (int, optional): Number of worker processes. Defaults to 1.
|
|
79
|
+
# log_level (str, optional): Log level for the server ('debug', 'info', etc.). Defaults to 'info'.
|
|
80
|
+
# debug (bool, optional): Run the server in debug mode with auto-reload. Defaults to False.
|
|
81
|
+
# no_auth (bool, optional): Disable authentication (no API keys required). Defaults to False.
|
|
82
|
+
# no_rate_limit (bool, optional): Disable rate limiting (unlimited requests). Defaults to False.
|
|
83
|
+
#
|
|
84
|
+
# Returns:
|
|
85
|
+
# None
|
|
86
|
+
# """
|
|
87
|
+
#
|
|
88
|
+
# run_api
|
|
89
|
+
# -------
|
|
90
|
+
# def run_api(
|
|
91
|
+
# host: str = '0.0.0.0',
|
|
92
|
+
# port: int = None,
|
|
93
|
+
# api_key: str = None,
|
|
94
|
+
# default_provider: str = None,
|
|
95
|
+
# base_url: str = None,
|
|
96
|
+
# debug: bool = False,
|
|
97
|
+
# workers: int = 1,
|
|
98
|
+
# log_level: str = 'info',
|
|
99
|
+
# show_available_providers: bool = True,
|
|
100
|
+
# no_auth: bool = False,
|
|
101
|
+
# no_rate_limit: bool = False,
|
|
102
|
+
# ) -> None:
|
|
103
|
+
# """
|
|
104
|
+
# Advanced server startup for the OpenAI-compatible API server.
|
|
105
|
+
#
|
|
106
|
+
# Parameters:
|
|
107
|
+
# host (str, optional): Host address to bind the server. Defaults to '0.0.0.0'.
|
|
108
|
+
# port (int, optional): Port to run the server on. Defaults to 8000 if not specified.
|
|
109
|
+
# api_key (str, optional): API key for authentication. If None, authentication is disabled.
|
|
110
|
+
# default_provider (str, optional): The default provider to use. If None, uses the package default.
|
|
111
|
+
# base_url (str, optional): Base URL prefix for the API (e.g., '/api/v1'). If None, no prefix is used.
|
|
112
|
+
# debug (bool, optional): Run the server in debug mode with auto-reload. Defaults to False.
|
|
113
|
+
# workers (int, optional): Number of worker processes. Defaults to 1.
|
|
114
|
+
# log_level (str, optional): Log level for the server ('debug', 'info', etc.). Defaults to 'info'.
|
|
115
|
+
# show_available_providers (bool, optional): Print available providers on startup. Defaults to True.
|
|
116
|
+
# no_auth (bool, optional): Disable authentication (no API keys required). Defaults to False.
|
|
117
|
+
# no_rate_limit (bool, optional): Disable rate limiting (unlimited requests). Defaults to False.
|
|
118
|
+
#
|
|
119
|
+
# Returns:
|
|
120
|
+
# None
|
|
121
|
+
# """
|
webscout/litagent/Readme.md
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# 🔥 LitAgent - The Lit User Agent Generator
|
|
2
2
|
|
|
3
|
-
LitAgent is a powerful
|
|
3
|
+
LitAgent is a powerful, modern user agent generator that keeps your requests fresh and undetectable! Built for web scraping, it helps you manage user agents with ease and style.
|
|
4
|
+
|
|
5
|
+
---
|
|
4
6
|
|
|
5
7
|
## 🚀 Quick Start
|
|
6
8
|
|
|
@@ -15,41 +17,46 @@ ua = agent.random()
|
|
|
15
17
|
print(ua) # Mozilla/5.0 (Windows NT 11.0) AppleWebKit/537.36 ...
|
|
16
18
|
```
|
|
17
19
|
|
|
20
|
+
---
|
|
21
|
+
|
|
18
22
|
## 🎯 Features
|
|
19
23
|
|
|
20
24
|
### Browser-Specific Agents
|
|
21
25
|
|
|
22
26
|
```python
|
|
23
27
|
# Get agents for specific browsers
|
|
24
|
-
chrome_ua = agent.chrome()
|
|
25
|
-
firefox_ua = agent.firefox()
|
|
26
|
-
safari_ua = agent.safari()
|
|
27
|
-
edge_ua = agent.edge()
|
|
28
|
-
opera_ua = agent.opera()
|
|
28
|
+
chrome_ua = agent.chrome() # Latest Chrome agent
|
|
29
|
+
firefox_ua = agent.firefox() # Latest Firefox agent
|
|
30
|
+
safari_ua = agent.safari() # Latest Safari agent
|
|
31
|
+
edge_ua = agent.edge() # Latest Edge agent
|
|
32
|
+
opera_ua = agent.opera() # Latest Opera agent
|
|
29
33
|
```
|
|
30
34
|
|
|
31
35
|
### Device-Specific Agents
|
|
32
36
|
|
|
33
37
|
```python
|
|
34
38
|
# Get mobile or desktop agents
|
|
35
|
-
mobile_ua = agent.mobile()
|
|
36
|
-
desktop_ua = agent.desktop()
|
|
39
|
+
mobile_ua = agent.mobile() # Mobile device agent
|
|
40
|
+
desktop_ua = agent.desktop() # Desktop device agent
|
|
37
41
|
|
|
38
42
|
# New - Get agents for specific device types
|
|
39
|
-
tablet_ua = agent.tablet()
|
|
40
|
-
tv_ua = agent.smart_tv()
|
|
41
|
-
console_ua = agent.gaming()
|
|
43
|
+
tablet_ua = agent.tablet() # Tablet device agent
|
|
44
|
+
tv_ua = agent.smart_tv() # Smart TV agent
|
|
45
|
+
console_ua = agent.gaming() # Gaming console agent
|
|
46
|
+
|
|
47
|
+
# Wearable device user agent support
|
|
48
|
+
wearable_ua = agent.wearable() # Get a wearable device user agent
|
|
42
49
|
```
|
|
43
50
|
|
|
44
51
|
### OS-Specific Agents
|
|
45
52
|
|
|
46
53
|
```python
|
|
47
54
|
# New - Get agents for specific operating systems
|
|
48
|
-
windows_ua = agent.windows()
|
|
49
|
-
mac_ua = agent.macos()
|
|
50
|
-
linux_ua = agent.linux()
|
|
51
|
-
android_ua = agent.android()
|
|
52
|
-
ios_ua = agent.ios()
|
|
55
|
+
windows_ua = agent.windows() # Windows agent
|
|
56
|
+
mac_ua = agent.macos() # macOS agent
|
|
57
|
+
linux_ua = agent.linux() # Linux agent`
|
|
58
|
+
android_ua = agent.android() # Android agent
|
|
59
|
+
ios_ua = agent.ios() # iOS agent
|
|
53
60
|
```
|
|
54
61
|
|
|
55
62
|
### Custom Agent Generation
|
|
@@ -83,6 +90,15 @@ ip = agent.rotate_ip()
|
|
|
83
90
|
print(ip) # 192.168.1.10 (example)
|
|
84
91
|
```
|
|
85
92
|
|
|
93
|
+
### Proxy Pool & Rotation
|
|
94
|
+
|
|
95
|
+
```python
|
|
96
|
+
agent.set_proxy_pool(["http://proxy1:8080", "http://proxy2:8080"])
|
|
97
|
+
proxy = agent.rotate_proxy() # Get next proxy from pool
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
---
|
|
101
|
+
|
|
86
102
|
## 💫 Real-World Examples
|
|
87
103
|
|
|
88
104
|
### With Requests
|
|
@@ -152,47 +168,30 @@ driver = create_driver()
|
|
|
152
168
|
driver.get('https://example.com')
|
|
153
169
|
```
|
|
154
170
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
```python
|
|
158
|
-
from playwright.sync_api import sync_playwright
|
|
159
|
-
from webscout import LitAgent
|
|
160
|
-
|
|
161
|
-
agent = LitAgent()
|
|
162
|
-
|
|
163
|
-
def browse_with_playwright():
|
|
164
|
-
with sync_playwright() as p:
|
|
165
|
-
browser_options = {
|
|
166
|
-
"user_agent": agent.chrome(),
|
|
167
|
-
"viewport": {"width": 1280, "height": 720}
|
|
168
|
-
}
|
|
169
|
-
browser = p.chromium.launch()
|
|
170
|
-
context = browser.new_context(**browser_options)
|
|
171
|
-
page = context.new_page()
|
|
172
|
-
page.goto('https://example.com')
|
|
173
|
-
# Continue with your scraping logic
|
|
174
|
-
browser.close()
|
|
175
|
-
```
|
|
171
|
+
---
|
|
176
172
|
|
|
177
173
|
## 🌟 Pro Tips
|
|
178
174
|
|
|
179
|
-
1. **Rotate Agents
|
|
175
|
+
1. **Rotate Agents Regularly**
|
|
176
|
+
Refresh your agent pool periodically to avoid detection.
|
|
177
|
+
|
|
180
178
|
```python
|
|
181
179
|
agent = LitAgent()
|
|
182
|
-
for
|
|
180
|
+
for i in range(10):
|
|
183
181
|
response = requests.get(url, headers={'User-Agent': agent.random()})
|
|
184
|
-
if
|
|
182
|
+
if i % 3 == 0:
|
|
185
183
|
agent.refresh()
|
|
186
184
|
```
|
|
187
185
|
|
|
188
|
-
2. **Device-Specific Scraping
|
|
186
|
+
2. **Device-Specific Scraping**
|
|
187
|
+
Use device-specific agents for different platforms.
|
|
188
|
+
|
|
189
189
|
```python
|
|
190
190
|
# Mobile site scraping
|
|
191
191
|
mobile_response = requests.get(
|
|
192
192
|
'https://m.example.com',
|
|
193
193
|
headers={'User-Agent': agent.mobile()}
|
|
194
194
|
)
|
|
195
|
-
|
|
196
195
|
# Desktop site scraping
|
|
197
196
|
desktop_response = requests.get(
|
|
198
197
|
'https://example.com',
|
|
@@ -200,7 +199,9 @@ def browse_with_playwright():
|
|
|
200
199
|
)
|
|
201
200
|
```
|
|
202
201
|
|
|
203
|
-
3. **Browser Consistency
|
|
202
|
+
3. **Browser Consistency**
|
|
203
|
+
Stick to one browser type per session for realism.
|
|
204
|
+
|
|
204
205
|
```python
|
|
205
206
|
chrome_agent = agent.chrome()
|
|
206
207
|
headers = {
|
|
@@ -210,11 +211,11 @@ def browse_with_playwright():
|
|
|
210
211
|
# Use these headers for all requests in this session
|
|
211
212
|
```
|
|
212
213
|
|
|
213
|
-
4. **
|
|
214
|
+
4. **Browser Fingerprinting & IP Rotation**
|
|
215
|
+
Generate consistent browser fingerprints and rotate IPs.
|
|
216
|
+
|
|
214
217
|
```python
|
|
215
|
-
# Create consistent browser fingerprinting
|
|
216
218
|
fingerprint = agent.generate_fingerprint(browser="chrome")
|
|
217
|
-
|
|
218
219
|
headers = {
|
|
219
220
|
'User-Agent': fingerprint['user_agent'],
|
|
220
221
|
'Accept-Language': fingerprint['accept_language'],
|
|
@@ -223,33 +224,37 @@ def browse_with_playwright():
|
|
|
223
224
|
'Sec-Ch-Ua-Platform': fingerprint['platform'],
|
|
224
225
|
'X-Forwarded-For': fingerprint['x-forwarded-for']
|
|
225
226
|
}
|
|
226
|
-
|
|
227
|
-
# Use this consistent set for all session requests while rotating IPs
|
|
228
227
|
```
|
|
229
228
|
|
|
230
|
-
5. **
|
|
229
|
+
5. **Multi-threading Support**
|
|
230
|
+
Use thread-safe mode for concurrent requests.
|
|
231
|
+
|
|
231
232
|
```python
|
|
232
233
|
import concurrent.futures
|
|
233
234
|
from webscout import LitAgent
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
235
|
+
import requests
|
|
236
|
+
|
|
237
|
+
agent = LitAgent(thread_safe=True)
|
|
238
|
+
|
|
237
239
|
def fetch_url(url):
|
|
238
240
|
headers = {'User-Agent': agent.random()}
|
|
239
241
|
return requests.get(url, headers=headers).text
|
|
240
|
-
|
|
242
|
+
|
|
241
243
|
urls = ['https://example1.com', 'https://example2.com', 'https://example3.com']
|
|
242
|
-
|
|
243
244
|
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
|
|
244
245
|
results = list(executor.map(fetch_url, urls))
|
|
245
246
|
```
|
|
246
247
|
|
|
248
|
+
---
|
|
249
|
+
|
|
247
250
|
## 🔧 Supported Browsers & Devices
|
|
248
251
|
|
|
249
252
|
- **Browsers**: Chrome, Firefox, Safari, Edge, Opera, Brave, Vivaldi
|
|
250
253
|
- **Operating Systems**: Windows, macOS, Linux, Android, iOS, Chrome OS
|
|
251
254
|
- **Devices**: Mobile phones, Tablets, Desktops, Game consoles, Smart TVs, Wearables
|
|
252
255
|
|
|
256
|
+
---
|
|
257
|
+
|
|
253
258
|
## 🎨 Why LitAgent?
|
|
254
259
|
|
|
255
260
|
- 🚀 Modern and up-to-date user agents
|
|
@@ -263,7 +268,11 @@ def browse_with_playwright():
|
|
|
263
268
|
- 🧵 Thread-safe operation
|
|
264
269
|
- 🕰️ Automatic refresh scheduling
|
|
265
270
|
|
|
266
|
-
|
|
271
|
+
---
|
|
272
|
+
|
|
273
|
+
## 📊 Analytics and Reporting
|
|
274
|
+
|
|
275
|
+
Get statistics on your agent usage and export your data.
|
|
267
276
|
|
|
268
277
|
```python
|
|
269
278
|
# Get statistics on your agent usage
|
|
@@ -276,10 +285,14 @@ print(f"Detection avoidance rate: {stats.avoidance_rate}%")
|
|
|
276
285
|
agent.export_stats('agent_usage.json')
|
|
277
286
|
```
|
|
278
287
|
|
|
288
|
+
---
|
|
289
|
+
|
|
279
290
|
## 📋 Installation
|
|
280
291
|
|
|
281
292
|
```bash
|
|
282
293
|
pip install webscout
|
|
283
294
|
```
|
|
284
295
|
|
|
296
|
+
---
|
|
297
|
+
|
|
285
298
|
Made with 💖 by the HelpingAI team
|
webscout/litagent/agent.py
CHANGED
|
@@ -93,15 +93,23 @@ class LitAgent:
|
|
|
93
93
|
self._stats["device_usage"][device_type] = self._stats["device_usage"].get(device_type, 0) + 1
|
|
94
94
|
|
|
95
95
|
def random(self) -> str:
|
|
96
|
-
"""Get a random user agent! 🎲"""
|
|
96
|
+
"""Get a random user agent! 🎲 (with blacklist/whitelist support)"""
|
|
97
|
+
if hasattr(self, '_whitelist') and self._whitelist:
|
|
98
|
+
pool = list(self._whitelist)
|
|
99
|
+
else:
|
|
100
|
+
pool = [a for a in self.agents if not hasattr(self, '_blacklist') or a not in self._blacklist]
|
|
101
|
+
if not pool:
|
|
102
|
+
pool = self.agents
|
|
97
103
|
if self.thread_safe and self.lock:
|
|
98
104
|
with self.lock:
|
|
99
|
-
agent = random.choice(
|
|
105
|
+
agent = random.choice(pool)
|
|
100
106
|
self._update_stats()
|
|
107
|
+
self._add_to_history(agent)
|
|
101
108
|
return agent
|
|
102
109
|
else:
|
|
103
|
-
agent = random.choice(
|
|
110
|
+
agent = random.choice(pool)
|
|
104
111
|
self._update_stats()
|
|
112
|
+
self._add_to_history(agent)
|
|
105
113
|
return agent
|
|
106
114
|
|
|
107
115
|
def browser(self, name: str) -> str:
|
|
@@ -330,7 +338,8 @@ class LitAgent:
|
|
|
330
338
|
self._update_stats(browser_type=browser, device_type=device_type)
|
|
331
339
|
return agent
|
|
332
340
|
|
|
333
|
-
|
|
341
|
+
@staticmethod
|
|
342
|
+
def generate_fingerprint(browser: Optional[str] = None) -> Dict[str, str]:
|
|
334
343
|
"""
|
|
335
344
|
Generate a consistent browser fingerprint for anti-fingerprinting purposes.
|
|
336
345
|
|
|
@@ -347,13 +356,14 @@ class LitAgent:
|
|
|
347
356
|
Dict[str, str]: A dictionary containing fingerprinting headers and values.
|
|
348
357
|
"""
|
|
349
358
|
# Get a random user agent using the random() method
|
|
350
|
-
|
|
359
|
+
agent = LitAgent()
|
|
360
|
+
user_agent = agent.random()
|
|
351
361
|
|
|
352
362
|
# If browser is specified, try to get a matching one
|
|
353
363
|
if browser:
|
|
354
364
|
browser = browser.lower()
|
|
355
365
|
if browser in BROWSERS:
|
|
356
|
-
user_agent =
|
|
366
|
+
user_agent = agent.browser(browser)
|
|
357
367
|
|
|
358
368
|
accept_language = random.choice(FINGERPRINTS["accept_language"])
|
|
359
369
|
accept = random.choice(FINGERPRINTS["accept"])
|
|
@@ -367,7 +377,7 @@ class LitAgent:
|
|
|
367
377
|
sec_ch_ua = FINGERPRINTS["sec_ch_ua"][browser_name].format(version, version)
|
|
368
378
|
break
|
|
369
379
|
|
|
370
|
-
ip =
|
|
380
|
+
ip = agent.rotate_ip()
|
|
371
381
|
fingerprint = {
|
|
372
382
|
"user_agent": user_agent,
|
|
373
383
|
"accept_language": accept_language,
|
|
@@ -379,10 +389,10 @@ class LitAgent:
|
|
|
379
389
|
"x-client-ip": ip,
|
|
380
390
|
"forwarded": f"for={ip};proto=https",
|
|
381
391
|
"x-forwarded-proto": "https",
|
|
382
|
-
"x-request-id":
|
|
392
|
+
"x-request-id": agent.random_id(8) if hasattr(agent, 'random_id') else ''.join(random.choices('0123456789abcdef', k=8)),
|
|
383
393
|
}
|
|
384
394
|
|
|
385
|
-
|
|
395
|
+
agent._update_stats(browser_type=browser)
|
|
386
396
|
return fingerprint
|
|
387
397
|
|
|
388
398
|
def refresh(self) -> None:
|
|
@@ -472,6 +482,63 @@ class LitAgent:
|
|
|
472
482
|
"""Generate a random identifier string."""
|
|
473
483
|
return ''.join(random.choices('0123456789abcdef', k=length)).lower()
|
|
474
484
|
|
|
485
|
+
def wearable(self) -> str:
|
|
486
|
+
"""Get a wearable device agent! ⌚"""
|
|
487
|
+
wearable_type = random.choice(DEVICES['wearable'])
|
|
488
|
+
# Example user agent for wearables (simplified)
|
|
489
|
+
if 'Apple Watch' in wearable_type:
|
|
490
|
+
agent = f"Mozilla/5.0 (AppleWatch; CPU WatchOS like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/9.0 Mobile/13S344 Safari/602.1"
|
|
491
|
+
elif 'Samsung' in wearable_type:
|
|
492
|
+
agent = f"Mozilla/5.0 (Linux; Tizen 3.0; {wearable_type}) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/1.0"
|
|
493
|
+
elif 'Fitbit' in wearable_type:
|
|
494
|
+
agent = f"Mozilla/5.0 (Linux; {wearable_type}) AppleWebKit/537.36 (KHTML, like Gecko)"
|
|
495
|
+
elif 'Garmin' in wearable_type:
|
|
496
|
+
agent = f"Mozilla/5.0 (Linux; {wearable_type}) AppleWebKit/537.36 (KHTML, like Gecko)"
|
|
497
|
+
else:
|
|
498
|
+
agent = self.random()
|
|
499
|
+
self._update_stats(device_type="wearable")
|
|
500
|
+
return agent
|
|
501
|
+
|
|
502
|
+
def set_proxy_pool(self, proxies: List[str]):
|
|
503
|
+
"""Set a pool of proxies for rotation."""
|
|
504
|
+
self._proxy_pool = proxies
|
|
505
|
+
self._proxy_index = 0
|
|
506
|
+
|
|
507
|
+
def rotate_proxy(self) -> Optional[str]:
|
|
508
|
+
"""Rotate through the proxy pool and return the next proxy."""
|
|
509
|
+
if not hasattr(self, '_proxy_pool') or not self._proxy_pool:
|
|
510
|
+
return None
|
|
511
|
+
proxy = self._proxy_pool[self._proxy_index]
|
|
512
|
+
self._proxy_index = (self._proxy_index + 1) % len(self._proxy_pool)
|
|
513
|
+
return proxy
|
|
514
|
+
|
|
515
|
+
def add_to_blacklist(self, agent: str):
|
|
516
|
+
"""Add a user agent to the blacklist."""
|
|
517
|
+
if not hasattr(self, '_blacklist'):
|
|
518
|
+
self._blacklist = set()
|
|
519
|
+
self._blacklist.add(agent)
|
|
520
|
+
|
|
521
|
+
def add_to_whitelist(self, agent: str):
|
|
522
|
+
"""Add a user agent to the whitelist."""
|
|
523
|
+
if not hasattr(self, '_whitelist'):
|
|
524
|
+
self._whitelist = set()
|
|
525
|
+
self._whitelist.add(agent)
|
|
526
|
+
|
|
527
|
+
def _add_to_history(self, agent: str):
|
|
528
|
+
if not hasattr(self, '_history'):
|
|
529
|
+
self._history = []
|
|
530
|
+
self._history.append(agent)
|
|
531
|
+
if len(self._history) > 50:
|
|
532
|
+
self._history.pop(0)
|
|
533
|
+
|
|
534
|
+
def get_history(self) -> List[str]:
|
|
535
|
+
"""Get the last 50 user agents served."""
|
|
536
|
+
return getattr(self, '_history', [])
|
|
537
|
+
|
|
538
|
+
def validate_agent(self, agent: str) -> bool:
|
|
539
|
+
"""Validate if a user agent string is realistic (basic check)."""
|
|
540
|
+
return agent.startswith("Mozilla/5.0") and any(b in agent for b in BROWSERS.keys())
|
|
541
|
+
|
|
475
542
|
if __name__ == "__main__":
|
|
476
543
|
# Test it out! 🧪
|
|
477
544
|
agent = LitAgent()
|
|
@@ -484,9 +551,32 @@ if __name__ == "__main__":
|
|
|
484
551
|
print("Tablet:", agent.tablet())
|
|
485
552
|
print("Smart TV:", agent.smart_tv())
|
|
486
553
|
print("Gaming:", agent.gaming())
|
|
554
|
+
print("Wearable:", agent.wearable())
|
|
487
555
|
|
|
488
556
|
# Test custom agent
|
|
489
557
|
print("Custom:", agent.custom(browser="chrome", os="windows", os_version="10.0"))
|
|
490
558
|
|
|
491
559
|
# Test fingerprinting
|
|
492
560
|
print("Fingerprint:", agent.generate_fingerprint("chrome"))
|
|
561
|
+
|
|
562
|
+
# Test proxy rotation
|
|
563
|
+
agent.set_proxy_pool(["http://proxy1.com", "http://proxy2.com"])
|
|
564
|
+
print("Proxy 1:", agent.rotate_proxy())
|
|
565
|
+
print("Proxy 2:", agent.rotate_proxy())
|
|
566
|
+
|
|
567
|
+
# Test blacklist/whitelist
|
|
568
|
+
agent.add_to_blacklist("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36")
|
|
569
|
+
agent.add_to_whitelist("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
|
|
570
|
+
print("Blacklisted:", agent.random())
|
|
571
|
+
print("Whitelisted:", agent.random())
|
|
572
|
+
|
|
573
|
+
# Test agent history
|
|
574
|
+
for _ in range(55):
|
|
575
|
+
agent.random()
|
|
576
|
+
print("History:", agent.get_history())
|
|
577
|
+
|
|
578
|
+
# Test agent validation
|
|
579
|
+
print("Valid agent:", agent.validate_agent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"))
|
|
580
|
+
print("Invalid agent:", agent.validate_agent("InvalidUserAgentString"))
|
|
581
|
+
ip = agent.rotate_ip()
|
|
582
|
+
print(ip) # 192.168.1.10 (example)
|
webscout/version.py
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
__version__ = "8.3.
|
|
1
|
+
__version__ = "8.3.3"
|
|
2
2
|
__prog__ = "webscout"
|