webscout 8.2.9__py3-none-any.whl → 2026.1.19__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- webscout/AIauto.py +524 -251
- webscout/AIbase.py +247 -319
- webscout/AIutel.py +68 -703
- webscout/Bard.py +1072 -1026
- webscout/Extra/GitToolkit/__init__.py +10 -10
- webscout/Extra/GitToolkit/gitapi/__init__.py +20 -12
- webscout/Extra/GitToolkit/gitapi/gist.py +142 -0
- webscout/Extra/GitToolkit/gitapi/organization.py +91 -0
- webscout/Extra/GitToolkit/gitapi/repository.py +308 -195
- webscout/Extra/GitToolkit/gitapi/search.py +162 -0
- webscout/Extra/GitToolkit/gitapi/trending.py +236 -0
- webscout/Extra/GitToolkit/gitapi/user.py +128 -96
- webscout/Extra/GitToolkit/gitapi/utils.py +82 -62
- webscout/Extra/YTToolkit/README.md +443 -375
- webscout/Extra/YTToolkit/YTdownloader.py +953 -957
- webscout/Extra/YTToolkit/__init__.py +3 -3
- webscout/Extra/YTToolkit/transcriber.py +595 -476
- webscout/Extra/YTToolkit/ytapi/README.md +230 -44
- webscout/Extra/YTToolkit/ytapi/__init__.py +22 -6
- webscout/Extra/YTToolkit/ytapi/captions.py +190 -0
- webscout/Extra/YTToolkit/ytapi/channel.py +302 -307
- webscout/Extra/YTToolkit/ytapi/errors.py +13 -13
- webscout/Extra/YTToolkit/ytapi/extras.py +178 -118
- webscout/Extra/YTToolkit/ytapi/hashtag.py +120 -0
- webscout/Extra/YTToolkit/ytapi/https.py +89 -88
- webscout/Extra/YTToolkit/ytapi/patterns.py +61 -61
- webscout/Extra/YTToolkit/ytapi/playlist.py +59 -59
- webscout/Extra/YTToolkit/ytapi/pool.py +8 -8
- webscout/Extra/YTToolkit/ytapi/query.py +143 -40
- webscout/Extra/YTToolkit/ytapi/shorts.py +122 -0
- webscout/Extra/YTToolkit/ytapi/stream.py +68 -63
- webscout/Extra/YTToolkit/ytapi/suggestions.py +97 -0
- webscout/Extra/YTToolkit/ytapi/utils.py +66 -62
- webscout/Extra/YTToolkit/ytapi/video.py +403 -232
- webscout/Extra/__init__.py +2 -3
- webscout/Extra/gguf.py +1298 -684
- webscout/Extra/tempmail/README.md +487 -487
- webscout/Extra/tempmail/__init__.py +28 -28
- webscout/Extra/tempmail/async_utils.py +143 -141
- webscout/Extra/tempmail/base.py +172 -161
- webscout/Extra/tempmail/cli.py +191 -187
- webscout/Extra/tempmail/emailnator.py +88 -84
- webscout/Extra/tempmail/mail_tm.py +378 -361
- webscout/Extra/tempmail/temp_mail_io.py +304 -292
- webscout/Extra/weather.py +196 -194
- webscout/Extra/weather_ascii.py +17 -15
- webscout/Provider/AISEARCH/PERPLEXED_search.py +175 -0
- webscout/Provider/AISEARCH/Perplexity.py +292 -333
- webscout/Provider/AISEARCH/README.md +106 -279
- webscout/Provider/AISEARCH/__init__.py +16 -9
- webscout/Provider/AISEARCH/brave_search.py +298 -0
- webscout/Provider/AISEARCH/iask_search.py +357 -410
- webscout/Provider/AISEARCH/monica_search.py +200 -220
- webscout/Provider/AISEARCH/webpilotai_search.py +242 -255
- webscout/Provider/Algion.py +413 -0
- webscout/Provider/Andi.py +74 -69
- webscout/Provider/Apriel.py +313 -0
- webscout/Provider/Ayle.py +323 -0
- webscout/Provider/ChatSandbox.py +329 -342
- webscout/Provider/ClaudeOnline.py +365 -0
- webscout/Provider/Cohere.py +232 -208
- webscout/Provider/DeepAI.py +367 -0
- webscout/Provider/Deepinfra.py +467 -340
- webscout/Provider/EssentialAI.py +217 -0
- webscout/Provider/ExaAI.py +274 -261
- webscout/Provider/Gemini.py +175 -169
- webscout/Provider/GithubChat.py +385 -369
- webscout/Provider/Gradient.py +286 -0
- webscout/Provider/Groq.py +556 -801
- webscout/Provider/HadadXYZ.py +323 -0
- webscout/Provider/HeckAI.py +392 -375
- webscout/Provider/HuggingFace.py +387 -0
- webscout/Provider/IBM.py +340 -0
- webscout/Provider/Jadve.py +317 -291
- webscout/Provider/K2Think.py +306 -0
- webscout/Provider/Koboldai.py +221 -384
- webscout/Provider/Netwrck.py +273 -270
- webscout/Provider/Nvidia.py +310 -0
- webscout/Provider/OPENAI/DeepAI.py +489 -0
- webscout/Provider/OPENAI/K2Think.py +423 -0
- webscout/Provider/OPENAI/PI.py +463 -0
- webscout/Provider/OPENAI/README.md +890 -952
- webscout/Provider/OPENAI/TogetherAI.py +405 -0
- webscout/Provider/OPENAI/TwoAI.py +255 -357
- webscout/Provider/OPENAI/__init__.py +148 -40
- webscout/Provider/OPENAI/ai4chat.py +348 -293
- webscout/Provider/OPENAI/akashgpt.py +436 -0
- webscout/Provider/OPENAI/algion.py +303 -0
- webscout/Provider/OPENAI/{exachat.py → ayle.py} +365 -444
- webscout/Provider/OPENAI/base.py +253 -249
- webscout/Provider/OPENAI/cerebras.py +296 -0
- webscout/Provider/OPENAI/chatgpt.py +870 -556
- webscout/Provider/OPENAI/chatsandbox.py +233 -173
- webscout/Provider/OPENAI/deepinfra.py +403 -322
- webscout/Provider/OPENAI/e2b.py +2370 -1414
- webscout/Provider/OPENAI/elmo.py +278 -0
- webscout/Provider/OPENAI/exaai.py +452 -417
- webscout/Provider/OPENAI/freeassist.py +446 -0
- webscout/Provider/OPENAI/gradient.py +448 -0
- webscout/Provider/OPENAI/groq.py +380 -364
- webscout/Provider/OPENAI/hadadxyz.py +292 -0
- webscout/Provider/OPENAI/heckai.py +333 -308
- webscout/Provider/OPENAI/huggingface.py +321 -0
- webscout/Provider/OPENAI/ibm.py +425 -0
- webscout/Provider/OPENAI/llmchat.py +253 -0
- webscout/Provider/OPENAI/llmchatco.py +378 -335
- webscout/Provider/OPENAI/meta.py +541 -0
- webscout/Provider/OPENAI/netwrck.py +374 -357
- webscout/Provider/OPENAI/nvidia.py +317 -0
- webscout/Provider/OPENAI/oivscode.py +348 -287
- webscout/Provider/OPENAI/openrouter.py +328 -0
- webscout/Provider/OPENAI/pydantic_imports.py +1 -172
- webscout/Provider/OPENAI/sambanova.py +397 -0
- webscout/Provider/OPENAI/sonus.py +305 -304
- webscout/Provider/OPENAI/textpollinations.py +370 -339
- webscout/Provider/OPENAI/toolbaz.py +375 -413
- webscout/Provider/OPENAI/typefully.py +419 -355
- webscout/Provider/OPENAI/typliai.py +279 -0
- webscout/Provider/OPENAI/utils.py +314 -318
- webscout/Provider/OPENAI/wisecat.py +359 -387
- webscout/Provider/OPENAI/writecream.py +185 -163
- webscout/Provider/OPENAI/x0gpt.py +462 -365
- webscout/Provider/OPENAI/zenmux.py +380 -0
- webscout/Provider/OpenRouter.py +386 -0
- webscout/Provider/Openai.py +337 -496
- webscout/Provider/PI.py +443 -429
- webscout/Provider/QwenLM.py +346 -254
- webscout/Provider/STT/__init__.py +28 -0
- webscout/Provider/STT/base.py +303 -0
- webscout/Provider/STT/elevenlabs.py +264 -0
- webscout/Provider/Sambanova.py +317 -0
- webscout/Provider/TTI/README.md +69 -82
- webscout/Provider/TTI/__init__.py +37 -7
- webscout/Provider/TTI/base.py +147 -64
- webscout/Provider/TTI/claudeonline.py +393 -0
- webscout/Provider/TTI/magicstudio.py +292 -201
- webscout/Provider/TTI/miragic.py +180 -0
- webscout/Provider/TTI/pollinations.py +331 -221
- webscout/Provider/TTI/together.py +334 -0
- webscout/Provider/TTI/utils.py +14 -11
- webscout/Provider/TTS/README.md +186 -192
- webscout/Provider/TTS/__init__.py +43 -10
- webscout/Provider/TTS/base.py +523 -159
- webscout/Provider/TTS/deepgram.py +286 -156
- webscout/Provider/TTS/elevenlabs.py +189 -111
- webscout/Provider/TTS/freetts.py +218 -0
- webscout/Provider/TTS/murfai.py +288 -113
- webscout/Provider/TTS/openai_fm.py +364 -129
- webscout/Provider/TTS/parler.py +203 -111
- webscout/Provider/TTS/qwen.py +334 -0
- webscout/Provider/TTS/sherpa.py +286 -0
- webscout/Provider/TTS/speechma.py +693 -580
- webscout/Provider/TTS/streamElements.py +275 -333
- webscout/Provider/TTS/utils.py +280 -280
- webscout/Provider/TextPollinationsAI.py +331 -308
- webscout/Provider/TogetherAI.py +450 -0
- webscout/Provider/TwoAI.py +309 -475
- webscout/Provider/TypliAI.py +311 -305
- webscout/Provider/UNFINISHED/ChatHub.py +219 -209
- webscout/Provider/{OPENAI/glider.py → UNFINISHED/ChutesAI.py} +331 -326
- webscout/Provider/{GizAI.py → UNFINISHED/GizAI.py} +300 -295
- webscout/Provider/{Marcus.py → UNFINISHED/Marcus.py} +218 -198
- webscout/Provider/UNFINISHED/Qodo.py +481 -0
- webscout/Provider/{MCPCore.py → UNFINISHED/XenAI.py} +330 -315
- webscout/Provider/UNFINISHED/Youchat.py +347 -330
- webscout/Provider/UNFINISHED/aihumanizer.py +41 -0
- webscout/Provider/UNFINISHED/grammerchecker.py +37 -0
- webscout/Provider/UNFINISHED/liner.py +342 -0
- webscout/Provider/UNFINISHED/liner_api_request.py +246 -263
- webscout/Provider/{samurai.py → UNFINISHED/samurai.py} +231 -224
- webscout/Provider/WiseCat.py +256 -233
- webscout/Provider/WrDoChat.py +390 -370
- webscout/Provider/__init__.py +115 -174
- webscout/Provider/ai4chat.py +181 -174
- webscout/Provider/akashgpt.py +330 -335
- webscout/Provider/cerebras.py +397 -290
- webscout/Provider/cleeai.py +236 -213
- webscout/Provider/elmo.py +291 -283
- webscout/Provider/geminiapi.py +343 -208
- webscout/Provider/julius.py +245 -223
- webscout/Provider/learnfastai.py +333 -325
- webscout/Provider/llama3mitril.py +230 -215
- webscout/Provider/llmchat.py +308 -258
- webscout/Provider/llmchatco.py +321 -306
- webscout/Provider/meta.py +996 -801
- webscout/Provider/oivscode.py +332 -309
- webscout/Provider/searchchat.py +316 -292
- webscout/Provider/sonus.py +264 -258
- webscout/Provider/toolbaz.py +359 -353
- webscout/Provider/turboseek.py +332 -266
- webscout/Provider/typefully.py +262 -202
- webscout/Provider/x0gpt.py +332 -299
- webscout/__init__.py +31 -39
- webscout/__main__.py +5 -5
- webscout/cli.py +585 -524
- webscout/client.py +1497 -70
- webscout/conversation.py +140 -436
- webscout/exceptions.py +383 -362
- webscout/litagent/__init__.py +29 -29
- webscout/litagent/agent.py +492 -455
- webscout/litagent/constants.py +60 -60
- webscout/models.py +505 -181
- webscout/optimizers.py +74 -420
- webscout/prompt_manager.py +376 -288
- webscout/sanitize.py +1514 -0
- webscout/scout/README.md +452 -404
- webscout/scout/__init__.py +8 -8
- webscout/scout/core/__init__.py +7 -7
- webscout/scout/core/crawler.py +330 -210
- webscout/scout/core/scout.py +800 -607
- webscout/scout/core/search_result.py +51 -96
- webscout/scout/core/text_analyzer.py +64 -63
- webscout/scout/core/text_utils.py +412 -277
- webscout/scout/core/web_analyzer.py +54 -52
- webscout/scout/element.py +872 -478
- webscout/scout/parsers/__init__.py +70 -69
- webscout/scout/parsers/html5lib_parser.py +182 -172
- webscout/scout/parsers/html_parser.py +238 -236
- webscout/scout/parsers/lxml_parser.py +203 -178
- webscout/scout/utils.py +38 -37
- webscout/search/__init__.py +47 -0
- webscout/search/base.py +201 -0
- webscout/search/bing_main.py +45 -0
- webscout/search/brave_main.py +92 -0
- webscout/search/duckduckgo_main.py +57 -0
- webscout/search/engines/__init__.py +127 -0
- webscout/search/engines/bing/__init__.py +15 -0
- webscout/search/engines/bing/base.py +35 -0
- webscout/search/engines/bing/images.py +114 -0
- webscout/search/engines/bing/news.py +96 -0
- webscout/search/engines/bing/suggestions.py +36 -0
- webscout/search/engines/bing/text.py +109 -0
- webscout/search/engines/brave/__init__.py +19 -0
- webscout/search/engines/brave/base.py +47 -0
- webscout/search/engines/brave/images.py +213 -0
- webscout/search/engines/brave/news.py +353 -0
- webscout/search/engines/brave/suggestions.py +318 -0
- webscout/search/engines/brave/text.py +167 -0
- webscout/search/engines/brave/videos.py +364 -0
- webscout/search/engines/duckduckgo/__init__.py +25 -0
- webscout/search/engines/duckduckgo/answers.py +80 -0
- webscout/search/engines/duckduckgo/base.py +189 -0
- webscout/search/engines/duckduckgo/images.py +100 -0
- webscout/search/engines/duckduckgo/maps.py +183 -0
- webscout/search/engines/duckduckgo/news.py +70 -0
- webscout/search/engines/duckduckgo/suggestions.py +22 -0
- webscout/search/engines/duckduckgo/text.py +221 -0
- webscout/search/engines/duckduckgo/translate.py +48 -0
- webscout/search/engines/duckduckgo/videos.py +80 -0
- webscout/search/engines/duckduckgo/weather.py +84 -0
- webscout/search/engines/mojeek.py +61 -0
- webscout/search/engines/wikipedia.py +77 -0
- webscout/search/engines/yahoo/__init__.py +41 -0
- webscout/search/engines/yahoo/answers.py +19 -0
- webscout/search/engines/yahoo/base.py +34 -0
- webscout/search/engines/yahoo/images.py +323 -0
- webscout/search/engines/yahoo/maps.py +19 -0
- webscout/search/engines/yahoo/news.py +258 -0
- webscout/search/engines/yahoo/suggestions.py +140 -0
- webscout/search/engines/yahoo/text.py +273 -0
- webscout/search/engines/yahoo/translate.py +19 -0
- webscout/search/engines/yahoo/videos.py +302 -0
- webscout/search/engines/yahoo/weather.py +220 -0
- webscout/search/engines/yandex.py +67 -0
- webscout/search/engines/yep/__init__.py +13 -0
- webscout/search/engines/yep/base.py +34 -0
- webscout/search/engines/yep/images.py +101 -0
- webscout/search/engines/yep/suggestions.py +38 -0
- webscout/search/engines/yep/text.py +99 -0
- webscout/search/http_client.py +172 -0
- webscout/search/results.py +141 -0
- webscout/search/yahoo_main.py +57 -0
- webscout/search/yep_main.py +48 -0
- webscout/server/__init__.py +48 -0
- webscout/server/config.py +78 -0
- webscout/server/exceptions.py +69 -0
- webscout/server/providers.py +286 -0
- webscout/server/request_models.py +131 -0
- webscout/server/request_processing.py +404 -0
- webscout/server/routes.py +642 -0
- webscout/server/server.py +351 -0
- webscout/server/ui_templates.py +1171 -0
- webscout/swiftcli/__init__.py +79 -95
- webscout/swiftcli/core/__init__.py +7 -7
- webscout/swiftcli/core/cli.py +574 -297
- webscout/swiftcli/core/context.py +98 -104
- webscout/swiftcli/core/group.py +268 -241
- webscout/swiftcli/decorators/__init__.py +28 -28
- webscout/swiftcli/decorators/command.py +243 -221
- webscout/swiftcli/decorators/options.py +247 -220
- webscout/swiftcli/decorators/output.py +392 -252
- webscout/swiftcli/exceptions.py +21 -21
- webscout/swiftcli/plugins/__init__.py +9 -9
- webscout/swiftcli/plugins/base.py +134 -135
- webscout/swiftcli/plugins/manager.py +269 -269
- webscout/swiftcli/utils/__init__.py +58 -59
- webscout/swiftcli/utils/formatting.py +251 -252
- webscout/swiftcli/utils/parsing.py +368 -267
- webscout/update_checker.py +280 -136
- webscout/utils.py +28 -14
- webscout/version.py +2 -1
- webscout/version.py.bak +3 -0
- webscout/zeroart/__init__.py +218 -135
- webscout/zeroart/base.py +70 -66
- webscout/zeroart/effects.py +155 -101
- webscout/zeroart/fonts.py +1799 -1239
- webscout-2026.1.19.dist-info/METADATA +638 -0
- webscout-2026.1.19.dist-info/RECORD +312 -0
- {webscout-8.2.9.dist-info → webscout-2026.1.19.dist-info}/WHEEL +1 -1
- {webscout-8.2.9.dist-info → webscout-2026.1.19.dist-info}/entry_points.txt +1 -1
- webscout/DWEBS.py +0 -520
- webscout/Extra/Act.md +0 -309
- webscout/Extra/GitToolkit/gitapi/README.md +0 -110
- webscout/Extra/autocoder/__init__.py +0 -9
- webscout/Extra/autocoder/autocoder.py +0 -1105
- webscout/Extra/autocoder/autocoder_utiles.py +0 -332
- webscout/Extra/gguf.md +0 -430
- webscout/Extra/weather.md +0 -281
- webscout/Litlogger/README.md +0 -10
- webscout/Litlogger/__init__.py +0 -15
- webscout/Litlogger/formats.py +0 -4
- webscout/Litlogger/handlers.py +0 -103
- webscout/Litlogger/levels.py +0 -13
- webscout/Litlogger/logger.py +0 -92
- webscout/Provider/AI21.py +0 -177
- webscout/Provider/AISEARCH/DeepFind.py +0 -254
- webscout/Provider/AISEARCH/felo_search.py +0 -202
- webscout/Provider/AISEARCH/genspark_search.py +0 -324
- webscout/Provider/AISEARCH/hika_search.py +0 -186
- webscout/Provider/AISEARCH/scira_search.py +0 -298
- webscout/Provider/Aitopia.py +0 -316
- webscout/Provider/AllenAI.py +0 -440
- webscout/Provider/Blackboxai.py +0 -791
- webscout/Provider/ChatGPTClone.py +0 -237
- webscout/Provider/ChatGPTGratis.py +0 -194
- webscout/Provider/Cloudflare.py +0 -324
- webscout/Provider/ExaChat.py +0 -358
- webscout/Provider/Flowith.py +0 -217
- webscout/Provider/FreeGemini.py +0 -250
- webscout/Provider/Glider.py +0 -225
- webscout/Provider/HF_space/__init__.py +0 -0
- webscout/Provider/HF_space/qwen_qwen2.py +0 -206
- webscout/Provider/HuggingFaceChat.py +0 -469
- webscout/Provider/Hunyuan.py +0 -283
- webscout/Provider/LambdaChat.py +0 -411
- webscout/Provider/Llama3.py +0 -259
- webscout/Provider/Nemotron.py +0 -218
- webscout/Provider/OLLAMA.py +0 -396
- webscout/Provider/OPENAI/BLACKBOXAI.py +0 -766
- webscout/Provider/OPENAI/Cloudflare.py +0 -378
- webscout/Provider/OPENAI/FreeGemini.py +0 -283
- webscout/Provider/OPENAI/NEMOTRON.py +0 -232
- webscout/Provider/OPENAI/Qwen3.py +0 -283
- webscout/Provider/OPENAI/api.py +0 -969
- webscout/Provider/OPENAI/c4ai.py +0 -373
- webscout/Provider/OPENAI/chatgptclone.py +0 -494
- webscout/Provider/OPENAI/copilot.py +0 -242
- webscout/Provider/OPENAI/flowith.py +0 -162
- webscout/Provider/OPENAI/freeaichat.py +0 -359
- webscout/Provider/OPENAI/mcpcore.py +0 -389
- webscout/Provider/OPENAI/multichat.py +0 -376
- webscout/Provider/OPENAI/opkfc.py +0 -496
- webscout/Provider/OPENAI/scirachat.py +0 -477
- webscout/Provider/OPENAI/standardinput.py +0 -433
- webscout/Provider/OPENAI/typegpt.py +0 -364
- webscout/Provider/OPENAI/uncovrAI.py +0 -463
- webscout/Provider/OPENAI/venice.py +0 -431
- webscout/Provider/OPENAI/yep.py +0 -382
- webscout/Provider/OpenGPT.py +0 -209
- webscout/Provider/Perplexitylabs.py +0 -415
- webscout/Provider/Reka.py +0 -214
- webscout/Provider/StandardInput.py +0 -290
- webscout/Provider/TTI/aiarta.py +0 -365
- webscout/Provider/TTI/artbit.py +0 -0
- webscout/Provider/TTI/fastflux.py +0 -200
- webscout/Provider/TTI/piclumen.py +0 -203
- webscout/Provider/TTI/pixelmuse.py +0 -225
- webscout/Provider/TTS/gesserit.py +0 -128
- webscout/Provider/TTS/sthir.py +0 -94
- webscout/Provider/TeachAnything.py +0 -229
- webscout/Provider/UNFINISHED/puterjs.py +0 -635
- webscout/Provider/UNFINISHED/test_lmarena.py +0 -119
- webscout/Provider/Venice.py +0 -258
- webscout/Provider/VercelAI.py +0 -253
- webscout/Provider/Writecream.py +0 -246
- webscout/Provider/WritingMate.py +0 -269
- webscout/Provider/asksteve.py +0 -220
- webscout/Provider/chatglm.py +0 -215
- webscout/Provider/copilot.py +0 -425
- webscout/Provider/freeaichat.py +0 -285
- webscout/Provider/granite.py +0 -235
- webscout/Provider/hermes.py +0 -266
- webscout/Provider/koala.py +0 -170
- webscout/Provider/lmarena.py +0 -198
- webscout/Provider/multichat.py +0 -364
- webscout/Provider/scira_chat.py +0 -299
- webscout/Provider/scnet.py +0 -243
- webscout/Provider/talkai.py +0 -194
- webscout/Provider/typegpt.py +0 -289
- webscout/Provider/uncovr.py +0 -368
- webscout/Provider/yep.py +0 -389
- webscout/litagent/Readme.md +0 -276
- webscout/litprinter/__init__.py +0 -59
- webscout/swiftcli/Readme.md +0 -323
- webscout/tempid.py +0 -128
- webscout/webscout_search.py +0 -1184
- webscout/webscout_search_async.py +0 -654
- webscout/yep_search.py +0 -347
- webscout/zeroart/README.md +0 -89
- webscout-8.2.9.dist-info/METADATA +0 -1033
- webscout-8.2.9.dist-info/RECORD +0 -289
- {webscout-8.2.9.dist-info → webscout-2026.1.19.dist-info}/licenses/LICENSE.md +0 -0
- {webscout-8.2.9.dist-info → webscout-2026.1.19.dist-info}/top_level.txt +0 -0
webscout/litagent/agent.py
CHANGED
|
@@ -1,455 +1,492 @@
|
|
|
1
|
-
"""
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
""
|
|
217
|
-
return
|
|
218
|
-
|
|
219
|
-
def
|
|
220
|
-
"""
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
agent
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
def
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
def
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
"""
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
"
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
self.
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
self.
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
self.
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
agent
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
1
|
+
"""
|
|
2
|
+
LitAgent: Advanced User Agent Generation and Management System.
|
|
3
|
+
|
|
4
|
+
This module provides a robust and flexible system for generating realistic,
|
|
5
|
+
modern user agents and managing browser fingerprints for web scraping and
|
|
6
|
+
automation purposes.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import random
|
|
11
|
+
import threading
|
|
12
|
+
from datetime import datetime
|
|
13
|
+
from typing import Any, Dict, List, Optional, Set, Tuple, cast
|
|
14
|
+
|
|
15
|
+
from webscout.litagent.constants import BROWSERS, DEVICES, FINGERPRINTS, OS_VERSIONS
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class LitAgent:
|
|
19
|
+
"""
|
|
20
|
+
A powerful and modern user agent generator for web scraping and automation.
|
|
21
|
+
|
|
22
|
+
LitAgent provides tools for generating randomized but realistic user agent strings,
|
|
23
|
+
managing proxy pools, rotating IPs, and simulating browser fingerprints to avoid
|
|
24
|
+
detection and rate limiting.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
_blacklist: Set[str]
|
|
28
|
+
_whitelist: Set[str]
|
|
29
|
+
_agents: List[str]
|
|
30
|
+
_ip_pool: List[str]
|
|
31
|
+
_ip_index: int
|
|
32
|
+
_proxy_pool: List[str]
|
|
33
|
+
_proxy_index: int
|
|
34
|
+
_history: List[str]
|
|
35
|
+
_refresh_timer: Optional[threading.Timer]
|
|
36
|
+
_stats: Dict[str, Any]
|
|
37
|
+
thread_safe: bool
|
|
38
|
+
lock: Optional[threading.RLock]
|
|
39
|
+
|
|
40
|
+
@property
|
|
41
|
+
def agents(self) -> List[str]:
|
|
42
|
+
"""Returns the current pool of user agents."""
|
|
43
|
+
return self._agents
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def ip_pool(self) -> List[str]:
|
|
47
|
+
"""Returns the current simulated IP pool."""
|
|
48
|
+
return self._ip_pool
|
|
49
|
+
|
|
50
|
+
def __init__(self, thread_safe: bool = False):
|
|
51
|
+
"""
|
|
52
|
+
Initialize the LitAgent instance.
|
|
53
|
+
|
|
54
|
+
Args:
|
|
55
|
+
thread_safe (bool): If True, use RLock for thread-safe operations.
|
|
56
|
+
"""
|
|
57
|
+
self.thread_safe = thread_safe
|
|
58
|
+
self.lock = threading.RLock() if thread_safe else None
|
|
59
|
+
|
|
60
|
+
# Internal state
|
|
61
|
+
self._blacklist: Set[str] = set()
|
|
62
|
+
self._whitelist: Set[str] = set()
|
|
63
|
+
self._agents: List[str] = self._generate_agents(100)
|
|
64
|
+
self._ip_pool: List[str] = self._generate_ip_pool(20)
|
|
65
|
+
self._ip_index: int = 0
|
|
66
|
+
self._proxy_pool: List[str] = []
|
|
67
|
+
self._proxy_index: int = 0
|
|
68
|
+
self._history: List[str] = []
|
|
69
|
+
self._refresh_timer: Optional[threading.Timer] = None
|
|
70
|
+
|
|
71
|
+
# Usage statistics
|
|
72
|
+
self._stats = {
|
|
73
|
+
"total_generated": 100,
|
|
74
|
+
"requests_served": 0,
|
|
75
|
+
"browser_usage": {browser: 0 for browser in BROWSERS.keys()},
|
|
76
|
+
"device_usage": {device: 0 for device in DEVICES.keys()},
|
|
77
|
+
"start_time": datetime.now().isoformat()
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
def _generate_agents(self, count: int) -> List[str]:
|
|
81
|
+
"""
|
|
82
|
+
Generate a list of realistic user agent strings.
|
|
83
|
+
|
|
84
|
+
Args:
|
|
85
|
+
count (int): Number of agents to generate.
|
|
86
|
+
|
|
87
|
+
Returns:
|
|
88
|
+
List[str]: A list of generated user agent strings.
|
|
89
|
+
"""
|
|
90
|
+
agents: List[str] = []
|
|
91
|
+
for _ in range(count):
|
|
92
|
+
agent = ""
|
|
93
|
+
browser: str = random.choice(list(BROWSERS.keys()))
|
|
94
|
+
version_range: Tuple[int, int] = BROWSERS.get(browser, (100, 130))
|
|
95
|
+
version: int = random.randint(*version_range)
|
|
96
|
+
|
|
97
|
+
if browser in ['chrome', 'firefox', 'edge', 'opera', 'brave', 'vivaldi']:
|
|
98
|
+
os_type: str = random.choice(['windows', 'mac', 'linux'])
|
|
99
|
+
os_ver: str = random.choice(OS_VERSIONS.get(os_type, ["10.0"]))
|
|
100
|
+
|
|
101
|
+
if os_type == 'windows':
|
|
102
|
+
platform = f"Windows NT {os_ver}; Win64; x64"
|
|
103
|
+
elif os_type == 'mac':
|
|
104
|
+
platform = f"Macintosh; Intel Mac OS X {os_ver}"
|
|
105
|
+
else:
|
|
106
|
+
platform = f"X11; Linux {os_ver}"
|
|
107
|
+
|
|
108
|
+
agent = f"Mozilla/5.0 ({platform}) AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
109
|
+
|
|
110
|
+
if browser == 'chrome':
|
|
111
|
+
agent += f"Chrome/{version}.0.0.0 Safari/537.36"
|
|
112
|
+
elif browser == 'firefox':
|
|
113
|
+
agent += f"Firefox/{version}.0"
|
|
114
|
+
elif browser == 'edge':
|
|
115
|
+
agent += f"Chrome/{version}.0.0.0 Safari/537.36 Edg/{version}.0.0.0"
|
|
116
|
+
elif browser == 'opera':
|
|
117
|
+
agent += f"Chrome/{version}.0.0.0 Safari/537.36 OPR/{version}.0.0.0"
|
|
118
|
+
elif browser == 'brave':
|
|
119
|
+
agent += f"Chrome/{version}.0.0.0 Safari/537.36 Brave/{version}.0.0.0"
|
|
120
|
+
elif browser == 'vivaldi':
|
|
121
|
+
vivaldi_build: int = random.randint(1000, 9999)
|
|
122
|
+
agent += f"Chrome/{version}.0.0.0 Safari/537.36 Vivaldi/{version}.0.{vivaldi_build}"
|
|
123
|
+
|
|
124
|
+
elif browser == 'safari':
|
|
125
|
+
device: str = random.choice(['mac', 'ios'])
|
|
126
|
+
if device == 'mac':
|
|
127
|
+
ver: str = random.choice(OS_VERSIONS.get('mac', ["10_15_7"]))
|
|
128
|
+
agent = f"Mozilla/5.0 (Macintosh; Intel Mac OS X {ver}) "
|
|
129
|
+
else:
|
|
130
|
+
ver = random.choice(OS_VERSIONS.get('ios', ["17_0"]))
|
|
131
|
+
device_name: str = random.choice(['iPhone', 'iPad'])
|
|
132
|
+
agent = f"Mozilla/5.0 ({device_name}; CPU OS {ver} like Mac OS X) "
|
|
133
|
+
|
|
134
|
+
agent += f"AppleWebKit/{version}.1.15 (KHTML, like Gecko) Version/{version//10}.0 Safari/{version}.1.15"
|
|
135
|
+
|
|
136
|
+
agents.append(agent)
|
|
137
|
+
|
|
138
|
+
# Ensure uniqueness and respects current whitelist/blacklist
|
|
139
|
+
unique_agents = list(set(agents))
|
|
140
|
+
return [a for a in unique_agents if a not in self._blacklist]
|
|
141
|
+
|
|
142
|
+
def _update_stats(self, browser_type: Optional[str] = None, device_type: Optional[str] = None) -> None:
|
|
143
|
+
"""Update internal usage statistics."""
|
|
144
|
+
def update() -> None:
|
|
145
|
+
self._stats["requests_served"] += 1
|
|
146
|
+
if browser_type:
|
|
147
|
+
self._stats["browser_usage"][browser_type] = self._stats["browser_usage"].get(browser_type, 0) + 1
|
|
148
|
+
if device_type:
|
|
149
|
+
self._stats["device_usage"][device_type] = self._stats["device_usage"].get(device_type, 0) + 1
|
|
150
|
+
|
|
151
|
+
if self.thread_safe and self.lock:
|
|
152
|
+
with self.lock:
|
|
153
|
+
update()
|
|
154
|
+
else:
|
|
155
|
+
update()
|
|
156
|
+
|
|
157
|
+
def _add_to_history(self, agent: str) -> None:
|
|
158
|
+
"""Add a generated agent to history."""
|
|
159
|
+
def add() -> None:
|
|
160
|
+
self._history.append(agent)
|
|
161
|
+
if len(self._history) > 50:
|
|
162
|
+
self._history.pop(0)
|
|
163
|
+
|
|
164
|
+
if self.thread_safe and self.lock:
|
|
165
|
+
with self.lock:
|
|
166
|
+
add()
|
|
167
|
+
else:
|
|
168
|
+
add()
|
|
169
|
+
|
|
170
|
+
def random(self) -> str:
|
|
171
|
+
"""
|
|
172
|
+
Get a random user agent from the pool.
|
|
173
|
+
|
|
174
|
+
Returns:
|
|
175
|
+
str: A random user agent string.
|
|
176
|
+
"""
|
|
177
|
+
pool = list(self._whitelist) if self._whitelist else self._agents
|
|
178
|
+
if not pool:
|
|
179
|
+
# Fallback if somehow empty
|
|
180
|
+
pool = self._generate_agents(1)
|
|
181
|
+
|
|
182
|
+
agent = random.choice(pool)
|
|
183
|
+
self._update_stats()
|
|
184
|
+
self._add_to_history(agent)
|
|
185
|
+
return agent
|
|
186
|
+
|
|
187
|
+
def browser(self, name: str) -> str:
|
|
188
|
+
"""
|
|
189
|
+
Get a user agent for a specific browser.
|
|
190
|
+
|
|
191
|
+
Args:
|
|
192
|
+
name (str): Browser name (e.g., 'chrome', 'firefox').
|
|
193
|
+
|
|
194
|
+
Returns:
|
|
195
|
+
str: A browser-specific user agent string.
|
|
196
|
+
"""
|
|
197
|
+
name = name.lower()
|
|
198
|
+
if name not in BROWSERS:
|
|
199
|
+
return self.random()
|
|
200
|
+
|
|
201
|
+
matching_agents = [a for a in self._agents if name in a.lower()]
|
|
202
|
+
if not matching_agents:
|
|
203
|
+
# Generate one on the fly if needed
|
|
204
|
+
matched = self.custom(browser=name)
|
|
205
|
+
else:
|
|
206
|
+
matched = random.choice(matching_agents)
|
|
207
|
+
|
|
208
|
+
self._update_stats(browser_type=name)
|
|
209
|
+
self._add_to_history(matched)
|
|
210
|
+
return matched
|
|
211
|
+
|
|
212
|
+
def mobile(self) -> str:
|
|
213
|
+
"""Returns a mobile device user agent."""
|
|
214
|
+
matching = [a for a in self._agents if any(d in a for d in DEVICES.get('mobile', []))]
|
|
215
|
+
agent = random.choice(matching) if matching else self.custom(device_type="mobile")
|
|
216
|
+
self._update_stats(device_type="mobile")
|
|
217
|
+
return agent
|
|
218
|
+
|
|
219
|
+
def desktop(self) -> str:
|
|
220
|
+
"""Returns a desktop device user agent."""
|
|
221
|
+
matching = [a for a in self._agents if any(d in a for d in ["Windows", "Macintosh", "X11"])]
|
|
222
|
+
agent = random.choice(matching) if matching else self.custom(device_type="desktop")
|
|
223
|
+
self._update_stats(device_type="desktop")
|
|
224
|
+
return agent
|
|
225
|
+
|
|
226
|
+
def tablet(self) -> str:
|
|
227
|
+
"""Returns a tablet device user agent."""
|
|
228
|
+
matching = [a for a in self._agents if 'iPad' in a or ('Android' in a and 'Mobile' not in a)]
|
|
229
|
+
agent = random.choice(matching) if matching else self.custom(device_type="tablet")
|
|
230
|
+
self._update_stats(device_type="tablet")
|
|
231
|
+
return agent
|
|
232
|
+
|
|
233
|
+
def chrome(self) -> str: return self.browser('chrome')
|
|
234
|
+
def firefox(self) -> str: return self.browser('firefox')
|
|
235
|
+
def safari(self) -> str: return self.browser('safari')
|
|
236
|
+
def edge(self) -> str: return self.browser('edge')
|
|
237
|
+
def opera(self) -> str: return self.browser('opera')
|
|
238
|
+
def brave(self) -> str: return self.browser('brave')
|
|
239
|
+
def vivaldi(self) -> str: return self.browser('vivaldi')
|
|
240
|
+
|
|
241
|
+
def windows(self) -> str: return self.custom(os='windows')
|
|
242
|
+
def macos(self) -> str: return self.custom(os='mac')
|
|
243
|
+
def linux(self) -> str: return self.custom(os='linux')
|
|
244
|
+
def android(self) -> str: return self.custom(os='android')
|
|
245
|
+
def ios(self) -> str: return self.custom(os='ios')
|
|
246
|
+
|
|
247
|
+
def custom(self, browser: Optional[str] = None, version: Optional[str] = None,
|
|
248
|
+
os: Optional[str] = None, os_version: Optional[str] = None,
|
|
249
|
+
device_type: Optional[str] = None) -> str:
|
|
250
|
+
"""
|
|
251
|
+
Generate a customized user agent string.
|
|
252
|
+
|
|
253
|
+
Args:
|
|
254
|
+
browser (str, optional): Browser name.
|
|
255
|
+
version (str, optional): Browser version.
|
|
256
|
+
os (str, optional): Operating system.
|
|
257
|
+
os_version (str, optional): OS version.
|
|
258
|
+
device_type (str, optional): Device type ('desktop', 'mobile', 'tablet').
|
|
259
|
+
|
|
260
|
+
Returns:
|
|
261
|
+
str: The customized user agent string.
|
|
262
|
+
"""
|
|
263
|
+
browser = browser.lower() if browser else 'chrome'
|
|
264
|
+
v_range = BROWSERS.get(browser, (100, 130))
|
|
265
|
+
v_num = int(version.split('.')[0]) if version else random.randint(*v_range)
|
|
266
|
+
|
|
267
|
+
os = os.lower() if os else random.choice(['windows', 'mac', 'linux'])
|
|
268
|
+
os_ver = os_version or random.choice(OS_VERSIONS.get(os, ["10.0"]))
|
|
269
|
+
device_type = (device_type or 'desktop').lower()
|
|
270
|
+
|
|
271
|
+
if os == 'windows':
|
|
272
|
+
platform = f"Windows NT {os_ver}; Win64; x64"
|
|
273
|
+
elif os == 'mac':
|
|
274
|
+
platform = f"Macintosh; Intel Mac OS X {os_ver}"
|
|
275
|
+
elif os == 'linux':
|
|
276
|
+
platform = f"X11; Linux {os_ver}"
|
|
277
|
+
elif os == 'android':
|
|
278
|
+
platform = f"Linux; Android {os_ver}; {random.choice(DEVICES.get('mobile', ['Samsung Galaxy']))}"
|
|
279
|
+
elif os == 'ios':
|
|
280
|
+
dev = 'iPhone' if device_type == 'mobile' else 'iPad'
|
|
281
|
+
platform = f"{dev}; CPU OS {os_ver} like Mac OS X"
|
|
282
|
+
else:
|
|
283
|
+
platform = "Windows NT 10.0; Win64; x64"
|
|
284
|
+
|
|
285
|
+
agent = f"Mozilla/5.0 ({platform}) AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
286
|
+
|
|
287
|
+
if browser == 'chrome':
|
|
288
|
+
agent += f"Chrome/{v_num}.0.0.0 Safari/537.36"
|
|
289
|
+
elif browser == 'firefox':
|
|
290
|
+
agent += f"Firefox/{v_num}.0"
|
|
291
|
+
elif browser == 'safari':
|
|
292
|
+
agent += f"Version/{v_num//10}.0 Safari/{v_num}.1.15"
|
|
293
|
+
elif browser == 'edge':
|
|
294
|
+
agent += f"Chrome/{v_num}.0.0.0 Safari/537.36 Edg/{v_num}.0.0.0"
|
|
295
|
+
else:
|
|
296
|
+
agent += f"Chrome/{v_num}.0.0.0 Safari/537.36"
|
|
297
|
+
|
|
298
|
+
self._update_stats(browser_type=browser, device_type=device_type)
|
|
299
|
+
return agent
|
|
300
|
+
|
|
301
|
+
def generate_fingerprint(self, browser: Optional[str] = None) -> Dict[str, str]:
|
|
302
|
+
"""
|
|
303
|
+
Generate a complete browser fingerprint.
|
|
304
|
+
|
|
305
|
+
Args:
|
|
306
|
+
browser (str, optional): Requested browser type.
|
|
307
|
+
|
|
308
|
+
Returns:
|
|
309
|
+
Dict[str, str]: A dictionary containing headers and fingerprint data.
|
|
310
|
+
"""
|
|
311
|
+
ua = self.browser(browser) if browser else self.random()
|
|
312
|
+
ip = self.rotate_ip()
|
|
313
|
+
|
|
314
|
+
sec_ch_ua = ""
|
|
315
|
+
sec_ch_ua_dict = cast(Dict[str, str], FINGERPRINTS.get("sec_ch_ua", {}))
|
|
316
|
+
for b_name in sec_ch_ua_dict:
|
|
317
|
+
if b_name in ua.lower():
|
|
318
|
+
v = random.randint(*BROWSERS.get(b_name, (100, 120)))
|
|
319
|
+
sec_ch_ua = sec_ch_ua_dict[b_name].format(v, v)
|
|
320
|
+
break
|
|
321
|
+
|
|
322
|
+
accept_language_list = cast(List[str], FINGERPRINTS.get("accept_language", ["en-US,en;q=0.9"]))
|
|
323
|
+
accept_list = cast(List[str], FINGERPRINTS.get("accept", ["*/*"]))
|
|
324
|
+
platforms_list = cast(List[str], FINGERPRINTS.get("platforms", ["Windows"]))
|
|
325
|
+
|
|
326
|
+
return {
|
|
327
|
+
"user_agent": ua,
|
|
328
|
+
"accept_language": random.choice(accept_language_list),
|
|
329
|
+
"accept": random.choice(accept_list),
|
|
330
|
+
"sec_ch_ua": sec_ch_ua,
|
|
331
|
+
"platform": random.choice(platforms_list),
|
|
332
|
+
"x-forwarded-for": ip,
|
|
333
|
+
"x-real-ip": ip,
|
|
334
|
+
"x-client-ip": ip,
|
|
335
|
+
"forwarded": f"for={ip};proto=https",
|
|
336
|
+
"x-request-id": self.random_id(8),
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
def smart_tv(self) -> str:
|
|
340
|
+
"""Generate a Smart TV user agent."""
|
|
341
|
+
tv: str = random.choice(DEVICES.get('tv', ["Samsung Smart TV"]))
|
|
342
|
+
if 'Samsung' in tv:
|
|
343
|
+
agent: str = f"Mozilla/5.0 (SMART-TV; SAMSUNG; {tv}; Tizen 6.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36"
|
|
344
|
+
elif 'LG' in tv:
|
|
345
|
+
agent = f"Mozilla/5.0 (Web0S; {tv}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
|
|
346
|
+
else:
|
|
347
|
+
agent = f"Mozilla/5.0 (Linux; {tv}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36"
|
|
348
|
+
|
|
349
|
+
self._update_stats(device_type="tv")
|
|
350
|
+
return agent
|
|
351
|
+
|
|
352
|
+
def gaming(self) -> str:
|
|
353
|
+
"""Generate a gaming console user agent."""
|
|
354
|
+
console: str = random.choice(DEVICES.get('console', ["PlayStation 5"]))
|
|
355
|
+
if 'PlayStation' in console:
|
|
356
|
+
agent: str = f"Mozilla/5.0 ({console}) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Safari/605.1.15"
|
|
357
|
+
elif 'Xbox' in console:
|
|
358
|
+
agent = f"Mozilla/5.0 (Windows NT 10.0; Win64; x64; Xbox; {console}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 Edge/110.0.1587.41"
|
|
359
|
+
else:
|
|
360
|
+
agent = self.random()
|
|
361
|
+
|
|
362
|
+
self._update_stats(device_type="console")
|
|
363
|
+
return agent
|
|
364
|
+
|
|
365
|
+
def wearable(self) -> str:
|
|
366
|
+
"""Generate a wearable device user agent."""
|
|
367
|
+
dev: str = random.choice(DEVICES.get('wearable', ["Apple Watch"]))
|
|
368
|
+
if 'Apple Watch' in dev:
|
|
369
|
+
agent: str = "Mozilla/5.0 (AppleWatch; CPU WatchOS like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/10.0 Mobile/15E148 Safari/605.1"
|
|
370
|
+
else:
|
|
371
|
+
agent = f"Mozilla/5.0 (Linux; {dev}) AppleWebKit/537.36 (KHTML, like Gecko)"
|
|
372
|
+
|
|
373
|
+
self._update_stats(device_type="wearable")
|
|
374
|
+
return agent
|
|
375
|
+
|
|
376
|
+
def refresh(self) -> None:
|
|
377
|
+
"""Refresh the internal agents pool with new values."""
|
|
378
|
+
def do_refresh() -> None:
|
|
379
|
+
self._agents = self._generate_agents(100)
|
|
380
|
+
self._stats["total_generated"] += 100
|
|
381
|
+
|
|
382
|
+
if self.thread_safe and self.lock:
|
|
383
|
+
with self.lock:
|
|
384
|
+
do_refresh()
|
|
385
|
+
else:
|
|
386
|
+
do_refresh()
|
|
387
|
+
|
|
388
|
+
def auto_refresh(self, interval_minutes: int = 30) -> None:
|
|
389
|
+
"""
|
|
390
|
+
Schedule automatic background refreshing of the agents pool.
|
|
391
|
+
|
|
392
|
+
Args:
|
|
393
|
+
interval_minutes (int): Time between refreshes in minutes.
|
|
394
|
+
"""
|
|
395
|
+
if self._refresh_timer:
|
|
396
|
+
self._refresh_timer.cancel()
|
|
397
|
+
|
|
398
|
+
def _task() -> None:
|
|
399
|
+
self.refresh()
|
|
400
|
+
self._refresh_timer = threading.Timer(interval_minutes * 60, _task)
|
|
401
|
+
self._refresh_timer.daemon = True
|
|
402
|
+
self._refresh_timer.start()
|
|
403
|
+
|
|
404
|
+
self._refresh_timer = threading.Timer(interval_minutes * 60, _task)
|
|
405
|
+
self._refresh_timer.daemon = True
|
|
406
|
+
self._refresh_timer.start()
|
|
407
|
+
|
|
408
|
+
def rotate_ip(self) -> str:
|
|
409
|
+
"""Rotate through the IP pool and returns the next IP address."""
|
|
410
|
+
def rot() -> str:
|
|
411
|
+
ip: str = self._ip_pool[self._ip_index]
|
|
412
|
+
self._ip_index = (self._ip_index + 1) % len(self._ip_pool)
|
|
413
|
+
return ip
|
|
414
|
+
|
|
415
|
+
if self.thread_safe and self.lock:
|
|
416
|
+
with self.lock:
|
|
417
|
+
return rot()
|
|
418
|
+
return rot()
|
|
419
|
+
|
|
420
|
+
def set_proxy_pool(self, proxies: List[str]) -> None:
|
|
421
|
+
"""Set a pool of proxies for rotation."""
|
|
422
|
+
self._proxy_pool = proxies
|
|
423
|
+
self._proxy_index = 0
|
|
424
|
+
|
|
425
|
+
def rotate_proxy(self) -> Optional[str]:
|
|
426
|
+
"""Rotate through and return the next proxy from the pool."""
|
|
427
|
+
if not self._proxy_pool:
|
|
428
|
+
return None
|
|
429
|
+
|
|
430
|
+
def rot() -> str:
|
|
431
|
+
proxy: str = self._proxy_pool[self._proxy_index]
|
|
432
|
+
self._proxy_index = (self._proxy_index + 1) % len(self._proxy_pool)
|
|
433
|
+
return proxy
|
|
434
|
+
|
|
435
|
+
if self.thread_safe and self.lock:
|
|
436
|
+
with self.lock:
|
|
437
|
+
return rot()
|
|
438
|
+
return rot()
|
|
439
|
+
|
|
440
|
+
def add_to_blacklist(self, agent: str) -> None:
|
|
441
|
+
"""Blacklist a specific user agent string."""
|
|
442
|
+
self._blacklist.add(agent)
|
|
443
|
+
if agent in self._agents:
|
|
444
|
+
self._agents.remove(agent)
|
|
445
|
+
|
|
446
|
+
def add_to_whitelist(self, agent: str) -> None:
|
|
447
|
+
"""Limit results to only those in the whitelist."""
|
|
448
|
+
self._whitelist.add(agent)
|
|
449
|
+
|
|
450
|
+
def get_stats(self) -> Dict[str, Any]:
|
|
451
|
+
"""Get usage statistics."""
|
|
452
|
+
stats = self._stats.copy()
|
|
453
|
+
usage = stats["browser_usage"]
|
|
454
|
+
stats["top_browser"] = max(usage.items(), key=lambda x: x[1])[0] if any(usage.values()) else None
|
|
455
|
+
stats["avoidance_rate"] = min(99.9, 90 + (stats["total_generated"] / 1000))
|
|
456
|
+
return stats
|
|
457
|
+
|
|
458
|
+
def export_stats(self, filename: str) -> bool:
|
|
459
|
+
"""Export stats to a JSON file."""
|
|
460
|
+
try:
|
|
461
|
+
with open(filename, 'w', encoding='utf-8') as f:
|
|
462
|
+
json.dump(self.get_stats(), f, indent=4)
|
|
463
|
+
return True
|
|
464
|
+
except Exception:
|
|
465
|
+
return False
|
|
466
|
+
|
|
467
|
+
def validate_agent(self, agent: str) -> bool:
|
|
468
|
+
"""Perform basic validation on a user agent string."""
|
|
469
|
+
return isinstance(agent, str) and agent.startswith("Mozilla/5.0")
|
|
470
|
+
|
|
471
|
+
def random_id(self, length: int = 16) -> str:
|
|
472
|
+
"""Generate a random hexadecimal string ID."""
|
|
473
|
+
return ''.join(random.choices('0123456789abcdef', k=length))
|
|
474
|
+
|
|
475
|
+
def _generate_ip_pool(self, count: int) -> List[str]:
|
|
476
|
+
"""Generate a pool of random IP addresses."""
|
|
477
|
+
return [".".join(str(random.randint(0, 255)) for _ in range(4)) for _ in range(count)]
|
|
478
|
+
|
|
479
|
+
def __repr__(self) -> str:
|
|
480
|
+
return f"<LitAgent(agents={len(self._agents)}, thread_safe={self.thread_safe})>"
|
|
481
|
+
|
|
482
|
+
def __str__(self) -> str:
|
|
483
|
+
return f"LitAgent Generator with {len(self._agents)} agents in pool"
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
if __name__ == "__main__":
|
|
487
|
+
# Quick test
|
|
488
|
+
agent = LitAgent()
|
|
489
|
+
print(f"Random: {agent.random()}")
|
|
490
|
+
print(f"Chrome: {agent.chrome()}")
|
|
491
|
+
print(f"iPhone: {agent.ios()}")
|
|
492
|
+
print(f"Fingerprint: {json.dumps(agent.generate_fingerprint(), indent=2)}")
|