PyPI - webscout - Versions diffs - 8.3.7__py3-none-any.whl → 2025.10.13__py3-none-any.whl - Mend

webscout 8.3.7py3-none-any.whl → 2025.10.13py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of webscout might be problematic. Click here for more details.

Files changed (306) hide show

webscout/AIauto.py +250 -250
webscout/AIbase.py +379 -379
webscout/AIutel.py +60 -60
webscout/Bard.py +1012 -1012
webscout/Bing_search.py +417 -417
webscout/DWEBS.py +529 -529
webscout/Extra/Act.md +309 -309
webscout/Extra/GitToolkit/__init__.py +10 -10
webscout/Extra/GitToolkit/gitapi/README.md +110 -110
webscout/Extra/GitToolkit/gitapi/__init__.py +11 -11
webscout/Extra/GitToolkit/gitapi/repository.py +195 -195
webscout/Extra/GitToolkit/gitapi/user.py +96 -96
webscout/Extra/GitToolkit/gitapi/utils.py +61 -61
webscout/Extra/YTToolkit/README.md +375 -375
webscout/Extra/YTToolkit/YTdownloader.py +956 -956
webscout/Extra/YTToolkit/__init__.py +2 -2
webscout/Extra/YTToolkit/transcriber.py +475 -475
webscout/Extra/YTToolkit/ytapi/README.md +44 -44
webscout/Extra/YTToolkit/ytapi/__init__.py +6 -6
webscout/Extra/YTToolkit/ytapi/channel.py +307 -307
webscout/Extra/YTToolkit/ytapi/errors.py +13 -13
webscout/Extra/YTToolkit/ytapi/extras.py +118 -118
webscout/Extra/YTToolkit/ytapi/https.py +88 -88
webscout/Extra/YTToolkit/ytapi/patterns.py +61 -61
webscout/Extra/YTToolkit/ytapi/playlist.py +58 -58
webscout/Extra/YTToolkit/ytapi/pool.py +7 -7
webscout/Extra/YTToolkit/ytapi/query.py +39 -39
webscout/Extra/YTToolkit/ytapi/stream.py +62 -62
webscout/Extra/YTToolkit/ytapi/utils.py +62 -62
webscout/Extra/YTToolkit/ytapi/video.py +232 -232
webscout/Extra/autocoder/__init__.py +9 -9
webscout/Extra/autocoder/autocoder.py +1105 -1105
webscout/Extra/autocoder/autocoder_utiles.py +332 -332
webscout/Extra/gguf.md +429 -429
webscout/Extra/gguf.py +1213 -1213
webscout/Extra/tempmail/README.md +487 -487
webscout/Extra/tempmail/__init__.py +27 -27
webscout/Extra/tempmail/async_utils.py +140 -140
webscout/Extra/tempmail/base.py +160 -160
webscout/Extra/tempmail/cli.py +186 -186
webscout/Extra/tempmail/emailnator.py +84 -84
webscout/Extra/tempmail/mail_tm.py +360 -360
webscout/Extra/tempmail/temp_mail_io.py +291 -291
webscout/Extra/weather.md +281 -281
webscout/Extra/weather.py +193 -193
webscout/Litlogger/README.md +10 -10
webscout/Litlogger/__init__.py +15 -15
webscout/Litlogger/formats.py +13 -13
webscout/Litlogger/handlers.py +121 -121
webscout/Litlogger/levels.py +13 -13
webscout/Litlogger/logger.py +134 -134
webscout/Provider/AISEARCH/Perplexity.py +332 -332
webscout/Provider/AISEARCH/README.md +279 -279
webscout/Provider/AISEARCH/__init__.py +16 -1
webscout/Provider/AISEARCH/felo_search.py +206 -206
webscout/Provider/AISEARCH/genspark_search.py +323 -323
webscout/Provider/AISEARCH/hika_search.py +185 -185
webscout/Provider/AISEARCH/iask_search.py +410 -410
webscout/Provider/AISEARCH/monica_search.py +219 -219
webscout/Provider/AISEARCH/scira_search.py +316 -316
webscout/Provider/AISEARCH/stellar_search.py +177 -177
webscout/Provider/AISEARCH/webpilotai_search.py +255 -255
webscout/Provider/Aitopia.py +314 -314
webscout/Provider/Andi.py +1 -1
webscout/Provider/Apriel.py +306 -0
webscout/Provider/ChatGPTClone.py +237 -236
webscout/Provider/ChatSandbox.py +343 -343
webscout/Provider/Cloudflare.py +324 -324
webscout/Provider/Cohere.py +208 -208
webscout/Provider/Deepinfra.py +370 -366
webscout/Provider/ExaAI.py +260 -260
webscout/Provider/ExaChat.py +308 -308
webscout/Provider/Flowith.py +221 -221
webscout/Provider/GMI.py +293 -0
webscout/Provider/Gemini.py +164 -164
webscout/Provider/GeminiProxy.py +167 -167
webscout/Provider/GithubChat.py +371 -372
webscout/Provider/Groq.py +800 -800
webscout/Provider/HeckAI.py +383 -383
webscout/Provider/Jadve.py +282 -282
webscout/Provider/K2Think.py +307 -307
webscout/Provider/Koboldai.py +205 -205
webscout/Provider/LambdaChat.py +423 -423
webscout/Provider/Nemotron.py +244 -244
webscout/Provider/Netwrck.py +248 -248
webscout/Provider/OLLAMA.py +395 -395
webscout/Provider/OPENAI/Cloudflare.py +393 -393
webscout/Provider/OPENAI/FalconH1.py +451 -451
webscout/Provider/OPENAI/FreeGemini.py +296 -296
webscout/Provider/OPENAI/K2Think.py +431 -431
webscout/Provider/OPENAI/NEMOTRON.py +240 -240
webscout/Provider/OPENAI/PI.py +427 -427
webscout/Provider/OPENAI/README.md +959 -959
webscout/Provider/OPENAI/TogetherAI.py +345 -345
webscout/Provider/OPENAI/TwoAI.py +465 -465
webscout/Provider/OPENAI/__init__.py +33 -18
webscout/Provider/OPENAI/base.py +248 -248
webscout/Provider/OPENAI/chatglm.py +528 -0
webscout/Provider/OPENAI/chatgpt.py +592 -592
webscout/Provider/OPENAI/chatgptclone.py +521 -521
webscout/Provider/OPENAI/chatsandbox.py +202 -202
webscout/Provider/OPENAI/deepinfra.py +318 -314
webscout/Provider/OPENAI/e2b.py +1665 -1665
webscout/Provider/OPENAI/exaai.py +420 -420
webscout/Provider/OPENAI/exachat.py +452 -452
webscout/Provider/OPENAI/friendli.py +232 -232
webscout/Provider/OPENAI/{refact.py → gmi.py} +324 -274
webscout/Provider/OPENAI/groq.py +364 -364
webscout/Provider/OPENAI/heckai.py +314 -314
webscout/Provider/OPENAI/llmchatco.py +337 -337
webscout/Provider/OPENAI/netwrck.py +355 -355
webscout/Provider/OPENAI/oivscode.py +290 -290
webscout/Provider/OPENAI/opkfc.py +518 -518
webscout/Provider/OPENAI/pydantic_imports.py +1 -1
webscout/Provider/OPENAI/scirachat.py +535 -535
webscout/Provider/OPENAI/sonus.py +308 -308
webscout/Provider/OPENAI/standardinput.py +442 -442
webscout/Provider/OPENAI/textpollinations.py +340 -340
webscout/Provider/OPENAI/toolbaz.py +419 -416
webscout/Provider/OPENAI/typefully.py +362 -362
webscout/Provider/OPENAI/utils.py +295 -295
webscout/Provider/OPENAI/venice.py +436 -436
webscout/Provider/OPENAI/wisecat.py +387 -387
webscout/Provider/OPENAI/writecream.py +166 -166
webscout/Provider/OPENAI/x0gpt.py +378 -378
webscout/Provider/OPENAI/yep.py +389 -389
webscout/Provider/OpenGPT.py +230 -230
webscout/Provider/Openai.py +243 -243
webscout/Provider/PI.py +405 -405
webscout/Provider/Perplexitylabs.py +430 -430
webscout/Provider/QwenLM.py +272 -272
webscout/Provider/STT/__init__.py +16 -1
webscout/Provider/Sambanova.py +257 -257
webscout/Provider/StandardInput.py +309 -309
webscout/Provider/TTI/README.md +82 -82
webscout/Provider/TTI/__init__.py +33 -18
webscout/Provider/TTI/aiarta.py +413 -413
webscout/Provider/TTI/base.py +136 -136
webscout/Provider/TTI/bing.py +243 -243
webscout/Provider/TTI/gpt1image.py +149 -149
webscout/Provider/TTI/imagen.py +196 -196
webscout/Provider/TTI/infip.py +211 -211
webscout/Provider/TTI/magicstudio.py +232 -232
webscout/Provider/TTI/monochat.py +219 -219
webscout/Provider/TTI/piclumen.py +214 -214
webscout/Provider/TTI/pixelmuse.py +232 -232
webscout/Provider/TTI/pollinations.py +232 -232
webscout/Provider/TTI/together.py +288 -288
webscout/Provider/TTI/utils.py +12 -12
webscout/Provider/TTI/venice.py +367 -367
webscout/Provider/TTS/README.md +192 -192
webscout/Provider/TTS/__init__.py +33 -18
webscout/Provider/TTS/parler.py +110 -110
webscout/Provider/TTS/streamElements.py +333 -333
webscout/Provider/TTS/utils.py +280 -280
webscout/Provider/TeachAnything.py +237 -237
webscout/Provider/TextPollinationsAI.py +310 -310
webscout/Provider/TogetherAI.py +356 -356
webscout/Provider/TwoAI.py +312 -312
webscout/Provider/TypliAI.py +311 -311
webscout/Provider/UNFINISHED/ChatHub.py +208 -208
webscout/Provider/UNFINISHED/ChutesAI.py +313 -313
webscout/Provider/UNFINISHED/GizAI.py +294 -294
webscout/Provider/UNFINISHED/Marcus.py +198 -198
webscout/Provider/UNFINISHED/Qodo.py +477 -477
webscout/Provider/UNFINISHED/VercelAIGateway.py +338 -338
webscout/Provider/UNFINISHED/XenAI.py +324 -324
webscout/Provider/UNFINISHED/Youchat.py +330 -330
webscout/Provider/UNFINISHED/liner.py +334 -0
webscout/Provider/UNFINISHED/liner_api_request.py +262 -262
webscout/Provider/UNFINISHED/puterjs.py +634 -634
webscout/Provider/UNFINISHED/samurai.py +223 -223
webscout/Provider/UNFINISHED/test_lmarena.py +119 -119
webscout/Provider/Venice.py +250 -250
webscout/Provider/VercelAI.py +256 -256
webscout/Provider/WiseCat.py +231 -231
webscout/Provider/WrDoChat.py +366 -366
webscout/Provider/__init__.py +33 -18
webscout/Provider/ai4chat.py +174 -174
webscout/Provider/akashgpt.py +331 -331
webscout/Provider/cerebras.py +446 -446
webscout/Provider/chatglm.py +394 -301
webscout/Provider/cleeai.py +211 -211
webscout/Provider/elmo.py +282 -282
webscout/Provider/geminiapi.py +208 -208
webscout/Provider/granite.py +261 -261
webscout/Provider/hermes.py +263 -263
webscout/Provider/julius.py +223 -223
webscout/Provider/learnfastai.py +309 -309
webscout/Provider/llama3mitril.py +214 -214
webscout/Provider/llmchat.py +243 -243
webscout/Provider/llmchatco.py +290 -290
webscout/Provider/meta.py +801 -801
webscout/Provider/oivscode.py +309 -309
webscout/Provider/scira_chat.py +383 -383
webscout/Provider/searchchat.py +292 -292
webscout/Provider/sonus.py +258 -258
webscout/Provider/toolbaz.py +370 -367
webscout/Provider/turboseek.py +273 -273
webscout/Provider/typefully.py +207 -207
webscout/Provider/yep.py +372 -372
webscout/__init__.py +27 -31
webscout/__main__.py +5 -5
webscout/auth/api_key_manager.py +189 -189
webscout/auth/config.py +175 -175
webscout/auth/models.py +185 -185
webscout/auth/routes.py +663 -664
webscout/auth/simple_logger.py +236 -236
webscout/cli.py +523 -523
webscout/conversation.py +438 -438
webscout/exceptions.py +361 -361
webscout/litagent/Readme.md +298 -298
webscout/litagent/__init__.py +28 -28
webscout/litagent/agent.py +581 -581
webscout/litagent/constants.py +59 -59
webscout/litprinter/__init__.py +58 -58
webscout/models.py +181 -181
webscout/optimizers.py +419 -419
webscout/prompt_manager.py +288 -288
webscout/sanitize.py +1078 -1078
webscout/scout/README.md +401 -401
webscout/scout/__init__.py +8 -8
webscout/scout/core/__init__.py +6 -6
webscout/scout/core/crawler.py +297 -297
webscout/scout/core/scout.py +706 -706
webscout/scout/core/search_result.py +95 -95
webscout/scout/core/text_analyzer.py +62 -62
webscout/scout/core/text_utils.py +277 -277
webscout/scout/core/web_analyzer.py +51 -51
webscout/scout/element.py +599 -599
webscout/scout/parsers/__init__.py +69 -69
webscout/scout/parsers/html5lib_parser.py +172 -172
webscout/scout/parsers/html_parser.py +236 -236
webscout/scout/parsers/lxml_parser.py +178 -178
webscout/scout/utils.py +37 -37
webscout/search/__init__.py +51 -0
webscout/search/base.py +195 -0
webscout/search/duckduckgo_main.py +54 -0
webscout/search/engines/__init__.py +48 -0
webscout/search/engines/bing.py +84 -0
webscout/search/engines/bing_news.py +52 -0
webscout/search/engines/brave.py +43 -0
webscout/search/engines/duckduckgo/__init__.py +25 -0
webscout/search/engines/duckduckgo/answers.py +78 -0
webscout/search/engines/duckduckgo/base.py +187 -0
webscout/search/engines/duckduckgo/images.py +97 -0
webscout/search/engines/duckduckgo/maps.py +168 -0
webscout/search/engines/duckduckgo/news.py +68 -0
webscout/search/engines/duckduckgo/suggestions.py +21 -0
webscout/search/engines/duckduckgo/text.py +211 -0
webscout/search/engines/duckduckgo/translate.py +47 -0
webscout/search/engines/duckduckgo/videos.py +63 -0
webscout/search/engines/duckduckgo/weather.py +74 -0
webscout/search/engines/mojeek.py +37 -0
webscout/search/engines/wikipedia.py +56 -0
webscout/search/engines/yahoo.py +65 -0
webscout/search/engines/yahoo_news.py +64 -0
webscout/search/engines/yandex.py +43 -0
webscout/search/engines/yep/__init__.py +13 -0
webscout/search/engines/yep/base.py +32 -0
webscout/search/engines/yep/images.py +99 -0
webscout/search/engines/yep/suggestions.py +35 -0
webscout/search/engines/yep/text.py +114 -0
webscout/search/http_client.py +156 -0
webscout/search/results.py +137 -0
webscout/search/yep_main.py +44 -0
webscout/swiftcli/Readme.md +323 -323
webscout/swiftcli/__init__.py +95 -95
webscout/swiftcli/core/__init__.py +7 -7
webscout/swiftcli/core/cli.py +308 -308
webscout/swiftcli/core/context.py +104 -104
webscout/swiftcli/core/group.py +241 -241
webscout/swiftcli/decorators/__init__.py +28 -28
webscout/swiftcli/decorators/command.py +221 -221
webscout/swiftcli/decorators/options.py +220 -220
webscout/swiftcli/decorators/output.py +302 -302
webscout/swiftcli/exceptions.py +21 -21
webscout/swiftcli/plugins/__init__.py +9 -9
webscout/swiftcli/plugins/base.py +135 -135
webscout/swiftcli/plugins/manager.py +269 -269
webscout/swiftcli/utils/__init__.py +59 -59
webscout/swiftcli/utils/formatting.py +252 -252
webscout/swiftcli/utils/parsing.py +267 -267
webscout/update_checker.py +117 -117
webscout/version.py +1 -1
webscout/version.py.bak +2 -0
webscout/zeroart/README.md +89 -89
webscout/zeroart/__init__.py +134 -134
webscout/zeroart/base.py +66 -66
webscout/zeroart/effects.py +100 -100
webscout/zeroart/fonts.py +1238 -1238
{webscout-8.3.7.dist-info → webscout-2025.10.13.dist-info}/METADATA +936 -937
webscout-2025.10.13.dist-info/RECORD +329 -0
webscout/Provider/AISEARCH/DeepFind.py +0 -254
webscout/Provider/OPENAI/Qwen3.py +0 -303
webscout/Provider/OPENAI/qodo.py +0 -630
webscout/Provider/OPENAI/xenai.py +0 -514
webscout/tempid.py +0 -134
webscout/webscout_search.py +0 -1183
webscout/webscout_search_async.py +0 -649
webscout/yep_search.py +0 -346
webscout-8.3.7.dist-info/RECORD +0 -301
{webscout-8.3.7.dist-info → webscout-2025.10.13.dist-info}/WHEEL +0 -0
{webscout-8.3.7.dist-info → webscout-2025.10.13.dist-info}/entry_points.txt +0 -0
{webscout-8.3.7.dist-info → webscout-2025.10.13.dist-info}/licenses/LICENSE.md +0 -0
{webscout-8.3.7.dist-info → webscout-2025.10.13.dist-info}/top_level.txt +0 -0

webscout/scout/README.md CHANGED Viewed

@@ -1,401 +1,401 @@
-**🚀 The Most Advanced HTML Parser & Web Crawler for AI/LLM Data Collection**
-**🌟 Built for the Future • Powered by Intelligence • Trusted by Developers**
-## 📋 Overview
-Scout is an ultra-powerful, enterprise-grade HTML parsing and web crawling library designed for the AI era. Built with LLM data collection in mind, Scout provides unparalleled capabilities for extracting, analyzing, and processing web content at scale. With its BeautifulSoup-compatible API enhanced with modern features, Scout is the go-to solution for serious web scraping projects.
-<details open>
-<summary><b>🌟 Why Scout is the Ultimate Choice</b></summary>
-- **🧠 LLM-Optimized Crawling**: Purpose-built for collecting high-quality training data for Large Language Models
-- **🌐 Subdomain Intelligence**: Automatically discovers and crawls subdomains (e.g., blog.example.com, docs.example.com)
-- **⚡ Lightning-Fast Performance**: Multi-threaded concurrent crawling with intelligent rate limiting
-- **🎯 Surgical Precision**: Advanced content extraction that preserves structure while removing noise
-- **🔍 Deep Analysis**: Built-in NLP capabilities for entity extraction, text analysis, and semantic understanding
-- **🛡️ Enterprise-Ready**: Robust error handling, retry mechanisms, and respect for robots.txt
-- **📊 Rich Data Extraction**: Captures metadata, structured data, semantic content, and more
-- **🔄 Format Flexibility**: Export to JSON, Markdown, CSV, or custom formats
-- **🎨 BeautifulSoup++ API**: Familiar interface with 10x more features
-</details>
-## 📑 Table of Contents
-- [Installation](#-installation)
-- [Quick Start](#-quick-start)
-- [Features](#-features)
-- [Advanced Usage](#-advanced-usage)
-- [API Reference](#-api-reference)
-- [Dependencies](#-dependencies)
-- [Supported Python Versions](#-supported-python-versions)
-- [Contributing](#-contributing)
-- [License](#-license)
-## 📦 Installation
-```bash
-pip install webscout
-```
-Or install the latest version from GitHub:
-```bash
-pip install git+https://github.com/OEvortex/Webscout.git
-```
-## 🚀 Quick Start
-### Basic Parsing
-```python
-from webscout.scout import Scout
-# Parse HTML content
-html_content = """
-<html>
-    <body>
-        <h1>Hello, Scout!</h1>
-        <div class="content">
-            <p>Web parsing made easy.</p>
-            <a href="https://example.com">Link</a>
-        </div>
-    </body>
-</html>
-"""
-scout = Scout(html_content)
-# Find elements
-title = scout.find('h1')
-links = scout.find_all('a')
-# Extract text
-print(title[0].get_text())  # Output: Hello, Scout!
-print(links.attrs('href'))  # Output: ['https://example.com']
-```
-### Web Crawling
-```python
-from webscout.scout import ScoutCrawler
-# Crawl a website with default settings
-crawler = ScoutCrawler('https://example.com')  # Default: max_pages=50
-# Or customize the crawler
-crawler = ScoutCrawler(
-    'https://example.com',                      # base_url
-    max_pages=100,                              # maximum pages to crawl
-    tags_to_remove=['script', 'style', 'nav']   # tags to remove from content
-)
-# Start crawling
-crawled_pages = crawler.crawl()
-for page in crawled_pages:
-    print(f"URL: {page['url']}")
-    print(f"Title: {page['title']}")
-    print(f"Links found: {len(page['links'])}")
-    print(f"Crawl depth: {page['depth']}")
-```
-### Text Analysis
-```python
-from webscout.scout import Scout
-# Parse a webpage
-html = """<div><h1>Climate Change</h1><p>Email us at info@example.com or call 555-123-4567.</p>
-<p>Visit https://climate-action.org for more information.</p></div>"""
-scout = Scout(html)
-# Analyze text and extract entities
-analysis = scout.analyze_text()
-print(f"Word frequencies: {analysis['word_count']}")
-print(f"Entities found: {analysis['entities']}")
-```
-## ✨ Features
-### 🔍 Multiple Parser Support
-Scout supports multiple HTML/XML parsers, allowing you to choose the best tool for your specific needs:
-| Parser | Description | Best For |
-|--------|-------------|----------|
-| `html.parser` | Python's built-in parser | General-purpose parsing, no dependencies |
-| `lxml` | Fast C-based parser | Performance-critical applications |
-| `html5lib` | Highly compliant HTML5 parser | Handling malformed HTML |
-| `lxml-xml` | XML parser | XML document parsing |
-```python
-# Choose your parser
-scout = Scout(html_content, features='lxml')  # For speed
-scout = Scout(html_content, features='html5lib')  # For compliance
-```
-### 🌐 Advanced Parsing Capabilities
-Scout provides powerful tools for navigating and manipulating HTML/XML documents:
-- **Element Selection**: Find elements by tag name, attributes, CSS selectors, and more
-- **Tree Traversal**: Navigate parent-child relationships and sibling elements
-- **Content Extraction**: Extract text, attributes, and structured data
-- **Document Manipulation**: Modify, replace, or remove elements
-- **Dynamic Building**: Easily append or insert new nodes
-```python
-# CSS selector support
-elements = scout.select('div.content > p')
-# Advanced find with attribute matching
-results = scout.find_all('a', attrs={'class': 'external', 'rel': 'nofollow'})
-# Tree traversal
-parent = element.find_parent('div')
-siblings = element.find_next_siblings('p')
-prev_sibling = element.find_previous_sibling('p')
-```
-### 🧠 Intelligent Analysis
-Scout includes built-in analysis tools for extracting insights from web content:
-#### Text Analysis
-```python
-# Extract and analyze text
-text = scout.get_text()
-word_counts = scout.text_analyzer.count_words(text)
-entities = scout.text_analyzer.extract_entities(text)
-```
-#### Web Structure Analysis
-```python
-# Analyze page structure
-structure = scout.analyze_page_structure()
-print(f"Most common tags: {structure['tag_distribution']}")
-print(f"Page depth: {max(structure['depth_analysis'].keys())}")
-```
-#### Semantic Information Extraction
-```python
-# Extract semantic information
-semantics = scout.extract_semantic_info()
-print(f"Headings: {semantics['headings']}")
-print(f"Lists: {len(semantics['lists']['ul']) + len(semantics['lists']['ol'])}")
-print(f"Tables: {semantics['tables']['count']}")
-```
-### 🕸️ Web Crawling
-Scout includes a powerful concurrent web crawler for fetching and analyzing multiple pages:
-```python
-from webscout.scout import ScoutCrawler
-# Create a crawler with default settings
-crawler = ScoutCrawler('https://example.com')  # Default: max_pages=50
-# Or customize the crawler with specific options
-crawler = ScoutCrawler(
-    'https://example.com',                      # base_url
-    max_pages=100,                              # maximum pages to crawl
-    tags_to_remove=['script', 'style', 'nav']   # tags to remove from content
-)
-# Start crawling
-pages = crawler.crawl()
-# Process results
-for page in pages:
-    print(f"URL: {page['url']}")
-    print(f"Title: {page['title']}")
-    print(f"Links: {len(page['links'])}")
-    print(f"Depth: {page['depth']}")
-```
-The crawler automatically:
-- Stays within the same domain as the base URL
-- Uses concurrent requests for faster crawling
-- Removes unwanted tags (like scripts and styles) for cleaner text extraction
-- Tracks crawl depth for each page
-### 📄 Format Conversion
-Scout can convert HTML to various formats:
-```python
-# Convert to JSON
-json_data = scout.to_json(indent=2)
-# Convert to Markdown
-markdown = scout.to_markdown(heading_style='ATX')
-# Pretty-print HTML
-pretty_html = scout.prettify()
-```
-## 🔬 Advanced Usage
-### Working with Search Results
-Scout's search methods return a `ScoutSearchResult` object with powerful methods for processing results:
-```python
-from webscout.scout import Scout
-scout = Scout(html_content)
-# Find all paragraphs
-paragraphs = scout.find_all('p')
-# Extract all text from results
-all_text = paragraphs.texts(separator='\n')
-# Extract specific attributes
-hrefs = paragraphs.attrs('href')
-# Filter results with a predicate function
-important = paragraphs.filter(lambda p: 'important' in p.get('class', []))
-# Transform results
-word_counts = paragraphs.map(lambda p: len(p.get_text().split()))
-# Analyze text in results
-analysis = paragraphs.analyze_text()
-```
-### URL Handling and Analysis
-```python
-from webscout.scout import Scout
-scout = Scout(html_content)
-# Parse and analyze URLs
-links = scout.extract_links(base_url='https://example.com')
-for link in links:
-    url_components = scout.url_parse(link['href'])
-    print(f"Domain: {url_components['netloc']}")
-    print(f"Path: {url_components['path']}")
-```
-### Metadata Extraction
-```python
-from webscout.scout import Scout
-scout = Scout(html_content)
-# Extract metadata
-metadata = scout.extract_metadata()
-print(f"Title: {metadata['title']}")
-print(f"Description: {metadata['description']}")
-print(f"Open Graph: {metadata['og_metadata']}")
-print(f"Twitter Card: {metadata['twitter_metadata']}")
-```
-### Content Hashing and Caching
-```python
-from webscout.scout import Scout
-scout = Scout(html_content)
-# Generate content hash
-content_hash = scout.hash_content(method='sha256')
-# Use caching for expensive operations
-if not scout.cache('parsed_data'):
-    data = scout.extract_semantic_info()
-    scout.cache('parsed_data', data)
-cached_data = scout.cache('parsed_data')
-```
-## 📚 API Reference
-### Core Classes
-| Class | Description |
-|-------|-------------|
-| `Scout` | Main class for HTML parsing and traversal |
-| `ScoutCrawler` | Web crawler for fetching and parsing multiple pages |
-| `ScoutTextAnalyzer` | Text analysis utilities |
-| `ScoutWebAnalyzer` | Web page analysis utilities |
-| `ScoutSearchResult` | Enhanced search results with filtering and analysis |
-| `Tag` | Represents an HTML/XML tag |
-| `NavigableString` | Represents text within an HTML/XML document |
-### Key Methods
-#### Scout Class
-- `__init__(markup, features='html.parser', from_encoding=None)`: Initialize with HTML content
-- `find(name, attrs={}, recursive=True, text=None)`: Find first matching element
-- `find_all(name, attrs={}, recursive=True, text=None, limit=None)`: Find all matching elements
-- `select(selector)`: Find elements using CSS selector
-- `get_text(separator=' ', strip=False)`: Extract text from document
-- `analyze_text()`: Perform text analysis
-- `analyze_page_structure()`: Analyze document structure
-- `extract_semantic_info()`: Extract semantic information
-- `extract_links(base_url=None)`: Extract all links
-- `extract_metadata()`: Extract metadata from document
-- `to_json(indent=2)`: Convert to JSON
-- `to_markdown(heading_style='ATX')`: Convert to Markdown
-- `prettify(formatter='minimal')`: Pretty-print HTML
-#### ScoutCrawler Class
-- `__init__(base_url, max_pages=50, tags_to_remove=None)`: Initialize the crawler
-- `crawl()`: Start crawling from the base URL
-- `_crawl_page(url, depth=0)`: Crawl a single page (internal method)
-- `_is_valid_url(url)`: Check if a URL is valid (internal method)
-For detailed API documentation, please refer to the [documentation](https://github.com/OEvortex/Webscout/wiki).
-## 🔧 Dependencies
-- `curl_cffi`: HTTP library used for web requests
-- `lxml`: XML and HTML processing library (optional, recommended)
-- `html5lib`: Standards-compliant HTML parser (optional)
-- `markdownify`: HTML to Markdown conversion
-- `concurrent.futures`: Asynchronous execution (standard library)
-## 🌈 Supported Python Versions
-- Python 3.8+
-## 🤝 Contributing
-Contributions are welcome! Here's how you can contribute:
-1. Fork the repository
-2. Create a feature branch (`git checkout -b feature/amazing-feature`)
-3. Commit your changes (`git commit -m 'Add some amazing feature'`)
-4. Push to the branch (`git push origin feature/amazing-feature`)
-5. Open a Pull Request
-Please make sure to update tests as appropriate.
-## 📄 License
-This project is licensed under the MIT License - see the LICENSE file for details.
----
-<div align="center">
-  <p>Made with ❤️ by the Webscout team</p>
-  <p>
-    <a href="https://github.com/OEvortex/Webscout">GitHub</a> •
-    <a href="https://github.com/OEvortex/Webscout/wiki">Documentation</a> •
-    <a href="https://github.com/OEvortex/Webscout/issues">Report Bug</a> •
-    <a href="https://github.com/OEvortex/Webscout/issues">Request Feature</a>
-  </p>
-</div>
+**🚀 The Most Advanced HTML Parser & Web Crawler for AI/LLM Data Collection**
+**🌟 Built for the Future • Powered by Intelligence • Trusted by Developers**
+## 📋 Overview
+Scout is an ultra-powerful, enterprise-grade HTML parsing and web crawling library designed for the AI era. Built with LLM data collection in mind, Scout provides unparalleled capabilities for extracting, analyzing, and processing web content at scale. With its BeautifulSoup-compatible API enhanced with modern features, Scout is the go-to solution for serious web scraping projects.
+<details open>
+<summary><b>🌟 Why Scout is the Ultimate Choice</b></summary>
+- **🧠 LLM-Optimized Crawling**: Purpose-built for collecting high-quality training data for Large Language Models
+- **🌐 Subdomain Intelligence**: Automatically discovers and crawls subdomains (e.g., blog.example.com, docs.example.com)
+- **⚡ Lightning-Fast Performance**: Multi-threaded concurrent crawling with intelligent rate limiting
+- **🎯 Surgical Precision**: Advanced content extraction that preserves structure while removing noise
+- **🔍 Deep Analysis**: Built-in NLP capabilities for entity extraction, text analysis, and semantic understanding
+- **🛡️ Enterprise-Ready**: Robust error handling, retry mechanisms, and respect for robots.txt
+- **📊 Rich Data Extraction**: Captures metadata, structured data, semantic content, and more
+- **🔄 Format Flexibility**: Export to JSON, Markdown, CSV, or custom formats
+- **🎨 BeautifulSoup++ API**: Familiar interface with 10x more features
+</details>
+## 📑 Table of Contents
+- [Installation](#-installation)
+- [Quick Start](#-quick-start)
+- [Features](#-features)
+- [Advanced Usage](#-advanced-usage)
+- [API Reference](#-api-reference)
+- [Dependencies](#-dependencies)
+- [Supported Python Versions](#-supported-python-versions)
+- [Contributing](#-contributing)
+- [License](#-license)
+## 📦 Installation
+```bash
+pip install webscout
+```
+Or install the latest version from GitHub:
+```bash
+pip install git+https://github.com/OEvortex/Webscout.git
+```
+## 🚀 Quick Start
+### Basic Parsing
+```python
+from webscout.scout import Scout
+# Parse HTML content
+html_content = """
+<html>
+    <body>
+        <h1>Hello, Scout!</h1>
+        <div class="content">
+            <p>Web parsing made easy.</p>
+            <a href="https://example.com">Link</a>
+        </div>
+    </body>
+</html>
+"""
+scout = Scout(html_content)
+# Find elements
+title = scout.find('h1')
+links = scout.find_all('a')
+# Extract text
+print(title[0].get_text())  # Output: Hello, Scout!
+print(links.attrs('href'))  # Output: ['https://example.com']
+```
+### Web Crawling
+```python
+from webscout.scout import ScoutCrawler
+# Crawl a website with default settings
+crawler = ScoutCrawler('https://example.com')  # Default: max_pages=50
+# Or customize the crawler
+crawler = ScoutCrawler(
+    'https://example.com',                      # base_url
+    max_pages=100,                              # maximum pages to crawl
+    tags_to_remove=['script', 'style', 'nav']   # tags to remove from content
+)
+# Start crawling
+crawled_pages = crawler.crawl()
+for page in crawled_pages:
+    print(f"URL: {page['url']}")
+    print(f"Title: {page['title']}")
+    print(f"Links found: {len(page['links'])}")
+    print(f"Crawl depth: {page['depth']}")
+```
+### Text Analysis
+```python
+from webscout.scout import Scout
+# Parse a webpage
+html = """<div><h1>Climate Change</h1><p>Email us at info@example.com or call 555-123-4567.</p>
+<p>Visit https://climate-action.org for more information.</p></div>"""
+scout = Scout(html)
+# Analyze text and extract entities
+analysis = scout.analyze_text()
+print(f"Word frequencies: {analysis['word_count']}")
+print(f"Entities found: {analysis['entities']}")
+```
+## ✨ Features
+### 🔍 Multiple Parser Support
+Scout supports multiple HTML/XML parsers, allowing you to choose the best tool for your specific needs:
+| Parser | Description | Best For |
+|--------|-------------|----------|
+| `html.parser` | Python's built-in parser | General-purpose parsing, no dependencies |
+| `lxml` | Fast C-based parser | Performance-critical applications |
+| `html5lib` | Highly compliant HTML5 parser | Handling malformed HTML |
+| `lxml-xml` | XML parser | XML document parsing |
+```python
+# Choose your parser
+scout = Scout(html_content, features='lxml')  # For speed
+scout = Scout(html_content, features='html5lib')  # For compliance
+```
+### 🌐 Advanced Parsing Capabilities
+Scout provides powerful tools for navigating and manipulating HTML/XML documents:
+- **Element Selection**: Find elements by tag name, attributes, CSS selectors, and more
+- **Tree Traversal**: Navigate parent-child relationships and sibling elements
+- **Content Extraction**: Extract text, attributes, and structured data
+- **Document Manipulation**: Modify, replace, or remove elements
+- **Dynamic Building**: Easily append or insert new nodes
+```python
+# CSS selector support
+elements = scout.select('div.content > p')
+# Advanced find with attribute matching
+results = scout.find_all('a', attrs={'class': 'external', 'rel': 'nofollow'})
+# Tree traversal
+parent = element.find_parent('div')
+siblings = element.find_next_siblings('p')
+prev_sibling = element.find_previous_sibling('p')
+```
+### 🧠 Intelligent Analysis
+Scout includes built-in analysis tools for extracting insights from web content:
+#### Text Analysis
+```python
+# Extract and analyze text
+text = scout.get_text()
+word_counts = scout.text_analyzer.count_words(text)
+entities = scout.text_analyzer.extract_entities(text)
+```
+#### Web Structure Analysis
+```python
+# Analyze page structure
+structure = scout.analyze_page_structure()
+print(f"Most common tags: {structure['tag_distribution']}")
+print(f"Page depth: {max(structure['depth_analysis'].keys())}")
+```
+#### Semantic Information Extraction
+```python
+# Extract semantic information
+semantics = scout.extract_semantic_info()
+print(f"Headings: {semantics['headings']}")
+print(f"Lists: {len(semantics['lists']['ul']) + len(semantics['lists']['ol'])}")
+print(f"Tables: {semantics['tables']['count']}")
+```
+### 🕸️ Web Crawling
+Scout includes a powerful concurrent web crawler for fetching and analyzing multiple pages:
+```python
+from webscout.scout import ScoutCrawler
+# Create a crawler with default settings
+crawler = ScoutCrawler('https://example.com')  # Default: max_pages=50
+# Or customize the crawler with specific options
+crawler = ScoutCrawler(
+    'https://example.com',                      # base_url
+    max_pages=100,                              # maximum pages to crawl
+    tags_to_remove=['script', 'style', 'nav']   # tags to remove from content
+)
+# Start crawling
+pages = crawler.crawl()
+# Process results
+for page in pages:
+    print(f"URL: {page['url']}")
+    print(f"Title: {page['title']}")
+    print(f"Links: {len(page['links'])}")
+    print(f"Depth: {page['depth']}")
+```
+The crawler automatically:
+- Stays within the same domain as the base URL
+- Uses concurrent requests for faster crawling
+- Removes unwanted tags (like scripts and styles) for cleaner text extraction
+- Tracks crawl depth for each page
+### 📄 Format Conversion
+Scout can convert HTML to various formats:
+```python
+# Convert to JSON
+json_data = scout.to_json(indent=2)
+# Convert to Markdown
+markdown = scout.to_markdown(heading_style='ATX')
+# Pretty-print HTML
+pretty_html = scout.prettify()
+```
+## 🔬 Advanced Usage
+### Working with Search Results
+Scout's search methods return a `ScoutSearchResult` object with powerful methods for processing results:
+```python
+from webscout.scout import Scout
+scout = Scout(html_content)
+# Find all paragraphs
+paragraphs = scout.find_all('p')
+# Extract all text from results
+all_text = paragraphs.texts(separator='\n')
+# Extract specific attributes
+hrefs = paragraphs.attrs('href')
+# Filter results with a predicate function
+important = paragraphs.filter(lambda p: 'important' in p.get('class', []))
+# Transform results
+word_counts = paragraphs.map(lambda p: len(p.get_text().split()))
+# Analyze text in results
+analysis = paragraphs.analyze_text()
+```
+### URL Handling and Analysis
+```python
+from webscout.scout import Scout
+scout = Scout(html_content)
+# Parse and analyze URLs
+links = scout.extract_links(base_url='https://example.com')
+for link in links:
+    url_components = scout.url_parse(link['href'])
+    print(f"Domain: {url_components['netloc']}")
+    print(f"Path: {url_components['path']}")
+```
+### Metadata Extraction
+```python
+from webscout.scout import Scout
+scout = Scout(html_content)
+# Extract metadata
+metadata = scout.extract_metadata()
+print(f"Title: {metadata['title']}")
+print(f"Description: {metadata['description']}")
+print(f"Open Graph: {metadata['og_metadata']}")
+print(f"Twitter Card: {metadata['twitter_metadata']}")
+```
+### Content Hashing and Caching
+```python
+from webscout.scout import Scout
+scout = Scout(html_content)
+# Generate content hash
+content_hash = scout.hash_content(method='sha256')
+# Use caching for expensive operations
+if not scout.cache('parsed_data'):
+    data = scout.extract_semantic_info()
+    scout.cache('parsed_data', data)
+cached_data = scout.cache('parsed_data')
+```
+## 📚 API Reference
+### Core Classes
+| Class | Description |
+|-------|-------------|
+| `Scout` | Main class for HTML parsing and traversal |
+| `ScoutCrawler` | Web crawler for fetching and parsing multiple pages |
+| `ScoutTextAnalyzer` | Text analysis utilities |
+| `ScoutWebAnalyzer` | Web page analysis utilities |
+| `ScoutSearchResult` | Enhanced search results with filtering and analysis |
+| `Tag` | Represents an HTML/XML tag |
+| `NavigableString` | Represents text within an HTML/XML document |
+### Key Methods
+#### Scout Class
+- `__init__(markup, features='html.parser', from_encoding=None)`: Initialize with HTML content
+- `find(name, attrs={}, recursive=True, text=None)`: Find first matching element
+- `find_all(name, attrs={}, recursive=True, text=None, limit=None)`: Find all matching elements
+- `select(selector)`: Find elements using CSS selector
+- `get_text(separator=' ', strip=False)`: Extract text from document
+- `analyze_text()`: Perform text analysis
+- `analyze_page_structure()`: Analyze document structure
+- `extract_semantic_info()`: Extract semantic information
+- `extract_links(base_url=None)`: Extract all links
+- `extract_metadata()`: Extract metadata from document
+- `to_json(indent=2)`: Convert to JSON
+- `to_markdown(heading_style='ATX')`: Convert to Markdown
+- `prettify(formatter='minimal')`: Pretty-print HTML
+#### ScoutCrawler Class
+- `__init__(base_url, max_pages=50, tags_to_remove=None)`: Initialize the crawler
+- `crawl()`: Start crawling from the base URL
+- `_crawl_page(url, depth=0)`: Crawl a single page (internal method)
+- `_is_valid_url(url)`: Check if a URL is valid (internal method)
+For detailed API documentation, please refer to the [documentation](https://github.com/OEvortex/Webscout/wiki).
+## 🔧 Dependencies
+- `curl_cffi`: HTTP library used for web requests
+- `lxml`: XML and HTML processing library (optional, recommended)
+- `html5lib`: Standards-compliant HTML parser (optional)
+- `markdownify`: HTML to Markdown conversion
+- `concurrent.futures`: Asynchronous execution (standard library)
+## 🌈 Supported Python Versions
+- Python 3.8+
+## 🤝 Contributing
+Contributions are welcome! Here's how you can contribute:
+1. Fork the repository
+2. Create a feature branch (`git checkout -b feature/amazing-feature`)
+3. Commit your changes (`git commit -m 'Add some amazing feature'`)
+4. Push to the branch (`git push origin feature/amazing-feature`)
+5. Open a Pull Request
+Please make sure to update tests as appropriate.
+## 📄 License
+This project is licensed under the MIT License - see the LICENSE file for details.
+---
+<div align="center">
+  <p>Made with ❤️ by the Webscout team</p>
+  <p>
+    <a href="https://github.com/OEvortex/Webscout">GitHub</a> •
+    <a href="https://github.com/OEvortex/Webscout/wiki">Documentation</a> •
+    <a href="https://github.com/OEvortex/Webscout/issues">Report Bug</a> •
+    <a href="https://github.com/OEvortex/Webscout/issues">Request Feature</a>
+  </p>
+</div>

webscout 8.3.7__py3-none-any.whl → 2025.10.13__py3-none-any.whl

Potentially problematic release.

webscout 8.3.7py3-none-any.whl → 2025.10.13py3-none-any.whl