PyPI - webscout - Versions diffs - 8.2.7__py3-none-any.whl → 8.2.9__py3-none-any.whl - Mend

webscout 8.2.7py3-none-any.whl → 8.2.9py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (281) hide show

webscout/AIauto.py +33 -15
webscout/AIbase.py +96 -37
webscout/AIutel.py +703 -250
webscout/Bard.py +441 -323
webscout/Extra/Act.md +309 -0
webscout/Extra/GitToolkit/__init__.py +10 -0
webscout/Extra/GitToolkit/gitapi/README.md +110 -0
webscout/Extra/GitToolkit/gitapi/__init__.py +12 -0
webscout/Extra/GitToolkit/gitapi/repository.py +195 -0
webscout/Extra/GitToolkit/gitapi/user.py +96 -0
webscout/Extra/GitToolkit/gitapi/utils.py +62 -0
webscout/Extra/YTToolkit/README.md +375 -0
webscout/Extra/YTToolkit/YTdownloader.py +957 -0
webscout/Extra/YTToolkit/__init__.py +3 -0
webscout/Extra/YTToolkit/transcriber.py +476 -0
webscout/Extra/YTToolkit/ytapi/README.md +44 -0
webscout/Extra/YTToolkit/ytapi/__init__.py +6 -0
webscout/Extra/YTToolkit/ytapi/channel.py +307 -0
webscout/Extra/YTToolkit/ytapi/errors.py +13 -0
webscout/Extra/YTToolkit/ytapi/extras.py +118 -0
webscout/Extra/YTToolkit/ytapi/https.py +88 -0
webscout/Extra/YTToolkit/ytapi/patterns.py +61 -0
webscout/Extra/YTToolkit/ytapi/playlist.py +59 -0
webscout/Extra/YTToolkit/ytapi/pool.py +8 -0
webscout/Extra/YTToolkit/ytapi/query.py +40 -0
webscout/Extra/YTToolkit/ytapi/stream.py +63 -0
webscout/Extra/YTToolkit/ytapi/utils.py +62 -0
webscout/Extra/YTToolkit/ytapi/video.py +232 -0
webscout/Extra/__init__.py +7 -0
webscout/Extra/autocoder/__init__.py +9 -0
webscout/Extra/autocoder/autocoder.py +1105 -0
webscout/Extra/autocoder/autocoder_utiles.py +332 -0
webscout/Extra/gguf.md +430 -0
webscout/Extra/gguf.py +684 -0
webscout/Extra/tempmail/README.md +488 -0
webscout/Extra/tempmail/__init__.py +28 -0
webscout/Extra/tempmail/async_utils.py +141 -0
webscout/Extra/tempmail/base.py +161 -0
webscout/Extra/tempmail/cli.py +187 -0
webscout/Extra/tempmail/emailnator.py +84 -0
webscout/Extra/tempmail/mail_tm.py +361 -0
webscout/Extra/tempmail/temp_mail_io.py +292 -0
webscout/Extra/weather.md +281 -0
webscout/Extra/weather.py +194 -0
webscout/Extra/weather_ascii.py +76 -0
webscout/Litlogger/README.md +10 -0
webscout/Litlogger/__init__.py +15 -0
webscout/Litlogger/formats.py +4 -0
webscout/Litlogger/handlers.py +103 -0
webscout/Litlogger/levels.py +13 -0
webscout/Litlogger/logger.py +92 -0
webscout/Provider/AI21.py +177 -0
webscout/Provider/AISEARCH/DeepFind.py +254 -0
webscout/Provider/AISEARCH/Perplexity.py +333 -0
webscout/Provider/AISEARCH/README.md +279 -0
webscout/Provider/AISEARCH/__init__.py +9 -0
webscout/Provider/AISEARCH/felo_search.py +202 -0
webscout/Provider/AISEARCH/genspark_search.py +324 -0
webscout/Provider/AISEARCH/hika_search.py +186 -0
webscout/Provider/AISEARCH/iask_search.py +410 -0
webscout/Provider/AISEARCH/monica_search.py +220 -0
webscout/Provider/AISEARCH/scira_search.py +298 -0
webscout/Provider/AISEARCH/webpilotai_search.py +255 -0
webscout/Provider/Aitopia.py +316 -0
webscout/Provider/AllenAI.py +440 -0
webscout/Provider/Andi.py +228 -0
webscout/Provider/Blackboxai.py +791 -0
webscout/Provider/ChatGPTClone.py +237 -0
webscout/Provider/ChatGPTGratis.py +194 -0
webscout/Provider/ChatSandbox.py +342 -0
webscout/Provider/Cloudflare.py +324 -0
webscout/Provider/Cohere.py +208 -0
webscout/Provider/Deepinfra.py +340 -0
webscout/Provider/ExaAI.py +261 -0
webscout/Provider/ExaChat.py +358 -0
webscout/Provider/Flowith.py +217 -0
webscout/Provider/FreeGemini.py +250 -0
webscout/Provider/Gemini.py +169 -0
webscout/Provider/GithubChat.py +369 -0
webscout/Provider/GizAI.py +295 -0
webscout/Provider/Glider.py +225 -0
webscout/Provider/Groq.py +801 -0
webscout/Provider/HF_space/__init__.py +0 -0
webscout/Provider/HF_space/qwen_qwen2.py +206 -0
webscout/Provider/HeckAI.py +375 -0
webscout/Provider/HuggingFaceChat.py +469 -0
webscout/Provider/Hunyuan.py +283 -0
webscout/Provider/Jadve.py +291 -0
webscout/Provider/Koboldai.py +384 -0
webscout/Provider/LambdaChat.py +411 -0
webscout/Provider/Llama3.py +259 -0
webscout/Provider/MCPCore.py +315 -0
webscout/Provider/Marcus.py +198 -0
webscout/Provider/Nemotron.py +218 -0
webscout/Provider/Netwrck.py +270 -0
webscout/Provider/OLLAMA.py +396 -0
webscout/Provider/OPENAI/BLACKBOXAI.py +766 -0
webscout/Provider/OPENAI/Cloudflare.py +378 -0
webscout/Provider/OPENAI/FreeGemini.py +283 -0
webscout/Provider/OPENAI/NEMOTRON.py +232 -0
webscout/Provider/OPENAI/Qwen3.py +283 -0
webscout/Provider/OPENAI/README.md +952 -0
webscout/Provider/OPENAI/TwoAI.py +357 -0
webscout/Provider/OPENAI/__init__.py +40 -0
webscout/Provider/OPENAI/ai4chat.py +293 -0
webscout/Provider/OPENAI/api.py +969 -0
webscout/Provider/OPENAI/base.py +249 -0
webscout/Provider/OPENAI/c4ai.py +373 -0
webscout/Provider/OPENAI/chatgpt.py +556 -0
webscout/Provider/OPENAI/chatgptclone.py +494 -0
webscout/Provider/OPENAI/chatsandbox.py +173 -0
webscout/Provider/OPENAI/copilot.py +242 -0
webscout/Provider/OPENAI/deepinfra.py +322 -0
webscout/Provider/OPENAI/e2b.py +1414 -0
webscout/Provider/OPENAI/exaai.py +417 -0
webscout/Provider/OPENAI/exachat.py +444 -0
webscout/Provider/OPENAI/flowith.py +162 -0
webscout/Provider/OPENAI/freeaichat.py +359 -0
webscout/Provider/OPENAI/glider.py +326 -0
webscout/Provider/OPENAI/groq.py +364 -0
webscout/Provider/OPENAI/heckai.py +308 -0
webscout/Provider/OPENAI/llmchatco.py +335 -0
webscout/Provider/OPENAI/mcpcore.py +389 -0
webscout/Provider/OPENAI/multichat.py +376 -0
webscout/Provider/OPENAI/netwrck.py +357 -0
webscout/Provider/OPENAI/oivscode.py +287 -0
webscout/Provider/OPENAI/opkfc.py +496 -0
webscout/Provider/OPENAI/pydantic_imports.py +172 -0
webscout/Provider/OPENAI/scirachat.py +477 -0
webscout/Provider/OPENAI/sonus.py +304 -0
webscout/Provider/OPENAI/standardinput.py +433 -0
webscout/Provider/OPENAI/textpollinations.py +339 -0
webscout/Provider/OPENAI/toolbaz.py +413 -0
webscout/Provider/OPENAI/typefully.py +355 -0
webscout/Provider/OPENAI/typegpt.py +364 -0
webscout/Provider/OPENAI/uncovrAI.py +463 -0
webscout/Provider/OPENAI/utils.py +318 -0
webscout/Provider/OPENAI/venice.py +431 -0
webscout/Provider/OPENAI/wisecat.py +387 -0
webscout/Provider/OPENAI/writecream.py +163 -0
webscout/Provider/OPENAI/x0gpt.py +365 -0
webscout/Provider/OPENAI/yep.py +382 -0
webscout/Provider/OpenGPT.py +209 -0
webscout/Provider/Openai.py +496 -0
webscout/Provider/PI.py +429 -0
webscout/Provider/Perplexitylabs.py +415 -0
webscout/Provider/QwenLM.py +254 -0
webscout/Provider/Reka.py +214 -0
webscout/Provider/StandardInput.py +290 -0
webscout/Provider/TTI/README.md +82 -0
webscout/Provider/TTI/__init__.py +7 -0
webscout/Provider/TTI/aiarta.py +365 -0
webscout/Provider/TTI/artbit.py +0 -0
webscout/Provider/TTI/base.py +64 -0
webscout/Provider/TTI/fastflux.py +200 -0
webscout/Provider/TTI/magicstudio.py +201 -0
webscout/Provider/TTI/piclumen.py +203 -0
webscout/Provider/TTI/pixelmuse.py +225 -0
webscout/Provider/TTI/pollinations.py +221 -0
webscout/Provider/TTI/utils.py +11 -0
webscout/Provider/TTS/README.md +192 -0
webscout/Provider/TTS/__init__.py +10 -0
webscout/Provider/TTS/base.py +159 -0
webscout/Provider/TTS/deepgram.py +156 -0
webscout/Provider/TTS/elevenlabs.py +111 -0
webscout/Provider/TTS/gesserit.py +128 -0
webscout/Provider/TTS/murfai.py +113 -0
webscout/Provider/TTS/openai_fm.py +129 -0
webscout/Provider/TTS/parler.py +111 -0
webscout/Provider/TTS/speechma.py +580 -0
webscout/Provider/TTS/sthir.py +94 -0
webscout/Provider/TTS/streamElements.py +333 -0
webscout/Provider/TTS/utils.py +280 -0
webscout/Provider/TeachAnything.py +229 -0
webscout/Provider/TextPollinationsAI.py +308 -0
webscout/Provider/TwoAI.py +475 -0
webscout/Provider/TypliAI.py +305 -0
webscout/Provider/UNFINISHED/ChatHub.py +209 -0
webscout/Provider/UNFINISHED/Youchat.py +330 -0
webscout/Provider/UNFINISHED/liner_api_request.py +263 -0
webscout/Provider/UNFINISHED/puterjs.py +635 -0
webscout/Provider/UNFINISHED/test_lmarena.py +119 -0
webscout/Provider/Venice.py +258 -0
webscout/Provider/VercelAI.py +253 -0
webscout/Provider/WiseCat.py +233 -0
webscout/Provider/WrDoChat.py +370 -0
webscout/Provider/Writecream.py +246 -0
webscout/Provider/WritingMate.py +269 -0
webscout/Provider/__init__.py +174 -0
webscout/Provider/ai4chat.py +174 -0
webscout/Provider/akashgpt.py +335 -0
webscout/Provider/asksteve.py +220 -0
webscout/Provider/cerebras.py +290 -0
webscout/Provider/chatglm.py +215 -0
webscout/Provider/cleeai.py +213 -0
webscout/Provider/copilot.py +425 -0
webscout/Provider/elmo.py +283 -0
webscout/Provider/freeaichat.py +285 -0
webscout/Provider/geminiapi.py +208 -0
webscout/Provider/granite.py +235 -0
webscout/Provider/hermes.py +266 -0
webscout/Provider/julius.py +223 -0
webscout/Provider/koala.py +170 -0
webscout/Provider/learnfastai.py +325 -0
webscout/Provider/llama3mitril.py +215 -0
webscout/Provider/llmchat.py +258 -0
webscout/Provider/llmchatco.py +306 -0
webscout/Provider/lmarena.py +198 -0
webscout/Provider/meta.py +801 -0
webscout/Provider/multichat.py +364 -0
webscout/Provider/oivscode.py +309 -0
webscout/Provider/samurai.py +224 -0
webscout/Provider/scira_chat.py +299 -0
webscout/Provider/scnet.py +243 -0
webscout/Provider/searchchat.py +292 -0
webscout/Provider/sonus.py +258 -0
webscout/Provider/talkai.py +194 -0
webscout/Provider/toolbaz.py +353 -0
webscout/Provider/turboseek.py +266 -0
webscout/Provider/typefully.py +202 -0
webscout/Provider/typegpt.py +289 -0
webscout/Provider/uncovr.py +368 -0
webscout/Provider/x0gpt.py +299 -0
webscout/Provider/yep.py +389 -0
webscout/__init__.py +4 -2
webscout/cli.py +3 -28
webscout/client.py +70 -0
webscout/conversation.py +35 -35
webscout/litagent/Readme.md +276 -0
webscout/litagent/__init__.py +29 -0
webscout/litagent/agent.py +455 -0
webscout/litagent/constants.py +60 -0
webscout/litprinter/__init__.py +59 -0
webscout/optimizers.py +419 -419
webscout/scout/README.md +404 -0
webscout/scout/__init__.py +8 -0
webscout/scout/core/__init__.py +7 -0
webscout/scout/core/crawler.py +210 -0
webscout/scout/core/scout.py +607 -0
webscout/scout/core/search_result.py +96 -0
webscout/scout/core/text_analyzer.py +63 -0
webscout/scout/core/text_utils.py +277 -0
webscout/scout/core/web_analyzer.py +52 -0
webscout/scout/element.py +478 -0
webscout/scout/parsers/__init__.py +69 -0
webscout/scout/parsers/html5lib_parser.py +172 -0
webscout/scout/parsers/html_parser.py +236 -0
webscout/scout/parsers/lxml_parser.py +178 -0
webscout/scout/utils.py +37 -0
webscout/swiftcli/Readme.md +323 -0
webscout/swiftcli/__init__.py +95 -0
webscout/swiftcli/core/__init__.py +7 -0
webscout/swiftcli/core/cli.py +297 -0
webscout/swiftcli/core/context.py +104 -0
webscout/swiftcli/core/group.py +241 -0
webscout/swiftcli/decorators/__init__.py +28 -0
webscout/swiftcli/decorators/command.py +221 -0
webscout/swiftcli/decorators/options.py +220 -0
webscout/swiftcli/decorators/output.py +252 -0
webscout/swiftcli/exceptions.py +21 -0
webscout/swiftcli/plugins/__init__.py +9 -0
webscout/swiftcli/plugins/base.py +135 -0
webscout/swiftcli/plugins/manager.py +269 -0
webscout/swiftcli/utils/__init__.py +59 -0
webscout/swiftcli/utils/formatting.py +252 -0
webscout/swiftcli/utils/parsing.py +267 -0
webscout/version.py +1 -1
webscout/webscout_search.py +2 -182
webscout/webscout_search_async.py +1 -179
webscout/zeroart/README.md +89 -0
webscout/zeroart/__init__.py +135 -0
webscout/zeroart/base.py +66 -0
webscout/zeroart/effects.py +101 -0
webscout/zeroart/fonts.py +1239 -0
{webscout-8.2.7.dist-info → webscout-8.2.9.dist-info}/METADATA +262 -83
webscout-8.2.9.dist-info/RECORD +289 -0
{webscout-8.2.7.dist-info → webscout-8.2.9.dist-info}/WHEEL +1 -1
{webscout-8.2.7.dist-info → webscout-8.2.9.dist-info}/entry_points.txt +1 -0
webscout-8.2.7.dist-info/RECORD +0 -26
{webscout-8.2.7.dist-info → webscout-8.2.9.dist-info}/licenses/LICENSE.md +0 -0
{webscout-8.2.7.dist-info → webscout-8.2.9.dist-info}/top_level.txt +0 -0

webscout/scout/README.md ADDED Viewed

@@ -0,0 +1,404 @@
+# 🕵️ Scout: Next-Gen Web Parsing Library
+<div align="center">
+[![Python](https://img.shields.io/badge/Python-3.8%2B-blue)](https://www.python.org/)
+[![License](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT)
+[![Maintenance](https://img.shields.io/badge/Maintained-Yes-brightgreen.svg)](https://github.com/OE-LUCIFER/Webscout)
+[![Documentation](https://img.shields.io/badge/Docs-Wiki-orange)](https://github.com/OE-LUCIFER/Webscout/wiki)
+[![PRs Welcome](https://img.shields.io/badge/PRs-Welcome-brightgreen.svg)](https://github.com/OE-LUCIFER/Webscout/pulls)
+</div>
+## 📋 Overview
+Scout is a powerful, flexible, and performant HTML parsing library that makes web scraping a breeze! It provides intelligent HTML/XML parsing with advanced features like web crawling, text analysis, semantic extraction, and Markdown conversion. Scout goes beyond traditional parsing libraries with its intuitive API and comprehensive feature set.
+<details open>
+<summary><b>Why Choose Scout?</b></summary>
+- **Powerful Parsing**: Multiple parser backends with intelligent markup handling
+- **Advanced Analysis**: Built-in text and web content analysis tools
+- **Concurrent Crawling**: Efficient multi-threaded web crawling
+- **Flexible API**: Intuitive interface similar to BeautifulSoup but with enhanced capabilities
+- **Format Conversion**: Convert HTML to JSON, Markdown, and more
+</details>
+## 📑 Table of Contents
+- [Installation](#-installation)
+- [Quick Start](#-quick-start)
+- [Features](#-features)
+- [Advanced Usage](#-advanced-usage)
+- [API Reference](#-api-reference)
+- [Dependencies](#-dependencies)
+- [Supported Python Versions](#-supported-python-versions)
+- [Contributing](#-contributing)
+- [License](#-license)
+## 📦 Installation
+```bash
+pip install webscout
+```
+Or install the latest version from GitHub:
+```bash
+pip install git+https://github.com/OE-LUCIFER/Webscout.git
+```
+## 🚀 Quick Start
+### Basic Parsing
+```python
+from webscout.scout import Scout
+# Parse HTML content
+html_content = """
+<html>
+    <body>
+        <h1>Hello, Scout!</h1>
+        <div class="content">
+            <p>Web parsing made easy.</p>
+            <a href="https://example.com">Link</a>
+        </div>
+    </body>
+</html>
+"""
+scout = Scout(html_content)
+# Find elements
+title = scout.find('h1')
+links = scout.find_all('a')
+# Extract text
+print(title[0].get_text())  # Output: Hello, Scout!
+print(links.attrs('href'))  # Output: ['https://example.com']
+```
+### Web Crawling
+```python
+from webscout.scout import ScoutCrawler
+# Crawl a website with default settings
+crawler = ScoutCrawler('https://example.com')  # Default: max_pages=50
+# Or customize the crawler
+crawler = ScoutCrawler(
+    'https://example.com',                      # base_url
+    max_pages=100,                              # maximum pages to crawl
+    tags_to_remove=['script', 'style', 'nav']   # tags to remove from content
+)
+# Start crawling
+crawled_pages = crawler.crawl()
+for page in crawled_pages:
+    print(f"URL: {page['url']}")
+    print(f"Title: {page['title']}")
+    print(f"Links found: {len(page['links'])}")
+    print(f"Crawl depth: {page['depth']}")
+```
+### Text Analysis
+```python
+from webscout.scout import Scout
+# Parse a webpage
+html = """<div><h1>Climate Change</h1><p>Email us at info@example.com or call 555-123-4567.</p>
+<p>Visit https://climate-action.org for more information.</p></div>"""
+scout = Scout(html)
+# Analyze text and extract entities
+analysis = scout.analyze_text()
+print(f"Word frequencies: {analysis['word_count']}")
+print(f"Entities found: {analysis['entities']}")
+```
+## ✨ Features
+### 🔍 Multiple Parser Support
+Scout supports multiple HTML/XML parsers, allowing you to choose the best tool for your specific needs:
+| Parser | Description | Best For |
+|--------|-------------|----------|
+| `html.parser` | Python's built-in parser | General-purpose parsing, no dependencies |
+| `lxml` | Fast C-based parser | Performance-critical applications |
+| `html5lib` | Highly compliant HTML5 parser | Handling malformed HTML |
+| `lxml-xml` | XML parser | XML document parsing |
+```python
+# Choose your parser
+scout = Scout(html_content, features='lxml')  # For speed
+scout = Scout(html_content, features='html5lib')  # For compliance
+```
+### 🌐 Advanced Parsing Capabilities
+Scout provides powerful tools for navigating and manipulating HTML/XML documents:
+- **Element Selection**: Find elements by tag name, attributes, CSS selectors, and more
+- **Tree Traversal**: Navigate parent-child relationships and sibling elements
+- **Content Extraction**: Extract text, attributes, and structured data
+- **Document Manipulation**: Modify, replace, or remove elements
+- **Dynamic Building**: Easily append or insert new nodes
+```python
+# CSS selector support
+elements = scout.select('div.content > p')
+# Advanced find with attribute matching
+results = scout.find_all('a', attrs={'class': 'external', 'rel': 'nofollow'})
+# Tree traversal
+parent = element.find_parent('div')
+siblings = element.find_next_siblings('p')
+prev_sibling = element.find_previous_sibling('p')
+```
+### 🧠 Intelligent Analysis
+Scout includes built-in analysis tools for extracting insights from web content:
+#### Text Analysis
+```python
+# Extract and analyze text
+text = scout.get_text()
+word_counts = scout.text_analyzer.count_words(text)
+entities = scout.text_analyzer.extract_entities(text)
+```
+#### Web Structure Analysis
+```python
+# Analyze page structure
+structure = scout.analyze_page_structure()
+print(f"Most common tags: {structure['tag_distribution']}")
+print(f"Page depth: {max(structure['depth_analysis'].keys())}")
+```
+#### Semantic Information Extraction
+```python
+# Extract semantic information
+semantics = scout.extract_semantic_info()
+print(f"Headings: {semantics['headings']}")
+print(f"Lists: {len(semantics['lists']['ul']) + len(semantics['lists']['ol'])}")
+print(f"Tables: {semantics['tables']['count']}")
+```
+### 🕸️ Web Crawling
+Scout includes a powerful concurrent web crawler for fetching and analyzing multiple pages:
+```python
+from webscout.scout import ScoutCrawler
+# Create a crawler with default settings
+crawler = ScoutCrawler('https://example.com')  # Default: max_pages=50
+# Or customize the crawler with specific options
+crawler = ScoutCrawler(
+    'https://example.com',                      # base_url
+    max_pages=100,                              # maximum pages to crawl
+    tags_to_remove=['script', 'style', 'nav']   # tags to remove from content
+)
+# Start crawling
+pages = crawler.crawl()
+# Process results
+for page in pages:
+    print(f"URL: {page['url']}")
+    print(f"Title: {page['title']}")
+    print(f"Links: {len(page['links'])}")
+    print(f"Depth: {page['depth']}")
+```
+The crawler automatically:
+- Stays within the same domain as the base URL
+- Uses concurrent requests for faster crawling
+- Removes unwanted tags (like scripts and styles) for cleaner text extraction
+- Tracks crawl depth for each page
+### 📄 Format Conversion
+Scout can convert HTML to various formats:
+```python
+# Convert to JSON
+json_data = scout.to_json(indent=2)
+# Convert to Markdown
+markdown = scout.to_markdown(heading_style='ATX')
+# Pretty-print HTML
+pretty_html = scout.prettify()
+```
+## 🔬 Advanced Usage
+### Working with Search Results
+Scout's search methods return a `ScoutSearchResult` object with powerful methods for processing results:
+```python
+from webscout.scout import Scout
+scout = Scout(html_content)
+# Find all paragraphs
+paragraphs = scout.find_all('p')
+# Extract all text from results
+all_text = paragraphs.texts(separator='\n')
+# Extract specific attributes
+hrefs = paragraphs.attrs('href')
+# Filter results with a predicate function
+important = paragraphs.filter(lambda p: 'important' in p.get('class', []))
+# Transform results
+word_counts = paragraphs.map(lambda p: len(p.get_text().split()))
+# Analyze text in results
+analysis = paragraphs.analyze_text()
+```
+### URL Handling and Analysis
+```python
+from webscout.scout import Scout
+scout = Scout(html_content)
+# Parse and analyze URLs
+links = scout.extract_links(base_url='https://example.com')
+for link in links:
+    url_components = scout.url_parse(link['href'])
+    print(f"Domain: {url_components['netloc']}")
+    print(f"Path: {url_components['path']}")
+```
+### Metadata Extraction
+```python
+from webscout.scout import Scout
+scout = Scout(html_content)
+# Extract metadata
+metadata = scout.extract_metadata()
+print(f"Title: {metadata['title']}")
+print(f"Description: {metadata['description']}")
+print(f"Open Graph: {metadata['og_metadata']}")
+print(f"Twitter Card: {metadata['twitter_metadata']}")
+```
+### Content Hashing and Caching
+```python
+from webscout.scout import Scout
+scout = Scout(html_content)
+# Generate content hash
+content_hash = scout.hash_content(method='sha256')
+# Use caching for expensive operations
+if not scout.cache('parsed_data'):
+    data = scout.extract_semantic_info()
+    scout.cache('parsed_data', data)
+cached_data = scout.cache('parsed_data')
+```
+## 📚 API Reference
+### Core Classes
+| Class | Description |
+|-------|-------------|
+| `Scout` | Main class for HTML parsing and traversal |
+| `ScoutCrawler` | Web crawler for fetching and parsing multiple pages |
+| `ScoutTextAnalyzer` | Text analysis utilities |
+| `ScoutWebAnalyzer` | Web page analysis utilities |
+| `ScoutSearchResult` | Enhanced search results with filtering and analysis |
+| `Tag` | Represents an HTML/XML tag |
+| `NavigableString` | Represents text within an HTML/XML document |
+### Key Methods
+#### Scout Class
+- `__init__(markup, features='html.parser', from_encoding=None)`: Initialize with HTML content
+- `find(name, attrs={}, recursive=True, text=None)`: Find first matching element
+- `find_all(name, attrs={}, recursive=True, text=None, limit=None)`: Find all matching elements
+- `select(selector)`: Find elements using CSS selector
+- `get_text(separator=' ', strip=False)`: Extract text from document
+- `analyze_text()`: Perform text analysis
+- `analyze_page_structure()`: Analyze document structure
+- `extract_semantic_info()`: Extract semantic information
+- `extract_links(base_url=None)`: Extract all links
+- `extract_metadata()`: Extract metadata from document
+- `to_json(indent=2)`: Convert to JSON
+- `to_markdown(heading_style='ATX')`: Convert to Markdown
+- `prettify(formatter='minimal')`: Pretty-print HTML
+#### ScoutCrawler Class
+- `__init__(base_url, max_pages=50, tags_to_remove=None)`: Initialize the crawler
+- `crawl()`: Start crawling from the base URL
+- `_crawl_page(url, depth=0)`: Crawl a single page (internal method)
+- `_is_valid_url(url)`: Check if a URL is valid (internal method)
+For detailed API documentation, please refer to the [documentation](https://github.com/OE-LUCIFER/Webscout/wiki).
+## 🔧 Dependencies
+- `curl_cffi`: HTTP library used for web requests
+- `lxml`: XML and HTML processing library (optional, recommended)
+- `html5lib`: Standards-compliant HTML parser (optional)
+- `markdownify`: HTML to Markdown conversion
+- `concurrent.futures`: Asynchronous execution (standard library)
+## 🌈 Supported Python Versions
+- Python 3.8+
+## 🤝 Contributing
+Contributions are welcome! Here's how you can contribute:
+1. Fork the repository
+2. Create a feature branch (`git checkout -b feature/amazing-feature`)
+3. Commit your changes (`git commit -m 'Add some amazing feature'`)
+4. Push to the branch (`git push origin feature/amazing-feature`)
+5. Open a Pull Request
+Please make sure to update tests as appropriate.
+## 📄 License
+This project is licensed under the MIT License - see the LICENSE file for details.
+---
+<div align="center">
+  <p>Made with ❤️ by the Webscout team</p>
+  <p>
+    <a href="https://github.com/OE-LUCIFER/Webscout">GitHub</a> •
+    <a href="https://github.com/OE-LUCIFER/Webscout/wiki">Documentation</a> •
+    <a href="https://github.com/OE-LUCIFER/Webscout/issues">Report Bug</a> •
+    <a href="https://github.com/OE-LUCIFER/Webscout/issues">Request Feature</a>
+  </p>
+</div>

webscout/scout/__init__.py ADDED Viewed

@@ -0,0 +1,8 @@
+"""
+Scout: A powerful, zero-dependency web scraping library
+"""
+from .core import Scout, ScoutCrawler, ScoutTextAnalyzer, ScoutWebAnalyzer, ScoutSearchResult
+from .element import Tag, NavigableString
+__all__ = ['Scout', 'ScoutCrawler', 'Tag', 'NavigableString','ScoutTextAnalyzer', 'ScoutWebAnalyzer', 'ScoutSearchResult']

webscout/scout/core/__init__.py ADDED Viewed

@@ -0,0 +1,7 @@
+from .text_analyzer import ScoutTextAnalyzer
+from .web_analyzer import ScoutWebAnalyzer
+from .search_result import ScoutSearchResult
+from .crawler import ScoutCrawler
+from .scout import Scout
+__all__ = ['ScoutTextAnalyzer', 'ScoutWebAnalyzer', 'ScoutSearchResult', 'ScoutCrawler', 'Scout']

webscout/scout/core/crawler.py ADDED Viewed

@@ -0,0 +1,210 @@
+"""
+Scout Crawler Module
+"""
+import concurrent.futures
+import urllib.parse
+import time
+import hashlib
+import re
+from urllib import robotparser
+from datetime import datetime
+from typing import Dict, List, Optional, Union
+from webscout.litagent import LitAgent
+from curl_cffi.requests import Session
+from .scout import Scout
+class ScoutCrawler:
+    """
+    Advanced web crawling utility for Scout library.
+    """
+    def __init__(self, base_url: str, max_pages: int = 50, tags_to_remove: List[str] = None, session: Optional[Session] = None, delay: float = 0.5, obey_robots: bool = True, allowed_domains: Optional[List[str]] = None):
+        """
+        Initialize the web crawler.
+        Args:
+            base_url (str): Starting URL to crawl
+            max_pages (int, optional): Maximum number of pages to crawl
+            tags_to_remove (List[str], optional): List of tags to remove
+        """
+        self.base_url = base_url
+        self.max_pages = max_pages
+        self.tags_to_remove = tags_to_remove if tags_to_remove is not None else [
+            "script",
+            "style",
+            "header",
+            "footer",
+            "nav",
+            "aside",
+            "form",
+            "button",
+        ]
+        self.visited_urls = set()
+        self.crawled_pages = []
+        self.session = session or Session()
+        self.agent = LitAgent()
+        # Use all headers and generate fingerprint
+        self.session.headers = self.agent.generate_fingerprint()
+        self.session.headers.setdefault("User-Agent", self.agent.chrome())
+        self.delay = delay
+        self.obey_robots = obey_robots
+        self.allowed_domains = allowed_domains or [urllib.parse.urlparse(base_url).netloc]
+        self.last_request_time = 0
+        self.url_hashes = set()
+        if obey_robots:
+            self.robots = robotparser.RobotFileParser()
+            robots_url = urllib.parse.urljoin(base_url, '/robots.txt')
+            try:
+                self.robots.set_url(robots_url)
+                self.robots.read()
+            except Exception:
+                self.robots = None
+        else:
+            self.robots = None
+    def _normalize_url(self, url: str) -> str:
+        url = url.split('#')[0]
+        url = re.sub(r'\?.*$', '', url)  # Remove query params
+        return url.rstrip('/')
+    def _is_valid_url(self, url: str) -> bool:
+        """
+        Check if a URL is valid and within the same domain.
+        Args:
+            url (str): URL to validate
+        Returns:
+            bool: Whether the URL is valid
+        """
+        try:
+            parsed_base = urllib.parse.urlparse(self.base_url)
+            parsed_url = urllib.parse.urlparse(url)
+            if parsed_url.scheme not in ["http", "https"]:
+                return False
+            if parsed_url.netloc not in self.allowed_domains:
+                return False
+            if self.obey_robots and self.robots:
+                return self.robots.can_fetch("*", url)
+            return True
+        except Exception:
+            return False
+    def _is_duplicate(self, url: str) -> bool:
+        norm = self._normalize_url(url)
+        url_hash = hashlib.md5(norm.encode()).hexdigest()
+        if url_hash in self.url_hashes:
+            return True
+        self.url_hashes.add(url_hash)
+        return False
+    def _extract_main_text(self, soup):
+        # Try to extract main content (simple heuristic)
+        main = soup.find('main')
+        if main:
+            return main.get_text(separator=" ", strip=True)
+        article = soup.find('article')
+        if article:
+            return article.get_text(separator=" ", strip=True)
+        # fallback to body
+        body = soup.find('body')
+        if body:
+            return body.get_text(separator=" ", strip=True)
+        return soup.get_text(separator=" ", strip=True)
+    def _crawl_page(self, url: str, depth: int = 0) -> Dict[str, Union[str, List[str]]]:
+        """
+        Crawl a single page and extract information.
+        Args:
+            url (str): URL to crawl
+            depth (int, optional): Current crawl depth
+        Returns:
+            Dict[str, Union[str, List[str]]]: Crawled page information
+        """
+        if url in self.visited_urls or self._is_duplicate(url):
+            return {}
+        # Throttle requests
+        now = time.time()
+        if self.last_request_time:
+            elapsed = now - self.last_request_time
+            if elapsed < self.delay:
+                time.sleep(self.delay - elapsed)
+        self.last_request_time = time.time()
+        try:
+            response = self.session.get(url, timeout=10)
+            response.raise_for_status()
+            if not response.headers.get('Content-Type', '').startswith('text/html'):
+                return {}
+            scout = Scout(response.content, features="lxml")
+            title_result = scout.find("title")
+            title = title_result[0].get_text() if title_result else ""
+            for tag_name in self.tags_to_remove:
+                for tag in scout._soup.find_all(tag_name):
+                    tag.extract()
+            visible_text = self._extract_main_text(scout._soup)
+            page_info = {
+                'url': url,
+                'title': title,
+                'links': [
+                    urllib.parse.urljoin(url, link.get('href'))
+                    for link in scout.find_all('a', href=True)
+                    if self._is_valid_url(urllib.parse.urljoin(url, link.get('href')))
+                ],
+                'text': visible_text,
+                'depth': depth,
+                'timestamp': datetime.utcnow().isoformat(),
+                'headers': dict(response.headers),
+            }
+            self.visited_urls.add(url)
+            self.crawled_pages.append(page_info)
+            return page_info
+        except Exception as e:
+            print(f"Error crawling {url}: {e}")
+            return {}
+    def crawl(self):
+        """
+        Start web crawling from base URL and yield each crawled page in real time.
+        Yields:
+            Dict[str, Union[str, List[str]]]: Crawled page information
+        """
+        with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
+            futures = {executor.submit(self._crawl_page, self.base_url, 0)}
+            submitted_links: set[str] = set()
+            while futures:
+                if len(self.visited_urls) >= self.max_pages:
+                    break
+                done, not_done = concurrent.futures.wait(
+                    futures, return_when=concurrent.futures.FIRST_COMPLETED
+                )
+                futures = not_done
+                for future in done:
+                    page_info = future.result()
+                    if page_info:
+                        yield page_info
+                    if len(self.visited_urls) >= self.max_pages:
+                        return
+                    for link in page_info.get("links", []):
+                        if (
+                            len(self.visited_urls) < self.max_pages
+                            and link not in self.visited_urls
+                            and link not in submitted_links
+                        ):
+                            submitted_links.add(link)
+                            futures.add(
+                                executor.submit(
+                                    self._crawl_page,
+                                    link,
+                                    page_info.get("depth", 0) + 1,
+                                )
+                            )

webscout 8.2.7__py3-none-any.whl → 8.2.9__py3-none-any.whl

webscout 8.2.7py3-none-any.whl → 8.2.9py3-none-any.whl