wet-mcp 1.2.0__tar.gz → 1.3.0__tar.gz

{wet_mcp-1.2.0 → wet_mcp-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wet-mcp
-Version: 1.2.0
+Version: 1.3.0
 Summary: Open-source MCP Server thay thế Tavily - Web search, extract, crawl với SearXNG
 Project-URL: Homepage, https://github.com/n24q02m/wet-mcp
 Project-URL: Repository, https://github.com/n24q02m/wet-mcp.git

{wet_mcp-1.2.0 → wet_mcp-1.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "wet-mcp"
-version = "1.2.0"
+version = "1.3.0"
 description = "Open-source MCP Server thay thế Tavily - Web search, extract, crawl với SearXNG"
 readme = "README.md"
 license = { text = "MIT" }

{wet_mcp-1.2.0 → wet_mcp-1.3.0}/src/wet_mcp/sources/crawler.py

@@ -28,16 +28,17 @@ async def extract(
     browser_config = BrowserConfig(
         headless=True,
         enable_stealth=stealth,
+        verbose=False,
     )
 
     results = []
 
-    async with AsyncWebCrawler(config=browser_config) as crawler:
+    async with AsyncWebCrawler(verbose=False, config=browser_config) as crawler:
         for url in urls:
             try:
                 result = await crawler.arun(
                     url,
-                    config=CrawlerRunConfig(),
+                    config=CrawlerRunConfig(verbose=False),
                 )
 
                 if result.success:
@@ -102,12 +103,13 @@ async def crawl(
     browser_config = BrowserConfig(
         headless=True,
         enable_stealth=stealth,
+        verbose=False,
     )
 
     all_results = []
     visited = set()
 
-    async with AsyncWebCrawler(config=browser_config) as crawler:
+    async with AsyncWebCrawler(verbose=False, config=browser_config) as crawler:
         for root_url in urls:
             to_crawl = [(root_url, 0)]
 
@@ -122,7 +124,7 @@ async def crawl(
                 try:
                     result = await crawler.arun(
                         url,
-                        config=CrawlerRunConfig(),
+                        config=CrawlerRunConfig(verbose=False),
                     )
 
                     if result.success:
@@ -179,12 +181,12 @@ async def sitemap(
 
     logger.info(f"Mapping {len(urls)} URLs")
 
-    browser_config = BrowserConfig(headless=True)
+    browser_config = BrowserConfig(headless=True, verbose=False)
 
     all_urls = []
     visited = set()
 
-    async with AsyncWebCrawler(config=browser_config) as crawler:
+    async with AsyncWebCrawler(verbose=False, config=browser_config) as crawler:
         for root_url in urls:
             to_visit = [(root_url, 0)]
             site_urls = []
@@ -201,7 +203,7 @@ async def sitemap(
                 try:
                     result = await crawler.arun(
                         url,
-                        config=CrawlerRunConfig(),
+                        config=CrawlerRunConfig(verbose=False),
                     )
 
                     if result.success and current_depth < depth:
@@ -237,12 +239,12 @@ async def list_media(
 
     logger.info(f"Listing media from: {url}")
 
-    browser_config = BrowserConfig(headless=True)
+    browser_config = BrowserConfig(headless=True, verbose=False)
 
-    async with AsyncWebCrawler(config=browser_config) as crawler:
+    async with AsyncWebCrawler(verbose=False, config=browser_config) as crawler:
         result = await crawler.arun(
             url,
-            config=CrawlerRunConfig(),
+            config=CrawlerRunConfig(verbose=False),
         )
 
         if not result.success:
@@ -286,10 +288,21 @@ async def download_media(
 
     results = []
 
-    async with httpx.AsyncClient(timeout=60) as client:
+    transport = httpx.AsyncHTTPTransport(retries=3)
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+    }
+
+    async with httpx.AsyncClient(
+        timeout=60, transport=transport, headers=headers
+    ) as client:
         for url in media_urls:
             try:
-                response = await client.get(url)
+                # Handle protocol-relative URLs
+                if url.startswith("//"):
+                    url = f"https:{url}"
+
+                response = await client.get(url, follow_redirects=True)
                 response.raise_for_status()
 
                 filename = url.split("/")[-1].split("?")[0] or "download"