intentkit 0.6.0.dev8__py3-none-any.whl → 0.6.0.dev9__py3-none-any.whl

intentkit/__init__.py

@@ -3,7 +3,7 @@
 A powerful platform for building AI agents with blockchain and cryptocurrency capabilities.
 """
 
-__version__ = "0.6.0-dev.8"
+__version__ = "0.6.0-dev.9"
 __author__ = "hyacinthus"
 __email__ = "hyacinthus@gmail.com"
 

intentkit/skills/web_scraper/scrape_and_index.py

@@ -23,7 +23,7 @@ class ScrapeAndIndexInput(BaseModel):
     urls: List[str] = Field(
         description="List of URLs to scrape and index. Each URL should be a valid web address starting with http:// or https://",
         min_items=1,
-        max_items=10,
+        max_items=25,
     )
     chunk_size: int = Field(
         description="Size of text chunks for indexing (default: 1000)",

intentkit/skills/web_scraper/utils.py

@@ -29,6 +29,31 @@ DEFAULT_REQUESTS_PER_SECOND = 2
 MAX_CONTENT_SIZE_MB = 10  # 10 MB limit
 MAX_CONTENT_SIZE_BYTES = MAX_CONTENT_SIZE_MB * 1024 * 1024
 
+# HTTP Headers to bypass Cloudflare and other bot protection
+DEFAULT_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Accept-Encoding": "gzip, deflate, br",
+    "DNT": "1",
+    "Connection": "keep-alive",
+    "Upgrade-Insecure-Requests": "1",
+    "Sec-Fetch-Dest": "document",
+    "Sec-Fetch-Mode": "navigate",
+    "Sec-Fetch-Site": "none",
+    "Sec-Fetch-User": "?1",
+    "Cache-Control": "max-age=0",
+}
+
+# Alternative headers for fallback when primary headers fail
+FALLBACK_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Accept": "*/*",
+    "Accept-Language": "en-US,en;q=0.5",
+    "Accept-Encoding": "gzip, deflate",
+    "Connection": "keep-alive",
+}
+
 # Storage keys
 VECTOR_STORE_KEY_PREFIX = "vector_store"
 METADATA_KEY_PREFIX = "indexed_urls"
@@ -523,20 +548,38 @@ async def scrape_and_index_urls(
         try:
             logger.info(f"[{agent_id}] Processing URL {i + 1}/{len(valid_urls)}: {url}")
 
-            # Load single URL
+            # Load single URL with enhanced headers
             loader = WebBaseLoader(
                 web_paths=[url],
                 requests_per_second=requests_per_second,
             )
 
-            # Configure loader
+            # Configure loader with enhanced headers to bypass bot protection
             loader.requests_kwargs = {
                 "verify": True,
                 "timeout": DEFAULT_REQUEST_TIMEOUT,
+                "headers": DEFAULT_HEADERS,
             }
 
-            # Scrape the URL
-            documents = await asyncio.to_thread(loader.load)
+            # Scrape the URL with retry logic
+            documents = None
+            try:
+                documents = await asyncio.to_thread(loader.load)
+            except Exception as primary_error:
+                # If primary headers fail, try fallback headers
+                logger.warning(
+                    f"[{agent_id}] Primary headers failed for {url}, trying fallback: {primary_error}"
+                )
+
+                loader.requests_kwargs["headers"] = FALLBACK_HEADERS
+                try:
+                    documents = await asyncio.to_thread(loader.load)
+                    logger.info(f"[{agent_id}] Fallback headers succeeded for {url}")
+                except Exception as fallback_error:
+                    logger.error(
+                        f"[{agent_id}] Both header sets failed for {url}: {fallback_error}"
+                    )
+                    raise fallback_error
 
             if not documents:
                 logger.warning(f"[{agent_id}] No content extracted from {url}")

intentkit/skills/web_scraper/website_indexer.py

@@ -81,7 +81,22 @@ class WebsiteIndexer(WebScraperBaseTool):
         """Fetch robots.txt content."""
         robots_url = urljoin(base_url, "/robots.txt")
 
-        async with httpx.AsyncClient(timeout=30) as client:
+        # Import headers from utils
+        from intentkit.skills.web_scraper.utils import DEFAULT_HEADERS, FALLBACK_HEADERS
+
+        # Try with primary headers first
+        async with httpx.AsyncClient(timeout=30, headers=DEFAULT_HEADERS) as client:
+            try:
+                response = await client.get(robots_url)
+                if response.status_code == 200:
+                    return response.text
+            except Exception as e:
+                logger.warning(
+                    f"Primary headers failed for robots.txt from {robots_url}: {e}"
+                )
+
+        # Try with fallback headers
+        async with httpx.AsyncClient(timeout=30, headers=FALLBACK_HEADERS) as client:
             try:
                 response = await client.get(robots_url)
                 if response.status_code == 200:
@@ -119,7 +134,22 @@ class WebsiteIndexer(WebScraperBaseTool):
 
     async def _fetch_sitemap_content(self, sitemap_url: str) -> str:
         """Fetch sitemap XML content."""
-        async with httpx.AsyncClient(timeout=30) as client:
+        # Import headers from utils
+        from intentkit.skills.web_scraper.utils import DEFAULT_HEADERS, FALLBACK_HEADERS
+
+        # Try with primary headers first
+        async with httpx.AsyncClient(timeout=30, headers=DEFAULT_HEADERS) as client:
+            try:
+                response = await client.get(sitemap_url)
+                if response.status_code == 200:
+                    return response.text
+            except Exception as e:
+                logger.warning(
+                    f"Primary headers failed for sitemap from {sitemap_url}: {e}"
+                )
+
+        # Try with fallback headers
+        async with httpx.AsyncClient(timeout=30, headers=FALLBACK_HEADERS) as client:
             try:
                 response = await client.get(sitemap_url)
                 if response.status_code == 200:

{intentkit-0.6.0.dev8.dist-info → intentkit-0.6.0.dev9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: intentkit
-Version: 0.6.0.dev8
+Version: 0.6.0.dev9
 Summary: Intent-based AI Agent Platform - Core Package
 Project-URL: Homepage, https://github.com/crestal-network/intentkit
 Project-URL: Repository, https://github.com/crestal-network/intentkit

{intentkit-0.6.0.dev8.dist-info → intentkit-0.6.0.dev9.dist-info}/RECORD

@@ -1,4 +1,4 @@
-intentkit/__init__.py,sha256=pGwXI_gV0Ck_IbiHjG9Pu7X2pcEipglnkuAd64veXhk,384
+intentkit/__init__.py,sha256=36psS49qk6ZnE5Y1kAewHaeiYnMBG5a6ukVF9pPQmt8,384
 intentkit/abstracts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 intentkit/abstracts/agent.py,sha256=108gb5W8Q1Sy4G55F2_ZFv2-_CnY76qrBtpIr0Oxxqk,1489
 intentkit/abstracts/api.py,sha256=ZUc24vaQvQVbbjznx7bV0lbbQxdQPfEV8ZxM2R6wZWo,166
@@ -350,9 +350,9 @@ intentkit/skills/web_scraper/base.py,sha256=R3P1H8PaJ-xk3wpEAWGtDvPVxdRpCXeKoLGk
 intentkit/skills/web_scraper/document_indexer.py,sha256=sZyz2WoXPc3g3SNMq4Fxe_U2q4sc9jcVwn9ZHUoVTtY,4860
 intentkit/skills/web_scraper/langchain.png,sha256=G_FHqaY0wmbJlqvcK_ndDcRfw42MDyGcdxwN7v4CsGE,9204
 intentkit/skills/web_scraper/schema.json,sha256=gSXihGJh8uG7b2Xbj8jZUYKP-_uMckB2n3Jx8NpZuZU,4514
-intentkit/skills/web_scraper/scrape_and_index.py,sha256=zaePi0EiHXkZx2d7AxFsZSQL3QfoUDizRRXfkbg9-O8,9903
-intentkit/skills/web_scraper/utils.py,sha256=pf5JtMu_ZdFc0XQxHSnyPcpwyLFjno88Mcv415yFnn4,21409
-intentkit/skills/web_scraper/website_indexer.py,sha256=T2Vi6VywICeqKP4qDiYCGeVdIsA8SNSX8pHTv85YwxI,16771
+intentkit/skills/web_scraper/scrape_and_index.py,sha256=j8hi9xEedNt8aAhlqN8HL75hjranCIkgPjoVHg-cfEs,9903
+intentkit/skills/web_scraper/utils.py,sha256=feGBTMWqpkoY7RFy2xDHVs5y8c2h8-XZ111jRo6cC3k,23349
+intentkit/skills/web_scraper/website_indexer.py,sha256=UmpOJ3BOJidzubOXtFbCtbEmoDTSNg_5VCVQ6l25Xis,18062
 intentkit/utils/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 intentkit/utils/chain.py,sha256=3GBHuAbXxQr_HlOvkbB2kruYSkweucfxI5u-swXzY40,15135
 intentkit/utils/error.py,sha256=JxnzDdKjwZX6Pa-bt_qaibcrWAeF6QAoHsu2uYPmqu4,4483
@@ -362,7 +362,7 @@ intentkit/utils/random.py,sha256=DymMxu9g0kuQLgJUqalvgksnIeLdS-v0aRk5nQU0mLI,452
 intentkit/utils/s3.py,sha256=9trQNkKQ5VgxWsewVsV8Y0q_pXzGRvsCYP8xauyUYkg,8549
 intentkit/utils/slack_alert.py,sha256=s7UpRgyzLW7Pbmt8cKzTJgMA9bm4EP-1rQ5KXayHu6E,2264
 intentkit/utils/tx.py,sha256=2yLLGuhvfBEY5n_GJ8wmIWLCzn0FsYKv5kRNzw_sLUI,1454
-intentkit-0.6.0.dev8.dist-info/METADATA,sha256=iHC3s23ay4ipDf_BjcHaR7kH60EmZqajtR3ruG2kDZk,7285
-intentkit-0.6.0.dev8.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-intentkit-0.6.0.dev8.dist-info/licenses/LICENSE,sha256=Bln6DhK-LtcO4aXy-PBcdZv2f24MlJFm_qn222biJtE,1071
-intentkit-0.6.0.dev8.dist-info/RECORD,,
+intentkit-0.6.0.dev9.dist-info/METADATA,sha256=5xhzxy_Yw4pFLa4NgEnMLs6-DY5N8hyOind6QCLOfX4,7285
+intentkit-0.6.0.dev9.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+intentkit-0.6.0.dev9.dist-info/licenses/LICENSE,sha256=Bln6DhK-LtcO4aXy-PBcdZv2f24MlJFm_qn222biJtE,1071
+intentkit-0.6.0.dev9.dist-info/RECORD,,