intentkit 0.6.0.dev8__tar.gz → 0.6.0.dev10__tar.gz

{intentkit-0.6.0.dev8 → intentkit-0.6.0.dev10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: intentkit
-Version: 0.6.0.dev8
+Version: 0.6.0.dev10
 Summary: Intent-based AI Agent Platform - Core Package
 Project-URL: Homepage, https://github.com/crestal-network/intentkit
 Project-URL: Repository, https://github.com/crestal-network/intentkit

{intentkit-0.6.0.dev8 → intentkit-0.6.0.dev10}/__init__.py

@@ -3,7 +3,7 @@
 A powerful platform for building AI agents with blockchain and cryptocurrency capabilities.
 """
 
-__version__ = "0.6.0-dev.8"
+__version__ = "0.6.0-dev.10"
 __author__ = "hyacinthus"
 __email__ = "hyacinthus@gmail.com"
 

{intentkit-0.6.0.dev8 → intentkit-0.6.0.dev10}/clients/cdp.py

@@ -1,6 +1,5 @@
 import json
 import logging
-import os
 from typing import Dict, Optional
 
 from cdp import EvmServerAccount
@@ -32,22 +31,10 @@ class CdpClient:
 
         logger = logging.getLogger(__name__)
 
-        # Collect credentials from system-config first, then env vars as fallback
-        api_key_id = (
-            self._skill_store.get_system_config("cdp_api_key_name")
-            or os.getenv("CDP_API_KEY_ID")
-            or os.getenv("CDP_API_KEY_NAME")
-        )
-
-        api_key_secret = (
-            self._skill_store.get_system_config("cdp_api_key_private_key")
-            or os.getenv("CDP_API_KEY_SECRET")
-            or os.getenv("CDP_API_KEY_PRIVATE_KEY")
-        )
-
-        wallet_secret = self._skill_store.get_system_config(
-            "cdp_wallet_secret"
-        ) or os.getenv("CDP_WALLET_SECRET")
+        # Get credentials from skill store system config
+        api_key_id = self._skill_store.get_system_config("cdp_api_key_id")
+        api_key_secret = self._skill_store.get_system_config("cdp_api_key_secret")
+        wallet_secret = self._skill_store.get_system_config("cdp_wallet_secret")
 
         address = None
 

{intentkit-0.6.0.dev8 → intentkit-0.6.0.dev10}/config/config.py

@@ -83,9 +83,10 @@ class Config:
         self.payment_enabled = self.load("PAYMENT_ENABLED", "false") == "true"
         # Open API for agent
         self.open_api_base_url = self.load("OPEN_API_BASE_URL", "http://localhost:8000")
-        # CDP
-        self.cdp_api_key_name = self.load("CDP_API_KEY_NAME")
-        self.cdp_api_key_private_key = self.load("CDP_API_KEY_PRIVATE_KEY")
+        # CDP - AgentKit 0.6.0 Configuration
+        self.cdp_api_key_id = self.load("CDP_API_KEY_ID")
+        self.cdp_api_key_secret = self.load("CDP_API_KEY_SECRET")
+        self.cdp_wallet_secret = self.load("CDP_WALLET_SECRET")
         # LLM providers
         self.openai_api_key = self.load("OPENAI_API_KEY")
         self.deepseek_api_key = self.load("DEEPSEEK_API_KEY")
@@ -149,6 +150,7 @@ class Config:
         # Now we know the env, set up logging
         setup_logging(self.env, self.debug)
         logger.info("config loaded")
+
         # If the slack alert token exists, init it
         if self.slack_alert_token and self.slack_alert_channel:
             init_slack(self.slack_alert_token, self.slack_alert_channel)

{intentkit-0.6.0.dev8 → intentkit-0.6.0.dev10}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "intentkit"
-version = "0.6.0-dev.8"
+version = "0.6.0-dev.10"
 description = "Intent-based AI Agent Platform - Core Package"
 authors = [{ name = "hyacinthus", email = "hyacinthus@gmail.com" }]
 requires-python = ">=3.12"

{intentkit-0.6.0.dev8 → intentkit-0.6.0.dev10}/skills/web_scraper/scrape_and_index.py

@@ -23,7 +23,7 @@ class ScrapeAndIndexInput(BaseModel):
     urls: List[str] = Field(
         description="List of URLs to scrape and index. Each URL should be a valid web address starting with http:// or https://",
         min_items=1,
-        max_items=10,
+        max_items=25,
     )
     chunk_size: int = Field(
         description="Size of text chunks for indexing (default: 1000)",

{intentkit-0.6.0.dev8 → intentkit-0.6.0.dev10}/skills/web_scraper/utils.py

@@ -29,6 +29,31 @@ DEFAULT_REQUESTS_PER_SECOND = 2
 MAX_CONTENT_SIZE_MB = 10  # 10 MB limit
 MAX_CONTENT_SIZE_BYTES = MAX_CONTENT_SIZE_MB * 1024 * 1024
 
+# HTTP Headers to bypass Cloudflare and other bot protection
+DEFAULT_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Accept-Encoding": "gzip, deflate, br",
+    "DNT": "1",
+    "Connection": "keep-alive",
+    "Upgrade-Insecure-Requests": "1",
+    "Sec-Fetch-Dest": "document",
+    "Sec-Fetch-Mode": "navigate",
+    "Sec-Fetch-Site": "none",
+    "Sec-Fetch-User": "?1",
+    "Cache-Control": "max-age=0",
+}
+
+# Alternative headers for fallback when primary headers fail
+FALLBACK_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Accept": "*/*",
+    "Accept-Language": "en-US,en;q=0.5",
+    "Accept-Encoding": "gzip, deflate",
+    "Connection": "keep-alive",
+}
+
 # Storage keys
 VECTOR_STORE_KEY_PREFIX = "vector_store"
 METADATA_KEY_PREFIX = "indexed_urls"
@@ -523,20 +548,38 @@ async def scrape_and_index_urls(
         try:
             logger.info(f"[{agent_id}] Processing URL {i + 1}/{len(valid_urls)}: {url}")
 
-            # Load single URL
+            # Load single URL with enhanced headers
             loader = WebBaseLoader(
                 web_paths=[url],
                 requests_per_second=requests_per_second,
             )
 
-            # Configure loader
+            # Configure loader with enhanced headers to bypass bot protection
             loader.requests_kwargs = {
                 "verify": True,
                 "timeout": DEFAULT_REQUEST_TIMEOUT,
+                "headers": DEFAULT_HEADERS,
             }
 
-            # Scrape the URL
-            documents = await asyncio.to_thread(loader.load)
+            # Scrape the URL with retry logic
+            documents = None
+            try:
+                documents = await asyncio.to_thread(loader.load)
+            except Exception as primary_error:
+                # If primary headers fail, try fallback headers
+                logger.warning(
+                    f"[{agent_id}] Primary headers failed for {url}, trying fallback: {primary_error}"
+                )
+
+                loader.requests_kwargs["headers"] = FALLBACK_HEADERS
+                try:
+                    documents = await asyncio.to_thread(loader.load)
+                    logger.info(f"[{agent_id}] Fallback headers succeeded for {url}")
+                except Exception as fallback_error:
+                    logger.error(
+                        f"[{agent_id}] Both header sets failed for {url}: {fallback_error}"
+                    )
+                    raise fallback_error
 
             if not documents:
                 logger.warning(f"[{agent_id}] No content extracted from {url}")

{intentkit-0.6.0.dev8 → intentkit-0.6.0.dev10}/skills/web_scraper/website_indexer.py

@@ -81,7 +81,22 @@ class WebsiteIndexer(WebScraperBaseTool):
         """Fetch robots.txt content."""
         robots_url = urljoin(base_url, "/robots.txt")
 
-        async with httpx.AsyncClient(timeout=30) as client:
+        # Import headers from utils
+        from intentkit.skills.web_scraper.utils import DEFAULT_HEADERS, FALLBACK_HEADERS
+
+        # Try with primary headers first
+        async with httpx.AsyncClient(timeout=30, headers=DEFAULT_HEADERS) as client:
+            try:
+                response = await client.get(robots_url)
+                if response.status_code == 200:
+                    return response.text
+            except Exception as e:
+                logger.warning(
+                    f"Primary headers failed for robots.txt from {robots_url}: {e}"
+                )
+
+        # Try with fallback headers
+        async with httpx.AsyncClient(timeout=30, headers=FALLBACK_HEADERS) as client:
             try:
                 response = await client.get(robots_url)
                 if response.status_code == 200:
@@ -119,7 +134,22 @@ class WebsiteIndexer(WebScraperBaseTool):
 
     async def _fetch_sitemap_content(self, sitemap_url: str) -> str:
         """Fetch sitemap XML content."""
-        async with httpx.AsyncClient(timeout=30) as client:
+        # Import headers from utils
+        from intentkit.skills.web_scraper.utils import DEFAULT_HEADERS, FALLBACK_HEADERS
+
+        # Try with primary headers first
+        async with httpx.AsyncClient(timeout=30, headers=DEFAULT_HEADERS) as client:
+            try:
+                response = await client.get(sitemap_url)
+                if response.status_code == 200:
+                    return response.text
+            except Exception as e:
+                logger.warning(
+                    f"Primary headers failed for sitemap from {sitemap_url}: {e}"
+                )
+
+        # Try with fallback headers
+        async with httpx.AsyncClient(timeout=30, headers=FALLBACK_HEADERS) as client:
             try:
                 response = await client.get(sitemap_url)
                 if response.status_code == 200: