intentkit 0.8.6.dev2__py3-none-any.whl → 0.8.17__py3-none-any.whl

intentkit/skills/web_scraper/utils.py

@@ -10,14 +10,14 @@ import base64
 import logging
 import os
 import tempfile
-from typing import Dict, List, Optional, Tuple
 
 from langchain_community.vectorstores import FAISS
 from langchain_core.documents import Document
 from langchain_openai import OpenAIEmbeddings
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 
-from intentkit.abstracts.skill import SkillStoreABC
+from intentkit.config.config import config
+from intentkit.models.skill import AgentSkillData, AgentSkillDataCreate
 
 logger = logging.getLogger(__name__)
 
@@ -62,21 +62,29 @@ METADATA_KEY_PREFIX = "indexed_urls"
 class VectorStoreManager:
     """Manages vector store operations including creation, saving, loading, and merging."""
 
-    def __init__(self, skill_store: SkillStoreABC):
-        self.skill_store = skill_store
+    def __init__(self, embedding_api_key: str | None = None):
+        self._embedding_api_key = embedding_api_key
+
+    def _resolve_api_key(self) -> str:
+        """Resolve the OpenAI API key to use for embeddings."""
+        if self._embedding_api_key:
+            return self._embedding_api_key
+        if config.openai_api_key:
+            return config.openai_api_key
+        raise ValueError("OpenAI API key is not configured")
 
     def create_embeddings(self) -> OpenAIEmbeddings:
-        """Create OpenAI embeddings using system API key."""
-        api_key = self.skill_store.get_system_config("openai_api_key")
+        """Create OpenAI embeddings using the resolved API key."""
+        api_key = self._resolve_api_key()
         return OpenAIEmbeddings(api_key=api_key)
 
-    def get_storage_keys(self, agent_id: str) -> Tuple[str, str]:
+    def get_storage_keys(self, agent_id: str) -> tuple[str, str]:
         """Get storage keys for vector store and metadata."""
         vector_store_key = f"{VECTOR_STORE_KEY_PREFIX}_{agent_id}"
         metadata_key = f"{METADATA_KEY_PREFIX}_{agent_id}"
         return vector_store_key, metadata_key
 
-    def encode_vector_store(self, vector_store: FAISS) -> Dict[str, str]:
+    def encode_vector_store(self, vector_store: FAISS) -> dict[str, str]:
         """Encode FAISS vector store to base64 for storage."""
         with tempfile.TemporaryDirectory() as temp_dir:
             vector_store.save_local(temp_dir)
@@ -93,7 +101,7 @@ class VectorStoreManager:
             return encoded_files
 
     def decode_vector_store(
-        self, encoded_files: Dict[str, str], embeddings: OpenAIEmbeddings
+        self, encoded_files: dict[str, str], embeddings: OpenAIEmbeddings
     ) -> FAISS:
         """Decode base64 files back to FAISS vector store."""
         with tempfile.TemporaryDirectory() as temp_dir:
@@ -110,20 +118,18 @@ class VectorStoreManager:
                 allow_dangerous_deserialization=True,
             )
 
-    async def get_existing_vector_store(self, agent_id: str) -> Optional[Dict]:
+    async def get_existing_vector_store(self, agent_id: str) -> dict | None:
         """Get existing vector store data if it exists."""
         vector_store_key, _ = self.get_storage_keys(agent_id)
-        return await self.skill_store.get_agent_skill_data(
-            agent_id, "web_scraper", vector_store_key
-        )
+        return await AgentSkillData.get(agent_id, "web_scraper", vector_store_key)
 
     async def merge_with_existing(
         self,
-        new_documents: List[Document],
+        new_documents: list[Document],
         agent_id: str,
         chunk_size: int = DEFAULT_CHUNK_SIZE,
         chunk_overlap: int = DEFAULT_CHUNK_OVERLAP,
-    ) -> Tuple[FAISS, bool]:
+    ) -> tuple[FAISS, bool]:
         """
         Merge new documents with existing vector store or create new one.
 
@@ -184,12 +190,13 @@ class VectorStoreManager:
 
         try:
             # Save to storage
-            await self.skill_store.save_agent_skill_data(
+            skill_data = AgentSkillDataCreate(
                 agent_id=agent_id,
                 skill="web_scraper",
                 key=vector_store_key,
                 data=storage_data,
             )
+            await skill_data.save()
 
             logger.info(f"[{agent_id}] Successfully saved vector store")
 
@@ -197,7 +204,7 @@ class VectorStoreManager:
             logger.error(f"[{agent_id}] Failed to save vector store: {e}")
             raise
 
-    async def load_vector_store(self, agent_id: str) -> Optional[FAISS]:
+    async def load_vector_store(self, agent_id: str) -> FAISS | None:
         """Load vector store for an agent."""
         stored_data = await self.get_existing_vector_store(agent_id)
 
@@ -226,7 +233,8 @@ class VectorStoreManager:
 
         return total_size
 
-    def format_size(self, size_bytes: int) -> str:
+    @staticmethod
+    def format_size(size_bytes: int) -> str:
         """Format size in bytes to human readable format."""
         if size_bytes < 1024:
             return f"{size_bytes} B"
@@ -241,10 +249,10 @@ class DocumentProcessor:
 
     @staticmethod
     def create_chunks(
-        documents: List[Document],
+        documents: list[Document],
         chunk_size: int = DEFAULT_CHUNK_SIZE,
         chunk_overlap: int = DEFAULT_CHUNK_OVERLAP,
-    ) -> List[Document]:
+    ) -> list[Document]:
         """Split documents into chunks."""
         text_splitter = RecursiveCharacterTextSplitter(
             chunk_size=chunk_size,
@@ -283,7 +291,7 @@ class DocumentProcessor:
         title: str,
         source: str,
         tags: str = "",
-        extra_metadata: Optional[Dict] = None,
+        extra_metadata: dict | None = None,
     ) -> Document:
         """Create a Document with standardized metadata."""
         cleaned_content = DocumentProcessor.clean_text(content)
@@ -312,27 +320,21 @@ class DocumentProcessor:
 class MetadataManager:
     """Manages metadata for indexed content."""
 
-    def __init__(self, skill_store: SkillStoreABC):
-        self.skill_store = skill_store
+    def __init__(self, vector_manager: VectorStoreManager):
+        self._vector_manager = vector_manager
 
-    async def get_existing_metadata(self, agent_id: str) -> Dict:
+    async def get_existing_metadata(self, agent_id: str) -> dict:
         """Get existing metadata for an agent."""
-        vs_manager = VectorStoreManager(self.skill_store)
-        _, metadata_key = vs_manager.get_storage_keys(agent_id)
-        return (
-            await self.skill_store.get_agent_skill_data(
-                agent_id, "web_scraper", metadata_key
-            )
-            or {}
-        )
+        _, metadata_key = self._vector_manager.get_storage_keys(agent_id)
+        return await AgentSkillData.get(agent_id, "web_scraper", metadata_key) or {}
 
     def create_url_metadata(
         self,
-        urls: List[str],
-        split_docs: List[Document],
+        urls: list[str],
+        split_docs: list[Document],
         source_type: str = "web_scraper",
-        extra_fields: Optional[Dict] = None,
-    ) -> Dict:
+        extra_fields: dict | None = None,
+    ) -> dict:
         """Create metadata for a list of URLs."""
         metadata = {}
         current_time = str(asyncio.get_event_loop().time())
@@ -358,9 +360,9 @@ class MetadataManager:
         title: str,
         source: str,
         tags: str,
-        split_docs: List[Document],
+        split_docs: list[Document],
         document_length: int,
-    ) -> Dict:
+    ) -> dict:
         """Create metadata for a document."""
         # Generate unique key
         key = f"document_{title.lower().replace(' ', '_')}"
@@ -379,10 +381,9 @@ class MetadataManager:
             }
         }
 
-    async def update_metadata(self, agent_id: str, new_metadata: Dict) -> None:
+    async def update_metadata(self, agent_id: str, new_metadata: dict) -> None:
         """Update metadata for an agent."""
-        vs_manager = VectorStoreManager(self.skill_store)
-        _, metadata_key = vs_manager.get_storage_keys(agent_id)
+        _, metadata_key = self._vector_manager.get_storage_keys(agent_id)
 
         # Get existing metadata
         existing_metadata = await self.get_existing_metadata(agent_id)
@@ -391,12 +392,13 @@ class MetadataManager:
         existing_metadata.update(new_metadata)
 
         # Save updated metadata
-        await self.skill_store.save_agent_skill_data(
+        skill_data = AgentSkillDataCreate(
             agent_id=agent_id,
             skill="web_scraper",
             key=metadata_key,
             data=existing_metadata,
         )
+        await skill_data.save()
 
 
 class ResponseFormatter:
@@ -405,12 +407,12 @@ class ResponseFormatter:
     @staticmethod
     def format_indexing_response(
         operation_type: str,
-        urls_or_content: List[str] | str,
+        urls_or_content: list[str] | str,
         total_chunks: int,
         chunk_size: int,
         chunk_overlap: int,
         was_merged: bool,
-        extra_info: Optional[Dict] = None,
+        extra_info: dict | None = None,
         current_size_bytes: int = 0,
         size_limit_reached: bool = False,
         total_requested_urls: int = 0,
@@ -457,9 +459,8 @@ class ResponseFormatter:
 
         # Add size information
         if current_size_bytes > 0:
-            vs_manager = VectorStoreManager(None)  # Just for formatting
-            formatted_size = vs_manager.format_size(current_size_bytes)
-            max_size = vs_manager.format_size(MAX_CONTENT_SIZE_BYTES)
+            formatted_size = VectorStoreManager.format_size(current_size_bytes)
+            max_size = VectorStoreManager.format_size(MAX_CONTENT_SIZE_BYTES)
             response_parts.append(
                 f"Current storage size: {formatted_size} / {max_size}"
             )
@@ -479,20 +480,20 @@ class ResponseFormatter:
 
 
 async def scrape_and_index_urls(
-    urls: List[str],
+    urls: list[str],
     agent_id: str,
-    skill_store: SkillStoreABC,
+    vector_manager: VectorStoreManager,
     chunk_size: int = DEFAULT_CHUNK_SIZE,
     chunk_overlap: int = DEFAULT_CHUNK_OVERLAP,
     requests_per_second: int = DEFAULT_REQUESTS_PER_SECOND,
-) -> Tuple[int, bool, List[str]]:
+) -> tuple[int, bool, list[str]]:
     """
     Scrape URLs and index their content into vector store with size limits.
 
     Args:
         urls: List of URLs to scrape
         agent_id: Agent identifier for storage
-        skill_store: Skill store instance
+        vector_manager: Manager for vector store operations
         chunk_size: Size of text chunks
         chunk_overlap: Overlap between chunks
         requests_per_second: Rate limiting for requests
@@ -520,16 +521,15 @@ async def scrape_and_index_urls(
         return 0, False, []
 
     # Check existing content size
-    vs_manager = VectorStoreManager(skill_store)
-    current_size = await vs_manager.get_content_size(agent_id)
+    current_size = await vector_manager.get_content_size(agent_id)
 
     logger.info(
-        f"[{agent_id}] Current storage size: {vs_manager.format_size(current_size)}"
+        f"[{agent_id}] Current storage size: {VectorStoreManager.format_size(current_size)}"
     )
 
     if current_size >= MAX_CONTENT_SIZE_BYTES:
         logger.warning(
-            f"[{agent_id}] Storage limit already reached: {vs_manager.format_size(current_size)}"
+            f"[{agent_id}] Storage limit already reached: {VectorStoreManager.format_size(current_size)}"
         )
         return 0, False, []
 
@@ -599,7 +599,7 @@ async def scrape_and_index_urls(
 
             # Process and index this URL's content
             chunks, merged = await index_documents(
-                documents, agent_id, skill_store, chunk_size, chunk_overlap
+                documents, agent_id, vector_manager, chunk_size, chunk_overlap
             )
 
             if chunks > 0:
@@ -609,7 +609,7 @@ async def scrape_and_index_urls(
                 current_size += content_size
 
                 logger.info(
-                    f"[{agent_id}] Processed {url}: {chunks} chunks, current size: {vs_manager.format_size(current_size)}"
+                    f"[{agent_id}] Processed {url}: {chunks} chunks, current size: {VectorStoreManager.format_size(current_size)}"
                 )
 
             # Add delay for rate limiting
@@ -635,12 +635,12 @@ async def scrape_and_index_urls(
 
 # Convenience function that combines all operations
 async def index_documents(
-    documents: List[Document],
+    documents: list[Document],
     agent_id: str,
-    skill_store: SkillStoreABC,
+    vector_manager: VectorStoreManager,
     chunk_size: int = DEFAULT_CHUNK_SIZE,
     chunk_overlap: int = DEFAULT_CHUNK_OVERLAP,
-) -> Tuple[int, bool]:
+) -> tuple[int, bool]:
     """
     Complete document indexing workflow.
 
@@ -654,13 +654,12 @@ async def index_documents(
         raise ValueError("No content could be processed into chunks")
 
     # Handle vector store
-    vs_manager = VectorStoreManager(skill_store)
-    vector_store, was_merged = await vs_manager.merge_with_existing(
+    vector_store, was_merged = await vector_manager.merge_with_existing(
         split_docs, agent_id, chunk_size, chunk_overlap
     )
 
     # Save vector store
-    await vs_manager.save_vector_store(
+    await vector_manager.save_vector_store(
         vector_store, agent_id, chunk_size, chunk_overlap
     )
 

intentkit/skills/web_scraper/website_indexer.py

@@ -1,5 +1,4 @@
 import logging
-from typing import List, Type
 from urllib.parse import urljoin, urlparse
 
 import httpx
@@ -44,11 +43,11 @@ class WebsiteIndexerInput(BaseModel):
         ge=0,
         le=1000,
     )
-    include_patterns: List[str] = Field(
+    include_patterns: list[str] = Field(
         description="URL patterns to include (e.g., ['/blog/', '/docs/']). If empty, all URLs are included",
         default=[],
     )
-    exclude_patterns: List[str] = Field(
+    exclude_patterns: list[str] = Field(
         description="URL patterns to exclude (e.g., ['/admin/', '/private/'])",
         default=[],
     )
@@ -68,7 +67,7 @@ class WebsiteIndexer(WebScraperBaseTool):
         "This tool finds sitemaps from robots.txt, parses the XML content to extract URLs, "
         "and then uses the reliable scrape_and_index functionality for content indexing."
     )
-    args_schema: Type[BaseModel] = WebsiteIndexerInput
+    args_schema: type[BaseModel] = WebsiteIndexerInput
 
     def _normalize_url(self, url: str) -> str:
         """Normalize URL by ensuring it has a proper scheme."""
@@ -106,7 +105,7 @@ class WebsiteIndexer(WebScraperBaseTool):
 
     def _extract_sitemaps_from_robots(
         self, robots_content: str, base_url: str
-    ) -> List[str]:
+    ) -> list[str]:
         """Extract sitemap URLs from robots.txt content."""
         sitemaps = []
 
@@ -121,7 +120,7 @@ class WebsiteIndexer(WebScraperBaseTool):
 
         return sitemaps
 
-    def _get_common_sitemap_patterns(self, base_url: str) -> List[str]:
+    def _get_common_sitemap_patterns(self, base_url: str) -> list[str]:
         """Generate common sitemap URL patterns."""
         return [
             urljoin(base_url, "/sitemap.xml"),
@@ -157,7 +156,7 @@ class WebsiteIndexer(WebScraperBaseTool):
                 logger.warning(f"Could not fetch sitemap from {sitemap_url}: {e}")
         return ""
 
-    async def _get_all_sitemap_content(self, base_url: str) -> tuple[str, List[str]]:
+    async def _get_all_sitemap_content(self, base_url: str) -> tuple[str, list[str]]:
         """Get all sitemap content for AI analysis."""
         all_content = []
         found_sitemaps = []
@@ -200,7 +199,7 @@ class WebsiteIndexer(WebScraperBaseTool):
         return combined_xml, found_sitemaps
 
     def _create_ai_extraction_prompt(
-        self, sitemap_xml: str, include_patterns: List[str], exclude_patterns: List[str]
+        self, sitemap_xml: str, include_patterns: list[str], exclude_patterns: list[str]
     ) -> str:
         """Create a prompt for AI to extract URLs from sitemap XML."""
         filter_instructions = ""
@@ -225,7 +224,7 @@ INSTRUCTIONS:
 
 Extract the URLs now:"""
 
-    def _parse_ai_response(self, ai_response: str) -> List[str]:
+    def _parse_ai_response(self, ai_response: str) -> list[str]:
         """Parse AI response to extract clean URLs."""
         urls = []
 
@@ -250,10 +249,7 @@ Extract the URLs now:"""
         """Call OpenAI GPT-4o-mini to extract URLs from sitemap content."""
         try:
             # Get OpenAI API key using the standard pattern
-            from intentkit.skills.openai.base import OpenAIBaseTool
-
-            temp_tool = OpenAIBaseTool(skill_store=self.skill_store)
-            api_key = temp_tool.get_api_key()
+            api_key = self.get_openai_api_key()
 
             # Initialize OpenAI client
             client = openai.AsyncOpenAI(api_key=api_key)
@@ -284,8 +280,8 @@ Extract the URLs now:"""
         max_urls: int = 50,
         chunk_size: int = DEFAULT_CHUNK_SIZE,
         chunk_overlap: int = DEFAULT_CHUNK_OVERLAP,
-        include_patterns: List[str] = None,
-        exclude_patterns: List[str] = None,
+        include_patterns: list[str] = None,
+        exclude_patterns: list[str] = None,
         **kwargs,
     ) -> str:
         """Discover website sitemaps, extract URLs with AI, and delegate to scrape_and_index."""
@@ -385,9 +381,12 @@ Extract the URLs now:"""
                 f"[{agent_id}] Extracted {len(unique_urls)} URLs from sitemaps. Scraping and indexing..."
             )
 
+            embedding_api_key = self.get_openai_api_key()
+            vector_manager = VectorStoreManager(embedding_api_key)
+
             # Use the utility function to scrape and index URLs directly
             total_chunks, was_merged, valid_urls = await scrape_and_index_urls(
-                unique_urls, agent_id, self.skill_store, chunk_size, chunk_overlap
+                unique_urls, agent_id, vector_manager, chunk_size, chunk_overlap
             )
 
             if total_chunks == 0:
@@ -397,12 +396,11 @@ Extract the URLs now:"""
                 return f"Error: No content could be extracted from the discovered URLs. Found sitemaps: {', '.join(found_sitemaps)}"
 
             # Get current storage size for response
-            vs_manager = VectorStoreManager(self.skill_store)
-            current_size = await vs_manager.get_content_size(agent_id)
+            current_size = await vector_manager.get_content_size(agent_id)
             size_limit_reached = len(valid_urls) < len(unique_urls)
 
             # Update metadata
-            metadata_manager = MetadataManager(self.skill_store)
+            metadata_manager = MetadataManager(vector_manager)
             new_metadata = metadata_manager.create_url_metadata(
                 valid_urls, [], "website_indexer"
             )

intentkit/skills/weth/__init__.py

@@ -1,10 +1,10 @@
 """WETH AgentKit skills."""
 
-from typing import TYPE_CHECKING, Optional, TypedDict
+from typing import TypedDict
 
 from coinbase_agentkit import weth_action_provider
 
-from intentkit.abstracts.skill import SkillStoreABC
+from intentkit.models.agent import Agent
 from intentkit.skills.base import (
     SkillConfig,
     SkillState,
@@ -13,9 +13,6 @@ from intentkit.skills.base import (
 )
 from intentkit.skills.weth.base import WethBaseTool
 
-if TYPE_CHECKING:
-    from intentkit.models.agent import Agent
-
 
 class SkillStates(TypedDict):
     WethActionProvider_wrap_eth: SkillState
@@ -28,11 +25,10 @@ class Config(SkillConfig):
 
 
 async def get_skills(
-    config: "Config",
+    config: Config,
     is_private: bool,
-    store: SkillStoreABC,
     agent_id: str,
-    agent: Optional["Agent"] = None,
+    agent: Agent | None = None,
     **_,
 ) -> list[WethBaseTool]:
     """Get all WETH skills."""
@@ -44,9 +40,7 @@ async def get_skills(
         if state == "public" or (state == "private" and is_private):
             available_skills.append(skill_name)
 
-    actions = await get_agentkit_actions(
-        agent_id, store, [weth_action_provider], agent=agent
-    )
+    actions = await get_agentkit_actions(agent_id, [weth_action_provider], agent=agent)
     tools: list[WethBaseTool] = []
     for skill in available_skills:
         for action in actions:

intentkit/skills/wow/__init__.py

@@ -1,10 +1,10 @@
 """WOW AgentKit skills."""
 
-from typing import TYPE_CHECKING, Optional, TypedDict
+from typing import TypedDict
 
 from coinbase_agentkit import wow_action_provider
 
-from intentkit.abstracts.skill import SkillStoreABC
+from intentkit.models.agent import Agent
 from intentkit.skills.base import (
     SkillConfig,
     SkillState,
@@ -13,9 +13,6 @@ from intentkit.skills.base import (
 )
 from intentkit.skills.wow.base import WowBaseTool
 
-if TYPE_CHECKING:
-    from intentkit.models.agent import Agent
-
 
 class SkillStates(TypedDict):
     WowActionProvider_buy_token: SkillState
@@ -30,11 +27,10 @@ class Config(SkillConfig):
 
 
 async def get_skills(
-    config: "Config",
+    config: Config,
     is_private: bool,
-    store: SkillStoreABC,
     agent_id: str,
-    agent: Optional["Agent"] = None,
+    agent: Agent | None = None,
     **_,
 ) -> list[WowBaseTool]:
     """Get all WOW skills."""
@@ -46,9 +42,7 @@ async def get_skills(
         if state == "public" or (state == "private" and is_private):
             available_skills.append(skill_name)
 
-    actions = await get_agentkit_actions(
-        agent_id, store, [wow_action_provider], agent=agent
-    )
+    actions = await get_agentkit_actions(agent_id, [wow_action_provider], agent=agent)
     tools: list[WowBaseTool] = []
     for skill in available_skills:
         for action in actions:

intentkit/skills/x402/__init__.py

@@ -0,0 +1,61 @@
+"""x402 skill category."""
+
+import logging
+from typing import TypedDict
+
+from intentkit.skills.base import SkillConfig, SkillState
+from intentkit.skills.x402.ask_agent import X402AskAgent
+from intentkit.skills.x402.base import X402BaseSkill
+from intentkit.skills.x402.http_request import X402HttpRequest
+
+logger = logging.getLogger(__name__)
+
+_cache: dict[str, X402BaseSkill] = {}
+
+
+class SkillStates(TypedDict):
+    x402_ask_agent: SkillState
+    x402_http_request: SkillState
+
+
+class Config(SkillConfig):
+    """Configuration for x402 skills."""
+
+    states: SkillStates
+
+
+_SKILL_BUILDERS: dict[str, type[X402BaseSkill]] = {
+    "x402_ask_agent": X402AskAgent,
+    "x402_http_request": X402HttpRequest,
+}
+
+
+async def get_skills(
+    config: "Config",
+    is_private: bool,
+    **_,
+) -> list[X402BaseSkill]:
+    """Return enabled x402 skills for the agent."""
+    enabled_skills = []
+    for skill_name, state in config["states"].items():
+        if state == "disabled":
+            continue
+        if state == "public" or (state == "private" and is_private):
+            enabled_skills.append(skill_name)
+
+    result: list[X402BaseSkill] = []
+    for name in enabled_skills:
+        skill = _get_skill(name)
+        if skill:
+            result.append(skill)
+    return result
+
+
+def _get_skill(name: str) -> X402BaseSkill | None:
+    builder = _SKILL_BUILDERS.get(name)
+    if builder:
+        if name not in _cache:
+            _cache[name] = builder()
+        return _cache[name]
+    logger.warning("Unknown x402 skill requested: %s", name)
+    return None