intentkit 0.6.0.dev12__tar.gz → 0.6.0.dev13__tar.gz

{intentkit-0.6.0.dev12 → intentkit-0.6.0.dev13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: intentkit
-Version: 0.6.0.dev12
+Version: 0.6.0.dev13
 Summary: Intent-based AI Agent Platform - Core Package
 Project-URL: Homepage, https://github.com/crestal-network/intentkit
 Project-URL: Repository, https://github.com/crestal-network/intentkit

{intentkit-0.6.0.dev12 → intentkit-0.6.0.dev13}/__init__.py

@@ -3,7 +3,7 @@
 A powerful platform for building AI agents with blockchain and cryptocurrency capabilities.
 """
 
-__version__ = "0.6.0-dev.12"
+__version__ = "0.6.0-dev.13"
 __author__ = "hyacinthus"
 __email__ = "hyacinthus@gmail.com"
 

{intentkit-0.6.0.dev12 → intentkit-0.6.0.dev13}/abstracts/skill.py

@@ -91,6 +91,18 @@ class SkillStoreABC(ABC):
         """
         pass
 
+    @staticmethod
+    @abstractmethod
+    async def delete_agent_skill_data(agent_id: str, skill: str, key: str) -> None:
+        """Delete skill data for an agent.
+
+        Args:
+            agent_id: ID of the agent
+            skill: Name of the skill
+            key: Data key
+        """
+        pass
+
     @staticmethod
     @abstractmethod
     async def get_thread_skill_data(

{intentkit-0.6.0.dev12 → intentkit-0.6.0.dev13}/core/skill.py

@@ -78,6 +78,17 @@ class SkillStore(SkillStoreABC):
         )
         await skill_data.save()
 
+    @staticmethod
+    async def delete_agent_skill_data(agent_id: str, skill: str, key: str) -> None:
+        """Delete skill data for an agent.
+
+        Args:
+            agent_id: ID of the agent
+            skill: Name of the skill
+            key: Data key
+        """
+        await AgentSkillData.delete(agent_id, skill, key)
+
     @staticmethod
     async def get_thread_skill_data(
         thread_id: str, skill: str, key: str

{intentkit-0.6.0.dev12 → intentkit-0.6.0.dev13}/models/skill.py

@@ -172,6 +172,25 @@ class AgentSkillData(AgentSkillDataCreate):
             )
             return result.data if result else None
 
+    @classmethod
+    async def delete(cls, agent_id: str, skill: str, key: str) -> None:
+        """Delete skill data for an agent.
+
+        Args:
+            agent_id: ID of the agent
+            skill: Name of the skill
+            key: Data key
+        """
+        async with get_session() as db:
+            await db.execute(
+                delete(AgentSkillDataTable).where(
+                    AgentSkillDataTable.agent_id == agent_id,
+                    AgentSkillDataTable.skill == skill,
+                    AgentSkillDataTable.key == key,
+                )
+            )
+            await db.commit()
+
     @classmethod
     async def clean_data(cls, agent_id: str):
         """Clean all skill data for an agent.

{intentkit-0.6.0.dev12 → intentkit-0.6.0.dev13}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "intentkit"
-version = "0.6.0-dev.12"
+version = "0.6.0-dev.13"
 description = "Intent-based AI Agent Platform - Core Package"
 authors = [{ name = "hyacinthus", email = "hyacinthus@gmail.com" }]
 requires-python = ">=3.12"

{intentkit-0.6.0.dev12 → intentkit-0.6.0.dev13}/skills/firecrawl/README.md

@@ -37,6 +37,14 @@ Queries previously indexed Firecrawl content using semantic search.
 - `query` (required): The search query
 - `limit` (optional): Maximum number of results to return (1-10, default: 4)
 
+### 4. firecrawl_clear_indexed_content
+Clears all previously indexed Firecrawl content from the vector store.
+
+**Parameters:**
+- `confirm` (required): Must be set to true to confirm the deletion (default: false)
+
+**Note:** This action is permanent and cannot be undone. Use when you want to start fresh with new content.
+
 ## API Key Configuration
 Set your Firecrawl API key as an environment variable:
 ```bash
@@ -53,7 +61,8 @@ export FIRECRAWL_API_KEY=fc-your-api-key-here
      "skills": [
        "firecrawl_scrape",
        "firecrawl_crawl", 
-       "firecrawl_query_indexed_content"
+       "firecrawl_query_indexed_content",
+       "firecrawl_clear_indexed_content"
      ]
    }
    ```
@@ -120,6 +129,30 @@ Prompt: "Use firecrawl_query_indexed_content to search for 'democratize finance'
 - Results tagged with [Firecrawl Scrape] or [Firecrawl Crawl]
 - Source URLs and metadata included
 
+### Step 7: Test Clear Indexed Content
+
+**Test clearing all indexed content:**
+```
+Prompt: "Use firecrawl_clear_indexed_content with confirm=true to clear all indexed content"
+```
+
+**Expected Result:**
+- All indexed content removed from vector store
+- Confirmation message displayed
+- Subsequent queries return no results
+
+### Step 8: Test Re-indexing After Clear
+
+**Test that content can be re-indexed after clearing:**
+```
+Prompt: "Use firecrawl_scrape to scrape https://example.com and index the content"
+```
+
+**Expected Result:**
+- Content successfully scraped and indexed
+- Fresh vector store created
+- Content available for querying again
+
 ## Common Use Cases
 
 ### Documentation Indexing
@@ -175,4 +208,4 @@ Prompt: "Use firecrawl_query_indexed_content to search for 'democratize finance'
 - **Metadata Rich**: Includes source URLs, timestamps, and content types
 - **Semantic Search**: Uses OpenAI embeddings for intelligent querying
 - **Batch Processing**: Efficient handling of multiple pages
-- **Content Filtering**: Flexible include/exclude options for targeted scraping
+- **Content Filtering**: Flexible include/exclude options for targeted scraping

{intentkit-0.6.0.dev12 → intentkit-0.6.0.dev13}/skills/firecrawl/__init__.py

@@ -6,6 +6,7 @@ from typing import TypedDict
 from intentkit.abstracts.skill import SkillStoreABC
 from intentkit.skills.base import SkillConfig, SkillState
 from intentkit.skills.firecrawl.base import FirecrawlBaseTool
+from intentkit.skills.firecrawl.clear import FirecrawlClearIndexedContent
 from intentkit.skills.firecrawl.crawl import FirecrawlCrawl
 from intentkit.skills.firecrawl.query import FirecrawlQueryIndexedContent
 from intentkit.skills.firecrawl.scrape import FirecrawlScrape
@@ -20,6 +21,7 @@ class SkillStates(TypedDict):
     firecrawl_scrape: SkillState
     firecrawl_crawl: SkillState
     firecrawl_query_indexed_content: SkillState
+    firecrawl_clear_indexed_content: SkillState
 
 
 class Config(SkillConfig):
@@ -95,5 +97,11 @@ def get_firecrawl_skill(
                 skill_store=store,
             )
         return _cache[name]
+    elif name == "firecrawl_clear_indexed_content":
+        if name not in _cache:
+            _cache[name] = FirecrawlClearIndexedContent(
+                skill_store=store,
+            )
+        return _cache[name]
     else:
         raise ValueError(f"Unknown Firecrawl skill: {name}")

intentkit-0.6.0.dev13/skills/firecrawl/clear.py

@@ -0,0 +1,87 @@
+import logging
+from typing import Type
+
+from langchain_core.runnables import RunnableConfig
+from pydantic import BaseModel, Field
+
+from intentkit.skills.firecrawl.base import FirecrawlBaseTool
+
+logger = logging.getLogger(__name__)
+
+
+class FirecrawlClearInput(BaseModel):
+    """Input for Firecrawl clear tool."""
+
+    confirm: bool = Field(
+        description="Confirmation to clear all indexed content (must be true)",
+        default=False,
+    )
+
+
+class FirecrawlClearIndexedContent(FirecrawlBaseTool):
+    """Tool for clearing all indexed Firecrawl content.
+
+    This tool removes all previously indexed content from the Firecrawl vector store,
+    allowing for a fresh start with new content.
+    """
+
+    name: str = "firecrawl_clear_indexed_content"
+    description: str = (
+        "Clear all previously indexed Firecrawl content from the vector store.\n"
+        "This will permanently delete all indexed content and cannot be undone.\n"
+        "Use this tool when you want to start fresh with new content."
+    )
+    args_schema: Type[BaseModel] = FirecrawlClearInput
+
+    async def _arun(
+        self,
+        confirm: bool = False,
+        config: RunnableConfig = None,
+        **kwargs,
+    ) -> str:
+        """Clear all indexed Firecrawl content for the agent.
+
+        Args:
+            confirm: Must be True to confirm the deletion
+            config: The configuration for the tool call
+
+        Returns:
+            str: Confirmation message
+        """
+        context = self.context_from_config(config)
+        agent_id = context.agent.id if context and context.agent else None
+
+        if not agent_id:
+            return "Error: Agent ID not available for clearing content."
+
+        if not confirm:
+            return "Error: You must set confirm=true to clear all indexed content."
+
+        logger.info(
+            f"firecrawl_clear: Starting clear indexed content operation for agent {agent_id}"
+        )
+
+        try:
+            # Delete vector store data (using web_scraper storage format for compatibility)
+            vector_store_key = f"vector_store_{agent_id}"
+            await self.skill_store.delete_agent_skill_data(
+                agent_id, "web_scraper", vector_store_key
+            )
+
+            # Delete metadata
+            metadata_key = f"indexed_urls_{agent_id}"
+            await self.skill_store.delete_agent_skill_data(
+                agent_id, "web_scraper", metadata_key
+            )
+
+            logger.info(
+                f"firecrawl_clear: Successfully cleared all indexed content for agent {agent_id}"
+            )
+            return "Successfully cleared all Firecrawl indexed content. The vector store is now empty and ready for new content."
+
+        except Exception as e:
+            logger.error(
+                f"firecrawl_clear: Error clearing indexed content for agent {agent_id}: {e}",
+                exc_info=True,
+            )
+            return f"Error clearing indexed content: {str(e)}"

{intentkit-0.6.0.dev12 → intentkit-0.6.0.dev13}/skills/firecrawl/schema.json

@@ -68,6 +68,22 @@
           ],
           "description": "Query previously indexed Firecrawl content to find relevant information and answer questions. Use this to search through content that was scraped and indexed using Firecrawl tools.",
           "default": "private"
+        },
+        "firecrawl_clear_indexed_content": {
+          "type": "string",
+          "title": "Clear Indexed Content",
+          "enum": [
+            "disabled",
+            "public",
+            "private"
+          ],
+          "x-enum-title": [
+            "Disabled",
+            "Agent Owner + All Users",
+            "Agent Owner Only"
+          ],
+          "description": "Clear all previously indexed Firecrawl content from the vector store. This will permanently delete all indexed content and cannot be undone. Use this tool when you want to start fresh with new content.",
+          "default": "private"
         }
       },
       "description": "States for each Firecrawl skill (disabled, public, or private)"

{intentkit-0.6.0.dev12 → intentkit-0.6.0.dev13}/skills/firecrawl/utils.py

@@ -1,11 +1,9 @@
 """Utilities for Firecrawl skill content indexing and querying."""
 
 import logging
-import pickle
 import re
 from typing import Any, Dict, List, Optional, Tuple
 
-import faiss
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_community.vectorstores import FAISS
 from langchain_core.documents import Document
@@ -77,46 +75,52 @@ class FirecrawlVectorStoreManager:
             openai_api_key=openai_api_key, model="text-embedding-3-small"
         )
 
-    def encode_vector_store(self, vector_store: FAISS) -> Dict[str, bytes]:
-        """Encode FAISS vector store to bytes for storage."""
-        try:
-            # Serialize the index
-            index_bytes = faiss.serialize_index(vector_store.index)
-
-            # Serialize the docstore and index_to_docstore_id
-            docstore_bytes = pickle.dumps(vector_store.docstore)
-            index_to_docstore_bytes = pickle.dumps(vector_store.index_to_docstore_id)
+    def encode_vector_store(self, vector_store: FAISS) -> Dict[str, str]:
+        """Encode FAISS vector store to base64 for storage (compatible with web_scraper)."""
+        import base64
+        import os
+        import tempfile
 
-            return {
-                "index": index_bytes,
-                "docstore": docstore_bytes,
-                "index_to_docstore": index_to_docstore_bytes,
-            }
+        try:
+            with tempfile.TemporaryDirectory() as temp_dir:
+                vector_store.save_local(temp_dir)
+
+                encoded_files = {}
+                for filename in os.listdir(temp_dir):
+                    file_path = os.path.join(temp_dir, filename)
+                    if os.path.isfile(file_path):
+                        with open(file_path, "rb") as f:
+                            encoded_files[filename] = base64.b64encode(f.read()).decode(
+                                "utf-8"
+                            )
+
+                return encoded_files
         except Exception as e:
             logger.error(f"Error encoding vector store: {e}")
             raise
 
     def decode_vector_store(
-        self, faiss_data: Dict[str, bytes], embeddings: OpenAIEmbeddings
+        self, encoded_files: Dict[str, str], embeddings: OpenAIEmbeddings
     ) -> FAISS:
-        """Decode FAISS vector store from stored bytes."""
-        try:
-            # Deserialize the index
-            index = faiss.deserialize_index(faiss_data["index"])
-
-            # Deserialize the docstore and index_to_docstore_id
-            docstore = pickle.loads(faiss_data["docstore"])
-            index_to_docstore_id = pickle.loads(faiss_data["index_to_docstore"])
-
-            # Create FAISS vector store
-            vector_store = FAISS(
-                embedding_function=embeddings,
-                index=index,
-                docstore=docstore,
-                index_to_docstore_id=index_to_docstore_id,
-            )
+        """Decode base64 files back to FAISS vector store (compatible with web_scraper)."""
+        import base64
+        import os
+        import tempfile
 
-            return vector_store
+        try:
+            with tempfile.TemporaryDirectory() as temp_dir:
+                # Decode and write files
+                for filename, encoded_content in encoded_files.items():
+                    file_path = os.path.join(temp_dir, filename)
+                    with open(file_path, "wb") as f:
+                        f.write(base64.b64decode(encoded_content))
+
+                # Load vector store
+                return FAISS.load_local(
+                    temp_dir,
+                    embeddings,
+                    allow_dangerous_deserialization=True,
+                )
         except Exception as e:
             logger.error(f"Error decoding vector store: {e}")
             raise
@@ -124,9 +128,9 @@ class FirecrawlVectorStoreManager:
     async def load_vector_store(self, agent_id: str) -> Optional[FAISS]:
         """Load existing vector store for an agent."""
         try:
-            vector_store_key = f"firecrawl_vector_store_{agent_id}"
+            vector_store_key = f"vector_store_{agent_id}"
             stored_data = await self.skill_store.get_agent_skill_data(
-                agent_id, "firecrawl", vector_store_key
+                agent_id, "web_scraper", vector_store_key
             )
 
             if not stored_data or "faiss_files" not in stored_data:
@@ -139,14 +143,27 @@ class FirecrawlVectorStoreManager:
             logger.error(f"Error loading vector store for agent {agent_id}: {e}")
             return None
 
-    async def save_vector_store(self, agent_id: str, vector_store: FAISS) -> None:
-        """Save vector store for an agent."""
+    async def save_vector_store(
+        self,
+        agent_id: str,
+        vector_store: FAISS,
+        chunk_size: int = 1000,
+        chunk_overlap: int = 200,
+    ) -> None:
+        """Save vector store for an agent (compatible with web_scraper format)."""
         try:
-            vector_store_key = f"firecrawl_vector_store_{agent_id}"
-            encoded_data = self.encode_vector_store(vector_store)
+            vector_store_key = f"vector_store_{agent_id}"
+            encoded_files = self.encode_vector_store(vector_store)
+
+            # Use the same data structure as web_scraper
+            storage_data = {
+                "faiss_files": encoded_files,
+                "chunk_size": chunk_size,
+                "chunk_overlap": chunk_overlap,
+            }
 
             await self.skill_store.save_agent_skill_data(
-                agent_id, "firecrawl", vector_store_key, {"faiss_files": encoded_data}
+                agent_id, "web_scraper", vector_store_key, storage_data
             )
 
         except Exception as e:
@@ -176,9 +193,9 @@ class FirecrawlMetadataManager:
     ) -> None:
         """Update metadata for an agent."""
         try:
-            metadata_key = f"firecrawl_metadata_{agent_id}"
+            metadata_key = f"indexed_urls_{agent_id}"
             await self.skill_store.save_agent_skill_data(
-                agent_id, "firecrawl", metadata_key, new_metadata
+                agent_id, "web_scraper", metadata_key, new_metadata
             )
         except Exception as e:
             logger.error(f"Error updating metadata for agent {agent_id}: {e}")
@@ -235,7 +252,9 @@ async def index_documents(
             was_merged = False
 
         # Save the vector store
-        await vs_manager.save_vector_store(agent_id, vector_store)
+        await vs_manager.save_vector_store(
+            agent_id, vector_store, chunk_size, chunk_overlap
+        )
 
         logger.info(
             f"Successfully indexed {len(split_docs)} chunks for agent {agent_id}"