intentkit 0.6.13.dev2__py3-none-any.whl → 0.8.17__py3-none-any.whl

intentkit/skills/firecrawl/scrape.py

@@ -1,5 +1,4 @@
 import logging
-from typing import List, Optional, Type
 
 import httpx
 from langchain_core.documents import Document
@@ -16,7 +15,7 @@ class FirecrawlScrapeInput(BaseModel):
     url: str = Field(
         description="The URL to scrape. Must be a valid HTTP or HTTPS URL."
     )
-    formats: List[str] = Field(
+    formats: list[str] = Field(
         description="Output formats to include in the response. Options: 'markdown', 'html', 'rawHtml', 'screenshot', 'links', 'json'",
         default=["markdown"],
     )
@@ -24,11 +23,11 @@ class FirecrawlScrapeInput(BaseModel):
         description="Whether to extract only the main content (excluding headers, footers, navigation, etc.)",
         default=True,
     )
-    include_tags: Optional[List[str]] = Field(
+    include_tags: list[str] | None = Field(
         description="HTML tags, classes, or IDs to include in the response (e.g., ['h1', 'p', '.main-content'])",
         default=None,
     )
-    exclude_tags: Optional[List[str]] = Field(
+    exclude_tags: list[str] | None = Field(
         description="HTML tags, classes, or IDs to exclude from the response (e.g., ['#ad', '#footer'])",
         default=None,
     )
@@ -62,10 +61,11 @@ class FirecrawlScrapeInput(BaseModel):
 
 
 class FirecrawlScrape(FirecrawlBaseTool):
-    """Tool for scraping web pages using Firecrawl.
+    """Tool for scraping web pages using Firecrawl with REPLACE behavior.
 
-    This tool uses Firecrawl's API to scrape web pages and convert them into clean,
-    LLM-ready formats like markdown, HTML, or structured JSON data.
+    This tool uses Firecrawl's API to scrape web pages and REPLACES any existing
+    indexed content for the same URL instead of appending to it. This prevents
+    duplicate content when re-scraping the same page.
 
     Attributes:
         name: The name of the tool.
@@ -75,20 +75,20 @@ class FirecrawlScrape(FirecrawlBaseTool):
 
     name: str = "firecrawl_scrape"
     description: str = (
-        "Scrape a single web page and extract its content in various formats (markdown, HTML, JSON, etc.). "
+        "Scrape a single web page and REPLACE any existing indexed content for that URL. "
+        "Unlike regular scrape, this tool removes old content before adding new content, preventing duplicates. "
         "This tool can handle JavaScript-rendered content, PDFs, and dynamic websites. "
-        "Optionally indexes the content for later querying using the firecrawl_query_indexed_content tool. "
-        "Use this when you need to extract clean, structured content from a specific URL."
+        "Use this when you want to refresh/update content from a URL that was previously scraped."
     )
-    args_schema: Type[BaseModel] = FirecrawlScrapeInput
+    args_schema: type[BaseModel] = FirecrawlScrapeInput
 
     async def _arun(
         self,
         url: str,
-        formats: List[str] = None,
+        formats: list[str] = None,
         only_main_content: bool = True,
-        include_tags: Optional[List[str]] = None,
-        exclude_tags: Optional[List[str]] = None,
+        include_tags: list[str] | None = None,
+        exclude_tags: list[str] | None = None,
         wait_for: int = 0,
         timeout: int = 30000,
         index_content: bool = True,
@@ -123,9 +123,8 @@ class FirecrawlScrape(FirecrawlBaseTool):
                 "rate_limit_minutes"
             ):
                 await self.user_rate_limit_by_category(
-                    context.user_id,
                     skill_config["rate_limit_number"],
-                    skill_config["rate_limit_minutes"],
+                    skill_config["rate_limit_minutes"] * 60,
                 )
 
         # Get the API key from the agent's configuration
@@ -187,7 +186,7 @@ class FirecrawlScrape(FirecrawlBaseTool):
                 result_data = data.get("data", {})
 
                 # Format the results based on requested formats
-                formatted_result = f"Successfully scraped: {url}\n\n"
+                formatted_result = f"Successfully scraped (REPLACE mode): {url}\n\n"
 
                 if "markdown" in formats and result_data.get("markdown"):
                     formatted_result += "## Markdown Content\n"
@@ -236,13 +235,16 @@ class FirecrawlScrape(FirecrawlBaseTool):
                         formatted_result += f"Language: {metadata['language']}\n"
                     formatted_result += "\n"
 
-                # Index content if requested
+                # Index content if requested - REPLACE MODE
                 if index_content and result_data.get("markdown"):
                     try:
-                        # Import indexing utilities from firecrawl utils
+                        # Import indexing utilities
+                        from langchain_community.vectorstores import FAISS
+
                         from intentkit.skills.firecrawl.utils import (
+                            FirecrawlDocumentProcessor,
                             FirecrawlMetadataManager,
-                            index_documents,
+                            FirecrawlVectorStoreManager,
                         )
 
                         # Create document from scraped content
@@ -261,38 +263,146 @@ class FirecrawlScrape(FirecrawlBaseTool):
                         # Get agent ID for indexing
                         agent_id = context.agent_id
                         if agent_id:
-                            # Index the document
-                            total_chunks, was_merged = await index_documents(
-                                [document],
-                                agent_id,
-                                self.skill_store,
-                                chunk_size,
-                                chunk_overlap,
+                            # Initialize vector store manager
+                            vs_manager = FirecrawlVectorStoreManager()
+
+                            # Load existing vector store
+                            existing_vector_store = await vs_manager.load_vector_store(
+                                agent_id
                             )
 
-                            # Update metadata
-                            metadata_manager = FirecrawlMetadataManager(
-                                self.skill_store
+                            # Split the new document into chunks
+                            split_docs = FirecrawlDocumentProcessor.split_documents(
+                                [document], chunk_size, chunk_overlap
                             )
-                            new_metadata = metadata_manager.create_url_metadata(
-                                [url], [document], "firecrawl_scrape"
+
+                            # Create embeddings
+                            embeddings = vs_manager.create_embeddings()
+
+                            if existing_vector_store:
+                                # Get all existing documents and filter out those from the same URL
+                                try:
+                                    # Try to access documents directly if available
+                                    if hasattr(
+                                        existing_vector_store, "docstore"
+                                    ) and hasattr(
+                                        existing_vector_store.docstore, "_dict"
+                                    ):
+                                        # Access FAISS documents directly
+                                        all_docs = list(
+                                            existing_vector_store.docstore._dict.values()
+                                        )
+                                    else:
+                                        # Fallback: use a reasonable k value for similarity search
+                                        # Use a dummy query to retrieve documents
+                                        all_docs = existing_vector_store.similarity_search(
+                                            "dummy",  # Use a dummy query instead of empty string
+                                            k=1000,  # Use reasonable upper bound
+                                        )
+
+                                    # Filter out documents from the same URL
+                                    preserved_docs = [
+                                        doc
+                                        for doc in all_docs
+                                        if doc.metadata.get("source") != url
+                                    ]
+
+                                    logger.info(
+                                        f"firecrawl_scrape: Preserving {len(preserved_docs)} docs from other URLs, "
+                                        f"replacing content from {url}"
+                                    )
+
+                                    # Create new vector store with preserved docs + new docs
+                                    if preserved_docs:
+                                        # Combine preserved and new documents
+                                        all_documents = preserved_docs + split_docs
+                                        new_vector_store = FAISS.from_documents(
+                                            all_documents, embeddings
+                                        )
+                                        formatted_result += "\n## Content Replacement\n"
+                                        formatted_result += f"Replaced existing content for URL: {url}\n"
+                                        num_preserved_urls = len(
+                                            set(
+                                                doc.metadata.get("source", "")
+                                                for doc in preserved_docs
+                                            )
+                                        )
+                                        formatted_result += f"Preserved content from {num_preserved_urls} other URLs\n"
+                                    else:
+                                        # No other documents to preserve, just create from new docs
+                                        new_vector_store = FAISS.from_documents(
+                                            split_docs, embeddings
+                                        )
+                                        formatted_result += "\n## Content Replacement\n"
+                                        formatted_result += f"Created new index with content from: {url}\n"
+                                except Exception as e:
+                                    logger.warning(
+                                        f"Could not preserve other URLs, creating fresh index: {e}"
+                                    )
+                                    # Fallback: create new store with just the new documents
+                                    new_vector_store = FAISS.from_documents(
+                                        split_docs, embeddings
+                                    )
+                                    formatted_result += "\n## Content Replacement\n"
+                                    formatted_result += f"Created fresh index with content from: {url}\n"
+                            else:
+                                # No existing store, create new one
+                                new_vector_store = FAISS.from_documents(
+                                    split_docs, embeddings
+                                )
+                                formatted_result += "\n## Content Indexing\n"
+                                formatted_result += (
+                                    f"Created new index with content from: {url}\n"
+                                )
+
+                            # Save the new vector store
+                            await vs_manager.save_vector_store(
+                                agent_id, new_vector_store, chunk_size, chunk_overlap
                             )
-                            await metadata_manager.update_metadata(
-                                agent_id, new_metadata
+
+                            # Update metadata to track all URLs
+                            # Get existing metadata to preserve other URLs
+                            metadata_key = f"indexed_urls_{agent_id}"
+                            existing_metadata = await self.get_agent_skill_data_raw(
+                                "firecrawl", metadata_key
                             )
 
-                            formatted_result += "\n## Content Indexing\n"
-                            formatted_result += (
-                                "Successfully indexed content into vector store:\n"
+                            if existing_metadata and existing_metadata.get("urls"):
+                                # Remove the current URL and add it back (to update timestamp)
+                                existing_urls = [
+                                    u for u in existing_metadata["urls"] if u != url
+                                ]
+                                existing_urls.append(url)
+                                updated_metadata = {
+                                    "urls": existing_urls,
+                                    "document_count": len(existing_urls),
+                                    "source_type": "firecrawl_mixed",
+                                    "indexed_at": str(len(existing_urls)),
+                                }
+                            else:
+                                # Create new metadata
+                                updated_metadata = (
+                                    FirecrawlMetadataManager.create_url_metadata(
+                                        [url], [document], "firecrawl_scrape"
+                                    )
+                                )
+
+                            await FirecrawlMetadataManager.update_metadata(
+                                agent_id, updated_metadata
                             )
-                            formatted_result += f"- Chunks created: {total_chunks}\n"
+
+                            formatted_result += "\n## Content Indexing (REPLACE MODE)\n"
+                            formatted_result += "Successfully REPLACED indexed content in vector store:\n"
+                            formatted_result += f"- Chunks created: {len(split_docs)}\n"
                             formatted_result += f"- Chunk size: {chunk_size}\n"
                             formatted_result += f"- Chunk overlap: {chunk_overlap}\n"
-                            formatted_result += f"- Content merged with existing: {'Yes' if was_merged else 'No'}\n"
+                            formatted_result += (
+                                "- Previous content for this URL: REPLACED\n"
+                            )
                             formatted_result += "Use the 'firecrawl_query_indexed_content' skill to search this content.\n"
 
                             logger.info(
-                                f"firecrawl_scrape: Successfully indexed {url} with {total_chunks} chunks"
+                                f"firecrawl_scrape: Successfully replaced content for {url} with {len(split_docs)} chunks"
                             )
                         else:
                             formatted_result += "\n## Content Indexing\n"

intentkit/skills/firecrawl/utils.py

@@ -2,14 +2,15 @@
 
 import logging
 import re
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any
 
-from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_community.vectorstores import FAISS
 from langchain_core.documents import Document
 from langchain_openai import OpenAIEmbeddings
+from langchain_text_splitters import RecursiveCharacterTextSplitter
 
-from intentkit.abstracts.skill import SkillStoreABC
+from intentkit.config.config import config
+from intentkit.models.skill import AgentSkillData, AgentSkillDataCreate
 
 logger = logging.getLogger(__name__)
 
@@ -35,8 +36,8 @@ class FirecrawlDocumentProcessor:
 
     @staticmethod
     def split_documents(
-        documents: List[Document], chunk_size: int = 1000, chunk_overlap: int = 200
-    ) -> List[Document]:
+        documents: list[Document], chunk_size: int = 1000, chunk_overlap: int = 200
+    ) -> list[Document]:
         """Split documents into smaller chunks for better indexing."""
         text_splitter = RecursiveCharacterTextSplitter(
             chunk_size=chunk_size,
@@ -62,20 +63,25 @@ class FirecrawlDocumentProcessor:
 class FirecrawlVectorStoreManager:
     """Manages vector store operations for Firecrawl content."""
 
-    def __init__(self, skill_store: SkillStoreABC):
-        self.skill_store = skill_store
+    def __init__(self, embedding_api_key: str | None = None):
+        self._embedding_api_key = embedding_api_key
+
+    def _resolve_api_key(self) -> str:
+        """Resolve the API key to use for embeddings."""
+        if self._embedding_api_key:
+            return self._embedding_api_key
+        if config.openai_api_key:
+            return config.openai_api_key
+        raise ValueError("OpenAI API key not found in system configuration")
 
     def create_embeddings(self) -> OpenAIEmbeddings:
         """Create OpenAI embeddings instance."""
-        openai_api_key = self.skill_store.get_system_config("openai_api_key")
-        if not openai_api_key:
-            raise ValueError("OpenAI API key not found in system configuration")
-
+        openai_api_key = self._resolve_api_key()
         return OpenAIEmbeddings(
             openai_api_key=openai_api_key, model="text-embedding-3-small"
         )
 
-    def encode_vector_store(self, vector_store: FAISS) -> Dict[str, str]:
+    def encode_vector_store(self, vector_store: FAISS) -> dict[str, str]:
         """Encode FAISS vector store to base64 for storage (compatible with web_scraper)."""
         import base64
         import os
@@ -100,7 +106,7 @@ class FirecrawlVectorStoreManager:
             raise
 
     def decode_vector_store(
-        self, encoded_files: Dict[str, str], embeddings: OpenAIEmbeddings
+        self, encoded_files: dict[str, str], embeddings: OpenAIEmbeddings
     ) -> FAISS:
         """Decode base64 files back to FAISS vector store (compatible with web_scraper)."""
         import base64
@@ -125,11 +131,11 @@ class FirecrawlVectorStoreManager:
             logger.error(f"Error decoding vector store: {e}")
             raise
 
-    async def load_vector_store(self, agent_id: str) -> Optional[FAISS]:
+    async def load_vector_store(self, agent_id: str) -> FAISS | None:
         """Load existing vector store for an agent."""
         try:
             vector_store_key = f"vector_store_{agent_id}"
-            stored_data = await self.skill_store.get_agent_skill_data(
+            stored_data = await AgentSkillData.get(
                 agent_id, "web_scraper", vector_store_key
             )
 
@@ -162,9 +168,13 @@ class FirecrawlVectorStoreManager:
                 "chunk_overlap": chunk_overlap,
             }
 
-            await self.skill_store.save_agent_skill_data(
-                agent_id, "web_scraper", vector_store_key, storage_data
+            skill_data = AgentSkillDataCreate(
+                agent_id=agent_id,
+                skill="web_scraper",
+                key=vector_store_key,
+                data=storage_data,
             )
+            await skill_data.save()
 
         except Exception as e:
             logger.error(f"Error saving vector store for agent {agent_id}: {e}")
@@ -174,12 +184,10 @@ class FirecrawlVectorStoreManager:
 class FirecrawlMetadataManager:
     """Manages metadata for Firecrawl indexed content."""
 
-    def __init__(self, skill_store: SkillStoreABC):
-        self.skill_store = skill_store
-
+    @staticmethod
     def create_url_metadata(
-        self, urls: List[str], documents: List[Document], source_type: str
-    ) -> Dict[str, Any]:
+        urls: list[str], documents: list[Document], source_type: str
+    ) -> dict[str, Any]:
         """Create metadata for indexed URLs."""
         return {
             "urls": urls,
@@ -188,34 +196,38 @@ class FirecrawlMetadataManager:
             "indexed_at": str(len(urls)),  # Simple counter
         }
 
-    async def update_metadata(
-        self, agent_id: str, new_metadata: Dict[str, Any]
-    ) -> None:
+    @staticmethod
+    @staticmethod
+    async def update_metadata(agent_id: str, new_metadata: dict[str, Any]) -> None:
         """Update metadata for an agent."""
         try:
             metadata_key = f"indexed_urls_{agent_id}"
-            await self.skill_store.save_agent_skill_data(
-                agent_id, "web_scraper", metadata_key, new_metadata
+            skill_data = AgentSkillDataCreate(
+                agent_id=agent_id,
+                skill="web_scraper",
+                key=metadata_key,
+                data=new_metadata,
             )
+            await skill_data.save()
         except Exception as e:
             logger.error(f"Error updating metadata for agent {agent_id}: {e}")
             raise
 
 
 async def index_documents(
-    documents: List[Document],
+    documents: list[Document],
     agent_id: str,
-    skill_store: SkillStoreABC,
+    vector_manager: FirecrawlVectorStoreManager,
     chunk_size: int = 1000,
     chunk_overlap: int = 200,
-) -> Tuple[int, bool]:
+) -> tuple[int, bool]:
     """
     Index documents into the Firecrawl vector store.
 
     Args:
         documents: List of documents to index
         agent_id: Agent ID for storage
-        skill_store: Skill store for persistence
+        vector_manager: Vector store manager
         chunk_size: Size of text chunks
         chunk_overlap: Overlap between chunks
 
@@ -224,8 +236,6 @@ async def index_documents(
     """
     try:
         # Initialize managers
-        vs_manager = FirecrawlVectorStoreManager(skill_store)
-
         # Split documents into chunks
         split_docs = FirecrawlDocumentProcessor.split_documents(
             documents, chunk_size, chunk_overlap
@@ -236,10 +246,10 @@ async def index_documents(
             return 0, False
 
         # Create embeddings
-        embeddings = vs_manager.create_embeddings()
+        embeddings = vector_manager.create_embeddings()
 
         # Try to load existing vector store
-        existing_vector_store = await vs_manager.load_vector_store(agent_id)
+        existing_vector_store = await vector_manager.load_vector_store(agent_id)
 
         if existing_vector_store:
             # Add to existing vector store
@@ -252,7 +262,7 @@ async def index_documents(
             was_merged = False
 
         # Save the vector store
-        await vs_manager.save_vector_store(
+        await vector_manager.save_vector_store(
             agent_id, vector_store, chunk_size, chunk_overlap
         )
 
@@ -269,16 +279,16 @@ async def index_documents(
 async def query_indexed_content(
     query: str,
     agent_id: str,
-    skill_store: SkillStoreABC,
+    vector_manager: FirecrawlVectorStoreManager,
     max_results: int = 4,
-) -> List[Document]:
+) -> list[Document]:
     """
     Query the Firecrawl indexed content.
 
     Args:
         query: Search query
         agent_id: Agent ID
-        skill_store: Skill store for persistence
+        vector_manager: Manager for vector store persistence
         max_results: Maximum number of results to return
 
     Returns:
@@ -286,10 +296,8 @@ async def query_indexed_content(
     """
     try:
         # Initialize vector store manager
-        vs_manager = FirecrawlVectorStoreManager(skill_store)
-
         # Load vector store
-        vector_store = await vs_manager.load_vector_store(agent_id)
+        vector_store = await vector_manager.load_vector_store(agent_id)
 
         if not vector_store:
             logger.warning(f"No vector store found for agent {agent_id}")

intentkit/skills/github/__init__.py

@@ -1,6 +1,5 @@
 from typing import TypedDict
 
-from intentkit.abstracts.skill import SkillStoreABC
 from intentkit.skills.base import SkillConfig, SkillState
 from intentkit.skills.github.base import GitHubBaseTool
 from intentkit.skills.github.github_search import GitHubSearch
@@ -22,7 +21,6 @@ class Config(SkillConfig):
 async def get_skills(
     config: "Config",
     is_private: bool,
-    store: SkillStoreABC,
     **_,
 ) -> list[GitHubBaseTool]:
     """Get all GitHub skills."""
@@ -36,19 +34,16 @@ async def get_skills(
             available_skills.append(skill_name)
 
     # Get each skill using the cached getter
-    return [get_github_skill(name, store) for name in available_skills]
+    return [get_github_skill(name) for name in available_skills]
 
 
 def get_github_skill(
     name: str,
-    store: SkillStoreABC,
 ) -> GitHubBaseTool:
     """Get a GitHub skill by name."""
     if name == "github_search":
         if name not in _cache:
-            _cache[name] = GitHubSearch(
-                skill_store=store,
-            )
+            _cache[name] = GitHubSearch()
         return _cache[name]
     else:
         raise ValueError(f"Unknown GitHub skill: {name}")

intentkit/skills/github/base.py

@@ -1,8 +1,5 @@
-from typing import Type
-
 from pydantic import BaseModel, Field
 
-from intentkit.abstracts.skill import SkillStoreABC
 from intentkit.skills.base import IntentKitSkill
 
 
@@ -11,10 +8,7 @@ class GitHubBaseTool(IntentKitSkill):
 
     name: str = Field(description="The name of the tool")
     description: str = Field(description="A description of what the tool does")
-    args_schema: Type[BaseModel]
-    skill_store: SkillStoreABC = Field(
-        description="The skill store for persisting data"
-    )
+    args_schema: type[BaseModel]
 
     @property
     def category(self) -> str:

intentkit/skills/github/github_search.py

@@ -1,6 +1,5 @@
 import logging
 from enum import Enum
-from typing import Type
 
 import httpx
 from pydantic import BaseModel, Field
@@ -54,7 +53,7 @@ class GitHubSearch(GitHubBaseTool):
         "- Code snippets across GitHub repositories\n"
         "You must call this tool whenever the user asks about finding something on GitHub."
     )
-    args_schema: Type[BaseModel] = GitHubSearchInput
+    args_schema: type[BaseModel] = GitHubSearchInput
 
     async def _arun(
         self,

intentkit/skills/github/schema.json

@@ -5,9 +5,8 @@
   "description": "Search capabilities for GitHub repositories, users, and code",
   "x-icon": "https://ai.service.crestal.dev/skills/github/github.jpg",
   "x-tags": [
-    "GitHub",
-    "Search",
-    "Code"
+    "Developer Tools",
+    "Search"
   ],
   "properties": {
     "enabled": {
@@ -56,4 +55,4 @@
     "enabled"
   ],
   "additionalProperties": true
-}
+}