intentkit 0.6.0.dev11__tar.gz → 0.6.0.dev13__tar.gz

{intentkit-0.6.0.dev11 → intentkit-0.6.0.dev13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: intentkit
-Version: 0.6.0.dev11
+Version: 0.6.0.dev13
 Summary: Intent-based AI Agent Platform - Core Package
 Project-URL: Homepage, https://github.com/crestal-network/intentkit
 Project-URL: Repository, https://github.com/crestal-network/intentkit

{intentkit-0.6.0.dev11 → intentkit-0.6.0.dev13}/__init__.py

@@ -3,7 +3,7 @@
 A powerful platform for building AI agents with blockchain and cryptocurrency capabilities.
 """
 
-__version__ = "0.6.0-dev.11"
+__version__ = "0.6.0-dev.13"
 __author__ = "hyacinthus"
 __email__ = "hyacinthus@gmail.com"
 

{intentkit-0.6.0.dev11 → intentkit-0.6.0.dev13}/abstracts/skill.py

@@ -91,6 +91,18 @@ class SkillStoreABC(ABC):
         """
         pass
 
+    @staticmethod
+    @abstractmethod
+    async def delete_agent_skill_data(agent_id: str, skill: str, key: str) -> None:
+        """Delete skill data for an agent.
+
+        Args:
+            agent_id: ID of the agent
+            skill: Name of the skill
+            key: Data key
+        """
+        pass
+
     @staticmethod
     @abstractmethod
     async def get_thread_skill_data(

{intentkit-0.6.0.dev11 → intentkit-0.6.0.dev13}/config/config.py

@@ -121,6 +121,7 @@ class Config:
         self.moralis_api_key = self.load("MORALIS_API_KEY")
         self.tavily_api_key = self.load("TAVILY_API_KEY")
         self.cookiefun_api_key = self.load("COOKIEFUN_API_KEY")
+        self.firecrawl_api_key = self.load("FIRECRAWL_API_KEY")
         # Sentry
         self.sentry_dsn = self.load("SENTRY_DSN")
         self.sentry_sample_rate = float(self.load("SENTRY_SAMPLE_RATE", "0.1"))

{intentkit-0.6.0.dev11 → intentkit-0.6.0.dev13}/core/skill.py

@@ -78,6 +78,17 @@ class SkillStore(SkillStoreABC):
         )
         await skill_data.save()
 
+    @staticmethod
+    async def delete_agent_skill_data(agent_id: str, skill: str, key: str) -> None:
+        """Delete skill data for an agent.
+
+        Args:
+            agent_id: ID of the agent
+            skill: Name of the skill
+            key: Data key
+        """
+        await AgentSkillData.delete(agent_id, skill, key)
+
     @staticmethod
     async def get_thread_skill_data(
         thread_id: str, skill: str, key: str

{intentkit-0.6.0.dev11 → intentkit-0.6.0.dev13}/models/agent_schema.json

@@ -539,6 +539,10 @@
           "title": "Web Scraper & Content Indexing",
           "$ref": "../skills/web_scraper/schema.json"
         },
+        "firecrawl": {
+          "title": "Firecrawl Web Scraping",
+          "$ref": "../skills/firecrawl/schema.json"
+        },
         "aixbt": {
           "title": "AIXBT",
           "$ref": "../skills/aixbt/schema.json"

{intentkit-0.6.0.dev11 → intentkit-0.6.0.dev13}/models/skill.py

@@ -172,6 +172,25 @@ class AgentSkillData(AgentSkillDataCreate):
             )
             return result.data if result else None
 
+    @classmethod
+    async def delete(cls, agent_id: str, skill: str, key: str) -> None:
+        """Delete skill data for an agent.
+
+        Args:
+            agent_id: ID of the agent
+            skill: Name of the skill
+            key: Data key
+        """
+        async with get_session() as db:
+            await db.execute(
+                delete(AgentSkillDataTable).where(
+                    AgentSkillDataTable.agent_id == agent_id,
+                    AgentSkillDataTable.skill == skill,
+                    AgentSkillDataTable.key == key,
+                )
+            )
+            await db.commit()
+
     @classmethod
     async def clean_data(cls, agent_id: str):
         """Clean all skill data for an agent.

{intentkit-0.6.0.dev11 → intentkit-0.6.0.dev13}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "intentkit"
-version = "0.6.0-dev.11"
+version = "0.6.0-dev.13"
 description = "Intent-based AI Agent Platform - Core Package"
 authors = [{ name = "hyacinthus", email = "hyacinthus@gmail.com" }]
 requires-python = ">=3.12"

intentkit-0.6.0.dev13/skills/firecrawl/README.md

@@ -0,0 +1,211 @@
+# Firecrawl Skills
+
+The Firecrawl skills provide advanced web scraping and content indexing capabilities using the Firecrawl API. These skills can handle JavaScript-heavy websites, PDFs, and provide automatic content indexing for intelligent querying.
+
+## Skills Overview
+
+### 1. firecrawl_scrape
+Scrapes a single webpage and optionally indexes the content for future querying.
+
+**Parameters:**
+- `url` (required): The URL to scrape
+- `formats` (optional): Output formats - markdown, html, rawHtml, screenshot, links, extract (default: ["markdown"])
+- `include_tags` (optional): HTML tags to include (e.g., ["h1", "h2", "p"])
+- `exclude_tags` (optional): HTML tags to exclude
+- `only_main_content` (optional): Extract only main content (default: true)
+- `index_content` (optional): Whether to index content for querying (default: true)
+- `chunk_size` (optional): Size of text chunks for indexing (default: 1000)
+- `chunk_overlap` (optional): Overlap between chunks (default: 200)
+
+### 2. firecrawl_crawl
+Crawls multiple pages from a website and indexes all content.
+
+**Parameters:**
+- `url` (required): The base URL to start crawling
+- `include_paths` (optional): URL patterns to include (e.g., ["/docs/*"])
+- `exclude_paths` (optional): URL patterns to exclude
+- `max_depth` (optional): Maximum crawl depth (default: 2)
+- `limit` (optional): Maximum number of pages to crawl (default: 5)
+- `index_content` (optional): Whether to index content for querying (default: true)
+- `chunk_size` (optional): Size of text chunks for indexing (default: 1000)
+- `chunk_overlap` (optional): Overlap between chunks (default: 200)
+
+### 3. firecrawl_query_indexed_content
+Queries previously indexed Firecrawl content using semantic search.
+
+**Parameters:**
+- `query` (required): The search query
+- `limit` (optional): Maximum number of results to return (1-10, default: 4)
+
+### 4. firecrawl_clear_indexed_content
+Clears all previously indexed Firecrawl content from the vector store.
+
+**Parameters:**
+- `confirm` (required): Must be set to true to confirm the deletion (default: false)
+
+**Note:** This action is permanent and cannot be undone. Use when you want to start fresh with new content.
+
+## API Key Configuration
+Set your Firecrawl API key as an environment variable:
+```bash
+export FIRECRAWL_API_KEY=fc-your-api-key-here
+```
+
+## Testing Instructions
+
+### Step 1: Create an Agent with Firecrawl Skills
+
+1. **Create a new agent** via the API or UI with the following skills:
+   ```json
+   {
+     "skills": [
+       "firecrawl_scrape",
+       "firecrawl_crawl", 
+       "firecrawl_query_indexed_content",
+       "firecrawl_clear_indexed_content"
+     ]
+   }
+   ```
+
+2. **Note the agent ID** for testing
+
+### Step 2: Test Single Page Scraping
+
+**Test scraping a documentation homepage:**
+```
+Prompt: "Use firecrawl_scrape to scrape https://docs.joincommonwealth.xyz/ and index the content for future querying"
+```
+
+**Expected Result:**
+- Content successfully scraped
+- Content automatically indexed with metadata
+- Confirmation of chunk creation and indexing
+
+### Step 3: Test Content Crawling
+
+**Test crawling multiple pages:**
+```
+Prompt: "Use firecrawl_crawl to crawl https://docs.joincommonwealth.xyz/ with max_depth=2 and limit=3 to index multiple documentation pages"
+```
+
+**Expected Result:**
+- Multiple pages crawled and scraped
+- Each page indexed separately
+- Batch processing confirmation
+
+### Step 4: Test Content Querying
+
+**Test querying indexed content:**
+```
+Prompt: "Use firecrawl_query_indexed_content to search for 'What is All Street and what is its purpose?' in the indexed content"
+```
+
+**Expected Result:**
+- Relevant content retrieved from indexed documents
+- Results tagged with [Firecrawl Scrape] or [Firecrawl Crawl]
+- Source URLs and metadata included
+
+### Step 5: Test Advanced Scraping Options
+
+**Test with specific formatting:**
+```
+Prompt: "Use firecrawl_scrape to scrape https://docs.joincommonwealth.xyz/all-street-manifesto with formats=['markdown', 'html'] and include_tags=['h1', 'h2', 'p'] and index_content=true"
+```
+
+**Expected Result:**
+- Content in both markdown and HTML formats
+- Only specified HTML tags included
+- Content indexed for querying
+
+### Step 6: Test Multiple Queries
+
+**Test different query types:**
+```
+Prompt: "Use firecrawl_query_indexed_content to search for 'democratize finance' in the indexed content"
+```
+
+**Expected Result:**
+- Relevant content retrieved from Firecrawl's independent vector store
+- Results tagged with [Firecrawl Scrape] or [Firecrawl Crawl]
+- Source URLs and metadata included
+
+### Step 7: Test Clear Indexed Content
+
+**Test clearing all indexed content:**
+```
+Prompt: "Use firecrawl_clear_indexed_content with confirm=true to clear all indexed content"
+```
+
+**Expected Result:**
+- All indexed content removed from vector store
+- Confirmation message displayed
+- Subsequent queries return no results
+
+### Step 8: Test Re-indexing After Clear
+
+**Test that content can be re-indexed after clearing:**
+```
+Prompt: "Use firecrawl_scrape to scrape https://example.com and index the content"
+```
+
+**Expected Result:**
+- Content successfully scraped and indexed
+- Fresh vector store created
+- Content available for querying again
+
+## Common Use Cases
+
+### Documentation Indexing
+```
+1. Scrape main documentation page
+2. Crawl related documentation sections
+3. Query for specific technical information
+```
+
+### Competitive Analysis
+```
+1. Scrape competitor websites
+2. Index product information and features
+3. Query for specific comparisons
+```
+
+### Research and Knowledge Base
+```
+1. Crawl research papers or articles
+2. Index academic or technical content
+3. Query for specific concepts or methodologies
+```
+
+## Troubleshooting
+
+### Common Issues
+
+1. **API Key Not Found**
+   - Ensure `FIRECRAWL_API_KEY` environment variable is set
+   - Restart the IntentKit server after setting the key
+
+2. **Scraping Failures**
+   - Check if the URL is accessible
+   - Verify Firecrawl API quota and limits
+   - Some websites may block scraping
+
+3. **Indexing Errors**
+   - Ensure OpenAI API key is configured for embeddings
+   - Check if content is too large for processing
+   - Verify vector store permissions
+
+4. **Query Returns No Results**
+   - Ensure content was successfully indexed
+   - Try broader or different search terms
+   - Check if vector store contains data
+
+## Features and Benefits
+
+- **JavaScript Rendering**: Handles SPAs and dynamic content
+- **PDF Support**: Can scrape and index PDF documents
+- **Intelligent Chunking**: Optimized text splitting for better search
+- **Independent Storage**: Uses its own dedicated vector store for Firecrawl content
+- **Metadata Rich**: Includes source URLs, timestamps, and content types
+- **Semantic Search**: Uses OpenAI embeddings for intelligent querying
+- **Batch Processing**: Efficient handling of multiple pages
+- **Content Filtering**: Flexible include/exclude options for targeted scraping

intentkit-0.6.0.dev13/skills/firecrawl/__init__.py

@@ -0,0 +1,107 @@
+"""Firecrawl skills for web scraping and crawling."""
+
+import logging
+from typing import TypedDict
+
+from intentkit.abstracts.skill import SkillStoreABC
+from intentkit.skills.base import SkillConfig, SkillState
+from intentkit.skills.firecrawl.base import FirecrawlBaseTool
+from intentkit.skills.firecrawl.clear import FirecrawlClearIndexedContent
+from intentkit.skills.firecrawl.crawl import FirecrawlCrawl
+from intentkit.skills.firecrawl.query import FirecrawlQueryIndexedContent
+from intentkit.skills.firecrawl.scrape import FirecrawlScrape
+
+# Cache skills at the system level, because they are stateless
+_cache: dict[str, FirecrawlBaseTool] = {}
+
+logger = logging.getLogger(__name__)
+
+
+class SkillStates(TypedDict):
+    firecrawl_scrape: SkillState
+    firecrawl_crawl: SkillState
+    firecrawl_query_indexed_content: SkillState
+    firecrawl_clear_indexed_content: SkillState
+
+
+class Config(SkillConfig):
+    """Configuration for Firecrawl skills."""
+
+    states: SkillStates
+    api_key: str = ""
+    api_key_provider: str = "agent_owner"
+    rate_limit_number: int = 100
+    rate_limit_minutes: int = 60
+
+
+async def get_skills(
+    config: "Config",
+    is_private: bool,
+    store: SkillStoreABC,
+    **_,
+) -> list[FirecrawlBaseTool]:
+    """Get all Firecrawl skills.
+
+    Args:
+        config: The configuration for Firecrawl skills.
+        is_private: Whether to include private skills.
+        store: The skill store for persisting data.
+
+    Returns:
+        A list of Firecrawl skills.
+    """
+    available_skills = []
+
+    # Include skills based on their state
+    for skill_name, state in config["states"].items():
+        if state == "disabled":
+            continue
+        elif state == "public" or (state == "private" and is_private):
+            available_skills.append(skill_name)
+
+    # Get each skill using the cached getter
+    return [get_firecrawl_skill(name, store) for name in available_skills]
+
+
+def get_firecrawl_skill(
+    name: str,
+    store: SkillStoreABC,
+) -> FirecrawlBaseTool:
+    """Get a Firecrawl skill by name.
+
+    Args:
+        name: The name of the skill to get
+        store: The skill store for persisting data
+
+    Returns:
+        The requested Firecrawl skill
+
+    Raises:
+        ValueError: If the skill name is unknown
+    """
+    if name == "firecrawl_scrape":
+        if name not in _cache:
+            _cache[name] = FirecrawlScrape(
+                skill_store=store,
+            )
+        return _cache[name]
+    elif name == "firecrawl_crawl":
+        if name not in _cache:
+            _cache[name] = FirecrawlCrawl(
+                skill_store=store,
+            )
+        return _cache[name]
+    elif name == "firecrawl_query_indexed_content":
+        if name not in _cache:
+            _cache[name] = FirecrawlQueryIndexedContent(
+                skill_store=store,
+            )
+        return _cache[name]
+    elif name == "firecrawl_clear_indexed_content":
+        if name not in _cache:
+            _cache[name] = FirecrawlClearIndexedContent(
+                skill_store=store,
+            )
+        return _cache[name]
+    else:
+        raise ValueError(f"Unknown Firecrawl skill: {name}")

intentkit-0.6.0.dev13/skills/firecrawl/base.py

@@ -0,0 +1,28 @@
+from typing import Type
+
+from pydantic import BaseModel, Field
+
+from intentkit.abstracts.skill import SkillStoreABC
+from intentkit.skills.base import IntentKitSkill, SkillContext
+
+
+class FirecrawlBaseTool(IntentKitSkill):
+    """Base class for Firecrawl tools."""
+
+    name: str = Field(description="The name of the tool")
+    description: str = Field(description="A description of what the tool does")
+    args_schema: Type[BaseModel]
+    skill_store: SkillStoreABC = Field(
+        description="The skill store for persisting data"
+    )
+
+    def get_api_key(self, context: SkillContext) -> str:
+        """Get the Firecrawl API key from configuration."""
+        skill_config = context.config
+        if skill_config.get("api_key_provider") == "agent_owner":
+            return skill_config.get("api_key")
+        return self.skill_store.get_system_config("firecrawl_api_key")
+
+    @property
+    def category(self) -> str:
+        return "firecrawl"

intentkit-0.6.0.dev13/skills/firecrawl/clear.py

@@ -0,0 +1,87 @@
+import logging
+from typing import Type
+
+from langchain_core.runnables import RunnableConfig
+from pydantic import BaseModel, Field
+
+from intentkit.skills.firecrawl.base import FirecrawlBaseTool
+
+logger = logging.getLogger(__name__)
+
+
+class FirecrawlClearInput(BaseModel):
+    """Input for Firecrawl clear tool."""
+
+    confirm: bool = Field(
+        description="Confirmation to clear all indexed content (must be true)",
+        default=False,
+    )
+
+
+class FirecrawlClearIndexedContent(FirecrawlBaseTool):
+    """Tool for clearing all indexed Firecrawl content.
+
+    This tool removes all previously indexed content from the Firecrawl vector store,
+    allowing for a fresh start with new content.
+    """
+
+    name: str = "firecrawl_clear_indexed_content"
+    description: str = (
+        "Clear all previously indexed Firecrawl content from the vector store.\n"
+        "This will permanently delete all indexed content and cannot be undone.\n"
+        "Use this tool when you want to start fresh with new content."
+    )
+    args_schema: Type[BaseModel] = FirecrawlClearInput
+
+    async def _arun(
+        self,
+        confirm: bool = False,
+        config: RunnableConfig = None,
+        **kwargs,
+    ) -> str:
+        """Clear all indexed Firecrawl content for the agent.
+
+        Args:
+            confirm: Must be True to confirm the deletion
+            config: The configuration for the tool call
+
+        Returns:
+            str: Confirmation message
+        """
+        context = self.context_from_config(config)
+        agent_id = context.agent.id if context and context.agent else None
+
+        if not agent_id:
+            return "Error: Agent ID not available for clearing content."
+
+        if not confirm:
+            return "Error: You must set confirm=true to clear all indexed content."
+
+        logger.info(
+            f"firecrawl_clear: Starting clear indexed content operation for agent {agent_id}"
+        )
+
+        try:
+            # Delete vector store data (using web_scraper storage format for compatibility)
+            vector_store_key = f"vector_store_{agent_id}"
+            await self.skill_store.delete_agent_skill_data(
+                agent_id, "web_scraper", vector_store_key
+            )
+
+            # Delete metadata
+            metadata_key = f"indexed_urls_{agent_id}"
+            await self.skill_store.delete_agent_skill_data(
+                agent_id, "web_scraper", metadata_key
+            )
+
+            logger.info(
+                f"firecrawl_clear: Successfully cleared all indexed content for agent {agent_id}"
+            )
+            return "Successfully cleared all Firecrawl indexed content. The vector store is now empty and ready for new content."
+
+        except Exception as e:
+            logger.error(
+                f"firecrawl_clear: Error clearing indexed content for agent {agent_id}: {e}",
+                exc_info=True,
+            )
+            return f"Error clearing indexed content: {str(e)}"