intentkit 0.6.0.dev11__py3-none-any.whl → 0.6.0.dev12__py3-none-any.whl

intentkit/__init__.py

@@ -3,7 +3,7 @@
 A powerful platform for building AI agents with blockchain and cryptocurrency capabilities.
 """
 
-__version__ = "0.6.0-dev.11"
+__version__ = "0.6.0-dev.12"
 __author__ = "hyacinthus"
 __email__ = "hyacinthus@gmail.com"
 

intentkit/config/config.py

@@ -121,6 +121,7 @@ class Config:
         self.moralis_api_key = self.load("MORALIS_API_KEY")
         self.tavily_api_key = self.load("TAVILY_API_KEY")
         self.cookiefun_api_key = self.load("COOKIEFUN_API_KEY")
+        self.firecrawl_api_key = self.load("FIRECRAWL_API_KEY")
         # Sentry
         self.sentry_dsn = self.load("SENTRY_DSN")
         self.sentry_sample_rate = float(self.load("SENTRY_SAMPLE_RATE", "0.1"))

intentkit/models/agent_schema.json

@@ -539,6 +539,10 @@
           "title": "Web Scraper & Content Indexing",
           "$ref": "../skills/web_scraper/schema.json"
         },
+        "firecrawl": {
+          "title": "Firecrawl Web Scraping",
+          "$ref": "../skills/firecrawl/schema.json"
+        },
         "aixbt": {
           "title": "AIXBT",
           "$ref": "../skills/aixbt/schema.json"

intentkit/skills/firecrawl/README.md

@@ -0,0 +1,178 @@
+# Firecrawl Skills
+
+The Firecrawl skills provide advanced web scraping and content indexing capabilities using the Firecrawl API. These skills can handle JavaScript-heavy websites, PDFs, and provide automatic content indexing for intelligent querying.
+
+## Skills Overview
+
+### 1. firecrawl_scrape
+Scrapes a single webpage and optionally indexes the content for future querying.
+
+**Parameters:**
+- `url` (required): The URL to scrape
+- `formats` (optional): Output formats - markdown, html, rawHtml, screenshot, links, extract (default: ["markdown"])
+- `include_tags` (optional): HTML tags to include (e.g., ["h1", "h2", "p"])
+- `exclude_tags` (optional): HTML tags to exclude
+- `only_main_content` (optional): Extract only main content (default: true)
+- `index_content` (optional): Whether to index content for querying (default: true)
+- `chunk_size` (optional): Size of text chunks for indexing (default: 1000)
+- `chunk_overlap` (optional): Overlap between chunks (default: 200)
+
+### 2. firecrawl_crawl
+Crawls multiple pages from a website and indexes all content.
+
+**Parameters:**
+- `url` (required): The base URL to start crawling
+- `include_paths` (optional): URL patterns to include (e.g., ["/docs/*"])
+- `exclude_paths` (optional): URL patterns to exclude
+- `max_depth` (optional): Maximum crawl depth (default: 2)
+- `limit` (optional): Maximum number of pages to crawl (default: 5)
+- `index_content` (optional): Whether to index content for querying (default: true)
+- `chunk_size` (optional): Size of text chunks for indexing (default: 1000)
+- `chunk_overlap` (optional): Overlap between chunks (default: 200)
+
+### 3. firecrawl_query_indexed_content
+Queries previously indexed Firecrawl content using semantic search.
+
+**Parameters:**
+- `query` (required): The search query
+- `limit` (optional): Maximum number of results to return (1-10, default: 4)
+
+## API Key Configuration
+Set your Firecrawl API key as an environment variable:
+```bash
+export FIRECRAWL_API_KEY=fc-your-api-key-here
+```
+
+## Testing Instructions
+
+### Step 1: Create an Agent with Firecrawl Skills
+
+1. **Create a new agent** via the API or UI with the following skills:
+   ```json
+   {
+     "skills": [
+       "firecrawl_scrape",
+       "firecrawl_crawl", 
+       "firecrawl_query_indexed_content"
+     ]
+   }
+   ```
+
+2. **Note the agent ID** for testing
+
+### Step 2: Test Single Page Scraping
+
+**Test scraping a documentation homepage:**
+```
+Prompt: "Use firecrawl_scrape to scrape https://docs.joincommonwealth.xyz/ and index the content for future querying"
+```
+
+**Expected Result:**
+- Content successfully scraped
+- Content automatically indexed with metadata
+- Confirmation of chunk creation and indexing
+
+### Step 3: Test Content Crawling
+
+**Test crawling multiple pages:**
+```
+Prompt: "Use firecrawl_crawl to crawl https://docs.joincommonwealth.xyz/ with max_depth=2 and limit=3 to index multiple documentation pages"
+```
+
+**Expected Result:**
+- Multiple pages crawled and scraped
+- Each page indexed separately
+- Batch processing confirmation
+
+### Step 4: Test Content Querying
+
+**Test querying indexed content:**
+```
+Prompt: "Use firecrawl_query_indexed_content to search for 'What is All Street and what is its purpose?' in the indexed content"
+```
+
+**Expected Result:**
+- Relevant content retrieved from indexed documents
+- Results tagged with [Firecrawl Scrape] or [Firecrawl Crawl]
+- Source URLs and metadata included
+
+### Step 5: Test Advanced Scraping Options
+
+**Test with specific formatting:**
+```
+Prompt: "Use firecrawl_scrape to scrape https://docs.joincommonwealth.xyz/all-street-manifesto with formats=['markdown', 'html'] and include_tags=['h1', 'h2', 'p'] and index_content=true"
+```
+
+**Expected Result:**
+- Content in both markdown and HTML formats
+- Only specified HTML tags included
+- Content indexed for querying
+
+### Step 6: Test Multiple Queries
+
+**Test different query types:**
+```
+Prompt: "Use firecrawl_query_indexed_content to search for 'democratize finance' in the indexed content"
+```
+
+**Expected Result:**
+- Relevant content retrieved from Firecrawl's independent vector store
+- Results tagged with [Firecrawl Scrape] or [Firecrawl Crawl]
+- Source URLs and metadata included
+
+## Common Use Cases
+
+### Documentation Indexing
+```
+1. Scrape main documentation page
+2. Crawl related documentation sections
+3. Query for specific technical information
+```
+
+### Competitive Analysis
+```
+1. Scrape competitor websites
+2. Index product information and features
+3. Query for specific comparisons
+```
+
+### Research and Knowledge Base
+```
+1. Crawl research papers or articles
+2. Index academic or technical content
+3. Query for specific concepts or methodologies
+```
+
+## Troubleshooting
+
+### Common Issues
+
+1. **API Key Not Found**
+   - Ensure `FIRECRAWL_API_KEY` environment variable is set
+   - Restart the IntentKit server after setting the key
+
+2. **Scraping Failures**
+   - Check if the URL is accessible
+   - Verify Firecrawl API quota and limits
+   - Some websites may block scraping
+
+3. **Indexing Errors**
+   - Ensure OpenAI API key is configured for embeddings
+   - Check if content is too large for processing
+   - Verify vector store permissions
+
+4. **Query Returns No Results**
+   - Ensure content was successfully indexed
+   - Try broader or different search terms
+   - Check if vector store contains data
+
+## Features and Benefits
+
+- **JavaScript Rendering**: Handles SPAs and dynamic content
+- **PDF Support**: Can scrape and index PDF documents
+- **Intelligent Chunking**: Optimized text splitting for better search
+- **Independent Storage**: Uses its own dedicated vector store for Firecrawl content
+- **Metadata Rich**: Includes source URLs, timestamps, and content types
+- **Semantic Search**: Uses OpenAI embeddings for intelligent querying
+- **Batch Processing**: Efficient handling of multiple pages
+- **Content Filtering**: Flexible include/exclude options for targeted scraping

intentkit/skills/firecrawl/__init__.py

@@ -0,0 +1,99 @@
+"""Firecrawl skills for web scraping and crawling."""
+
+import logging
+from typing import TypedDict
+
+from intentkit.abstracts.skill import SkillStoreABC
+from intentkit.skills.base import SkillConfig, SkillState
+from intentkit.skills.firecrawl.base import FirecrawlBaseTool
+from intentkit.skills.firecrawl.crawl import FirecrawlCrawl
+from intentkit.skills.firecrawl.query import FirecrawlQueryIndexedContent
+from intentkit.skills.firecrawl.scrape import FirecrawlScrape
+
+# Cache skills at the system level, because they are stateless
+_cache: dict[str, FirecrawlBaseTool] = {}
+
+logger = logging.getLogger(__name__)
+
+
+class SkillStates(TypedDict):
+    firecrawl_scrape: SkillState
+    firecrawl_crawl: SkillState
+    firecrawl_query_indexed_content: SkillState
+
+
+class Config(SkillConfig):
+    """Configuration for Firecrawl skills."""
+
+    states: SkillStates
+    api_key: str = ""
+    api_key_provider: str = "agent_owner"
+    rate_limit_number: int = 100
+    rate_limit_minutes: int = 60
+
+
+async def get_skills(
+    config: "Config",
+    is_private: bool,
+    store: SkillStoreABC,
+    **_,
+) -> list[FirecrawlBaseTool]:
+    """Get all Firecrawl skills.
+
+    Args:
+        config: The configuration for Firecrawl skills.
+        is_private: Whether to include private skills.
+        store: The skill store for persisting data.
+
+    Returns:
+        A list of Firecrawl skills.
+    """
+    available_skills = []
+
+    # Include skills based on their state
+    for skill_name, state in config["states"].items():
+        if state == "disabled":
+            continue
+        elif state == "public" or (state == "private" and is_private):
+            available_skills.append(skill_name)
+
+    # Get each skill using the cached getter
+    return [get_firecrawl_skill(name, store) for name in available_skills]
+
+
+def get_firecrawl_skill(
+    name: str,
+    store: SkillStoreABC,
+) -> FirecrawlBaseTool:
+    """Get a Firecrawl skill by name.
+
+    Args:
+        name: The name of the skill to get
+        store: The skill store for persisting data
+
+    Returns:
+        The requested Firecrawl skill
+
+    Raises:
+        ValueError: If the skill name is unknown
+    """
+    if name == "firecrawl_scrape":
+        if name not in _cache:
+            _cache[name] = FirecrawlScrape(
+                skill_store=store,
+            )
+        return _cache[name]
+    elif name == "firecrawl_crawl":
+        if name not in _cache:
+            _cache[name] = FirecrawlCrawl(
+                skill_store=store,
+            )
+        return _cache[name]
+    elif name == "firecrawl_query_indexed_content":
+        if name not in _cache:
+            _cache[name] = FirecrawlQueryIndexedContent(
+                skill_store=store,
+            )
+        return _cache[name]
+    else:
+        raise ValueError(f"Unknown Firecrawl skill: {name}")

intentkit/skills/firecrawl/base.py

@@ -0,0 +1,28 @@
+from typing import Type
+
+from pydantic import BaseModel, Field
+
+from intentkit.abstracts.skill import SkillStoreABC
+from intentkit.skills.base import IntentKitSkill, SkillContext
+
+
+class FirecrawlBaseTool(IntentKitSkill):
+    """Base class for Firecrawl tools."""
+
+    name: str = Field(description="The name of the tool")
+    description: str = Field(description="A description of what the tool does")
+    args_schema: Type[BaseModel]
+    skill_store: SkillStoreABC = Field(
+        description="The skill store for persisting data"
+    )
+
+    def get_api_key(self, context: SkillContext) -> str:
+        """Get the Firecrawl API key from configuration."""
+        skill_config = context.config
+        if skill_config.get("api_key_provider") == "agent_owner":
+            return skill_config.get("api_key")
+        return self.skill_store.get_system_config("firecrawl_api_key")
+
+    @property
+    def category(self) -> str:
+        return "firecrawl"