intentkit 0.5.0__py3-none-any.whl → 0.5.2__py3-none-any.whl

intentkit/skills/web_scraper/README.md

@@ -0,0 +1,82 @@
+# Web Scraper & Content Indexing Skills
+
+Intelligent web scraping and content indexing using LangChain's WebBaseLoader with vector search capabilities.
+
+## Skills
+
+### 🔍 `scrape_and_index`
+Scrape content from URLs and index into a searchable vector store with configurable chunking and persistent storage.
+
+### 🔎 `query_indexed_content`
+Search indexed content using semantic similarity to answer questions and retrieve relevant information.
+
+## Key Features
+
+- **Multi-URL Support**: Scrape up to 10 URLs simultaneously
+- **Smart Chunking**: Configurable text splitting (100-4000 chars) with overlap
+- **Vector Search**: FAISS + OpenAI embeddings for semantic retrieval
+- **Agent Storage**: Persistent, per-agent content indexing
+- **Rate Limiting**: Respectful scraping (0.1-10 req/sec)
+
+## Testing Examples
+
+### 1. Basic Scraping & Indexing
+
+**Agent Prompt:**
+```
+Please scrape and index this URL: https://docs.crestal.network/introduction
+```
+
+**Expected Response:**
+- Confirmation of successful scraping
+- Number of URLs processed and chunks created
+- Storage confirmation message
+
+### 2. Custom Chunking
+
+**Agent Prompt:**
+```
+Scrape and index https://docs.crestal.network/introduction with chunk size 500 and overlap 100.
+```
+
+### 3. Content Querying
+
+**Agent Prompt (after indexing):**
+```
+Based on the indexed documentation, what are the main items in it?
+```
+
+
+## Testing Workflow
+
+1. **Setup**: Configure the skill in your agent
+2. **Index Content**: Use `scrape_and_index` with test URLs
+3. **Query Content**: Use `query_indexed_content` with questions
+4. **Verify**: Check responses include source attribution and relevant content
+
+## API Testing
+
+```bash
+# Test scraping via API
+curl -X POST "http://localhost:8000/agents/your-agent-id/chat" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "message": "Scrape and index https://docs.crestal.network/introduction"
+  }'
+
+# Test querying via API  
+curl -X POST "http://localhost:8000/agents/your-agent-id/chat" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "message": "What information did you find?"
+  }'
+```
+
+## Dependencies
+
+Required packages (add to `pyproject.toml` if missing):
+- `langchain-community` - WebBaseLoader
+- `langchain-openai` - Embeddings
+- `langchain-text-splitters` - Document chunking  
+- `faiss-cpu` - Vector storage
+- `beautifulsoup4` - HTML parsing

intentkit/skills/web_scraper/__init__.py

@@ -0,0 +1,92 @@
+"""Web scraper skills for content indexing and retrieval."""
+
+import logging
+from typing import TypedDict
+
+from intentkit.abstracts.skill import SkillStoreABC
+from intentkit.skills.base import SkillConfig, SkillOwnerState, SkillState
+from intentkit.skills.web_scraper.base import WebScraperBaseTool
+from intentkit.skills.web_scraper.scrape_and_index import (
+    QueryIndexedContent,
+    ScrapeAndIndex,
+)
+
+# Cache skills at the system level, because they are stateless
+_cache: dict[str, WebScraperBaseTool] = {}
+
+logger = logging.getLogger(__name__)
+
+
+class SkillStates(TypedDict):
+    scrape_and_index: SkillOwnerState
+    query_indexed_content: SkillState
+
+
+class Config(SkillConfig):
+    """Configuration for web scraper skills."""
+
+    states: SkillStates
+
+
+async def get_skills(
+    config: "Config",
+    is_private: bool,
+    store: SkillStoreABC,
+    **_,
+) -> list[WebScraperBaseTool]:
+    """Get all web scraper skills.
+
+    Args:
+        config: The configuration for web scraper skills.
+        is_private: Whether to include private skills.
+        store: The skill store for persisting data.
+
+    Returns:
+        A list of web scraper skills.
+    """
+    available_skills = []
+
+    # Include skills based on their state
+    for skill_name, state in config["states"].items():
+        if state == "disabled":
+            continue
+        elif state == "public" or (state == "private" and is_private):
+            available_skills.append(skill_name)
+
+    # Get each skill using the cached getter
+    result = []
+    for name in available_skills:
+        skill = get_web_scraper_skill(name, store)
+        if skill:
+            result.append(skill)
+    return result
+
+
+def get_web_scraper_skill(
+    name: str,
+    store: SkillStoreABC,
+) -> WebScraperBaseTool:
+    """Get a web scraper skill by name.
+
+    Args:
+        name: The name of the skill to get
+        store: The skill store for persisting data
+
+    Returns:
+        The requested web scraper skill
+    """
+    if name == "scrape_and_index":
+        if name not in _cache:
+            _cache[name] = ScrapeAndIndex(
+                skill_store=store,
+            )
+        return _cache[name]
+    elif name == "query_indexed_content":
+        if name not in _cache:
+            _cache[name] = QueryIndexedContent(
+                skill_store=store,
+            )
+        return _cache[name]
+    else:
+        logger.warning(f"Unknown web scraper skill: {name}")
+        return None

intentkit/skills/web_scraper/base.py

@@ -0,0 +1,21 @@
+from typing import Type
+
+from pydantic import BaseModel, Field
+
+from intentkit.abstracts.skill import SkillStoreABC
+from intentkit.skills.base import IntentKitSkill
+
+
+class WebScraperBaseTool(IntentKitSkill):
+    """Base class for web scraper tools."""
+
+    name: str = Field(description="The name of the tool")
+    description: str = Field(description="A description of what the tool does")
+    args_schema: Type[BaseModel]
+    skill_store: SkillStoreABC = Field(
+        description="The skill store for persisting data"
+    )
+
+    @property
+    def category(self) -> str:
+        return "web_scraper"

intentkit/skills/web_scraper/schema.json

@@ -0,0 +1,115 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "title": "Web Scraper & Content Indexing",
+  "description": "Scrape web content and index it for intelligent querying and retrieval",
+  "x-icon": "https://ai.service.crestal.dev/skills/web_scraper/langchain.png",
+  "x-tags": [
+    "Web Scraping",
+    "Content Indexing",
+    "Vector Search",
+    "LangChain",
+    "Document Retrieval"
+  ],
+  "properties": {
+    "enabled": {
+      "type": "boolean",
+      "title": "Enabled",
+      "description": "Whether this skill is enabled",
+      "default": false
+    },
+    "states": {
+      "type": "object",
+      "properties": {
+        "scrape_and_index": {
+          "type": "string",
+          "title": "Scrape & Index Content",
+          "enum": [
+            "disabled",
+            "private"
+          ],
+          "x-enum-title": [
+            "Disabled",
+            "Agent Owner Only"
+          ],
+          "description": "Scrape content from web URLs and index it into a searchable vector store for later retrieval. Supports multiple URLs, customizable chunking, and persistent storage.",
+          "default": "private"
+        },
+        "query_indexed_content": {
+          "type": "string",
+          "title": "Query Indexed Content",
+          "enum": [
+            "disabled",
+            "public",
+            "private"
+          ],
+          "x-enum-title": [
+            "Disabled",
+            "Agent Owner + All Users",
+            "Agent Owner Only"
+          ],
+          "description": "Search and retrieve relevant information from previously indexed web content using semantic similarity. Perfect for answering questions based on scraped documents.",
+          "default": "private"
+        }
+      },
+      "description": "Configure the availability of each web scraper skill (disabled, public, or private)"
+    },
+    "max_urls_per_request": {
+      "type": "integer",
+      "title": "Max URLs per Request",
+      "description": "Maximum number of URLs that can be scraped in a single request",
+      "default": 10,
+      "minimum": 1,
+      "maximum": 20
+    },
+    "default_chunk_size": {
+      "type": "integer",
+      "title": "Default Chunk Size",
+      "description": "Default size of text chunks for document indexing (characters)",
+      "default": 1000,
+      "minimum": 100,
+      "maximum": 4000
+    },
+    "default_chunk_overlap": {
+      "type": "integer",
+      "title": "Default Chunk Overlap",
+      "description": "Default overlap between chunks to maintain context (characters)",
+      "default": 200,
+      "minimum": 0,
+      "maximum": 1000
+    },
+    "requests_per_second": {
+      "type": "number",
+      "title": "Requests per Second",
+      "description": "Rate limit for web scraping to be respectful to target servers",
+      "default": 2,
+      "minimum": 0.1,
+      "maximum": 10
+    },
+    "request_timeout": {
+      "type": "integer",
+      "title": "Request Timeout",
+      "description": "Timeout for web requests in seconds",
+      "default": 30,
+      "minimum": 5,
+      "maximum": 120
+    },
+    "api_key_provider": {
+      "type": "string",
+      "title": "API Key Provider",
+      "description": "Who provides the API key for embeddings",
+      "enum": [
+        "platform"
+      ],
+      "x-enum-title": [
+        "Platform Hosted"
+      ],
+      "default": "platform"
+    }
+  },
+  "required": [
+    "states",
+    "enabled"
+  ],
+  "additionalProperties": true
+}

intentkit/skills/web_scraper/scrape_and_index.py

@@ -0,0 +1,327 @@
+import asyncio
+import base64
+import logging
+import os
+import tempfile
+from typing import List, Type
+from urllib.parse import urlparse
+
+from langchain_community.document_loaders import WebBaseLoader
+from langchain_community.vectorstores import FAISS
+from langchain_core.runnables import RunnableConfig
+from langchain_openai import OpenAIEmbeddings
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from pydantic import BaseModel, Field
+
+from intentkit.skills.web_scraper.base import WebScraperBaseTool
+
+logger = logging.getLogger(__name__)
+
+
+class ScrapeAndIndexInput(BaseModel):
+    """Input for ScrapeAndIndex tool."""
+
+    urls: List[str] = Field(
+        description="List of URLs to scrape and index. Each URL should be a valid web address starting with http:// or https://",
+        min_items=1,
+        max_items=10,
+    )
+    chunk_size: int = Field(
+        description="Size of text chunks for indexing (default: 1000)",
+        default=1000,
+        ge=100,
+        le=4000,
+    )
+    chunk_overlap: int = Field(
+        description="Overlap between chunks (default: 200)",
+        default=200,
+        ge=0,
+        le=1000,
+    )
+
+
+class QueryIndexInput(BaseModel):
+    """Input for QueryIndex tool."""
+
+    query: str = Field(
+        description="Question or query to search in the indexed content",
+        min_length=1,
+        max_length=500,
+    )
+    max_results: int = Field(
+        description="Maximum number of relevant documents to return (default: 4)",
+        default=4,
+        ge=1,
+        le=10,
+    )
+
+
+class ScrapeAndIndex(WebScraperBaseTool):
+    """Tool for scraping web content and indexing it into a searchable vector store.
+
+    This tool can scrape multiple URLs, process the content into chunks,
+    and store it in a vector database for later retrieval and question answering.
+    """
+
+    name: str = "web_scraper_scrape_and_index"
+    description: str = (
+        "Scrape content from one or more web URLs and index them into a vector store for later querying.\n"
+        "Use this tool to collect and index web content that you want to reference later.\n"
+        "The indexed content can then be queried using the query_indexed_content tool."
+    )
+    args_schema: Type[BaseModel] = ScrapeAndIndexInput
+
+    def _validate_urls(self, urls: List[str]) -> List[str]:
+        """Validate and filter URLs."""
+        valid_urls = []
+        for url in urls:
+            try:
+                parsed = urlparse(url)
+                if parsed.scheme in ["http", "https"] and parsed.netloc:
+                    valid_urls.append(url)
+                else:
+                    logger.warning(f"Invalid URL format: {url}")
+            except Exception as e:
+                logger.warning(f"Error parsing URL {url}: {e}")
+        return valid_urls
+
+    async def _arun(
+        self,
+        urls: List[str],
+        chunk_size: int = 1000,
+        chunk_overlap: int = 200,
+        config: RunnableConfig = None,
+        **kwargs,
+    ) -> str:
+        """Scrape URLs and index content into vector store."""
+        try:
+            # Validate URLs
+            valid_urls = self._validate_urls(urls)
+            if not valid_urls:
+                return "Error: No valid URLs provided. URLs must start with http:// or https://"
+
+            # Get agent context for storage
+            context = self.context_from_config(config) if config else None
+            agent_id = context.agent.id if context else "default"
+
+            # Load documents from URLs
+            logger.info(f"Scraping {len(valid_urls)} URLs...")
+            loader = WebBaseLoader(
+                web_paths=valid_urls,
+                requests_per_second=2,  # Be respectful to servers
+                show_progress=True,
+            )
+
+            # Configure loader for better content extraction
+            loader.requests_kwargs = {
+                "verify": True,
+                "timeout": 30,
+            }
+
+            documents = await asyncio.to_thread(loader.load)
+
+            if not documents:
+                return "Error: No content could be extracted from the provided URLs."
+
+            # Split documents into chunks
+            text_splitter = RecursiveCharacterTextSplitter(
+                chunk_size=chunk_size,
+                chunk_overlap=chunk_overlap,
+                length_function=len,
+            )
+            split_docs = text_splitter.split_documents(documents)
+
+            if not split_docs:
+                return "Error: No content could be processed into chunks."
+
+            # Create embeddings and vector store
+            api_key = self.skill_store.get_system_config("openai_api_key")
+            embeddings = OpenAIEmbeddings(api_key=api_key)
+
+            # Create vector store
+            vector_store = FAISS.from_documents(split_docs, embeddings)
+
+            # Store the vector store for this agent using a temporary directory
+            vector_store_key = f"vector_store_{agent_id}"
+            metadata_key = f"indexed_urls_{agent_id}"
+
+            # Save vector store to temporary directory and encode to base64
+            with tempfile.TemporaryDirectory() as temp_dir:
+                vector_store.save_local(temp_dir)
+
+                # Read and encode all files in the temporary directory
+                encoded_files = {}
+                for filename in os.listdir(temp_dir):
+                    file_path = os.path.join(temp_dir, filename)
+                    if os.path.isfile(file_path):
+                        with open(file_path, "rb") as f:
+                            encoded_files[filename] = base64.b64encode(f.read()).decode(
+                                "utf-8"
+                            )
+
+            # Store vector store data
+            await self.skill_store.save_agent_skill_data(
+                agent_id=agent_id,
+                skill="web_scraper",
+                key=vector_store_key,
+                data={
+                    "faiss_files": encoded_files,
+                    "chunk_size": chunk_size,
+                    "chunk_overlap": chunk_overlap,
+                },
+            )
+
+            # Store metadata about indexed URLs
+            existing_metadata = (
+                await self.skill_store.get_agent_skill_data(
+                    agent_id, "web_scraper", metadata_key
+                )
+                or {}
+            )
+            existing_metadata.update(
+                {
+                    url: {
+                        "indexed_at": str(asyncio.get_event_loop().time()),
+                        "chunks": len(
+                            [
+                                doc
+                                for doc in split_docs
+                                if doc.metadata.get("source") == url
+                            ]
+                        ),
+                    }
+                    for url in valid_urls
+                }
+            )
+
+            await self.skill_store.save_agent_skill_data(
+                agent_id=agent_id,
+                skill="web_scraper",
+                key=metadata_key,
+                data=existing_metadata,
+            )
+
+            total_chunks = len(split_docs)
+            successful_urls = len(valid_urls)
+
+            return (
+                f"Successfully scraped and indexed {successful_urls} URLs:\n"
+                f"{'• ' + chr(10) + '• '.join(valid_urls)}\n\n"
+                f"Total chunks created: {total_chunks}\n"
+                f"Chunk size: {chunk_size} characters\n"
+                f"Chunk overlap: {chunk_overlap} characters\n\n"
+                f"The content is now indexed and can be queried using the query_indexed_content tool."
+            )
+
+        except Exception as e:
+            logger.error(f"Error in scrape_and_index: {e}")
+            return f"Error scraping and indexing URLs: {str(e)}"
+
+
+class QueryIndexedContent(WebScraperBaseTool):
+    """Tool for querying previously indexed web content.
+
+    This tool searches through content that was previously scraped and indexed
+    using the scrape_and_index tool to answer questions or find relevant information.
+    """
+
+    name: str = "web_scraper_query_indexed_content"
+    description: str = (
+        "Query previously indexed web content to find relevant information and answer questions.\n"
+        "Use this tool to search through content that was previously scraped and indexed.\n"
+        "This tool can help answer questions based on the indexed web content."
+    )
+    args_schema: Type[BaseModel] = QueryIndexInput
+
+    async def _arun(
+        self,
+        query: str,
+        max_results: int = 4,
+        config: RunnableConfig = None,
+        **kwargs,
+    ) -> str:
+        """Query the indexed content."""
+        try:
+            # Get agent context for storage
+            context = self.context_from_config(config) if config else None
+            agent_id = context.agent.id if context else "default"
+
+            # Retrieve vector store
+            vector_store_key = f"vector_store_{agent_id}"
+            metadata_key = f"indexed_urls_{agent_id}"
+
+            stored_data = await self.skill_store.get_agent_skill_data(
+                agent_id, "web_scraper", vector_store_key
+            )
+            if not stored_data or "faiss_files" not in stored_data:
+                return (
+                    "No indexed content found. Please use the scrape_and_index tool first "
+                    "to scrape and index some web content before querying."
+                )
+
+            # Restore vector store from base64 encoded files
+            api_key = self.skill_store.get_system_config("openai_api_key")
+            embeddings = OpenAIEmbeddings(api_key=api_key)
+
+            with tempfile.TemporaryDirectory() as temp_dir:
+                # Decode and write files to temporary directory
+                for filename, encoded_content in stored_data["faiss_files"].items():
+                    file_path = os.path.join(temp_dir, filename)
+                    with open(file_path, "wb") as f:
+                        f.write(base64.b64decode(encoded_content))
+
+                # Load the vector store from the temporary directory
+                vector_store = FAISS.load_local(
+                    temp_dir,
+                    embeddings,
+                    allow_dangerous_deserialization=True,  # Safe since we control the serialization
+                )
+
+                # Perform similarity search
+                relevant_docs = vector_store.similarity_search(query, k=max_results)
+
+            if not relevant_docs:
+                return f"No relevant content found for query: '{query}'"
+
+            # Get metadata about indexed URLs
+            metadata = (
+                await self.skill_store.get_agent_skill_data(
+                    agent_id, "web_scraper", metadata_key
+                )
+                or {}
+            )
+
+            # Format response
+            response_parts = [
+                f"Found {len(relevant_docs)} relevant pieces of content for: '{query}'\n",
+                "=" * 50,
+            ]
+
+            for i, doc in enumerate(relevant_docs, 1):
+                source_url = doc.metadata.get("source", "Unknown source")
+                title = doc.metadata.get("title", "No title")
+
+                response_parts.extend(
+                    [
+                        f"\n{i}. Source: {source_url}",
+                        f"   Title: {title}",
+                        f"   Content:\n   {doc.page_content[:500]}{'...' if len(doc.page_content) > 500 else ''}",
+                        "",
+                    ]
+                )
+
+            # Add summary of indexed content
+            response_parts.extend(
+                [
+                    "\n" + "=" * 50,
+                    f"Total indexed URLs: {len(metadata)}",
+                    "Indexed sources:",
+                    *[f"• {url}" for url in metadata.keys()],
+                ]
+            )
+
+            return "\n".join(response_parts)
+
+        except Exception as e:
+            logger.error(f"Error in query_indexed_content: {e}")
+            return f"Error querying indexed content: {str(e)}"

intentkit/utils/__init__.py

@@ -0,0 +1 @@
+