intentkit 0.6.0.dev7__tar.gz → 0.6.0.dev8__tar.gz

{intentkit-0.6.0.dev7 → intentkit-0.6.0.dev8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: intentkit
-Version: 0.6.0.dev7
+Version: 0.6.0.dev8
 Summary: Intent-based AI Agent Platform - Core Package
 Project-URL: Homepage, https://github.com/crestal-network/intentkit
 Project-URL: Repository, https://github.com/crestal-network/intentkit

{intentkit-0.6.0.dev7 → intentkit-0.6.0.dev8}/__init__.py

@@ -3,7 +3,7 @@
 A powerful platform for building AI agents with blockchain and cryptocurrency capabilities.
 """
 
-__version__ = "0.6.0-dev.7"
+__version__ = "0.6.0-dev.8"
 __author__ = "hyacinthus"
 __email__ = "hyacinthus@gmail.com"
 

{intentkit-0.6.0.dev7 → intentkit-0.6.0.dev8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "intentkit"
-version = "0.6.0-dev.7"
+version = "0.6.0-dev.8"
 description = "Intent-based AI Agent Platform - Core Package"
 authors = [{ name = "hyacinthus", email = "hyacinthus@gmail.com" }]
 requires-python = ">=3.12"

{intentkit-0.6.0.dev7 → intentkit-0.6.0.dev8}/skills/web_scraper/README.md

@@ -10,12 +10,22 @@ Scrape content from URLs and index into a searchable vector store with configura
 ### 🔎 `query_indexed_content`
 Search indexed content using semantic similarity to answer questions and retrieve relevant information.
 
+### `website_indexer`
+Index entire websites by discovering and scraping all pages using sitemaps. Automatically finds sitemaps from robots.txt, extracts all URLs, and comprehensively indexes website content.
+
+### `document_indexer`
+Import and index document content directly to the vector database. Perfect for adding content from Google Docs, Notion pages, PDFs, or any other document sources by copy-pasting.
+
 ## Key Features
 
-- **Multi-URL Support**: Scrape up to 10 URLs simultaneously
+- **Multi-URL Support**: Scrape up to 10 URLs simultaneously 
+- **Sitemap Discovery**: Automatic sitemap detection from robots.txt with common patterns
+- **Direct Text Input**: Add content directly without web scraping
 - **Smart Chunking**: Configurable text splitting (100-4000 chars) with overlap
 - **Vector Search**: FAISS + OpenAI embeddings for semantic retrieval
 - **Agent Storage**: Persistent, per-agent content indexing
+- **Content Filtering**: Include/exclude URL patterns for targeted scraping
+- **Tagging System**: Organize content with custom tags
 - **Rate Limiting**: Respectful scraping (0.1-10 req/sec)
 
 ## Testing Examples
@@ -39,7 +49,27 @@ Please scrape and index this URL: https://docs.crestal.network/introduction
 Scrape and index https://docs.crestal.network/introduction with chunk size 500 and overlap 100.
 ```
 
-### 3. Content Querying
+### 3. Complete Website Indexing
+
+**Agent Prompt:**
+```
+Index the entire documentation site at https://docs.crestal.network using its sitemap. Include only pages with '/docs/' and '/guides/' in the URL, exclude '/admin/' pages, and limit to 50 URLs.
+```
+
+### 4. Document Content Import
+
+**Agent Prompt:**
+```
+I'm going to paste some content from my Google Doc. Please add it to the knowledge base:
+
+Title: "Meeting Notes - Q4 Strategy"
+Source: "Google Docs"
+Tags: "meeting, strategy, q4, planning"
+
+[Paste your document content here...]
+```
+
+### 5. Content Querying
 
 **Agent Prompt (after indexing):**
 ```
@@ -75,8 +105,9 @@ curl -X POST "http://localhost:8000/agents/your-agent-id/chat" \
 ## Dependencies
 
 Required packages (add to `pyproject.toml` if missing):
-- `langchain-community` - WebBaseLoader
+- `langchain-community` - WebBaseLoader and document processing
 - `langchain-openai` - Embeddings
 - `langchain-text-splitters` - Document chunking  
 - `faiss-cpu` - Vector storage
-- `beautifulsoup4` - HTML parsing
+- `beautifulsoup4` - HTML parsing
+- `httpx` - Async HTTP client for sitemap discovery

{intentkit-0.6.0.dev7 → intentkit-0.6.0.dev8}/skills/web_scraper/__init__.py

@@ -6,10 +6,12 @@ from typing import TypedDict
 from intentkit.abstracts.skill import SkillStoreABC
 from intentkit.skills.base import SkillConfig, SkillOwnerState, SkillState
 from intentkit.skills.web_scraper.base import WebScraperBaseTool
+from intentkit.skills.web_scraper.document_indexer import DocumentIndexer
 from intentkit.skills.web_scraper.scrape_and_index import (
     QueryIndexedContent,
     ScrapeAndIndex,
 )
+from intentkit.skills.web_scraper.website_indexer import WebsiteIndexer
 
 # Cache skills at the system level, because they are stateless
 _cache: dict[str, WebScraperBaseTool] = {}
@@ -20,6 +22,8 @@ logger = logging.getLogger(__name__)
 class SkillStates(TypedDict):
     scrape_and_index: SkillOwnerState
     query_indexed_content: SkillState
+    website_indexer: SkillOwnerState
+    document_indexer: SkillOwnerState
 
 
 class Config(SkillConfig):
@@ -87,6 +91,18 @@ def get_web_scraper_skill(
                 skill_store=store,
             )
         return _cache[name]
+    elif name == "website_indexer":
+        if name not in _cache:
+            _cache[name] = WebsiteIndexer(
+                skill_store=store,
+            )
+        return _cache[name]
+    elif name == "document_indexer":
+        if name not in _cache:
+            _cache[name] = DocumentIndexer(
+                skill_store=store,
+            )
+        return _cache[name]
     else:
         logger.warning(f"Unknown web scraper skill: {name}")
         return None

intentkit-0.6.0.dev8/skills/web_scraper/document_indexer.py

@@ -0,0 +1,143 @@
+import logging
+from typing import Type
+
+from langchain_core.runnables import RunnableConfig
+from pydantic import BaseModel, Field
+
+from intentkit.skills.web_scraper.base import WebScraperBaseTool
+from intentkit.skills.web_scraper.utils import (
+    DocumentProcessor,
+    MetadataManager,
+    ResponseFormatter,
+    VectorStoreManager,
+    index_documents,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class DocumentIndexerInput(BaseModel):
+    """Input for DocumentIndexer tool."""
+
+    text_content: str = Field(
+        description="The text content to add to the vector database. Can be content from Google Docs, Notion, or any other text source",
+        min_length=10,
+        max_length=100000,
+    )
+    title: str = Field(
+        description="Title or name for this text content (will be used as metadata)",
+        max_length=200,
+    )
+    source: str = Field(
+        description="Source of the text content (e.g., 'Google Doc', 'Notion Page', 'Manual Entry')",
+        default="Manual Entry",
+        max_length=100,
+    )
+    chunk_size: int = Field(
+        description="Size of text chunks for indexing (default: 1000)",
+        default=1000,
+        ge=100,
+        le=4000,
+    )
+    chunk_overlap: int = Field(
+        description="Overlap between chunks (default: 200)",
+        default=200,
+        ge=0,
+        le=1000,
+    )
+    tags: str = Field(
+        description="Optional tags for categorizing the content (comma-separated)",
+        default="",
+        max_length=500,
+    )
+
+
+class DocumentIndexer(WebScraperBaseTool):
+    """Tool for importing and indexing document content to the vector database.
+
+    This tool allows users to copy and paste document content from various sources
+    (like Google Docs, Notion, PDFs, etc.) and index it directly into the vector store
+    for later querying and retrieval.
+    """
+
+    name: str = "web_scraper_document_indexer"
+    description: str = (
+        "Import and index document content directly to the vector database. "
+        "Perfect for adding content from Google Docs, Notion pages, PDFs, or any other document sources. "
+        "The indexed content can then be queried using the query_indexed_content tool."
+    )
+    args_schema: Type[BaseModel] = DocumentIndexerInput
+
+    async def _arun(
+        self,
+        text_content: str,
+        title: str,
+        source: str = "Manual Entry",
+        chunk_size: int = 1000,
+        chunk_overlap: int = 200,
+        tags: str = "",
+        config: RunnableConfig = None,
+        **kwargs,
+    ) -> str:
+        """Add text content to the vector database."""
+        # Get agent context - throw error if not available
+        if not config:
+            raise ValueError("Configuration is required but not provided")
+
+        context = self.context_from_config(config)
+        if not context or not context.agent or not context.agent.id:
+            raise ValueError("Agent ID is required but not found in configuration")
+
+        agent_id = context.agent.id
+
+        logger.info(f"[{agent_id}] Starting document indexing for title: '{title}'")
+
+        # Validate content
+        if not DocumentProcessor.validate_content(text_content):
+            logger.error(f"[{agent_id}] Content validation failed - too short")
+            return "Error: Text content is too short. Please provide at least 10 characters of content."
+
+        # Create document with metadata
+        document = DocumentProcessor.create_document(
+            text_content,
+            title,
+            source,
+            tags,
+            extra_metadata={"source_type": "document_indexer"},
+        )
+
+        logger.info(
+            f"[{agent_id}] Document created, length: {len(document.page_content)} chars"
+        )
+
+        # Index the document
+        total_chunks, was_merged = await index_documents(
+            [document], agent_id, self.skill_store, chunk_size, chunk_overlap
+        )
+
+        # Get current storage size for response
+        vs_manager = VectorStoreManager(self.skill_store)
+        current_size = await vs_manager.get_content_size(agent_id)
+
+        # Update metadata
+        metadata_manager = MetadataManager(self.skill_store)
+        new_metadata = metadata_manager.create_document_metadata(
+            title, source, tags, [document], len(text_content)
+        )
+        await metadata_manager.update_metadata(agent_id, new_metadata)
+
+        logger.info(f"[{agent_id}] Document indexing completed successfully")
+
+        # Format response
+        response = ResponseFormatter.format_indexing_response(
+            "indexed",
+            f"Document: {title}",
+            total_chunks,
+            chunk_size,
+            chunk_overlap,
+            was_merged,
+            current_size_bytes=current_size,
+        )
+
+        logger.info(f"[{agent_id}] Document indexing completed successfully")
+        return response

{intentkit-0.6.0.dev7 → intentkit-0.6.0.dev8}/skills/web_scraper/schema.json

@@ -50,6 +50,34 @@
           ],
           "description": "Search and retrieve relevant information from previously indexed web content using semantic similarity. Perfect for answering questions based on scraped documents.",
           "default": "private"
+        },
+        "website_indexer": {
+          "type": "string",
+          "title": "Complete Website Indexer",
+          "enum": [
+            "disabled",
+            "private"
+          ],
+          "x-enum-title": [
+            "Disabled",
+            "Agent Owner Only"
+          ],
+          "description": "Index entire websites by discovering and scraping all pages using sitemaps. Automatically finds sitemaps from robots.txt, extracts all URLs, and comprehensively indexes website content.",
+          "default": "private"
+        },
+        "document_indexer": {
+          "type": "string",
+          "title": "Document Content Indexer",
+          "enum": [
+            "disabled",
+            "private"
+          ],
+          "x-enum-title": [
+            "Disabled",
+            "Agent Owner Only"
+          ],
+          "description": "Import and index document content directly to the vector database. Perfect for adding content from Google Docs, Notion pages, PDFs, or any other document sources by copy-pasting.",
+          "default": "private"
         }
       },
       "description": "Configure the availability of each web scraper skill (disabled, public, or private)"

intentkit-0.6.0.dev8/skills/web_scraper/scrape_and_index.py

@@ -0,0 +1,262 @@
+import logging
+from typing import List, Type
+
+from langchain_core.runnables import RunnableConfig
+from pydantic import BaseModel, Field
+
+from intentkit.skills.web_scraper.base import WebScraperBaseTool
+from intentkit.skills.web_scraper.utils import (
+    DEFAULT_CHUNK_OVERLAP,
+    DEFAULT_CHUNK_SIZE,
+    MetadataManager,
+    ResponseFormatter,
+    VectorStoreManager,
+    scrape_and_index_urls,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class ScrapeAndIndexInput(BaseModel):
+    """Input for ScrapeAndIndex tool."""
+
+    urls: List[str] = Field(
+        description="List of URLs to scrape and index. Each URL should be a valid web address starting with http:// or https://",
+        min_items=1,
+        max_items=10,
+    )
+    chunk_size: int = Field(
+        description="Size of text chunks for indexing (default: 1000)",
+        default=DEFAULT_CHUNK_SIZE,
+        ge=100,
+        le=4000,
+    )
+    chunk_overlap: int = Field(
+        description="Overlap between chunks (default: 200)",
+        default=DEFAULT_CHUNK_OVERLAP,
+        ge=0,
+        le=1000,
+    )
+
+
+class QueryIndexInput(BaseModel):
+    """Input for QueryIndex tool."""
+
+    query: str = Field(
+        description="Question or query to search in the indexed content",
+        min_length=1,
+        max_length=500,
+    )
+    max_results: int = Field(
+        description="Maximum number of relevant documents to return (default: 4)",
+        default=4,
+        ge=1,
+        le=10,
+    )
+
+
+class ScrapeAndIndex(WebScraperBaseTool):
+    """Tool for scraping web content and indexing it into a searchable vector store.
+
+    This tool can scrape multiple URLs, process the content into chunks,
+    and store it in a vector database for later retrieval and question answering.
+    """
+
+    name: str = "web_scraper_scrape_and_index"
+    description: str = (
+        "Scrape content from one or more web URLs and index them into a vector store for later querying.\n"
+        "Use this tool to collect and index web content that you want to reference later.\n"
+        "The indexed content can then be queried using the query_indexed_content tool."
+    )
+    args_schema: Type[BaseModel] = ScrapeAndIndexInput
+
+    async def _arun(
+        self,
+        urls: List[str],
+        chunk_size: int = DEFAULT_CHUNK_SIZE,
+        chunk_overlap: int = DEFAULT_CHUNK_OVERLAP,
+        config: RunnableConfig = None,
+        **kwargs,
+    ) -> str:
+        """Scrape URLs and index content into vector store."""
+        try:
+            # Get agent context - throw error if not available
+            if not config:
+                raise ValueError("Configuration is required but not provided")
+
+            context = self.context_from_config(config)
+            if not context or not context.agent or not context.agent.id:
+                raise ValueError("Agent ID is required but not found in configuration")
+
+            agent_id = context.agent.id
+
+            logger.info(
+                f"[{agent_id}] Starting scrape and index operation with {len(urls)} URLs"
+            )
+
+            # Use the utility function to scrape and index URLs
+            total_chunks, was_merged, valid_urls = await scrape_and_index_urls(
+                urls, agent_id, self.skill_store, chunk_size, chunk_overlap
+            )
+
+            logger.info(
+                f"[{agent_id}] Scraping completed: {total_chunks} chunks indexed, merged: {was_merged}"
+            )
+
+            if not valid_urls:
+                logger.error(f"[{agent_id}] No valid URLs provided")
+                return "Error: No valid URLs provided. URLs must start with http:// or https://"
+
+            if total_chunks == 0:
+                logger.error(f"[{agent_id}] No content extracted from URLs")
+                return "Error: No content could be extracted from the provided URLs."
+
+            # Get current storage size for response
+            vs_manager = VectorStoreManager(self.skill_store)
+            current_size = await vs_manager.get_content_size(agent_id)
+            size_limit_reached = len(valid_urls) < len(urls)
+
+            # Update metadata
+            metadata_manager = MetadataManager(self.skill_store)
+            new_metadata = metadata_manager.create_url_metadata(
+                valid_urls, [], "scrape_and_index"
+            )
+            await metadata_manager.update_metadata(agent_id, new_metadata)
+
+            logger.info(f"[{agent_id}] Metadata updated successfully")
+
+            # Format response
+            response = ResponseFormatter.format_indexing_response(
+                "scraped and indexed",
+                valid_urls,
+                total_chunks,
+                chunk_size,
+                chunk_overlap,
+                was_merged,
+                current_size_bytes=current_size,
+                size_limit_reached=size_limit_reached,
+                total_requested_urls=len(urls),
+            )
+
+            logger.info(
+                f"[{agent_id}] Scrape and index operation completed successfully"
+            )
+            return response
+
+        except Exception as e:
+            # Extract agent_id for error logging if possible
+            agent_id = "UNKNOWN"
+            try:
+                if config:
+                    context = self.context_from_config(config)
+                    if context and context.agent and context.agent.id:
+                        agent_id = context.agent.id
+            except Exception:
+                pass
+
+            logger.error(f"[{agent_id}] Error in ScrapeAndIndex: {e}", exc_info=True)
+            raise type(e)(f"[agent:{agent_id}]: {e}") from e
+
+
+class QueryIndexedContent(WebScraperBaseTool):
+    """Tool for querying previously indexed web content.
+
+    This tool searches through content that was previously scraped and indexed
+    using the scrape_and_index tool to answer questions or find relevant information.
+    """
+
+    name: str = "web_scraper_query_indexed_content"
+    description: str = (
+        "Query previously indexed web content to find relevant information and answer questions.\n"
+        "Use this tool to search through content that was previously scraped and indexed.\n"
+        "This tool can help answer questions based on the indexed web content."
+    )
+    args_schema: Type[BaseModel] = QueryIndexInput
+
+    async def _arun(
+        self,
+        query: str,
+        max_results: int = 4,
+        config: RunnableConfig = None,
+        **kwargs,
+    ) -> str:
+        """Query the indexed content."""
+        try:
+            # Get agent context - throw error if not available
+            if not config:
+                raise ValueError("Configuration is required but not provided")
+
+            context = self.context_from_config(config)
+            if not context or not context.agent or not context.agent.id:
+                raise ValueError("Agent ID is required but not found in configuration")
+
+            agent_id = context.agent.id
+
+            logger.info(f"[{agent_id}] Starting query operation: '{query}'")
+
+            # Retrieve vector store
+            vector_store_key = f"vector_store_{agent_id}"
+
+            logger.info(f"[{agent_id}] Looking for vector store: {vector_store_key}")
+
+            stored_data = await self.skill_store.get_agent_skill_data(
+                agent_id, "web_scraper", vector_store_key
+            )
+
+            if not stored_data:
+                logger.warning(f"[{agent_id}] No vector store found")
+                return "No indexed content found. Please use the scrape_and_index tool first to scrape and index some web content before querying."
+
+            if not stored_data or "faiss_files" not in stored_data:
+                logger.warning(f"[{agent_id}] Invalid stored data structure")
+                return "No indexed content found. Please use the scrape_and_index tool first to scrape and index some web content before querying."
+
+            # Create embeddings and decode vector store
+            logger.info(f"[{agent_id}] Decoding vector store")
+            vs_manager = VectorStoreManager(self.skill_store)
+            embeddings = vs_manager.create_embeddings()
+            vector_store = vs_manager.decode_vector_store(
+                stored_data["faiss_files"], embeddings
+            )
+
+            logger.info(
+                f"[{agent_id}] Vector store loaded, index count: {vector_store.index.ntotal}"
+            )
+
+            # Perform similarity search
+            docs = vector_store.similarity_search(query, k=max_results)
+            logger.info(f"[{agent_id}] Found {len(docs)} similar documents")
+
+            if not docs:
+                logger.info(f"[{agent_id}] No relevant documents found for query")
+                return f"No relevant information found for your query: '{query}'. The indexed content may not contain information related to your search."
+
+            # Format results
+            results = []
+            for i, doc in enumerate(docs, 1):
+                content = doc.page_content.strip()
+                source = doc.metadata.get("source", "Unknown")
+                results.append(f"**Source {i}:** {source}\n{content}")
+
+            response = "\n\n".join(results)
+            logger.info(
+                f"[{agent_id}] Query completed successfully, returning {len(response)} chars"
+            )
+
+            return response
+
+        except Exception as e:
+            # Extract agent_id for error logging if possible
+            agent_id = "UNKNOWN"
+            try:
+                if config:
+                    context = self.context_from_config(config)
+                    if context and context.agent and context.agent.id:
+                        agent_id = context.agent.id
+            except Exception:
+                pass
+
+            logger.error(
+                f"[{agent_id}] Error in QueryIndexedContent: {e}", exc_info=True
+            )
+            raise type(e)(f"[agent:{agent_id}]: {e}") from e