intentkit 0.6.19.dev1__tar.gz → 0.6.20.dev1__tar.gz

{intentkit-0.6.19.dev1 → intentkit-0.6.20.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: intentkit
-Version: 0.6.19.dev1
+Version: 0.6.20.dev1
 Summary: Intent-based AI Agent Platform - Core Package
 Project-URL: Homepage, https://github.com/crestal-network/intentkit
 Project-URL: Repository, https://github.com/crestal-network/intentkit

{intentkit-0.6.19.dev1 → intentkit-0.6.20.dev1}/__init__.py

@@ -3,7 +3,7 @@
 A powerful platform for building AI agents with blockchain and cryptocurrency capabilities.
 """
 
-__version__ = "0.6.19-dev1"
+__version__ = "0.6.20-dev.1"
 __author__ = "hyacinthus"
 __email__ = "hyacinthus@gmail.com"
 

{intentkit-0.6.19.dev1 → intentkit-0.6.20.dev1}/clients/cdp.py

@@ -144,6 +144,17 @@ class CdpClient:
             wallet_secret=wallet_secret,
         )
         self._wallet_provider = CdpEvmServerWalletProvider(self._wallet_provider_config)
+        # hack for cdp bug
+        if network_id == "base-mainnet":
+            self._wallet_provider._network.network_id = "base"
+        elif network_id == "arbitrum-mainnet":
+            self._wallet_provider._network.network_id = "arbitrum"
+        elif network_id == "optimism-mainnet":
+            self._wallet_provider._network.network_id = "optimism"
+        elif network_id == "polygon-mainnet":
+            self._wallet_provider._network.network_id = "polygon"
+        elif network_id == "ethereum-mainnet":
+            self._wallet_provider._network.network_id = "ethereum"
         return self._wallet_provider
 
     async def get_account(self) -> EvmServerAccount:

{intentkit-0.6.19.dev1 → intentkit-0.6.20.dev1}/core/credit.py

@@ -150,6 +150,9 @@ async def recharge(
         + user_account.reward_credits,
         base_amount=amount,
         base_original_amount=amount,
+        base_free_amount=Decimal("0"),  # No free credits involved in base amount
+        base_reward_amount=Decimal("0"),  # No reward credits involved in base amount
+        base_permanent_amount=amount,  # All base amount is permanent for recharge
         permanent_amount=amount,  # Set permanent_amount since this is a permanent credit
         free_amount=Decimal("0"),  # No free credits involved
         reward_amount=Decimal("0"),  # No reward credits involved
@@ -277,6 +280,11 @@ async def reward(
         + user_account.reward_credits,
         base_amount=amount,
         base_original_amount=amount,
+        base_free_amount=Decimal("0"),  # No free credits involved in base amount
+        base_reward_amount=amount,  # All base amount is reward for reward events
+        base_permanent_amount=Decimal(
+            "0"
+        ),  # No permanent credits involved in base amount
         reward_amount=amount,  # Set reward_amount since this is a reward credit
         free_amount=Decimal("0"),  # No free credits involved
         permanent_amount=Decimal("0"),  # No permanent credits involved
@@ -451,6 +459,9 @@ async def adjustment(
         + user_account.reward_credits,
         base_amount=abs_amount,
         base_original_amount=abs_amount,
+        base_free_amount=free_amount,
+        base_reward_amount=reward_amount,
+        base_permanent_amount=permanent_amount,
         free_amount=free_amount,
         reward_amount=reward_amount,
         permanent_amount=permanent_amount,
@@ -984,6 +995,9 @@ async def expense_message(
         + user_account.reward_credits,
         base_amount=base_amount,
         base_original_amount=base_original_amount,
+        base_free_amount=base_free_amount,
+        base_reward_amount=base_reward_amount,
+        base_permanent_amount=base_permanent_amount,
         base_llm_amount=base_llm_amount,
         fee_platform_amount=fee_platform_amount,
         fee_platform_free_amount=fee_platform_free_amount,
@@ -1415,6 +1429,9 @@ async def expense_skill(
         base_amount=skill_cost_info.base_amount,
         base_original_amount=skill_cost_info.base_original_amount,
         base_skill_amount=skill_cost_info.base_skill_amount,
+        base_free_amount=base_free_amount,
+        base_reward_amount=base_reward_amount,
+        base_permanent_amount=base_permanent_amount,
         fee_platform_amount=skill_cost_info.fee_platform_amount,
         fee_platform_free_amount=fee_platform_free_amount,
         fee_platform_reward_amount=fee_platform_reward_amount,
@@ -1589,6 +1606,9 @@ async def refill_free_credits_for_account(
         + updated_account.reward_credits,
         base_amount=amount_to_add,
         base_original_amount=amount_to_add,
+        base_free_amount=amount_to_add,
+        base_reward_amount=Decimal("0"),
+        base_permanent_amount=Decimal("0"),
         free_amount=amount_to_add,  # Set free_amount since this is a free credit refill
         reward_amount=Decimal("0"),  # No reward credits involved
         permanent_amount=Decimal("0"),  # No permanent credits involved
@@ -1864,6 +1884,9 @@ async def expense_summarize(
         base_amount=base_amount,
         base_original_amount=base_original_amount,
         base_llm_amount=base_llm_amount,
+        base_free_amount=base_free_amount,
+        base_reward_amount=base_reward_amount,
+        base_permanent_amount=base_permanent_amount,
         fee_platform_amount=fee_platform_amount,
         fee_platform_free_amount=fee_platform_free_amount,
         fee_platform_reward_amount=fee_platform_reward_amount,

{intentkit-0.6.19.dev1 → intentkit-0.6.20.dev1}/models/agent_schema.json

@@ -685,7 +685,7 @@
     "network_id": {
       "title": "Default Network",
       "type": "string",
-      "description": "Default Network",
+      "description": "Default Network, please note that some CDP Wallet native skills like swap only support the base network.",
       "default": "base-mainnet",
       "enum": [
         "ethereum-mainnet",
@@ -697,8 +697,7 @@
         "arbitrum-mainnet",
         "arbitrum-sepolia",
         "optimism-mainnet",
-        "optimism-sepolia",
-        "solana"
+        "optimism-sepolia"
       ],
       "x-group": "onchain"
     }
@@ -715,7 +714,7 @@
       "readonly_wallet_address": {
         "title": "Readonly Wallet Address",
         "type": "string",
-        "description": "Wallet address for readonly wallet provider",
+        "description": "Set the wallet address as agent wallet, then it can only be analyzed by the agent.",
         "maxLength": 100,
         "x-group": "onchain"
       }

{intentkit-0.6.19.dev1 → intentkit-0.6.20.dev1}/models/chat.py

@@ -503,18 +503,17 @@ class ChatMessage(ChatMessageCreate):
     def sanitize_privacy(self) -> "ChatMessage":
         """Remove sensitive information from the chat message.
 
-        This method clears the message content and removes parameters and response
+        This method clears the skill parameters and response
         from skill calls while preserving the structure and metadata.
 
         Returns:
             ChatMessage: A new ChatMessage instance with sensitive data removed
         """
+        if self.author_type != AuthorType.SKILL:
+            return self
         # Create a copy of the current message
         sanitized_data = self.model_dump()
 
-        # Clear the message content
-        sanitized_data["message"] = ""
-
         # Clear sensitive data from skill calls
         if sanitized_data.get("skill_calls"):
             for skill_call in sanitized_data["skill_calls"]:

{intentkit-0.6.19.dev1 → intentkit-0.6.20.dev1}/models/credit.py

@@ -520,6 +520,7 @@ class CreditAccount(BaseModel):
                 balance_after=free_quota,
                 base_amount=free_quota,
                 base_original_amount=free_quota,
+                base_free_amount=free_quota,
                 free_amount=free_quota,  # Set free_amount since this is a free credit refill
                 reward_amount=Decimal("0"),  # No reward credits involved
                 permanent_amount=Decimal("0"),  # No permanent credits involved
@@ -805,6 +806,21 @@ class CreditEventTable(Base):
         default=0,
         nullable=True,
     )
+    base_free_amount = Column(
+        Numeric(22, 4),
+        default=0,
+        nullable=True,
+    )
+    base_reward_amount = Column(
+        Numeric(22, 4),
+        default=0,
+        nullable=True,
+    )
+    base_permanent_amount = Column(
+        Numeric(22, 4),
+        default=0,
+        nullable=True,
+    )
     fee_platform_amount = Column(
         Numeric(22, 4),
         default=0,
@@ -979,6 +995,18 @@ class CreditEvent(BaseModel):
         Optional[Decimal],
         Field(default=Decimal("0"), description="Base skill cost amount"),
     ]
+    base_free_amount: Annotated[
+        Optional[Decimal],
+        Field(default=Decimal("0"), description="Base free credit amount"),
+    ]
+    base_reward_amount: Annotated[
+        Optional[Decimal],
+        Field(default=Decimal("0"), description="Base reward credit amount"),
+    ]
+    base_permanent_amount: Annotated[
+        Optional[Decimal],
+        Field(default=Decimal("0"), description="Base permanent credit amount"),
+    ]
     fee_platform_amount: Annotated[
         Optional[Decimal],
         Field(default=Decimal("0"), description="Platform fee amount"),
@@ -1073,6 +1101,9 @@ class CreditEvent(BaseModel):
         "base_original_amount",
         "base_llm_amount",
         "base_skill_amount",
+        "base_free_amount",
+        "base_reward_amount",
+        "base_permanent_amount",
         "fee_platform_amount",
         "fee_platform_free_amount",
         "fee_platform_reward_amount",

{intentkit-0.6.19.dev1 → intentkit-0.6.20.dev1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "intentkit"
-version = "0.6.19-dev1"
+version = "0.6.20-dev.1"
 description = "Intent-based AI Agent Platform - Core Package"
 authors = [{ name = "hyacinthus", email = "hyacinthus@gmail.com" }]
 requires-python = ">=3.12"

{intentkit-0.6.19.dev1 → intentkit-0.6.20.dev1}/skills/firecrawl/README.md

@@ -5,18 +5,22 @@ The Firecrawl skills provide advanced web scraping and content indexing capabili
 ## Skills Overview
 
 ### 1. firecrawl_scrape
-Scrapes a single webpage and optionally indexes the content for future querying.
+Scrapes a single webpage and REPLACES any existing indexed content for that URL, preventing duplicates.
 
 **Parameters:**
 - `url` (required): The URL to scrape
-- `formats` (optional): Output formats - markdown, html, rawHtml, screenshot, links, extract (default: ["markdown"])
+- `formats` (optional): Output formats - markdown, html, rawHtml, screenshot, links, json (default: ["markdown"])
+- `only_main_content` (optional): Extract only main content (default: true)
 - `include_tags` (optional): HTML tags to include (e.g., ["h1", "h2", "p"])
 - `exclude_tags` (optional): HTML tags to exclude
-- `only_main_content` (optional): Extract only main content (default: true)
+- `wait_for` (optional): Wait time in milliseconds before scraping
+- `timeout` (optional): Maximum timeout in milliseconds (default: 30000)
 - `index_content` (optional): Whether to index content for querying (default: true)
 - `chunk_size` (optional): Size of text chunks for indexing (default: 1000)
 - `chunk_overlap` (optional): Overlap between chunks (default: 200)
 
+**Use Case:** Use this when you want to refresh/update content from a URL that was previously scraped, ensuring no duplicate or stale content remains.
+
 ### 2. firecrawl_crawl
 Crawls multiple pages from a website and indexes all content.
 
@@ -158,8 +162,9 @@ Prompt: "Use firecrawl_scrape to scrape https://example.com and index the conten
 ### Documentation Indexing
 ```
 1. Scrape main documentation page
-2. Crawl related documentation sections
-3. Query for specific technical information
+2. Crawl related documentation sections  
+3. Use scrape again to update changed pages (replaces old content)
+4. Query for specific technical information
 ```
 
 ### Competitive Analysis
@@ -205,6 +210,7 @@ Prompt: "Use firecrawl_scrape to scrape https://example.com and index the conten
 - **PDF Support**: Can scrape and index PDF documents
 - **Intelligent Chunking**: Optimized text splitting for better search
 - **Independent Storage**: Uses its own dedicated vector store for Firecrawl content
+- **Content Replacement**: Replace mode prevents duplicate/stale content
 - **Metadata Rich**: Includes source URLs, timestamps, and content types
 - **Semantic Search**: Uses OpenAI embeddings for intelligent querying
 - **Batch Processing**: Efficient handling of multiple pages

{intentkit-0.6.19.dev1 → intentkit-0.6.20.dev1}/skills/firecrawl/schema.json

@@ -34,7 +34,7 @@
             "Agent Owner + All Users",
             "Agent Owner Only"
           ],
-          "description": "Scrape single web pages and extract content in various formats (markdown, HTML, JSON, etc.). Handles JavaScript-rendered content, PDFs, and dynamic websites.",
+          "description": "Scrape single web pages and REPLACE any existing indexed content for that URL. Unlike regular scrape, this prevents duplicate content when re-scraping the same page. Use this to refresh/update content from a previously scraped URL.",
           "default": "private"
         },
         "firecrawl_crawl": {

{intentkit-0.6.19.dev1 → intentkit-0.6.20.dev1}/skills/firecrawl/scrape.py

@@ -62,10 +62,11 @@ class FirecrawlScrapeInput(BaseModel):
 
 
 class FirecrawlScrape(FirecrawlBaseTool):
-    """Tool for scraping web pages using Firecrawl.
+    """Tool for scraping web pages using Firecrawl with REPLACE behavior.
 
-    This tool uses Firecrawl's API to scrape web pages and convert them into clean,
-    LLM-ready formats like markdown, HTML, or structured JSON data.
+    This tool uses Firecrawl's API to scrape web pages and REPLACES any existing
+    indexed content for the same URL instead of appending to it. This prevents
+    duplicate content when re-scraping the same page.
 
     Attributes:
         name: The name of the tool.
@@ -75,10 +76,10 @@ class FirecrawlScrape(FirecrawlBaseTool):
 
     name: str = "firecrawl_scrape"
     description: str = (
-        "Scrape a single web page and extract its content in various formats (markdown, HTML, JSON, etc.). "
+        "Scrape a single web page and REPLACE any existing indexed content for that URL. "
+        "Unlike regular scrape, this tool removes old content before adding new content, preventing duplicates. "
         "This tool can handle JavaScript-rendered content, PDFs, and dynamic websites. "
-        "Optionally indexes the content for later querying using the firecrawl_query_indexed_content tool. "
-        "Use this when you need to extract clean, structured content from a specific URL."
+        "Use this when you want to refresh/update content from a URL that was previously scraped."
     )
     args_schema: Type[BaseModel] = FirecrawlScrapeInput
 
@@ -187,7 +188,7 @@ class FirecrawlScrape(FirecrawlBaseTool):
                 result_data = data.get("data", {})
 
                 # Format the results based on requested formats
-                formatted_result = f"Successfully scraped: {url}\n\n"
+                formatted_result = f"Successfully scraped (REPLACE mode): {url}\n\n"
 
                 if "markdown" in formats and result_data.get("markdown"):
                     formatted_result += "## Markdown Content\n"
@@ -236,13 +237,16 @@ class FirecrawlScrape(FirecrawlBaseTool):
                         formatted_result += f"Language: {metadata['language']}\n"
                     formatted_result += "\n"
 
-                # Index content if requested
+                # Index content if requested - REPLACE MODE
                 if index_content and result_data.get("markdown"):
                     try:
-                        # Import indexing utilities from firecrawl utils
+                        # Import indexing utilities
+                        from langchain_community.vectorstores import FAISS
+
                         from intentkit.skills.firecrawl.utils import (
+                            FirecrawlDocumentProcessor,
                             FirecrawlMetadataManager,
-                            index_documents,
+                            FirecrawlVectorStoreManager,
                         )
 
                         # Create document from scraped content
@@ -261,38 +265,149 @@ class FirecrawlScrape(FirecrawlBaseTool):
                         # Get agent ID for indexing
                         agent_id = context.agent_id
                         if agent_id:
-                            # Index the document
-                            total_chunks, was_merged = await index_documents(
-                                [document],
-                                agent_id,
-                                self.skill_store,
-                                chunk_size,
-                                chunk_overlap,
-                            )
-
-                            # Update metadata
+                            # Initialize managers
+                            vs_manager = FirecrawlVectorStoreManager(self.skill_store)
                             metadata_manager = FirecrawlMetadataManager(
                                 self.skill_store
                             )
-                            new_metadata = metadata_manager.create_url_metadata(
-                                [url], [document], "firecrawl_scrape"
+
+                            # Load existing vector store
+                            existing_vector_store = await vs_manager.load_vector_store(
+                                agent_id
                             )
-                            await metadata_manager.update_metadata(
-                                agent_id, new_metadata
+
+                            # Split the new document into chunks
+                            split_docs = FirecrawlDocumentProcessor.split_documents(
+                                [document], chunk_size, chunk_overlap
                             )
 
-                            formatted_result += "\n## Content Indexing\n"
-                            formatted_result += (
-                                "Successfully indexed content into vector store:\n"
+                            # Create embeddings
+                            embeddings = vs_manager.create_embeddings()
+
+                            if existing_vector_store:
+                                # Get all existing documents and filter out those from the same URL
+                                try:
+                                    # Try to access documents directly if available
+                                    if hasattr(
+                                        existing_vector_store, "docstore"
+                                    ) and hasattr(
+                                        existing_vector_store.docstore, "_dict"
+                                    ):
+                                        # Access FAISS documents directly
+                                        all_docs = list(
+                                            existing_vector_store.docstore._dict.values()
+                                        )
+                                    else:
+                                        # Fallback: use a reasonable k value for similarity search
+                                        # Use a dummy query to retrieve documents
+                                        all_docs = existing_vector_store.similarity_search(
+                                            "dummy",  # Use a dummy query instead of empty string
+                                            k=1000,  # Use reasonable upper bound
+                                        )
+
+                                    # Filter out documents from the same URL
+                                    preserved_docs = [
+                                        doc
+                                        for doc in all_docs
+                                        if doc.metadata.get("source") != url
+                                    ]
+
+                                    logger.info(
+                                        f"firecrawl_scrape: Preserving {len(preserved_docs)} docs from other URLs, "
+                                        f"replacing content from {url}"
+                                    )
+
+                                    # Create new vector store with preserved docs + new docs
+                                    if preserved_docs:
+                                        # Combine preserved and new documents
+                                        all_documents = preserved_docs + split_docs
+                                        new_vector_store = FAISS.from_documents(
+                                            all_documents, embeddings
+                                        )
+                                        formatted_result += "\n## Content Replacement\n"
+                                        formatted_result += f"Replaced existing content for URL: {url}\n"
+                                        num_preserved_urls = len(
+                                            set(
+                                                doc.metadata.get("source", "")
+                                                for doc in preserved_docs
+                                            )
+                                        )
+                                        formatted_result += f"Preserved content from {num_preserved_urls} other URLs\n"
+                                    else:
+                                        # No other documents to preserve, just create from new docs
+                                        new_vector_store = FAISS.from_documents(
+                                            split_docs, embeddings
+                                        )
+                                        formatted_result += "\n## Content Replacement\n"
+                                        formatted_result += f"Created new index with content from: {url}\n"
+                                except Exception as e:
+                                    logger.warning(
+                                        f"Could not preserve other URLs, creating fresh index: {e}"
+                                    )
+                                    # Fallback: create new store with just the new documents
+                                    new_vector_store = FAISS.from_documents(
+                                        split_docs, embeddings
+                                    )
+                                    formatted_result += "\n## Content Replacement\n"
+                                    formatted_result += f"Created fresh index with content from: {url}\n"
+                            else:
+                                # No existing store, create new one
+                                new_vector_store = FAISS.from_documents(
+                                    split_docs, embeddings
+                                )
+                                formatted_result += "\n## Content Indexing\n"
+                                formatted_result += (
+                                    f"Created new index with content from: {url}\n"
+                                )
+
+                            # Save the new vector store
+                            await vs_manager.save_vector_store(
+                                agent_id, new_vector_store, chunk_size, chunk_overlap
                             )
-                            formatted_result += f"- Chunks created: {total_chunks}\n"
+
+                            # Update metadata to track all URLs
+                            # Get existing metadata to preserve other URLs
+                            metadata_key = f"indexed_urls_{agent_id}"
+                            existing_metadata = (
+                                await self.skill_store.get_agent_skill_data(
+                                    agent_id, "firecrawl", metadata_key
+                                )
+                            )
+
+                            if existing_metadata and existing_metadata.get("urls"):
+                                # Remove the current URL and add it back (to update timestamp)
+                                existing_urls = [
+                                    u for u in existing_metadata["urls"] if u != url
+                                ]
+                                existing_urls.append(url)
+                                updated_metadata = {
+                                    "urls": existing_urls,
+                                    "document_count": len(existing_urls),
+                                    "source_type": "firecrawl_mixed",
+                                    "indexed_at": str(len(existing_urls)),
+                                }
+                            else:
+                                # Create new metadata
+                                updated_metadata = metadata_manager.create_url_metadata(
+                                    [url], [document], "firecrawl_scrape"
+                                )
+
+                            await metadata_manager.update_metadata(
+                                agent_id, updated_metadata
+                            )
+
+                            formatted_result += "\n## Content Indexing (REPLACE MODE)\n"
+                            formatted_result += "Successfully REPLACED indexed content in vector store:\n"
+                            formatted_result += f"- Chunks created: {len(split_docs)}\n"
                             formatted_result += f"- Chunk size: {chunk_size}\n"
                             formatted_result += f"- Chunk overlap: {chunk_overlap}\n"
-                            formatted_result += f"- Content merged with existing: {'Yes' if was_merged else 'No'}\n"
+                            formatted_result += (
+                                "- Previous content for this URL: REPLACED\n"
+                            )
                             formatted_result += "Use the 'firecrawl_query_indexed_content' skill to search this content.\n"
 
                             logger.info(
-                                f"firecrawl_scrape: Successfully indexed {url} with {total_chunks} chunks"
+                                f"firecrawl_scrape: Successfully replaced content for {url} with {len(split_docs)} chunks"
                             )
                         else:
                             formatted_result += "\n## Content Indexing\n"