intentkit 0.6.13.dev2__py3-none-any.whl → 0.8.17__py3-none-any.whl

intentkit/skills/venice_image/utils.py

@@ -1,78 +1,77 @@
-import base64
-import io
-import logging
-from typing import Optional
-
-import filetype
-import httpx
-from PIL import Image
-from pydantic import HttpUrl
-
-from intentkit.skills.base import ToolException
-
-logger = logging.getLogger(__name__)
-
-
-async def fetch_image_as_bytes(image_url: HttpUrl) -> bytes:
-    """Fetches image bytes from a given URL. Converts unsupported formats to PNG using Pillow.
-
-    Raises:
-        ToolException: If fetching or converting the image fails.
-    """
-    try:
-        async with httpx.AsyncClient(timeout=90) as client:
-            response = await client.get(str(image_url), follow_redirects=True)
-            response.raise_for_status()
-
-            original_bytes = response.content
-
-            # Guess file type from content
-            kind = filetype.guess(original_bytes)
-            detected_ext = kind.extension if kind else None
-            detected_mime = kind.mime if kind else "unknown"
-
-            if not detected_ext or not detected_mime.startswith("image/"):
-                msg = f"URL {image_url} did not return a recognizable image format. Detected: {detected_mime}"
-                logger.error(msg)
-                raise ToolException(msg)
-
-            if detected_ext in ("jpg", "jpeg", "png"):
-                return original_bytes
-
-            # Convert unsupported image to PNG
-            try:
-                img = Image.open(io.BytesIO(original_bytes)).convert("RGBA")
-                with io.BytesIO() as output:
-                    img.save(output, format="PNG")
-                    logger.info(
-                        f"Converted unsupported image type '{detected_ext}' to PNG."
-                    )
-                    return output.getvalue()
-            except Exception as e:
-                msg = f"Failed to convert image ({detected_ext}) to PNG: {e}"
-                logger.error(msg, exc_info=True)
-                raise ToolException(msg) from e
-
-    except httpx.HTTPStatusError as e:
-        msg = f"HTTP error fetching image {image_url}: Status {e.response.status_code}"
-        logger.error(msg)
-        raise ToolException(msg) from e
-    except httpx.RequestError as e:
-        msg = f"Network error fetching image {image_url}: {e}"
-        logger.error(msg)
-        raise ToolException(msg) from e
-    except Exception as e:
-        msg = f"Unexpected error fetching image {image_url}: {e}"
-        logger.error(msg, exc_info=True)
-        raise ToolException(msg) from e
-
-
-async def fetch_image_as_base64(image_url: HttpUrl) -> Optional[str]:
-    """Fetches an image from the URL and returns the image as a Base64-encoded string."""
-    image_bytes = await fetch_image_as_bytes(image_url)
-
-    if image_bytes is None:
-        return None
-
-    # Convert image bytes to a Base64-encoded string
-    return base64.b64encode(image_bytes).decode("utf-8")
+import base64
+import io
+import logging
+
+import filetype
+import httpx
+from PIL import Image
+from pydantic import HttpUrl
+
+from intentkit.skills.base import ToolException
+
+logger = logging.getLogger(__name__)
+
+
+async def fetch_image_as_bytes(image_url: HttpUrl) -> bytes:
+    """Fetches image bytes from a given URL. Converts unsupported formats to PNG using Pillow.
+
+    Raises:
+        ToolException: If fetching or converting the image fails.
+    """
+    try:
+        async with httpx.AsyncClient(timeout=90) as client:
+            response = await client.get(str(image_url), follow_redirects=True)
+            response.raise_for_status()
+
+            original_bytes = response.content
+
+            # Guess file type from content
+            kind = filetype.guess(original_bytes)
+            detected_ext = kind.extension if kind else None
+            detected_mime = kind.mime if kind else "unknown"
+
+            if not detected_ext or not detected_mime.startswith("image/"):
+                msg = f"URL {image_url} did not return a recognizable image format. Detected: {detected_mime}"
+                logger.error(msg)
+                raise ToolException(msg)
+
+            if detected_ext in ("jpg", "jpeg", "png"):
+                return original_bytes
+
+            # Convert unsupported image to PNG
+            try:
+                img = Image.open(io.BytesIO(original_bytes)).convert("RGBA")
+                with io.BytesIO() as output:
+                    img.save(output, format="PNG")
+                    logger.info(
+                        f"Converted unsupported image type '{detected_ext}' to PNG."
+                    )
+                    return output.getvalue()
+            except Exception as e:
+                msg = f"Failed to convert image ({detected_ext}) to PNG: {e}"
+                logger.error(msg, exc_info=True)
+                raise ToolException(msg) from e
+
+    except httpx.HTTPStatusError as e:
+        msg = f"HTTP error fetching image {image_url}: Status {e.response.status_code}"
+        logger.error(msg)
+        raise ToolException(msg) from e
+    except httpx.RequestError as e:
+        msg = f"Network error fetching image {image_url}: {e}"
+        logger.error(msg)
+        raise ToolException(msg) from e
+    except Exception as e:
+        msg = f"Unexpected error fetching image {image_url}: {e}"
+        logger.error(msg, exc_info=True)
+        raise ToolException(msg) from e
+
+
+async def fetch_image_as_base64(image_url: HttpUrl) -> str | None:
+    """Fetches an image from the URL and returns the image as a Base64-encoded string."""
+    image_bytes = await fetch_image_as_bytes(image_url)
+
+    if image_bytes is None:
+        return None
+
+    # Convert image bytes to a Base64-encoded string
+    return base64.b64encode(image_bytes).decode("utf-8")

intentkit/skills/web_scraper/__init__.py

@@ -3,7 +3,6 @@
 import logging
 from typing import TypedDict
 
-from intentkit.abstracts.skill import SkillStoreABC
 from intentkit.skills.base import SkillConfig, SkillOwnerState, SkillState
 from intentkit.skills.web_scraper.base import WebScraperBaseTool
 from intentkit.skills.web_scraper.document_indexer import DocumentIndexer
@@ -35,7 +34,6 @@ class Config(SkillConfig):
 async def get_skills(
     config: "Config",
     is_private: bool,
-    store: SkillStoreABC,
     **_,
 ) -> list[WebScraperBaseTool]:
     """Get all web scraper skills.
@@ -43,7 +41,6 @@ async def get_skills(
     Args:
         config: The configuration for web scraper skills.
         is_private: Whether to include private skills.
-        store: The skill store for persisting data.
 
     Returns:
         A list of web scraper skills.
@@ -60,7 +57,7 @@ async def get_skills(
     # Get each skill using the cached getter
     result = []
     for name in available_skills:
-        skill = get_web_scraper_skill(name, store)
+        skill = get_web_scraper_skill(name)
         if skill:
             result.append(skill)
     return result
@@ -68,40 +65,30 @@ async def get_skills(
 
 def get_web_scraper_skill(
     name: str,
-    store: SkillStoreABC,
 ) -> WebScraperBaseTool:
     """Get a web scraper skill by name.
 
     Args:
         name: The name of the skill to get
-        store: The skill store for persisting data
 
     Returns:
         The requested web scraper skill
     """
     if name == "scrape_and_index":
         if name not in _cache:
-            _cache[name] = ScrapeAndIndex(
-                skill_store=store,
-            )
+            _cache[name] = ScrapeAndIndex()
         return _cache[name]
     elif name == "query_indexed_content":
         if name not in _cache:
-            _cache[name] = QueryIndexedContent(
-                skill_store=store,
-            )
+            _cache[name] = QueryIndexedContent()
         return _cache[name]
     elif name == "website_indexer":
         if name not in _cache:
-            _cache[name] = WebsiteIndexer(
-                skill_store=store,
-            )
+            _cache[name] = WebsiteIndexer()
         return _cache[name]
     elif name == "document_indexer":
         if name not in _cache:
-            _cache[name] = DocumentIndexer(
-                skill_store=store,
-            )
+            _cache[name] = DocumentIndexer()
         return _cache[name]
     else:
         logger.warning(f"Unknown web scraper skill: {name}")

intentkit/skills/web_scraper/base.py

@@ -1,8 +1,7 @@
-from typing import Type
-
+from langchain_core.tools.base import ToolException
 from pydantic import BaseModel, Field
 
-from intentkit.abstracts.skill import SkillStoreABC
+from intentkit.config.config import config
 from intentkit.skills.base import IntentKitSkill
 
 
@@ -11,11 +10,26 @@ class WebScraperBaseTool(IntentKitSkill):
 
     name: str = Field(description="The name of the tool")
     description: str = Field(description="A description of what the tool does")
-    args_schema: Type[BaseModel]
-    skill_store: SkillStoreABC = Field(
-        description="The skill store for persisting data"
-    )
+    args_schema: type[BaseModel]
 
     @property
     def category(self) -> str:
         return "web_scraper"
+
+    def get_openai_api_key(self) -> str:
+        """Retrieve the OpenAI API key for embedding operations."""
+        context = self.get_context()
+        skill_config = context.agent.skill_config(self.category)
+        api_key_provider = skill_config.get("api_key_provider")
+
+        if api_key_provider == "platform":
+            if not config.openai_api_key:
+                raise ToolException("OpenAI API key is not configured")
+            return config.openai_api_key
+
+        if skill_config.get("api_key"):
+            return skill_config["api_key"]
+
+        raise ToolException(
+            f"Invalid API key provider: {api_key_provider}, or no api_key in config"
+        )

intentkit/skills/web_scraper/document_indexer.py

@@ -1,5 +1,4 @@
 import logging
-from typing import Type
 
 from pydantic import BaseModel, Field
 
@@ -65,7 +64,7 @@ class DocumentIndexer(WebScraperBaseTool):
         "Perfect for adding content from Google Docs, Notion pages, PDFs, or any other document sources. "
         "The indexed content can then be queried using the query_indexed_content tool."
     )
-    args_schema: Type[BaseModel] = DocumentIndexerInput
+    args_schema: type[BaseModel] = DocumentIndexerInput
 
     async def _arun(
         self,
@@ -108,17 +107,19 @@ class DocumentIndexer(WebScraperBaseTool):
             f"[{agent_id}] Document created, length: {len(document.page_content)} chars"
         )
 
+        embedding_api_key = self.get_openai_api_key()
+        vector_manager = VectorStoreManager(embedding_api_key)
+
         # Index the document
         total_chunks, was_merged = await index_documents(
-            [document], agent_id, self.skill_store, chunk_size, chunk_overlap
+            [document], agent_id, vector_manager, chunk_size, chunk_overlap
         )
 
         # Get current storage size for response
-        vs_manager = VectorStoreManager(self.skill_store)
-        current_size = await vs_manager.get_content_size(agent_id)
+        current_size = await vector_manager.get_content_size(agent_id)
 
         # Update metadata
-        metadata_manager = MetadataManager(self.skill_store)
+        metadata_manager = MetadataManager(vector_manager)
         new_metadata = metadata_manager.create_document_metadata(
             title, source, tags, [document], len(text_content)
         )

intentkit/skills/web_scraper/schema.json

@@ -5,11 +5,7 @@
   "description": "Scrape web content and index it for intelligent querying and retrieval",
   "x-icon": "https://ai.service.crestal.dev/skills/web_scraper/langchain.png",
   "x-tags": [
-    "Web Scraping",
-    "Content Indexing",
-    "Vector Search",
-    "LangChain",
-    "Document Retrieval"
+    "Knowledge Base"
   ],
   "properties": {
     "enabled": {
@@ -140,4 +136,4 @@
     "enabled"
   ],
   "additionalProperties": true
-}
+}

intentkit/skills/web_scraper/scrape_and_index.py

@@ -1,5 +1,4 @@
 import logging
-from typing import List, Type
 
 from pydantic import BaseModel, Field
 
@@ -19,7 +18,7 @@ logger = logging.getLogger(__name__)
 class ScrapeAndIndexInput(BaseModel):
     """Input for ScrapeAndIndex tool."""
 
-    urls: List[str] = Field(
+    urls: list[str] = Field(
         description="List of URLs to scrape and index. Each URL should be a valid web address starting with http:// or https://",
         min_items=1,
         max_items=25,
@@ -67,11 +66,11 @@ class ScrapeAndIndex(WebScraperBaseTool):
         "Use this tool to collect and index web content that you want to reference later.\n"
         "The indexed content can then be queried using the query_indexed_content tool."
     )
-    args_schema: Type[BaseModel] = ScrapeAndIndexInput
+    args_schema: type[BaseModel] = ScrapeAndIndexInput
 
     async def _arun(
         self,
-        urls: List[str],
+        urls: list[str],
         chunk_size: int = DEFAULT_CHUNK_SIZE,
         chunk_overlap: int = DEFAULT_CHUNK_OVERLAP,
         **kwargs,
@@ -92,9 +91,12 @@ class ScrapeAndIndex(WebScraperBaseTool):
                 f"[{agent_id}] Starting scrape and index operation with {len(urls)} URLs"
             )
 
+            embedding_api_key = self.get_openai_api_key()
+            vector_manager = VectorStoreManager(embedding_api_key)
+
             # Use the utility function to scrape and index URLs
             total_chunks, was_merged, valid_urls = await scrape_and_index_urls(
-                urls, agent_id, self.skill_store, chunk_size, chunk_overlap
+                urls, agent_id, vector_manager, chunk_size, chunk_overlap
             )
 
             logger.info(
@@ -110,12 +112,11 @@ class ScrapeAndIndex(WebScraperBaseTool):
                 return "Error: No content could be extracted from the provided URLs."
 
             # Get current storage size for response
-            vs_manager = VectorStoreManager(self.skill_store)
-            current_size = await vs_manager.get_content_size(agent_id)
+            current_size = await vector_manager.get_content_size(agent_id)
             size_limit_reached = len(valid_urls) < len(urls)
 
             # Update metadata
-            metadata_manager = MetadataManager(self.skill_store)
+            metadata_manager = MetadataManager(vector_manager)
             new_metadata = metadata_manager.create_url_metadata(
                 valid_urls, [], "scrape_and_index"
             )
@@ -169,7 +170,7 @@ class QueryIndexedContent(WebScraperBaseTool):
         "Use this tool to search through content that was previously scraped and indexed.\n"
         "This tool can help answer questions based on the indexed web content."
     )
-    args_schema: Type[BaseModel] = QueryIndexInput
+    args_schema: type[BaseModel] = QueryIndexInput
 
     async def _arun(
         self,
@@ -196,9 +197,9 @@ class QueryIndexedContent(WebScraperBaseTool):
 
             logger.info(f"[{agent_id}] Looking for vector store: {vector_store_key}")
 
-            stored_data = await self.skill_store.get_agent_skill_data(
-                agent_id, "web_scraper", vector_store_key
-            )
+            embedding_api_key = self.get_openai_api_key()
+            vector_manager = VectorStoreManager(embedding_api_key)
+            stored_data = await vector_manager.get_existing_vector_store(agent_id)
 
             if not stored_data:
                 logger.warning(f"[{agent_id}] No vector store found")
@@ -210,9 +211,8 @@ class QueryIndexedContent(WebScraperBaseTool):
 
             # Create embeddings and decode vector store
             logger.info(f"[{agent_id}] Decoding vector store")
-            vs_manager = VectorStoreManager(self.skill_store)
-            embeddings = vs_manager.create_embeddings()
-            vector_store = vs_manager.decode_vector_store(
+            embeddings = vector_manager.create_embeddings()
+            vector_store = vector_manager.decode_vector_store(
                 stored_data["faiss_files"], embeddings
             )