versionhq 1.2.2.3__tar.gz → 1.2.2.5__tar.gz

{versionhq-1.2.2.3 → versionhq-1.2.2.5}/.github/workflows/publish.yml

@@ -5,6 +5,7 @@ on:
     types: [released]
 
 permissions:
+  id-token: write
   contents: read
 
 jobs:
@@ -58,5 +59,6 @@ jobs:
           packages-dir: dist/
           repository-url: https://upload.pypi.org/legacy/
           verbose: true
-          user: __token__
-          password: ${{ secrets.PYPI_API_TOKEN }}
+          # user: krik8235
+          # user: __token__
+          # password: ${{ secrets.PYPI_API_TOKEN }}

{versionhq-1.2.2.3 → versionhq-1.2.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: versionhq
-Version: 1.2.2.3
+Version: 1.2.2.5
 Summary: An agentic orchestration framework for building agent networks that handle task automation.
 Author-email: Kuriko Iwai <kuriko@versi0n.io>
 License: MIT License
@@ -50,7 +50,7 @@ Requires-Dist: werkzeug>=3.1.3
 Requires-Dist: typing
 Requires-Dist: json-repair
 Requires-Dist: litellm>=1.55.8
-Requires-Dist: openai>=1.57.0
+Requires-Dist: openai>=1.64.0
 Requires-Dist: composio-openai>=0.6.9
 Requires-Dist: composio>=0.1.0
 Requires-Dist: setuptools>=75.6.0
@@ -69,7 +69,7 @@ Requires-Dist: matplotlib>=3.10.0
 Provides-Extra: docling
 Requires-Dist: docling>=2.17.0; extra == "docling"
 Provides-Extra: mem0ai
-Requires-Dist: mem0ai>=0.1.48; extra == "mem0ai"
+Requires-Dist: mem0ai>=0.1.55; extra == "mem0ai"
 Provides-Extra: pdfplumber
 Requires-Dist: pdfplumber>=0.11.5; extra == "pdfplumber"
 Provides-Extra: pandas
@@ -78,6 +78,11 @@ Provides-Extra: numpy
 Requires-Dist: numpy>=1.26.4; extra == "numpy"
 Provides-Extra: pygraphviz
 Requires-Dist: pygraphviz>=1.14; extra == "pygraphviz"
+Provides-Extra: tools
+Requires-Dist: html2text>=2024.2.26; extra == "tools"
+Requires-Dist: sec-api>=1.0.28; extra == "tools"
+Provides-Extra: eval
+Requires-Dist: scikit-learn>=1.6.1; extra == "eval"
 
 # Overview
 
@@ -222,10 +227,7 @@ The following code snippet demonstrates agent customization:
 ```python
 import versionhq as vhq
 
-agent = vhq.Agent(
-   role="Marketing Analyst",
-   goal="my amazing goal"
-) # assuming this agent was created during the network formation
+agent = vhq.Agent(role="Marketing Analyst")
 
 # update the agent
 agent.update(
@@ -316,8 +318,8 @@ To create an agent network with one or more manager agents, designate members us
 ```python
 import versionhq as vhq
 
-agent_a = vhq.Agent(role="agent a", goal="My amazing goals", llm="llm-of-your-choice")
-agent_b = vhq.Agent(role="agent b", goal="My amazing goals", llm="llm-of-your-choice")
+agent_a = vhq.Agent(role="agent a", llm="llm-of-your-choice")
+agent_b = vhq.Agent(role="agent b", llm="llm-of-your-choice")
 
 task_1 = vhq.Task(
    description="Analyze the client's business model.",

{versionhq-1.2.2.3 → versionhq-1.2.2.5}/README.md

@@ -141,10 +141,7 @@ The following code snippet demonstrates agent customization:
 ```python
 import versionhq as vhq
 
-agent = vhq.Agent(
-   role="Marketing Analyst",
-   goal="my amazing goal"
-) # assuming this agent was created during the network formation
+agent = vhq.Agent(role="Marketing Analyst")
 
 # update the agent
 agent.update(
@@ -235,8 +232,8 @@ To create an agent network with one or more manager agents, designate members us
 ```python
 import versionhq as vhq
 
-agent_a = vhq.Agent(role="agent a", goal="My amazing goals", llm="llm-of-your-choice")
-agent_b = vhq.Agent(role="agent b", goal="My amazing goals", llm="llm-of-your-choice")
+agent_a = vhq.Agent(role="agent a", llm="llm-of-your-choice")
+agent_b = vhq.Agent(role="agent b", llm="llm-of-your-choice")
 
 task_1 = vhq.Task(
    description="Analyze the client's business model.",

{versionhq-1.2.2.3 → versionhq-1.2.2.5}/docs/core/agent/index.md

@@ -7,7 +7,7 @@ tags:
 
 <class>`class` versionhq.agent.model.<bold>Agent<bold></class>
 
-A Pydantic class to store `Agent` objects and handle `Task` execution as well as `LLM` configuration.
+A Pydantic class to store an `Agent` object that handles `Task` execution.
 
 
 ## Quick Start

{versionhq-1.2.2.3 → versionhq-1.2.2.5}/docs/index.md

@@ -113,10 +113,7 @@ The following code snippet demonstrates agent customization:
 ```python
 import versionhq as vhq
 
-agent = vhq.Agent(
-   role="Marketing Analyst",
-   goal="my amazing goal"
-) # assuming this agent was created during the network formation
+agent = vhq.Agent(role="Marketing Analyst")
 
 # update the agent
 agent.update(

{versionhq-1.2.2.3 → versionhq-1.2.2.5}/docs/quickstart.md

@@ -48,7 +48,7 @@ def dummy_func(message: str, test1: str, test2: list[str]) -> str:
    return f"""{message}: {test1}, {", ".join(test2)}"""
 
 
-agent = vhq.Agent(role="demo", goal="amazing project goal")
+agent = vhq.Agent(role="demo manager")
 
 task = vhq.Task(
    description="Amazing task",

{versionhq-1.2.2.3 → versionhq-1.2.2.5}/pyproject.toml

@@ -15,7 +15,7 @@ exclude = ["test*", "__pycache__", "*.egg-info"]
 
 [project]
 name = "versionhq"
-version = "1.2.2.3"
+version = "1.2.2.5"
 authors = [{ name = "Kuriko Iwai", email = "kuriko@versi0n.io" }]
 description = "An agentic orchestration framework for building agent networks that handle task automation."
 readme = "README.md"
@@ -30,7 +30,7 @@ dependencies = [
   "typing",
   "json-repair",
   "litellm>=1.55.8",
-  "openai>=1.57.0",
+  "openai>=1.64.0",
   "composio-openai>=0.6.9",
   "composio>=0.1.0",
   "setuptools>=75.6.0",
@@ -71,7 +71,7 @@ docling = [
     "docling>=2.17.0",
 ]
 mem0ai = [
-    "mem0ai>=0.1.48",
+    "mem0ai>=0.1.55",
 ]
 pdfplumber = [
     "pdfplumber>=0.11.5",
@@ -85,6 +85,13 @@ numpy = [
 pygraphviz = [
     "pygraphviz>=1.14",
 ]
+tools = [
+    "html2text>=2024.2.26",
+    "sec-api>=1.0.28",
+]
+eval = [
+    "scikit-learn>=1.6.1",
+]
 
 [tool.uv]
 dev-dependencies = [

{versionhq-1.2.2.3 → versionhq-1.2.2.5}/src/versionhq/__init__.py

@@ -21,6 +21,7 @@ from versionhq.task_graph.model import TaskStatus, TaskGraph, Node, Edge, Depend
 from versionhq.task.model import Task, TaskOutput, ResponseField, TaskExecutionType
 from versionhq.task.evaluation import Evaluation, EvaluationItem
 from versionhq.tool.model import Tool, ToolSet
+from versionhq.tool.rag_tool import RagTool
 from versionhq.tool.cache_handler import CacheHandler
 from versionhq.tool.tool_handler import ToolHandler
 from versionhq.tool.composio_tool import ComposioHandler
@@ -31,7 +32,7 @@ from versionhq.agent_network.formation import form_agent_network
 from versionhq.task_graph.draft import workflow
 
 
-__version__ = "1.2.2.3"
+__version__ = "1.2.2.5"
 __all__ = [
     "Agent",
 
@@ -80,6 +81,7 @@ __all__ = [
 
     "Tool",
     "ToolSet",
+    "RagTool",
     "CacheHandler",
     "ToolHandler",
     "ComposioHandler",

versionhq-1.2.2.5/src/versionhq/_utils/llm_as_a_judge.py

@@ -0,0 +1,65 @@
+import json
+import numpy as np
+from sklearn.metrics import precision_score, recall_score, roc_auc_score, cohen_kappa_score
+from typing import List, Tuple, Dict, Any
+from pathlib import Path
+
+
+class LLMJudge:
+
+    class MockLLM:
+        def _generate(self, prompt: str) -> str:
+            return str(np.random.random())
+
+
+    def __init__(self, model: MockLLM = None):
+        self.model = model if model else self.MockLLM()
+
+
+    def judge_summary(self, original_text: str, summary: str) -> float:
+        prompt = f"""Evaluate the quality of the following summary on a scale of 0 to 1, where 0 is poor and 1 is excellent.
+Consider accuracy, completeness, and conciseness.
+Original text: {original_text}
+Summary: {summary}
+Quality score:"""
+        response = self.model._generate(prompt)
+        score = float(response.strip())
+        return score
+
+
+def generate_summaries(file_path: str, data: List[Dict[str, Any]] = None, summarizer: Any = None) -> List[Tuple[str, str, float]]:
+    """Generates a list of tuple with an original text, summary text, and human judge score."""
+    if not data:
+        with open(file_path, 'r') as file:
+            data = json.load(file)
+    summaries = []
+    for item in data:
+        original_text = item['text']
+        summary = summarizer.summarize(original_text)
+        human_score = item['human_score']
+        summaries.append((original_text, summary, human_score))
+
+    return summaries
+
+
+def validate(judge: LLMJudge, data: List[Tuple[str, str, float]], threshold: float = 0.5):
+    human_scores = []
+    predicted_scores = []
+
+    for original_text, summary, human_score in data:
+        predicted_score = judge.judge_summary(original_text=original_text, summary=summary)
+        human_scores.append(human_score)
+        predicted_scores.append(predicted_score)
+
+    human_binary = [1 if score >= threshold else 0 for score in human_scores]
+    pred_binary = [1 if score >= threshold else 0 for score in predicted_scores]
+    precision = precision_score(human_binary, pred_binary, zero_division=0)
+    recall = recall_score(human_binary, pred_binary, zero_division=0)
+    auroc = roc_auc_score(human_binary, pred_binary, average='weighted')
+    kappa = cohen_kappa_score(human_binary, pred_binary)
+    return {
+        "precision": precision,
+        "recall": recall,
+        "auroc": auroc,
+        "cohen_kappa": kappa
+    }

{versionhq-1.2.2.3 → versionhq-1.2.2.5}/src/versionhq/agent/inhouse_agents.py

@@ -38,7 +38,7 @@ vhq_formation_planner = Agent(
         "Solo is a formation where a single agent with tools, knowledge, and memory handles tasks indivudually. When self-learning mode is on - it will turn into Random formation. Typical usecase is an email agent drafts promo message for the given audience using their own knowledge.",
         "Supervising is a formation where the leader agent gives directions, while sharing its knowledge and memory with subbordinates.Subordinates can be solo agents or networks. Typical usecase is that the leader agent strategizes an outbound campaign plan and assigns components such as media mix or message creation to subordinate agents.",
         "Network is a formation where multple agents can share tasks, knowledge, and memory among network members without hierarchy. Typical usecase is that an email agent and social media agent share the product knowledge and deploy multi-channel outbound campaign. ",
-        "Random is a formation where a single agent handles tasks, asking help from other agents without sharing its memory or knowledge. Typical usecase is that an email agent drafts promo message for the given audience, asking insights on tones from other email agents which oversee other customer clusters, or an agent calls the external, third party agent to deploy the campaign. ",
+        "Random is a formation where a single agent handles tasks, asking help from other agents without sharing its memory or knowledge. Typical usecase is that an email agent drafts promo message for the given audience, asking insights on tones from other email agents which oversee other customer clusters, or an agent calls the external, third party agent to deploy the campaign.",
     ]
 )
 

{versionhq-1.2.2.3 → versionhq-1.2.2.5}/src/versionhq/agent/model.py

@@ -8,7 +8,7 @@ from pydantic import UUID4, BaseModel, Field, InstanceOf, PrivateAttr, model_val
 from pydantic_core import PydanticCustomError
 
 from versionhq.llm.model import LLM, DEFAULT_CONTEXT_WINDOW_SIZE, DEFAULT_MODEL_NAME, PROVIDERS
-from versionhq.tool.model import Tool, ToolSet
+from versionhq.tool.model import Tool, ToolSet, BaseTool
 from versionhq.knowledge.model import BaseKnowledgeSource, Knowledge
 from versionhq.memory.contextual_memory import ContextualMemory
 from versionhq.memory.model import ShortTermMemory, LongTermMemory, UserMemory
@@ -35,11 +35,11 @@ class Agent(BaseModel):
     config: Optional[Dict[str, Any]] = Field(default=None, exclude=True, description="values to add to the Agent class")
 
     id: UUID4 = Field(default_factory=uuid.uuid4, frozen=True)
-    role: str = Field(description="role of the agent - used in summary and logs")
-    goal: str = Field(description="concise goal of the agent (details are set in the Task instance)")
+    role: str = Field(description="required. agent's role")
+    goal: Optional[str] = Field(default=None)
     backstory: Optional[str] = Field(default=None, description="developer prompt to the llm")
     skillsets: Optional[List[str]] = Field(default_factory=list)
-    tools: Optional[List[InstanceOf[Tool | ToolSet] | Type[Tool] | Any]] = Field(default_factory=list)
+    tools: Optional[List[Any]] = Field(default_factory=list)
 
     # knowledge
     knowledge_sources: Optional[List[BaseKnowledgeSource | Any]] = Field(default=None)
@@ -92,7 +92,7 @@ class Agent(BaseModel):
 
     @model_validator(mode="after")
     def validate_required_fields(self) -> Self:
-        required_fields = ["role", "goal"]
+        required_fields = ["role",]
         for field in required_fields:
             if getattr(self, field) is None:
                 raise ValueError(f"{field} must be provided either directly or through config")
@@ -122,29 +122,41 @@ class Agent(BaseModel):
         """
         Similar to the LLM set up, when the agent has tools, we will declare them using the Tool class.
         """
-        if not self.tools:
-            pass
+        from versionhq.tool.rag_tool import RagTool
 
-        else:
-            tool_list = []
+        if not self.tools:
+            return self
 
-            for item in self.tools:
-                if isinstance(item, Tool) or isinstance(item, ToolSet):
+        tool_list = []
+        for item in self.tools:
+            match item:
+                case RagTool() | BaseTool():
                     tool_list.append(item)
 
-                elif isinstance(item, dict) and "func" in item:
-                    tool = Tool(**item)
-                    tool_list.append(tool)
+                case Tool():
+                    if item.func is not None:
+                        tool_list.append(item)
 
-                elif type(item) is Tool and hasattr(item, "func"):
-                    tool_list.append(item)
+                case ToolSet():
+                    if item.tool and item.tool.func is not None:
+                        tool_list.append(item)
 
-                else:
-                    Logger(**self.loger_config, filename=self.key).log(level="error", message=f"Tool {str(item)} is missing a function.", color="red")
-                    raise PydanticCustomError("invalid_tool", f"The tool {str(item)} is missing a function.", {})
+                case dict():
+                    if "func" in item:
+                        tool = Tool(func=item["func"])
+                        for k, v in item.items():
+                            if k in Tool.model_fields.keys() and k != "func" and  v is not None:
+                                setattr(tool, k, v)
+                        tool_list.append(tool)
 
-            self.tools = tool_list
+                case _:
+                    if item.__base__ == BaseTool or item.__base__ == RagTool or item.__base__ == Tool:
+                        tool_list.append(item)
+                    else:
+                        Logger(**self._logger_config, filename=self.key).log(level="error", message=f"Tool {str(item)} is missing a function.", color="red")
+                        raise PydanticCustomError("invalid_tool", f"The tool {str(item)} is missing a function.", {})
 
+        self.tools = tool_list
         return self
 
 
@@ -158,9 +170,9 @@ class Agent(BaseModel):
             from versionhq.agent.TEMPLATES.Backstory import BACKSTORY_FULL, BACKSTORY_SHORT
             backstory = ""
             skills = ", ".join([item for item in self.skillsets]) if self.skillsets else ""
-            tools = ", ".join([item.name for item in self.tools if hasattr(item, "name")]) if self.tools else ""
+            tools = ", ".join([item.name for item in self.tools if hasattr(item, "name") and item.name is not None]) if self.tools else ""
             role = self.role.lower()
-            goal = self.goal.lower()
+            goal = self.goal.lower() if self.goal else ""
 
             if self.tools or self.skillsets:
                 backstory = BACKSTORY_FULL.format(role=role, goal=goal, skills=skills, tools=tools)
@@ -199,7 +211,7 @@ class Agent(BaseModel):
                     if isinstance(item, BaseKnowledgeSource):
                         knowledge_sources.append(item)
 
-                    elif isinstance(item, str) and "http" in item:
+                    elif isinstance(item, str) and "http" in item and DoclingSource._validate_url(url=item) == True:
                         docling_fp.append(item)
 
                     elif isinstance(item, str):
@@ -223,8 +235,8 @@ class Agent(BaseModel):
 
                 self._knowledge = Knowledge(sources=knowledge_sources, embedder_config=self.embedder_config, collection_name=collection_name)
 
-            except:
-                Logger(**self._logger_config, filename=self.key).log(level="warning", message="We cannot find the format for the source. Add BaseKnowledgeSource objects instead.", color="yellow")
+            except Exception as e:
+                Logger(**self._logger_config, filename=self.key).log(level="warning", message=f"We cannot find the format for the source. Add BaseKnowledgeSource objects instead. {str(e)}", color="yellow")
 
         return self
 
@@ -482,7 +494,7 @@ class Agent(BaseModel):
         Defines and executes a task when it is not given and returns TaskOutput object.
         """
 
-        if not self.goal or not self.role:
+        if not self.role or not self.goal:
             return None
 
         from versionhq.task.model import Task
@@ -506,10 +518,13 @@ class Agent(BaseModel):
         """
 
         from versionhq.task.model import Task
+        from versionhq.tool.rag_tool import RagTool
         from versionhq.knowledge._utils import extract_knowledge_context
 
         task: InstanceOf[Task] = task
-        tools: Optional[List[InstanceOf[Tool | ToolSet] | Type[Tool]]] = task_tools + self.tools if task.can_use_agent_tools else task_tools
+        all_tools: Optional[List[Tool | ToolSet | RagTool | Type[BaseTool]]] = task_tools + self.tools if task.can_use_agent_tools else task_tools
+        rag_tools: Optional[List[RagTool]] = [item for item in all_tools if isinstance(item, RagTool)] if all_tools else None
+        tools: Optional[List[Tool | ToolSet | RagTool | Type[BaseTool]]] = [item for item in all_tools if not isinstance(item, RagTool)] if all_tools else None
 
         if self.max_rpm and self._rpm_controller:
             self._rpm_controller._reset_request_count()
@@ -523,6 +538,12 @@ class Agent(BaseModel):
                 if agent_knowledge_context:
                     task_prompt += agent_knowledge_context
 
+        if rag_tools:
+            for item in rag_tools:
+                rag_tool_context = item.run(agent=self, query=task.description)
+                if rag_tool_context:
+                    task_prompt += ",".join(rag_tool_context) if isinstance(rag_tool_context, list) else str(rag_tool_context)
+
         if self.with_memory == True:
             contextual_memory = ContextualMemory(
                 memory_config=self.memory_config, stm=self.short_term_memory, ltm=self.long_term_memory, um=self.user_memory
@@ -533,7 +554,6 @@ class Agent(BaseModel):
             if memory.strip() != "":
                 task_prompt += memory.strip()
 
-
         ## comment out for now
         # if self.networks and self.networks._train:
         #     task_prompt = self._training_handler(task_prompt=task_prompt)
@@ -575,7 +595,7 @@ class Agent(BaseModel):
 
 
     def __repr__(self):
-        return f"Agent(role={self.role}, goal={self.goal}"
+        return f"Agent(role={self.role}, id={str(self.id)}"
 
     def __str__(self):
         return super().__str__()

{versionhq-1.2.2.3 → versionhq-1.2.2.5}/src/versionhq/knowledge/source_docling.py

@@ -83,7 +83,8 @@ class DoclingSource(BaseKnowledgeSource):
             yield chunk.text
 
 
-    def _validate_url(self, url: str) -> bool:
+    @staticmethod
+    def _validate_url(url: str) -> bool:
         try:
             result = urlparse(url)
             return all(

{versionhq-1.2.2.3 → versionhq-1.2.2.5}/src/versionhq/knowledge/storage.py

@@ -125,7 +125,8 @@ class KnowledgeStorage(BaseKnowledgeStorage):
     def search(self, query: List[str], limit: int = 3, filter: Optional[dict] = None, score_threshold: float = 0.35) -> List[Dict[str, Any]]:
         with suppress_logging():
             if self.collection:
-                fetched = self.collection.query(query_texts=query, n_results=limit, where=filter)
+                query_texts = ", ".join(query) if isinstance(query, list) else str(query)
+                fetched = self.collection.query(query_texts=query_texts, n_results=limit, where=filter)
                 results = []
                 for i in range(len(fetched["ids"][0])):
                     result = {

{versionhq-1.2.2.3 → versionhq-1.2.2.5}/src/versionhq/tool/model.py

@@ -43,7 +43,6 @@ class BaseTool(ABC, BaseModel):
             { "__annotations__": { k: v for k, v in cls._run.__annotations__.items() if k != "return" }},
         )
 
-
     @field_validator("properties", mode="before")
     @classmethod
     def _default_properties(cls, v: Dict[str, Any]) -> Dict[str, Any]:
@@ -81,10 +80,9 @@ class BaseTool(ABC, BaseModel):
         return self
 
     @abstractmethod
-    def _run(self, *args: Any, **kwargs: Any,) -> Any:
+    def _run(self, *args: Any, **kwargs: Any) -> Any:
         """any handling"""
 
-
     @staticmethod
     def _get_arg_annotations(annotation: type[Any] | None) -> str:
         if annotation is None:
@@ -143,11 +141,9 @@ class BaseTool(ABC, BaseModel):
         return create_model(schema_name, **fields)
 
 
-
 class Tool(BaseTool):
     func: Callable = Field(default=None)
 
-
     @model_validator(mode="after")
     def validate_func(self) -> Self:
         if not self.func and not self._run:

versionhq-1.2.2.5/src/versionhq/tool/rag_tool.py

@@ -0,0 +1,112 @@
+import re
+import requests
+import html2text
+import gzip
+import http.client
+import urllib.request
+from urllib.request import Request
+from textwrap import dedent
+from typing import Any, Optional, List, Dict
+
+from pydantic import Field
+
+from versionhq.agent.model import Agent
+from versionhq.tool.model import BaseTool
+from versionhq._utils.logger import Logger
+
+
+
+class RagTool(BaseTool):
+    """A Pydantic class to store a RAG tool object. Inherited from BaseTool"""
+
+    api_key_name: str = Field(default=None)
+    api_endpoint: Optional[str] = Field(default=None)
+
+    url: Optional[str] = Field(default=None, description="url to scrape")
+    headers: Optional[Dict[str, Any]] = Field(default_factory=dict, description="request headers")
+
+    sources: Optional[List[Any]] = Field(default_factory=list, description="indexed data sources")
+    query: Optional[str] = Field(default=None)
+    text: Optional[str] = Field(default=None, description="text data source")
+
+
+    def _sanitize_source_code(self, source_code: str | bytes = None) -> str | None:
+        if not source_code:
+            return None
+
+        if isinstance(source_code, bytes):
+            source_code = source_code.decode('utf-8')
+
+        h = html2text.HTML2Text()
+        h.ignore_links = False
+        text = h.handle(source_code)
+        text = re.sub(r"[^a-zA-Z$0-9\s\n]", "", text)
+        return dedent(text)
+
+
+    def _scrape_url(self, url: str = None) -> str | None:
+        url = url if url else self.url
+
+        if not url:
+            return None
+
+        http.client.HTTPConnection.debuglevel = 1
+
+        try:
+            req = Request(url=url, headers=self.headers, origin_req_host=url, method="GET")
+            res = ""
+
+            with urllib.request.urlopen(req) as url:
+                if url.info().get("Content-Encoding") == "gzip":
+                    res = gzip.decompress(url.read())
+                else:
+                    res = url.read()
+
+            text = self._sanitize_source_code(source_code=res)
+            return text
+
+        except requests.exceptions.HTTPError as e:
+            Logger().log(level="error", message=f"HTTP error occurred: {str(e)}", color="red")
+            return None
+
+        except Exception as e:
+            Logger().log(level="error", message=f"Error fetching URL {self.api_endpoint}: {str(e)}", color="red")
+            return None
+
+
+    def store_data(self, agent: Agent = None) -> None:
+        """Stores retrieved data in the storage"""
+        if not agent:
+            return
+
+        text = self.text if self.text else self._scrape_url(self.url)
+        self.text = text
+        knowledge_sources = [*agent.knowledge_sources, str(text), ] if agent.knowledge_sources else [str(text),]
+        agent.update(knowledge_sources=knowledge_sources)
+
+
+    def _run(self, agent: Agent = None, query: str = None) -> List[str]:
+        query = query if query else self.query
+
+        if not query or not agent:
+            text = self.text if self.text else self._scrape_url(self.url)
+            self.text = text
+            return [text,]
+
+        else:
+            results, res = [], []
+            if agent._knowledge:
+                res = agent._knowledge.query(query=[query], limit=5)
+            else:
+                self.store_data(agent=agent)
+                res = agent._knowledge.query(query=[query], limit=5)
+
+            for item in res:
+                if isinstance(item, dict):
+                    results.append(item["context"])
+                else:
+                    results.append(str(item))
+            return results
+
+    def run(self, *args, **kwargs):
+        return self._run(*args, **kwargs)