versionhq 1.2.4.1__py3-none-any.whl → 1.2.4.3__py3-none-any.whl

versionhq/__init__.py

@@ -17,7 +17,7 @@ from versionhq.clients.workflow.model import MessagingWorkflow, MessagingCompone
 from versionhq.knowledge.model import Knowledge, KnowledgeStorage
 from versionhq.knowledge.source import PDFKnowledgeSource, CSVKnowledgeSource, JSONKnowledgeSource, TextFileKnowledgeSource, ExcelKnowledgeSource, StringKnowledgeSource
 from versionhq.knowledge.source_docling import DoclingSource
-from versionhq.task_graph.model import TaskStatus, TaskGraph, Node, Edge, DependencyType, Condition, ConditionType
+from versionhq.task_graph.model import TaskStatus, TaskGraph, Node, Edge, DependencyType, Condition, ConditionType, ReformTriggerEvent
 from versionhq.task.model import Task, TaskOutput, ResponseField, TaskExecutionType
 from versionhq.task.evaluation import Evaluation, EvaluationItem
 from versionhq.tool.model import Tool, ToolSet
@@ -32,7 +32,7 @@ from versionhq.agent_network.formation import form_agent_network
 from versionhq.task_graph.draft import workflow
 
 
-__version__ = "1.2.4.1"
+__version__ = "1.2.4.3"
 __all__ = [
     "Agent",
 
@@ -70,6 +70,7 @@ __all__ = [
     "DependencyType",
     "Condition",
     "ConditionType",
+    "ReformTriggerEvent",
 
     "Task",
     "TaskOutput",

versionhq/_prompt/auto_feedback.py

@@ -0,0 +1,103 @@
+from typing import List, Optional, Dict
+from typing_extensions import Self
+
+from pydantic import InstanceOf, Field
+
+from versionhq.agent.model import Agent
+from versionhq.task.model import Task
+from versionhq.task_graph.model import TaskGraph, Node, DependencyType, ReformTriggerEvent
+from versionhq._prompt.model import Prompt
+from versionhq._prompt.constants import REFLECT, INTEGRATE, parameter_sets
+
+
+class PromptFeedbackGraph(TaskGraph):
+    """A Pydantic class to handle auto prompt feedback cycle."""
+
+    _times_iteration: int = 0
+    user_prompts: Optional[Dict[str, str]] = Field(default_factory=dict) # { "0": "...", "1": "..."}
+    dev_prompts: Optional[Dict[str, str]] = Field(default_factory=dict)
+    prompts: Optional[Dict[str, InstanceOf[Prompt]]] = Field(default_factory=dict)
+
+
+    def __init__(self, prompt: InstanceOf[Prompt] = None, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        if prompt:
+            user_prompt, dev_prompt, _ = prompt.format_core()
+            self.prompts = { self.key: prompt }
+            self.user_prompts = { self.key: user_prompt }
+            self.dev_prompts = { self.key: dev_prompt }
+
+
+    def _fetch_latest_prompt(self) -> InstanceOf[Prompt] | None:
+        return self.prompts[self.key] if self.key in self.prompts else None
+
+
+    def _generate_agents(self) -> List[Agent] | None:
+        agents = []
+        prompt = self._fetch_latest_prompt()
+
+        if not prompt:
+            return None
+
+        agent = prompt.agent
+        agent_params = agent.model_dump(exclude={"id", "llm", "llm_config", "self_learning"})
+        for params in parameter_sets:
+            agent = Agent(**agent_params, llm=agent.llm.model, llm_config={**params}, self_learning=True)
+            agents.append(agent)
+        return agents
+
+
+    def _reflect(self, original_response: str) -> Task:
+        description = REFLECT.format(original_prompt=self.original_prompt, original_response=original_response)
+        return Task(description=description)
+
+
+    def set_up_graph(self, **attributes) -> Self:
+        """Sets up a TaskGraph object with nodes and edges."""
+
+        prompt = self._fetch_latest_prompt()
+        base_task = prompt.task if prompt else None
+        base_agent = prompt.agent if prompt else None
+
+        if not base_task or not base_agent:
+            return None
+
+        agents = self._generate_agents()
+        if not agents:
+            return None
+
+        self.concl_template = base_task.pydantic_output if base_task.pydantic_output else base_task.response_fields if base_task.response_fields else None
+        base_agent.callbacks.append(self._reflect)
+        init_node = Node(task=base_task, assigned_to=base_agent)
+        self.add_node(init_node)
+
+        final_task = Task(description=INTEGRATE.format(original_prompt=self.original_prompt, responses=""))
+        final_node = Node(task=final_task, agent=base_agent)
+        self.add_node(node=final_node)
+
+        for agent in agents:
+            node = Node(task=base_task, assigned_to=agent)
+            self.add_node(node=node)
+            self.add_dependency(source=init_node.identifier, target=node.identifier, dependency_type=DependencyType.FINISH_TO_START, required=True)
+            self.add_dependency(source=node.identifier, target=final_node.identifier, dependency_type=DependencyType.FINISH_TO_START, required=True)
+
+        if attributes:
+            for k, v in attributes.items():
+                if hasattr(self, k):
+                    setattr(self, k, v)
+
+        return self
+
+    @property
+    def index(self) -> str:
+        """Returns an index to add new item."""
+        return str(len([k for k in self.user_prompts.keys()]))
+
+    @property
+    def original_prompt(self) -> str:
+        return str(self.user_prompts["0"]) + str(self.dev_prompts["0"])
+
+    @property
+    def key(self):
+        return str(self._times_iteration)

versionhq/_prompt/constants.py

@@ -0,0 +1,30 @@
+REFLECT = "Here is the orignal prompt: {original_prompt}\nHere is the original response: {original_response}\nAnalyze the original prompt and repsonse, check for any pontential issue, and create an improved response."
+
+INTEGRATE = "Here is the original prompt: {original_prompt}\nHere are responses: {responses}. Help integrate them as a single response."
+
+parameter_sets = [
+    {
+        "temperature": 0.2,
+        "top_p": 0.5,
+        "max_tokens": 5000,
+        "frequency_penalty": 0.5,
+        "presence_penalty": 0.5,
+        "stop": ["\n\n", "###"],
+    },
+    {
+        "temperature": 0.7,
+        "top_p": 0.8,
+        "max_tokens": 8000,
+        "frequency_penalty": 0.3,
+        "presence_penalty": 0.3,
+        "stop": ["\n\n"],
+    },
+    {
+        "temperature": 1.0,
+        "top_p": 0.95,
+        "max_tokens": 12000,
+        "frequency_penalty": 0.0,
+        "presence_penalty": 0.0,
+        "stop": [],
+    }
+]

versionhq/_prompt/model.py

@@ -1,34 +1,50 @@
-from typing import Any, Dict, List
+
+from typing import Dict, List, Tuple, Any
 from textwrap import dedent
 
+from pydantic import InstanceOf
+
+from versionhq._utils import is_valid_url
+
 
 class Prompt:
     """A class to format, store, and manage a prompt."""
 
     task: Any = None
     agent: Any = None
-    context: str = None
+    context: Any = None
 
 
-    def __init__(self, task, agent):
-        from versionhq.task.model import Task
+    def __init__(self, task, agent, context):
         from versionhq.agent.model import Agent
+        from versionhq.task.model import Task
 
         self.task = task if isinstance(task, Task) else Task(description=str(task))
         self.agent = agent if isinstance(agent, Agent) else Agent(role=str(agent))
+        self.context = context
 
 
     def _draft_output_prompt(self) -> str:
+        """Drafts prompt for output either from `pydantic_output` or `response_fields`"""
+
+        from versionhq.llm.model import DEFAULT_MODEL_PROVIDER_NAME
+
         output_prompt = ""
+        model_provider = self.agent.llm.provider if self.agent else DEFAULT_MODEL_PROVIDER_NAME
 
         if self.task.pydantic_output:
-            output_prompt = f"""Your response MUST STRICTLY follow the given repsonse format:
-JSON schema: {str(self.task.pydantic_output)}
-"""
+            output_prompt, output_formats_to_follow = "", dict()
+            response_format = str(self.task._structure_response_format(model_provider=model_provider))
+            for k, v in self.task.pydantic_output.model_fields.items():
+                output_formats_to_follow[k] = f"<Return your answer in {v.annotation}>"
 
+            output_prompt = f"""Your response MUST be a valid JSON string that strictly follows the response format. Use double quotes for all keys and string values. Do not use single quotes, trailing commas, or any other non-standard JSON syntax.
+Response format: {response_format}
+Ref. Output image: {output_formats_to_follow}
+"""
         elif self.task.response_fields:
             output_prompt, output_formats_to_follow = "", dict()
-            response_format = str(self.task._structure_response_format(model_provider=self.agent.llm.provider))
+            response_format = str(self.task._structure_response_format(model_provider=model_provider))
             for item in self.task.response_fields:
                 if item:
                     output_formats_to_follow[item.title] = f"<Return your answer in {item.data_type.__name__}>"
@@ -38,43 +54,42 @@ Response format: {response_format}
 Ref. Output image: {output_formats_to_follow}
 """
         else:
-            output_prompt = "You MUST Return your response as a valid JSON serializable string, enclosed in double quotes. Do not use single quotes, trailing commas, or other non-standard JSON syntax."
+            output_prompt = "You MUST return your response as a valid JSON serializable string, enclosed in double quotes. Use double quotes for all keys and string values. Do NOT use single quotes, trailing commas, or other non-standard JSON syntax."
 
         return dedent(output_prompt)
 
 
-    def _draft_context_prompt(self) -> str:
+    def _draft_context_prompt(self, context: Any = None) -> str:
         """
         Create a context prompt from the given context in any format: a task object, task output object, list, dict.
         """
         from versionhq.task.model import Task, TaskOutput
 
         context_to_add = None
-        if not self.context:
-            # Logger().log(level="error", color="red", message="Missing a context to add to the prompt. We'll return ''.")
-            return
+        if not context:
+            return context_to_add
 
-        match self.context:
+        match context:
             case str():
-                context_to_add = self.context
+                context_to_add = context
 
             case Task():
-                if not self.context.output:
-                    res = self.context.execute()
+                if not context.output:
+                    res = context.execute()
                     context_to_add = res._to_context_prompt()
 
                 else:
-                    context_to_add = self.context.output.raw
+                    context_to_add = context.output.raw
 
             case TaskOutput():
-                context_to_add = self.context._to_context_prompt()
+                context_to_add = context._to_context_prompt()
 
 
             case dict():
-                context_to_add = str(self.context)
+                context_to_add = str(context)
 
             case list():
-                res = ", ".join([self._draft_context_prompt(context=item) for item in self.context])
+                res = ", ".join([self._draft_context_prompt(context=item) for item in context])
                 context_to_add = res
 
             case _:
@@ -83,65 +98,121 @@ Ref. Output image: {output_formats_to_follow}
         return dedent(context_to_add)
 
 
-    def _draft_user_prompt(self) -> str:
+    def _format_content_prompt(self) -> Dict[str, str]:
+        """Formats content (file, image, audio) prompt message."""
+
+        import base64
+        from pathlib import Path
+
+        content_messages = {}
+
+        if self.task.image:
+            with open(self.task.image, "rb") as file:
+                content = file.read()
+                if content:
+                    encoded_file = base64.b64encode(content).decode("utf-8")
+                    img_url = f"data:image/jpeg;base64,{encoded_file}"
+                    content_messages.update({ "type": "image_url", "image_url": { "url": img_url }})
+
+        if self.task.file:
+            if is_valid_url(self.task.file):
+                content_messages.update({ "type": "image_url", "image_url": self.file })
+
+        if self.task.audio and self.agent.llm.provider == "gemini":
+            audio_bytes = Path(self.task.audio).read_bytes()
+            encoded_data = base64.b64encode(audio_bytes).decode("utf-8")
+            content_messages.update({ "type": "image_url", "image_url": "data:audio/mp3;base64,{}".format(encoded_data)})
+
+        return content_messages
+
+
+    def _find_rag_tools(self) -> List[InstanceOf[Any]]:
+        """Find RAG tools from the agent and task object."""
+
+        from versionhq.tool.rag_tool import RagTool
+
+        tools = []
+        if self.task.tools:
+            [tools.append(item) for item in self.task.tools if isinstance(item, RagTool)]
+
+        if self.agent.tools and self.task.can_use_agent_tools:
+            [tools.append(item) for item in self.agent.tools if isinstance(item, RagTool)]
+
+        return tools
+
+
+    def draft_user_prompt(self) -> str:
+        """Draft task prompts from its description and context."""
+
         output_prompt = self._draft_output_prompt()
         task_slices = [self.task.description, output_prompt, ]
 
         if self.context:
-            context_prompt = self._draft_context_prompt()
+            context_prompt = self._draft_context_prompt(context=self.context)
             task_slices.insert(len(task_slices), f"Consider the following context when responding: {context_prompt}")
 
         return "\n".join(task_slices)
 
 
-    def _format_content_prompt(self) -> Dict[str, str]:
-        """Formats and returns image_url content added to the messages."""
+    def format_core(self, rag_tools: List[Any] = None) -> Tuple[str, str, List[Dict[str, str]]]:
+        """Formats prompt messages sent to the LLM, then returns task prompt, developer prompt, and messages."""
 
-        from versionhq._utils import is_valid_url
+        from versionhq.knowledge._utils import extract_knowledge_context
+        from versionhq.memory.contextual_memory import ContextualMemory
 
-        content_messages = {}
+        user_prompt = self.draft_user_prompt()
+        rag_tools = rag_tools if rag_tools else self._find_rag_tools()
 
-        if self.task.image:
-            if is_valid_url(self.task.image):
-                content_messages.update({ "type": "image_url", "image_url": self.task.image })
-            else:
-                content_messages.update({ "type": "image_url", "image_url": { "url":  self.task.image }})
+        if self.agent._knowledge:
+            agent_knowledge = self.agent._knowledge.query(query=[user_prompt,], limit=5)
+            if agent_knowledge:
+                agent_knowledge_context = extract_knowledge_context(knowledge_snippets=agent_knowledge)
+                if agent_knowledge_context:
+                    user_prompt += agent_knowledge_context
 
-        if self.task.file:
-            if is_valid_url(self.task.file):
-                content_messages.update({ "type": "image_url", "image_url": self.task.file })
-            else:
-                content_messages.update({ "type": "image_url", "image_url": { "url":  self.task.file }})
+        if rag_tools:
+            for item in rag_tools:
+                rag_tool_context = item.run(agent=self.agent, query=self.task.description)
+                if rag_tool_context:
+                    user_prompt += ",".join(rag_tool_context) if isinstance(rag_tool_context, list) else str(rag_tool_context)
 
-        if self.task.audio:
-            from pathlib import Path
-            import base64
+        if self.agent.with_memory == True:
+            contextual_memory = ContextualMemory(
+                memory_config=self.agent.memory_config, stm=self.agent.short_term_memory, ltm=self.agent.long_term_memory, um=self.agent.user_memory
+            )
+            context_str = self._draft_context_prompt(context=self.context)
+            query = f"{self.task.description} {context_str}".strip()
+            memory = contextual_memory.build_context_for_task(query=query)
+            if memory.strip() != "":
+                user_prompt += memory.strip()
 
-            audio_bytes = Path(self.audio_file_path).read_bytes()
-            encoded_data = base64.b64encode(audio_bytes).decode("utf-8")
-            content_messages.update({  "type": "image_url", "image_url": "data:audio/mp3;base64,{}".format(encoded_data)})
 
-        return content_messages
+        ## comment out - training
+        # if self.agent.networks and self.agent.networks._train:
+        #     user_prompt = self.agent._training_handler(user_prompt=user_prompt)
+        # else:
+        #     user_prompt = self.agent._use_trained_data(user_prompt=user_prompt)
+
 
-    @property
-    def messages(self) -> List[Dict[str, str]]:
-        user_prompt = self._draft_user_prompt()
         content_prompt = self._format_content_prompt()
 
         messages = []
-        messages.append(
-            {
-                "role": "user",
-                "content": [
-                    {
-                        "type": "text",
-                        "text": user_prompt
-                    },
-                    content_prompt,
-                ]
-            })
-
-        if self.use_developer_prompt:
-            messages.append({ "role": "developer", "content": self.backstory })
-
-        return messages
+        if content_prompt:
+            messages.append(
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": user_prompt
+                        },
+                        content_prompt,
+                    ]
+                })
+        else:
+            messages.append({ "role": "user", "content": user_prompt })
+
+        if self.agent.use_developer_prompt:
+            messages.append({ "role": "developer", "content": self.agent.backstory })
+
+        return user_prompt, self.agent.backstory if self.agent.use_developer_prompt else None, messages

versionhq/_utils/__init__.py

@@ -2,3 +2,4 @@ from versionhq._utils.logger import Logger
 from versionhq._utils.process_config import process_config
 from versionhq._utils.vars import KNOWLEDGE_DIRECTORY, MAX_FILE_NAME_LENGTH
 from versionhq._utils.is_valid_url import is_valid_url
+from versionhq._utils.usage_metrics import UsageMetrics, ErrorType

versionhq/_utils/usage_metrics.py

@@ -1,55 +1,72 @@
-from pydantic import BaseModel, Field
+import uuid
+import enum
+import datetime
+from typing import Dict, List
+from typing_extensions import Self
+
+from pydantic import BaseModel, UUID4, InstanceOf
+
+
+class ErrorType(enum.Enum):
+    FORMAT = 1
+    TOOL = 2
+    API = 3
+    OVERFITTING = 4
+    HUMAN_INTERACTION = 5
 
 
 class UsageMetrics(BaseModel):
-    """
-    Model to track usage
-    """
-
-    total_tokens: int = Field(default=0, description="total number of tokens used")
-    prompt_tokens: int = Field(default=0, description="number of tokens used in prompts")
-    cached_prompt_tokens: int = Field(default=0, description="number of cached prompt tokens used")
-    completion_tokens: int = Field(default=0, description="number of tokens used in completions")
-    successful_requests: int = Field(default=0, description="number of successful requests made")
-
-    def add_usage_metrics(self, usage_metrics: "UsageMetrics") -> None:
-        """
-        Add the usage metrics from another UsageMetrics object.
-        """
-        self.total_tokens += usage_metrics.total_tokens
-        self.prompt_tokens += usage_metrics.prompt_tokens
-        self.cached_prompt_tokens += usage_metrics.cached_prompt_tokens
-        self.completion_tokens += usage_metrics.completion_tokens
-        self.successful_requests += usage_metrics.successful_requests
-
-
-
-# class TokenProcess:
-#     total_tokens: int = 0
-#     prompt_tokens: int = 0
-#     cached_prompt_tokens: int = 0
-#     completion_tokens: int = 0
-#     successful_requests: int = 0
-
-#     def sum_prompt_tokens(self, tokens: int) -> None:
-#         self.prompt_tokens = self.prompt_tokens + tokens
-#         self.total_tokens = self.total_tokens + tokens
-
-#     def sum_completion_tokens(self, tokens: int) -> None:
-#         self.completion_tokens = self.completion_tokens + tokens
-#         self.total_tokens = self.total_tokens + tokens
-
-#     def sum_cached_prompt_tokens(self, tokens: int) -> None:
-#         self.cached_prompt_tokens = self.cached_prompt_tokens + tokens
-
-#     def sum_successful_requests(self, requests: int) -> None:
-#         self.successful_requests = self.successful_requests + requests
-
-#     def get_summary(self) -> UsageMetrics:
-#         return UsageMetrics(
-#             total_tokens=self.total_tokens,
-#             prompt_tokens=self.prompt_tokens,
-#             cached_prompt_tokens=self.cached_prompt_tokens,
-#             completion_tokens=self.completion_tokens,
-#             successful_requests=self.successful_requests,
-#         )
+    """A Pydantic model to manage token usage, errors, job latency."""
+
+    id: UUID4 = uuid.uuid4() # stores task id or task graph id
+    total_tokens: int = 0
+    prompt_tokens: int = 0
+    completion_tokens: int = 0
+    successful_requests: int = 0
+    total_errors: int = 0
+    error_breakdown: Dict[ErrorType, int] = dict()
+    latency: float = 0.0  # in ms
+
+    def record_token_usage(self, token_usages: List[Dict[str, int]]) -> None:
+        """Records usage metrics from the raw response of the model."""
+
+        if token_usages:
+            for item in token_usages:
+                self.total_tokens += int(item["total_tokens"]) if "total_tokens" in item else 0
+                self.completion_tokens += int(item["completion_tokens"])  if "completion_tokens" in item else 0
+                self.prompt_tokens += int(item["prompt_tokens"]) if "prompt_tokens" in item else 0
+
+
+    def record_errors(self, type: ErrorType = None) -> None:
+        self.total_errors += 1
+        if type:
+            if type in self.error_breakdown:
+                self.error_breakdown[type] += 1
+            else:
+                self.error_breakdown[type] = 1
+
+
+    def record_latency(self, start_dt: datetime.datetime, end_dt: datetime.datetime) -> None:
+        self.latency += round((end_dt - start_dt).total_seconds() * 1000, 3)
+
+
+    def aggregate(self, metrics: InstanceOf["UsageMetrics"]) -> Self:
+        if not metrics:
+            return self
+
+        self.total_tokens += metrics.total_tokens if metrics.total_tokens else 0
+        self.prompt_tokens += metrics.prompt_tokens if metrics.prompt_tokens else 0
+        self.completion_tokens += metrics.completion_tokens if metrics.completion_tokens else 0
+        self.successful_requests += metrics.successful_requests  if metrics.successful_requests else 0
+        self.total_errors += metrics.total_errors if metrics.total_errors else 0
+        self.latency += metrics.latency if metrics.latency else 0.0
+        self.latency = round(self.latency, 3)
+
+        if metrics.error_breakdown:
+            for k, v in metrics.error_breakdown.items():
+                if self.error_breakdown and k in self.error_breakdown:
+                    self.error_breakdown[k] += int(v)
+                else:
+                    self.error_breakdown.update({ k: v })
+
+        return self