versionhq 1.2.4.1__py3-none-any.whl → 1.2.4.3__py3-none-any.whl

versionhq/task/model.py

@@ -4,7 +4,6 @@ import datetime
 import uuid
 import inspect
 import enum
-from textwrap import dedent
 from concurrent.futures import Future
 from hashlib import md5
 from typing import Any, Dict, List, Set, Optional, Callable, Type
@@ -16,7 +15,7 @@ from pydantic_core import PydanticCustomError
 import versionhq as vhq
 from versionhq.task.evaluation import Evaluation, EvaluationItem
 from versionhq.tool.model import Tool, ToolSet
-from versionhq._utils import process_config, Logger, is_valid_url
+from versionhq._utils import process_config, Logger, UsageMetrics, ErrorType
 
 
 class TaskExecutionType(enum.Enum):
@@ -175,7 +174,6 @@ class TaskOutput(BaseModel):
     """
     A class to store the final output of the given task in raw (string), json_dict, and pydantic class formats.
     """
-    _tokens: int = PrivateAttr(default=0)
 
     task_id: UUID4 = Field(default_factory=uuid.uuid4, frozen=True, description="store Task ID")
     raw: str = Field(default="", description="Raw output of the task")
@@ -183,23 +181,38 @@ class TaskOutput(BaseModel):
     pydantic: Optional[Any] = Field(default=None)
     tool_output: Optional[Any] = Field(default=None, description="stores tool result when the task takes tool output as its final output")
     callback_output: Optional[Any] = Field(default=None, description="stores task or agent callback outcome")
-    latency: float = Field(default=None, description="job latency in ms")
     evaluation: Optional[InstanceOf[Evaluation]] = Field(default=None, description="stores overall evaluation of the task output. stored in ltm")
 
 
+    def _fetch_value_of(self, key: str = None) -> Any:
+        """Returns a value to the given key."""
+
+        if not key:
+            return None
+
+        if self.pydantic and hasattr(self.pydantic, key):
+            return getattr(self.pydantic, key)
+
+        elif self.json_dict and key in self.json_dict:
+            return self.json_dict[key]
+
+        else:
+            return None
+
+
     def _to_context_prompt(self) -> str:
-        """
-        Returns response in string as a prompt context.
-        """
+        """Formats prompt context in text formats from the final response."""
+
         context = ""
-        try:
-            context = json.dumps(self.json_dict)
-        except:
-            try:
-                if self.pydantic:
-                    context = self.pydantic.model_dump()
-            except:
-                context = self.raw
+        match self.final:
+            case dict() | self.pydantic:
+                try:
+                    context = json.dumps(self.final)
+                except:
+                    context = str(self.final)
+            case _:
+                context = str(self.final)
+
         return context
 
 
@@ -225,7 +238,6 @@ class TaskOutput(BaseModel):
 
             task_eval = Task(description=description, pydantic_output=EvaluationItem)
             res = task_eval.execute(agent=self.evaluation.eval_by)
-            self._tokens += task_eval._tokens
 
             if res.pydantic:
                 item = EvaluationItem(
@@ -252,6 +264,24 @@ class TaskOutput(BaseModel):
         return self.evaluation
 
 
+    @property
+    def final(self) -> Any:
+        """Returns final output from the task."""
+
+        output = None
+
+        if self.callback_output:
+            output = self.callback_output
+
+        elif self.tool_output and str(self.tool_output) == self.raw: # tool_output_as_final
+            output = self.tool_output
+
+        else:
+            output = self.pydantic if self.pydantic else self.json_dict if self.json_dict else self.raw
+
+        return output
+
+
     @property
     def aggregate_score(self) -> float | int:
         return self.evaluation.aggregate_score if self.evaluation is not None else 0
@@ -280,6 +310,7 @@ class Task(BaseModel):
     description: str = Field(description="Description of the actual task")
 
     # response format
+    response_schema: Optional[Type[BaseModel] | List[ResponseField]] = Field(default=None)
     pydantic_output: Optional[Type[BaseModel]] = Field(default=None, description="store Pydantic class as structured response format")
     response_fields: Optional[List[ResponseField]] = Field(default_factory=list, description="store list of ResponseField as structured response format")
 
@@ -292,6 +323,11 @@ class Task(BaseModel):
     file: Optional[str] = Field(default=None, description="absolute file path or url in string")
     audio: Optional[str] = Field(default=None,  description="absolute file path or url in string")
 
+    # test run
+    should_test_run: bool = Field(default=False)
+    human: bool = Field(default=False)
+    _pfg: Any = None
+
     # executing
     execution_type: TaskExecutionType = Field(default=TaskExecutionType.SYNC)
     allow_delegation: bool = Field(default=False, description="whether to delegate the task to another agent")
@@ -304,9 +340,7 @@ class Task(BaseModel):
     fsls: Optional[list[str]] = Field(default=None, description="stores ideal/weak responses")
 
     # recording
-    _tokens: int = 0
-    _tool_errors: int = 0
-    _format_errors: int = 0
+    _usage: UsageMetrics = PrivateAttr(default=None)
     _delegations: int = 0
     processed_agents: Set[str] = Field(default_factory=set, description="store keys of the agents that executed the task")
     output: Optional[TaskOutput] = Field(default=None, description="store the final TaskOutput object")
@@ -331,6 +365,8 @@ class Task(BaseModel):
         for field in required_fields:
             if getattr(self, field) is None:
                 raise ValueError( f"{field} must be provided either directly or through config")
+
+        self._usage = UsageMetrics(id=self.id)
         return self
 
 
@@ -351,121 +387,6 @@ class Task(BaseModel):
         return self
 
 
-    def _draft_output_prompt(self, model_provider: str = None) -> str:
-        output_prompt = ""
-
-        if self.pydantic_output:
-            output_prompt, output_formats_to_follow = "", dict()
-            response_format = str(self._structure_response_format(model_provider=model_provider))
-            for k, v in self.pydantic_output.model_fields.items():
-                output_formats_to_follow[k] = f"<Return your answer in {v.annotation}>"
-
-            output_prompt = f"""Your response MUST be a valid JSON string that strictly follows the response format. Use double quotes for all keys and string values. Do not use single quotes, trailing commas, or any other non-standard JSON syntax.
-Response format: {response_format}
-Ref. Output image: {output_formats_to_follow}
-"""
-        elif self.response_fields:
-            output_prompt, output_formats_to_follow = "", dict()
-            response_format = str(self._structure_response_format(model_provider=model_provider))
-            for item in self.response_fields:
-                if item:
-                    output_formats_to_follow[item.title] = f"<Return your answer in {item.data_type.__name__}>"
-
-            output_prompt = f"""Your response MUST be a valid JSON string that strictly follows the response format. Use double quotes for all keys and string values. Do not use single quotes, trailing commas, or any other non-standard JSON syntax.
-Response format: {response_format}
-Ref. Output image: {output_formats_to_follow}
-"""
-        # elif not self.tools or self.can_use_agent_tools == False:
-        else:
-            output_prompt = "You MUST return your response as a valid JSON serializable string, enclosed in double quotes. Use double quotes for all keys and string values. Do NOT use single quotes, trailing commas, or other non-standard JSON syntax."
-
-        # else:
-        #     output_prompt = "You will return a response in a concise manner."
-
-        return dedent(output_prompt)
-
-
-    def _draft_context_prompt(self, context: Any) -> str:
-        """
-        Create a context prompt from the given context in any format: a task object, task output object, list, dict.
-        """
-
-        context_to_add = None
-        if not context:
-            # Logger().log(level="error", color="red", message="Missing a context to add to the prompt. We'll return ''.")
-            return context_to_add
-
-        match context:
-            case str():
-                context_to_add = context
-
-            case Task():
-                if not context.output:
-                    res = context.execute()
-                    context_to_add = res._to_context_prompt()
-
-                else:
-                    context_to_add = context.output.raw
-
-            case TaskOutput():
-                context_to_add = context._to_context_prompt()
-
-
-            case dict():
-                context_to_add = str(context)
-
-            case list():
-                res = ", ".join([self._draft_context_prompt(context=item) for item in context])
-                context_to_add = res
-
-            case _:
-                pass
-
-        return dedent(context_to_add)
-
-
-    def _user_prompt(self, model_provider: str = None, context: Optional[Any] = None) -> str:
-        """
-        Format the task prompt and cascade it to the agent.
-        """
-        output_prompt = self._draft_output_prompt(model_provider=model_provider)
-        task_slices = [self.description, output_prompt, ]
-
-        if context:
-            context_prompt = self._draft_context_prompt(context=context)
-            task_slices.insert(len(task_slices), f"Consider the following context when responding: {context_prompt}")
-
-        return "\n".join(task_slices)
-
-
-    def _format_content_prompt(self) -> Dict[str, str]:
-        """Formats content (file, image, audio) prompts that added to the messages sent to the LLM."""
-
-        from pathlib import Path
-        import base64
-
-        content_messages = {}
-
-        if self.image:
-            with open(self.image, "rb") as file:
-                content = file.read()
-                if content:
-                    encoded_file = base64.b64encode(content).decode("utf-8")
-                    img_url = f"data:image/jpeg;base64,{encoded_file}"
-                    content_messages.update({ "type": "image_url", "image_url": { "url": img_url }})
-
-        if self.file:
-            if is_valid_url(self.file):
-                content_messages.update({ "type": "image_url", "image_url": self.file })
-
-        if self.audio:
-            audio_bytes = Path(self.audio).read_bytes()
-            encoded_data = base64.b64encode(audio_bytes).decode("utf-8")
-            content_messages.update({  "type": "image_url", "image_url": "data:audio/mp3;base64,{}".format(encoded_data)})
-
-        return content_messages
-
-
     def _structure_response_format(self, data_type: str = "object", model_provider: str = "gemini") -> Dict[str, Any] | None:
         """Structures `response_fields` or `pydantic_output` to a LLM response format."""
 
@@ -495,7 +416,6 @@ Ref. Output image: {output_formats_to_follow}
                     "json_schema": { "name": "outcome", "schema": response_schema }
                 }
 
-
             elif self.pydantic_output:
                 response_format = StructuredOutput(response_format=self.pydantic_output, provider=model_provider)._format()
 
@@ -525,14 +445,15 @@ Ref. Output image: {output_formats_to_follow}
             output = json.loads(j)
 
         if isinstance(output, dict):
-            return output
+            return output["json_schema"] if "json_schema" in output else output
         else:
             try:
                 output = ast.literal_eval(j)
             except:
                 output = ast.literal_eval(r)
 
-            return output if isinstance(output, dict) else { "output": str(r) }
+
+            return output["json_schema"] if isinstance(output, dict) and "json_schema" in output else output if isinstance(output, dict) else { "output": str(r) }
 
 
     def _create_json_output(self, raw: str) -> Dict[str, Any]:
@@ -548,12 +469,13 @@ Ref. Output image: {output_formats_to_follow}
         try:
             output = json.loads(raw)
             if isinstance(output, dict):
-                return output
+                return output["json_schema"] if "json_schema" in output else output
             else:
                output = self._sanitize_raw_output(raw=raw)
                return output
         except:
             output = self._sanitize_raw_output(raw=raw)
+            self._usage.record_errors(type=ErrorType.FORMAT)
             return output
 
 
@@ -673,23 +595,25 @@ Ref. Output image: {output_formats_to_follow}
 
 
     # task execution
-    def execute(
-            self, type: TaskExecutionType = None, agent: Optional["vhq.Agent"] = None, context: Optional[Any] = None
-        ) -> TaskOutput | Future[TaskOutput]:
-        """
-        A main method to handle task execution. Build an agent when the agent is not given.
-        """
+    def execute(self, type: TaskExecutionType = None, agent: "vhq.Agent" = None, context: Any = None) -> TaskOutput | Future[TaskOutput]:
+        """A main method to handle task execution."""
+
         type = type if type else  self.execution_type if self.execution_type else TaskExecutionType.SYNC
+        agent = agent if agent else self._build_agent_from_task(task_description=self.description)
+        res = None
 
-        if not agent:
-            agent = self._build_agent_from_task(task_description=self.description)
+        if (self.should_test_run or agent.self_learn) and not self._pfg:
+            res = self._test_time_computation(agent=agent, context=context)
+            return res
 
         match type:
             case TaskExecutionType.SYNC:
-                return self._execute_sync(agent=agent, context=context)
+                res = self._execute_sync(agent=agent, context=context)
 
             case TaskExecutionType.ASYNC:
-                return self._execute_async(agent=agent, context=context)
+                res = self._execute_async(agent=agent, context=context)
+
+        return res
 
 
     def _execute_sync(self, agent, context: Optional[Any] = None) -> TaskOutput:
@@ -710,14 +634,14 @@ Ref. Output image: {output_formats_to_follow}
 
 
     def _execute_core(self, agent, context: Optional[Any]) -> TaskOutput:
-        """
-        A core method to execute a task.
-        """
+        """A core method to execute a single task."""
+
+        start_dt = datetime.datetime.now()
         task_output: InstanceOf[TaskOutput] = None
         raw_output: str = None
         tool_output: str | list = None
         task_tools: List[List[InstanceOf[Tool]| InstanceOf[ToolSet] | Type[Tool]]] = []
-        started_at, ended_at = datetime.datetime.now(), datetime.datetime.now()
+        user_prompt, dev_prompt = None, None
 
         if self.tools:
             for item in self.tools:
@@ -730,17 +654,14 @@ Ref. Output image: {output_formats_to_follow}
             self._delegations += 1
 
         if self.tool_res_as_final == True:
-            started_at = datetime.datetime.now()
-            tool_output = agent.execute_task(task=self, context=context, task_tools=task_tools)
+            user_prompt, dev_prompt, tool_output = agent.execute_task(task=self, context=context, task_tools=task_tools)
             raw_output = str(tool_output) if tool_output else ""
-            ended_at = datetime.datetime.now()
+            if not raw_output:
+                self._usage.record_errors(type=ErrorType.TOOL)
             task_output = TaskOutput(task_id=self.id, tool_output=tool_output, raw=raw_output)
 
         else:
-            started_at = datetime.datetime.now()
-            raw_output = agent.execute_task(task=self, context=context, task_tools=task_tools)
-            ended_at = datetime.datetime.now()
-
+            user_prompt, dev_prompt, raw_output = agent.execute_task(task=self, context=context, task_tools=task_tools)
             json_dict_output = self._create_json_output(raw=raw_output)
             if "outcome" in json_dict_output:
                 json_dict_output = self._create_json_output(raw=str(json_dict_output["outcome"]))
@@ -754,8 +675,6 @@ Ref. Output image: {output_formats_to_follow}
                 json_dict=json_dict_output,
             )
 
-        task_output.latency = round((ended_at - started_at).total_seconds() * 1000, 3)
-        task_output._tokens = self._tokens
         self.output = task_output
         self.processed_agents.add(agent.key)
 
@@ -767,6 +686,11 @@ Ref. Output image: {output_formats_to_follow}
         #     )
         #     self._save_file(content)
 
+        if self._pfg:
+            index = self._pfg.index
+            self._pfg.user_prompts.update({ index: user_prompt })
+            self._pfg.dev_prompts.update({ index: dev_prompt })
+
         if raw_output:
             if self.should_evaluate:
                 task_output.evaluate(task=self)
@@ -784,9 +708,36 @@ Ref. Output image: {output_formats_to_follow}
                 self.output = task_output
             self._store_logs()
 
+        end_dt = datetime.datetime.now()
+        self._usage.record_latency(start_dt=start_dt, end_dt=end_dt)
         return task_output
 
 
+    def _test_time_computation(self, agent, context: Optional[Any]) -> TaskOutput | None:
+        """Handles test-time computation."""
+
+        from versionhq.task_graph.model import ReformTriggerEvent
+        from versionhq._prompt.model import Prompt
+        from versionhq._prompt.auto_feedback import PromptFeedbackGraph
+
+        # self._usage = None
+        prompt = Prompt(task=self, agent=agent, context=context)
+        pfg = PromptFeedbackGraph(prompt=prompt, should_reform=self.human, reform_trigger_event=ReformTriggerEvent.USER_INPUT if self.human else None)
+        pfg = pfg.set_up_graph()
+        self._pfg = pfg
+
+        try:
+            if self._pfg and self.output is None:
+                res, all_outputs = self._pfg.activate()
+                if all_outputs: self._usage = self._pfg._usage
+                return res
+
+        except:
+            self._usage.record_errors(type=ErrorType.API)
+            Logger().log(level="error", message="Failed to execute the task.", color="red")
+            return None
+
+
     @property
     def key(self) -> str:
         output_format = "json" if self.response_fields else "pydantic" if self.pydantic_output is not None else "raw"

versionhq/task_graph/draft.py

@@ -10,7 +10,7 @@ sys.modules['pydantic.main'].ModelMetaclass = ModelMetaclass
 
 from versionhq.agent.model import Agent
 from versionhq.task.model import ResponseField
-from versionhq.task_graph.model import TaskGraph, Task, DependencyType, Node
+from versionhq.task_graph.model import TaskGraph, Task, DependencyType, Node, ReformTriggerEvent
 from versionhq._utils.logger import Logger
 
 
@@ -81,7 +81,8 @@ def workflow(final_output: Type[BaseModel], context: Any = None, human: bool = F
         nodes={node.identifier: node for node in nodes},
         concl_format=final_output,
         concl=None,
-        should_reform=True,
+        should_reform=human,
+        reform_trigger_event=ReformTriggerEvent.USER_INPUT if human else None,
     )
 
     for res in task_items:
@@ -95,17 +96,6 @@ def workflow(final_output: Type[BaseModel], context: Any = None, human: bool = F
                 task_graph.add_dependency(
                     source=source.identifier, target=target.identifier, dependency_type=dependency_type)
 
-    task_graph.visualize()
-
-    if human:
-        print('Proceed? Y/n:')
-        x = input()
-
-        if x.lower() == "y":
-            print("ok. generating agent network")
-
-        else:
-            request = input("request?")
-            print('ok. regenerating the graph based on your input: ', request)
+    # task_graph.visualize()
 
     return task_graph