flock-core 0.4.520__py3-none-any.whl → 0.5.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of flock-core might be problematic. Click here for more details.
- flock/cli/manage_agents.py +3 -3
- flock/components/__init__.py +28 -0
- flock/components/evaluation/__init__.py +9 -0
- flock/components/evaluation/declarative_evaluation_component.py +198 -0
- flock/components/routing/__init__.py +15 -0
- flock/{routers/conditional/conditional_router.py → components/routing/conditional_routing_component.py} +60 -49
- flock/components/routing/default_routing_component.py +103 -0
- flock/components/routing/llm_routing_component.py +208 -0
- flock/components/utility/__init__.py +15 -0
- flock/{modules/enterprise_memory/enterprise_memory_module.py → components/utility/memory_utility_component.py} +195 -173
- flock/{modules/performance/metrics_module.py → components/utility/metrics_utility_component.py} +101 -86
- flock/{modules/output/output_module.py → components/utility/output_utility_component.py} +49 -49
- flock/core/__init__.py +2 -8
- flock/core/agent/__init__.py +16 -0
- flock/core/agent/flock_agent_components.py +104 -0
- flock/core/agent/flock_agent_execution.py +101 -0
- flock/core/agent/flock_agent_integration.py +147 -0
- flock/core/agent/flock_agent_lifecycle.py +177 -0
- flock/core/agent/flock_agent_serialization.py +378 -0
- flock/core/component/__init__.py +15 -0
- flock/core/{flock_module.py → component/agent_component_base.py} +136 -35
- flock/core/component/evaluation_component_base.py +56 -0
- flock/core/component/routing_component_base.py +75 -0
- flock/core/component/utility_component_base.py +69 -0
- flock/core/config/flock_agent_config.py +49 -2
- flock/core/evaluation/utils.py +1 -1
- flock/core/execution/evaluation_executor.py +1 -1
- flock/core/flock.py +137 -483
- flock/core/flock_agent.py +151 -1018
- flock/core/flock_factory.py +94 -73
- flock/core/{flock_registry.py → flock_registry.py.backup} +3 -17
- flock/core/logging/logging.py +1 -0
- flock/core/mcp/flock_mcp_server.py +42 -37
- flock/core/mixin/dspy_integration.py +5 -5
- flock/core/orchestration/__init__.py +18 -0
- flock/core/orchestration/flock_batch_processor.py +94 -0
- flock/core/orchestration/flock_evaluator.py +113 -0
- flock/core/orchestration/flock_execution.py +288 -0
- flock/core/orchestration/flock_initialization.py +125 -0
- flock/core/orchestration/flock_server_manager.py +65 -0
- flock/core/orchestration/flock_web_server.py +117 -0
- flock/core/registry/__init__.py +39 -0
- flock/core/registry/agent_registry.py +69 -0
- flock/core/registry/callable_registry.py +139 -0
- flock/core/registry/component_discovery.py +142 -0
- flock/core/registry/component_registry.py +64 -0
- flock/core/registry/config_mapping.py +64 -0
- flock/core/registry/decorators.py +137 -0
- flock/core/registry/registry_hub.py +202 -0
- flock/core/registry/server_registry.py +57 -0
- flock/core/registry/type_registry.py +86 -0
- flock/core/serialization/flock_serializer.py +33 -30
- flock/core/serialization/serialization_utils.py +28 -25
- flock/core/util/input_resolver.py +29 -2
- flock/platform/docker_tools.py +3 -3
- flock/tools/markdown_tools.py +1 -2
- flock/tools/text_tools.py +1 -2
- flock/webapp/app/main.py +9 -5
- flock/workflow/activities.py +59 -84
- flock/workflow/activities_unified.py +230 -0
- flock/workflow/agent_execution_activity.py +6 -6
- flock/workflow/flock_workflow.py +1 -1
- {flock_core-0.4.520.dist-info → flock_core-0.5.0b1.dist-info}/METADATA +2 -2
- {flock_core-0.4.520.dist-info → flock_core-0.5.0b1.dist-info}/RECORD +67 -68
- flock/core/flock_evaluator.py +0 -60
- flock/core/flock_router.py +0 -83
- flock/evaluators/__init__.py +0 -1
- flock/evaluators/declarative/__init__.py +0 -1
- flock/evaluators/declarative/declarative_evaluator.py +0 -194
- flock/evaluators/memory/memory_evaluator.py +0 -90
- flock/evaluators/test/test_case_evaluator.py +0 -38
- flock/evaluators/zep/zep_evaluator.py +0 -59
- flock/modules/__init__.py +0 -1
- flock/modules/assertion/__init__.py +0 -1
- flock/modules/assertion/assertion_module.py +0 -286
- flock/modules/callback/__init__.py +0 -1
- flock/modules/callback/callback_module.py +0 -91
- flock/modules/enterprise_memory/README.md +0 -99
- flock/modules/mem0/__init__.py +0 -1
- flock/modules/mem0/mem0_module.py +0 -126
- flock/modules/mem0_async/__init__.py +0 -1
- flock/modules/mem0_async/async_mem0_module.py +0 -126
- flock/modules/memory/__init__.py +0 -1
- flock/modules/memory/memory_module.py +0 -429
- flock/modules/memory/memory_parser.py +0 -125
- flock/modules/memory/memory_storage.py +0 -736
- flock/modules/output/__init__.py +0 -1
- flock/modules/performance/__init__.py +0 -1
- flock/modules/zep/__init__.py +0 -1
- flock/modules/zep/zep_module.py +0 -192
- flock/routers/__init__.py +0 -1
- flock/routers/agent/__init__.py +0 -1
- flock/routers/agent/agent_router.py +0 -236
- flock/routers/agent/handoff_agent.py +0 -58
- flock/routers/default/__init__.py +0 -1
- flock/routers/default/default_router.py +0 -80
- flock/routers/feedback/feedback_router.py +0 -114
- flock/routers/list_generator/list_generator_router.py +0 -166
- flock/routers/llm/__init__.py +0 -1
- flock/routers/llm/llm_router.py +0 -365
- {flock_core-0.4.520.dist-info → flock_core-0.5.0b1.dist-info}/WHEEL +0 -0
- {flock_core-0.4.520.dist-info → flock_core-0.5.0b1.dist-info}/entry_points.txt +0 -0
- {flock_core-0.4.520.dist-info → flock_core-0.5.0b1.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# src/flock/core/agent/flock_agent_execution.py
|
|
2
|
+
"""Execution management functionality for FlockAgent."""
|
|
3
|
+
|
|
4
|
+
import asyncio
|
|
5
|
+
from typing import TYPE_CHECKING, Any
|
|
6
|
+
|
|
7
|
+
from opentelemetry import trace
|
|
8
|
+
from flock.core.logging.logging import get_logger
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from flock.core.flock_agent import FlockAgent
|
|
12
|
+
|
|
13
|
+
logger = get_logger("agent.execution")
|
|
14
|
+
tracer = trace.get_tracer(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class FlockAgentExecution:
|
|
18
|
+
"""Handles execution management for FlockAgent including run, run_async, and run_temporal."""
|
|
19
|
+
|
|
20
|
+
def __init__(self, agent: "FlockAgent"):
|
|
21
|
+
self.agent = agent
|
|
22
|
+
|
|
23
|
+
def run(self, inputs: dict[str, Any]) -> dict[str, Any]:
|
|
24
|
+
"""Synchronous wrapper for run_async."""
|
|
25
|
+
try:
|
|
26
|
+
loop = asyncio.get_running_loop()
|
|
27
|
+
except (
|
|
28
|
+
RuntimeError
|
|
29
|
+
): # 'RuntimeError: There is no current event loop...'
|
|
30
|
+
loop = asyncio.new_event_loop()
|
|
31
|
+
asyncio.set_event_loop(loop)
|
|
32
|
+
return loop.run_until_complete(self.run_async(inputs))
|
|
33
|
+
|
|
34
|
+
async def run_async(self, inputs: dict[str, Any]) -> dict[str, Any]:
|
|
35
|
+
"""Asynchronous execution logic with lifecycle hooks."""
|
|
36
|
+
with tracer.start_as_current_span("agent.run") as span:
|
|
37
|
+
span.set_attribute("agent.name", self.agent.name)
|
|
38
|
+
span.set_attribute("inputs", str(inputs))
|
|
39
|
+
try:
|
|
40
|
+
# Initialize lifecycle system if not already present
|
|
41
|
+
if not hasattr(self.agent, '_lifecycle'):
|
|
42
|
+
from flock.core.agent.flock_agent_lifecycle import FlockAgentLifecycle
|
|
43
|
+
self.agent._lifecycle = FlockAgentLifecycle(self.agent)
|
|
44
|
+
|
|
45
|
+
await self.agent._lifecycle.initialize(inputs)
|
|
46
|
+
result = await self.agent._lifecycle.evaluate(inputs)
|
|
47
|
+
await self.agent._lifecycle.terminate(inputs, result)
|
|
48
|
+
span.set_attribute("result", str(result))
|
|
49
|
+
logger.info("Agent run completed", agent=self.agent.name)
|
|
50
|
+
return result
|
|
51
|
+
except Exception as run_error:
|
|
52
|
+
logger.error(
|
|
53
|
+
"Error running agent", agent=self.agent.name, error=str(run_error)
|
|
54
|
+
)
|
|
55
|
+
if "evaluate" not in str(
|
|
56
|
+
run_error
|
|
57
|
+
): # Simple check, might need refinement
|
|
58
|
+
await self.agent._lifecycle.on_error(run_error, inputs)
|
|
59
|
+
logger.error(
|
|
60
|
+
f"Agent '{self.agent.name}' run failed: {run_error}",
|
|
61
|
+
exc_info=True,
|
|
62
|
+
)
|
|
63
|
+
span.record_exception(run_error)
|
|
64
|
+
raise # Re-raise after handling
|
|
65
|
+
|
|
66
|
+
async def run_temporal(self, inputs: dict[str, Any]) -> dict[str, Any]:
|
|
67
|
+
"""Execute agent using Temporal workflow orchestration."""
|
|
68
|
+
with tracer.start_as_current_span("agent.run_temporal") as span:
|
|
69
|
+
span.set_attribute("agent.name", self.agent.name)
|
|
70
|
+
span.set_attribute("inputs", str(inputs))
|
|
71
|
+
try:
|
|
72
|
+
from temporalio.client import Client
|
|
73
|
+
|
|
74
|
+
from flock.workflow.agent_activities import (
|
|
75
|
+
run_flock_agent_activity,
|
|
76
|
+
)
|
|
77
|
+
from flock.workflow.temporal_setup import run_activity
|
|
78
|
+
|
|
79
|
+
client = await Client.connect(
|
|
80
|
+
"localhost:7233", namespace="default"
|
|
81
|
+
)
|
|
82
|
+
agent_data = self.agent._serialization.to_dict()
|
|
83
|
+
inputs_data = inputs
|
|
84
|
+
|
|
85
|
+
result = await run_activity(
|
|
86
|
+
client,
|
|
87
|
+
self.agent.name,
|
|
88
|
+
run_flock_agent_activity,
|
|
89
|
+
{"agent_data": agent_data, "inputs": inputs_data},
|
|
90
|
+
)
|
|
91
|
+
span.set_attribute("result", str(result))
|
|
92
|
+
logger.info("Temporal run successful", agent=self.agent.name)
|
|
93
|
+
return result
|
|
94
|
+
except Exception as temporal_error:
|
|
95
|
+
logger.error(
|
|
96
|
+
"Error in Temporal workflow",
|
|
97
|
+
agent=self.agent.name,
|
|
98
|
+
error=str(temporal_error),
|
|
99
|
+
)
|
|
100
|
+
span.record_exception(temporal_error)
|
|
101
|
+
raise
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
# src/flock/core/agent/flock_agent_integration.py
|
|
2
|
+
"""Tool and server integration functionality for FlockAgent."""
|
|
3
|
+
|
|
4
|
+
from typing import TYPE_CHECKING, Any
|
|
5
|
+
|
|
6
|
+
from flock.core.context.context import FlockContext
|
|
7
|
+
from flock.core.logging.logging import get_logger
|
|
8
|
+
from flock.core.mcp.flock_mcp_server import FlockMCPServerBase
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from flock.core.flock_agent import FlockAgent
|
|
12
|
+
|
|
13
|
+
logger = get_logger("agent.integration")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class FlockAgentIntegration:
|
|
17
|
+
"""Handles tool and server integration for FlockAgent including MCP servers and callable tools."""
|
|
18
|
+
|
|
19
|
+
def __init__(self, agent: "FlockAgent"):
|
|
20
|
+
self.agent = agent
|
|
21
|
+
|
|
22
|
+
def resolve_callables(self, context: FlockContext | None = None) -> None:
|
|
23
|
+
"""Resolves callable fields (description, input, output) using context."""
|
|
24
|
+
if callable(self.agent.description):
|
|
25
|
+
self.agent.description = self.agent.description(
|
|
26
|
+
context
|
|
27
|
+
) # Pass context if needed by callable
|
|
28
|
+
if callable(self.agent.input):
|
|
29
|
+
self.agent.input = self.agent.input(context)
|
|
30
|
+
if callable(self.agent.output):
|
|
31
|
+
self.agent.output = self.agent.output(context)
|
|
32
|
+
|
|
33
|
+
def resolve_description(self, context: FlockContext | None = None) -> str:
|
|
34
|
+
if callable(self.agent.description):
|
|
35
|
+
try:
|
|
36
|
+
# Attempt to call without context first.
|
|
37
|
+
return self.agent.description(context)
|
|
38
|
+
except Exception as e:
|
|
39
|
+
logger.error(
|
|
40
|
+
f"Error resolving callable description for agent '{self.agent.name}': {e}"
|
|
41
|
+
)
|
|
42
|
+
return None
|
|
43
|
+
elif isinstance(self.agent.description, str):
|
|
44
|
+
return self.agent.description
|
|
45
|
+
return None
|
|
46
|
+
|
|
47
|
+
async def get_mcp_tools(self) -> list[Any]:
|
|
48
|
+
"""Get tools from registered MCP servers."""
|
|
49
|
+
mcp_tools = []
|
|
50
|
+
if self.agent.servers:
|
|
51
|
+
from flock.core.registry import get_registry
|
|
52
|
+
|
|
53
|
+
registry = get_registry() # Get the registry
|
|
54
|
+
for server in self.agent.servers:
|
|
55
|
+
registered_server: FlockMCPServerBase | None = None
|
|
56
|
+
server_tools = []
|
|
57
|
+
if isinstance(server, FlockMCPServerBase):
|
|
58
|
+
# check if registered
|
|
59
|
+
server_name = server.config.name
|
|
60
|
+
registered_server = registry.get_server(
|
|
61
|
+
server_name
|
|
62
|
+
)
|
|
63
|
+
else:
|
|
64
|
+
# servers must be registered.
|
|
65
|
+
registered_server = registry.get_server(
|
|
66
|
+
name=server
|
|
67
|
+
)
|
|
68
|
+
if registered_server:
|
|
69
|
+
server_tools = await registered_server.get_tools(
|
|
70
|
+
agent_id=self.agent.agent_id,
|
|
71
|
+
run_id=self.agent.context.run_id,
|
|
72
|
+
)
|
|
73
|
+
else:
|
|
74
|
+
logger.warning(
|
|
75
|
+
f"No Server with name '{server.config.name if isinstance(server, FlockMCPServerBase) else server}' registered! Skipping."
|
|
76
|
+
)
|
|
77
|
+
mcp_tools = mcp_tools + server_tools
|
|
78
|
+
return mcp_tools
|
|
79
|
+
|
|
80
|
+
async def execute_with_middleware(
|
|
81
|
+
self,
|
|
82
|
+
current_inputs: dict[str, Any],
|
|
83
|
+
registered_tools: list[Any],
|
|
84
|
+
mcp_tools: list[Any]
|
|
85
|
+
) -> dict[str, Any]:
|
|
86
|
+
"""Execute evaluator with optional DI middleware pipeline."""
|
|
87
|
+
container = None
|
|
88
|
+
if self.agent.context is not None:
|
|
89
|
+
container = self.agent.context.get_variable("di.container")
|
|
90
|
+
|
|
91
|
+
# If a MiddlewarePipeline is registered in DI, wrap the evaluator
|
|
92
|
+
result: dict[str, Any] | None = None
|
|
93
|
+
|
|
94
|
+
if container is not None:
|
|
95
|
+
try:
|
|
96
|
+
from wd.di.middleware import (
|
|
97
|
+
MiddlewarePipeline,
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
pipeline: MiddlewarePipeline | None = None
|
|
101
|
+
try:
|
|
102
|
+
pipeline = container.get_service(MiddlewarePipeline)
|
|
103
|
+
except Exception:
|
|
104
|
+
pipeline = None
|
|
105
|
+
|
|
106
|
+
if pipeline is not None:
|
|
107
|
+
# Build execution chain where the evaluator is the terminal handler
|
|
108
|
+
|
|
109
|
+
async def _final_handler():
|
|
110
|
+
return await self.agent.evaluator.evaluate_core(
|
|
111
|
+
self.agent, current_inputs, self.agent.context, registered_tools, mcp_tools
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
idx = 0
|
|
115
|
+
|
|
116
|
+
async def _invoke_next():
|
|
117
|
+
nonlocal idx
|
|
118
|
+
|
|
119
|
+
if idx < len(pipeline._middleware):
|
|
120
|
+
mw = pipeline._middleware[idx]
|
|
121
|
+
idx += 1
|
|
122
|
+
return await mw(self.agent.context, _invoke_next) # type: ignore[arg-type]
|
|
123
|
+
return await _final_handler()
|
|
124
|
+
|
|
125
|
+
# Execute pipeline
|
|
126
|
+
result = await _invoke_next()
|
|
127
|
+
else:
|
|
128
|
+
# No pipeline registered, direct evaluation
|
|
129
|
+
result = await self.agent.evaluator.evaluate_core(
|
|
130
|
+
self.agent, current_inputs, self.agent.context, registered_tools, mcp_tools
|
|
131
|
+
)
|
|
132
|
+
except ImportError:
|
|
133
|
+
# wd.di not installed – fall back
|
|
134
|
+
result = await self.agent.evaluator.evaluate_core(
|
|
135
|
+
self.agent, current_inputs, self.agent.context, registered_tools, mcp_tools
|
|
136
|
+
)
|
|
137
|
+
else:
|
|
138
|
+
# No DI container – standard execution
|
|
139
|
+
result = await self.agent.evaluator.evaluate_core(
|
|
140
|
+
self.agent,
|
|
141
|
+
current_inputs,
|
|
142
|
+
self.agent.context,
|
|
143
|
+
registered_tools,
|
|
144
|
+
mcp_tools,
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
return result
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
# src/flock/core/agent/flock_agent_lifecycle.py
|
|
2
|
+
"""Lifecycle management functionality for FlockAgent."""
|
|
3
|
+
|
|
4
|
+
from typing import TYPE_CHECKING, Any
|
|
5
|
+
|
|
6
|
+
from opentelemetry import trace
|
|
7
|
+
|
|
8
|
+
from flock.core.logging.logging import get_logger
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from flock.core.flock_agent import FlockAgent
|
|
12
|
+
|
|
13
|
+
logger = get_logger("agent.lifecycle")
|
|
14
|
+
tracer = trace.get_tracer(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class FlockAgentLifecycle:
|
|
18
|
+
"""Handles lifecycle management for FlockAgent including initialization, evaluation, and termination."""
|
|
19
|
+
|
|
20
|
+
def __init__(self, agent: "FlockAgent"):
|
|
21
|
+
self.agent = agent
|
|
22
|
+
|
|
23
|
+
async def initialize(self, inputs: dict[str, Any]) -> None:
|
|
24
|
+
"""Initialize agent and run module initializers."""
|
|
25
|
+
logger.debug(f"Initializing agent '{self.agent.name}'")
|
|
26
|
+
with tracer.start_as_current_span("agent.initialize") as span:
|
|
27
|
+
span.set_attribute("agent.name", self.agent.name)
|
|
28
|
+
span.set_attribute("inputs", str(inputs))
|
|
29
|
+
logger.info(
|
|
30
|
+
f"agent.initialize",
|
|
31
|
+
agent=self.agent.name,
|
|
32
|
+
)
|
|
33
|
+
try:
|
|
34
|
+
for component in self.agent.get_enabled_components():
|
|
35
|
+
await component.on_initialize(self.agent, inputs, self.agent.context)
|
|
36
|
+
except Exception as component_error:
|
|
37
|
+
logger.error(
|
|
38
|
+
"Error during initialize",
|
|
39
|
+
agent=self.agent.name,
|
|
40
|
+
error=str(component_error),
|
|
41
|
+
)
|
|
42
|
+
span.record_exception(component_error)
|
|
43
|
+
|
|
44
|
+
async def terminate(
|
|
45
|
+
self, inputs: dict[str, Any], result: dict[str, Any]
|
|
46
|
+
) -> None:
|
|
47
|
+
"""Terminate agent and run module terminators."""
|
|
48
|
+
logger.debug(f"Terminating agent '{self.agent.name}'")
|
|
49
|
+
with tracer.start_as_current_span("agent.terminate") as span:
|
|
50
|
+
span.set_attribute("agent.name", self.agent.name)
|
|
51
|
+
span.set_attribute("inputs", str(inputs))
|
|
52
|
+
span.set_attribute("result", str(result))
|
|
53
|
+
logger.info(
|
|
54
|
+
f"agent.terminate",
|
|
55
|
+
agent=self.agent.name,
|
|
56
|
+
)
|
|
57
|
+
try:
|
|
58
|
+
current_result = result
|
|
59
|
+
for component in self.agent.get_enabled_components():
|
|
60
|
+
tmp_result = await component.on_terminate(
|
|
61
|
+
self.agent, inputs, self.agent.context, current_result
|
|
62
|
+
)
|
|
63
|
+
# If the component returns a result, use it
|
|
64
|
+
if tmp_result:
|
|
65
|
+
current_result = tmp_result
|
|
66
|
+
|
|
67
|
+
if self.agent.config.write_to_file:
|
|
68
|
+
self.agent._serialization._save_output(self.agent.name, current_result)
|
|
69
|
+
|
|
70
|
+
if self.agent.config.wait_for_input:
|
|
71
|
+
# simple input prompt
|
|
72
|
+
input("Press Enter to continue...")
|
|
73
|
+
|
|
74
|
+
except Exception as component_error:
|
|
75
|
+
logger.error(
|
|
76
|
+
"Error during terminate",
|
|
77
|
+
agent=self.agent.name,
|
|
78
|
+
error=str(component_error),
|
|
79
|
+
)
|
|
80
|
+
span.record_exception(component_error)
|
|
81
|
+
|
|
82
|
+
async def on_error(self, error: Exception, inputs: dict[str, Any]) -> None:
|
|
83
|
+
"""Handle errors and run component error handlers."""
|
|
84
|
+
logger.error(f"Error occurred in agent '{self.agent.name}': {error}")
|
|
85
|
+
with tracer.start_as_current_span("agent.on_error") as span:
|
|
86
|
+
span.set_attribute("agent.name", self.agent.name)
|
|
87
|
+
span.set_attribute("inputs", str(inputs))
|
|
88
|
+
try:
|
|
89
|
+
for component in self.agent.get_enabled_components():
|
|
90
|
+
await component.on_error(self.agent, inputs, self.agent.context, error)
|
|
91
|
+
except Exception as component_error:
|
|
92
|
+
logger.error(
|
|
93
|
+
"Error during on_error",
|
|
94
|
+
agent=self.agent.name,
|
|
95
|
+
error=str(component_error),
|
|
96
|
+
)
|
|
97
|
+
span.record_exception(component_error)
|
|
98
|
+
|
|
99
|
+
async def evaluate(self, inputs: dict[str, Any]) -> dict[str, Any]:
|
|
100
|
+
"""Core evaluation logic, calling the assigned evaluator and components."""
|
|
101
|
+
if not self.agent.evaluator:
|
|
102
|
+
raise RuntimeError(
|
|
103
|
+
f"Agent '{self.agent.name}' has no evaluator assigned."
|
|
104
|
+
)
|
|
105
|
+
with tracer.start_as_current_span("agent.evaluate") as span:
|
|
106
|
+
span.set_attribute("agent.name", self.agent.name)
|
|
107
|
+
span.set_attribute("inputs", str(inputs))
|
|
108
|
+
logger.info(
|
|
109
|
+
f"agent.evaluate",
|
|
110
|
+
agent=self.agent.name,
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
logger.debug(f"Evaluating agent '{self.agent.name}'")
|
|
114
|
+
current_inputs = inputs
|
|
115
|
+
|
|
116
|
+
# Pre-evaluate hooks
|
|
117
|
+
for component in self.agent.get_enabled_components():
|
|
118
|
+
current_inputs = await component.on_pre_evaluate(
|
|
119
|
+
self.agent, current_inputs, self.agent.context
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
# Actual evaluation
|
|
123
|
+
try:
|
|
124
|
+
# Get tools and MCP tools through integration handler
|
|
125
|
+
registered_tools = []
|
|
126
|
+
if self.agent.tools:
|
|
127
|
+
registered_tools = self.agent.tools
|
|
128
|
+
|
|
129
|
+
# Retrieve available mcp_tools if the evaluator needs them
|
|
130
|
+
mcp_tools = []
|
|
131
|
+
if self.agent.servers:
|
|
132
|
+
mcp_tools = await self.agent._integration.get_mcp_tools()
|
|
133
|
+
|
|
134
|
+
# --------------------------------------------------
|
|
135
|
+
# Use evaluator component's evaluate_core method
|
|
136
|
+
# --------------------------------------------------
|
|
137
|
+
result = await self.agent.evaluator.evaluate_core(
|
|
138
|
+
self.agent, current_inputs, self.agent.context, registered_tools, mcp_tools
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
except Exception as eval_error:
|
|
142
|
+
logger.error(
|
|
143
|
+
"Error during evaluate",
|
|
144
|
+
agent=self.agent.name,
|
|
145
|
+
error=str(eval_error),
|
|
146
|
+
)
|
|
147
|
+
span.record_exception(eval_error)
|
|
148
|
+
await self.on_error(
|
|
149
|
+
eval_error, current_inputs
|
|
150
|
+
) # Call error hook
|
|
151
|
+
raise # Re-raise the exception
|
|
152
|
+
|
|
153
|
+
# Post-evaluate hooks
|
|
154
|
+
current_result = result
|
|
155
|
+
for component in self.agent.get_enabled_components():
|
|
156
|
+
tmp_result = await component.on_post_evaluate(
|
|
157
|
+
self.agent,
|
|
158
|
+
current_inputs,
|
|
159
|
+
self.agent.context,
|
|
160
|
+
current_result,
|
|
161
|
+
)
|
|
162
|
+
# If the component returns a result, use it
|
|
163
|
+
if tmp_result:
|
|
164
|
+
current_result = tmp_result
|
|
165
|
+
|
|
166
|
+
# Handle routing logic
|
|
167
|
+
router = self.agent.router
|
|
168
|
+
if router:
|
|
169
|
+
try:
|
|
170
|
+
self.agent.next_agent = await router.determine_next_step(
|
|
171
|
+
self.agent, current_result, self.agent.context
|
|
172
|
+
)
|
|
173
|
+
except Exception as e:
|
|
174
|
+
logger.error(f"Error in routing: {e}")
|
|
175
|
+
|
|
176
|
+
logger.debug(f"Evaluation completed for agent '{self.agent.name}'")
|
|
177
|
+
return current_result
|