aiqtoolkit 1.1.0a20250503__py3-none-any.whl

aiq/profiler/callbacks/agno_callback_handler.py

@@ -0,0 +1,295 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+import logging
+import threading
+import time
+from collections.abc import Callable
+from typing import Any
+from uuid import uuid4
+
+import litellm
+
+from aiq.builder.context import AIQContext
+from aiq.builder.framework_enum import LLMFrameworkEnum
+from aiq.data_models.intermediate_step import IntermediateStepPayload
+from aiq.data_models.intermediate_step import IntermediateStepType
+from aiq.data_models.intermediate_step import StreamEventData
+from aiq.data_models.intermediate_step import TraceMetadata
+from aiq.data_models.intermediate_step import UsageInfo
+from aiq.profiler.callbacks.base_callback_class import BaseProfilerCallback
+from aiq.profiler.callbacks.token_usage_base_model import TokenUsageBaseModel
+
+logger = logging.getLogger(__name__)
+
+
+class AgnoProfilerHandler(BaseProfilerCallback):
+    """
+    A callback manager/handler for Agno that intercepts calls to:
+
+      - Tool execution
+      - LLM Calls
+
+    to collect usage statistics (tokens, inputs, outputs, time intervals, etc.)
+    and store them in AIQ Toolkit's usage_stats queue for subsequent analysis.
+    """
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._lock = threading.Lock()
+        self.last_call_ts = time.time()
+        self.step_manager = AIQContext.get().intermediate_step_manager
+
+        # Original references to Agno methods (for uninstrumenting if needed)
+        self._original_tool_execute = None
+        self._original_llm_call = None
+
+    def instrument(self) -> None:
+        """
+        Monkey-patch the relevant Agno methods with usage-stat collection logic.
+        """
+        # Save the originals and apply patches
+        self._original_llm_call = getattr(litellm, "completion", None)
+
+        # Patch LLM completion if available
+        if self._original_llm_call:
+            litellm.completion = self._llm_call_monkey_patch()
+            logger.debug("AgnoProfilerHandler LLM call instrumentation applied successfully.")
+        else:
+            logger.debug("Could not patch Agno LLM calls: litellm.completion not found")
+
+        # Note: Agno doesn't have a class-based tool structure to patch directly.
+        # Instead, it uses decorators to convert functions to tools.
+        # In AIQ Toolkit, tool executions are captured at the execute_agno_tool level
+        # in packages/aiqtoolkit_agno/src/aiq/plugins/agno/tool_wrapper.py
+
+        # To properly monitor Agno tool executions, we would need to either:
+        # 1. Patch the execute_agno_tool function in tool_wrapper.py
+        # 2. Add explicit instrumentation in that function to push events to the step manager
+        # 3. Or, if Agno updates to have a class-based tool structure, update this handler
+        #    to patch those classes
+
+        # Recommended future enhancement:
+        # The execute_agno_tool function in packages/aiqtoolkit_agno/src/aiq/plugins/agno/tool_wrapper.py
+        # should be updated to directly push IntermediateStepPayload events to the step manager
+        # at the beginning and end of tool execution, similar to what this handler does for LLM calls.
+
+        logger.debug("AgnoProfilerHandler instrumentation completed.")
+
+    def _tool_execute_monkey_patch(self) -> Callable[..., Any]:
+        """
+        Returns a function that wraps tool execution calls with usage-logging.
+
+        Note: This method is currently not used in the instrument() function since
+        Agno doesn't have a class-based tool structure to patch. It's kept for
+        reference or future use if Agno changes its architecture.
+        """
+        original_func = self._original_tool_execute
+
+        def wrapped_tool_execute(*args, **kwargs) -> Any:
+            """
+            Collects usage stats for tool execution, calls the original, and captures output stats.
+            """
+            now = time.time()
+            tool_name = kwargs.get("tool_name", "")
+            uuid = str(uuid4())
+
+            try:
+                # Pre-call usage event
+                stats = IntermediateStepPayload(event_type=IntermediateStepType.TOOL_START,
+                                                framework=LLMFrameworkEnum.AGNO,
+                                                name=tool_name,
+                                                UUID=uuid,
+                                                data=StreamEventData(),
+                                                metadata=TraceMetadata(tool_inputs={
+                                                    "args": args, "kwargs": dict(kwargs)
+                                                }),
+                                                usage_info=UsageInfo(token_usage=TokenUsageBaseModel()))
+
+                self.step_manager.push_intermediate_step(stats)
+                self.last_call_ts = now
+
+                # Call the original execute
+                result = original_func(*args, **kwargs)
+                now = time.time()
+
+                # Post-call usage stats
+                usage_stat = IntermediateStepPayload(
+                    event_type=IntermediateStepType.TOOL_END,
+                    span_event_timestamp=now,
+                    framework=LLMFrameworkEnum.AGNO,
+                    name=tool_name,
+                    UUID=uuid,
+                    data=StreamEventData(input={
+                        "args": args, "kwargs": dict(kwargs)
+                    }, output=str(result)),
+                    metadata=TraceMetadata(tool_outputs={"result": str(result)}),
+                    usage_info=UsageInfo(token_usage=TokenUsageBaseModel()),
+                )
+
+                self.step_manager.push_intermediate_step(usage_stat)
+                return result
+
+            except Exception as e:
+                logger.exception("Tool execution error: %s", e)
+                raise
+
+        return wrapped_tool_execute
+
+    def _llm_call_monkey_patch(self) -> Callable[..., Any]:
+        """
+        Returns a function that wraps calls to litellm.completion(...) with usage-logging.
+        """
+        original_func = self._original_llm_call
+
+        def wrapped_llm_call(*args, **kwargs) -> Any:
+            """
+            Collects usage stats for LLM calls, calls the original, and captures output stats.
+            """
+            now = time.time()
+            seconds_between_calls = int(now - self.last_call_ts)
+            model_name = kwargs.get('model', "")
+
+            model_input = ""
+            try:
+                for message in kwargs.get('messages', []):
+                    model_input += message.get('content', "")
+            except Exception as e:
+                logger.exception("Error getting model input: %s", e)
+
+            uuid = str(uuid4())
+
+            # Record the start event
+            input_stats = IntermediateStepPayload(
+                event_type=IntermediateStepType.LLM_START,
+                framework=LLMFrameworkEnum.AGNO,
+                name=model_name,
+                UUID=uuid,
+                data=StreamEventData(input=model_input),
+                metadata=TraceMetadata(chat_inputs=copy.deepcopy(kwargs.get('messages', []))),
+                usage_info=UsageInfo(token_usage=TokenUsageBaseModel(),
+                                     num_llm_calls=1,
+                                     seconds_between_calls=seconds_between_calls))
+
+            self.step_manager.push_intermediate_step(input_stats)
+
+            # Verify we have a valid original function before calling it
+            if original_func is None:
+                logger.error("Original litellm.completion function is None - cannot call it")
+                output = None
+            else:
+                # Call the original litellm.completion(...)
+                logger.debug(
+                    f"Calling litellm.completion for {model_name} with {len(args)} args and {len(kwargs)} kwargs")
+                try:
+                    output = original_func(*args, **kwargs)
+                    logger.debug(f"Original litellm.completion returned: {type(output)}")
+                except Exception as e:
+                    logger.exception(f"Error calling original litellm.completion: {e}")
+                    output = None
+
+            # Initialize default values
+            model_output = ""
+            chat_responses = None
+            token_usage = TokenUsageBaseModel()
+
+            # Log what we received to help with debugging
+            logger.debug(f"LLM call to {model_name} received output type: {type(output)}")
+
+            # Safely process the output if it's not None
+            if output is not None:
+                try:
+                    # Extract model output text from choices
+                    if hasattr(output, 'choices') and output.choices:
+                        logger.debug(f"Output has {len(output.choices)} choices")
+                        for i, choice in enumerate(output.choices):
+                            logger.debug(f"Processing choice {i} of type {type(choice)}")
+                            if hasattr(choice, 'model_extra') and 'message' in choice.model_extra:
+                                msg = choice.model_extra["message"]
+                                content = msg.get('content', "")
+                                logger.debug(f"Got content from model_extra.message: {content[:50]}...")
+                                model_output += content
+                            elif hasattr(choice, 'message') and hasattr(choice.message, 'content'):
+                                content = choice.message.content or ""
+                                logger.debug(f"Got content from message.content: {content[:50]}...")
+                                model_output += content
+                            else:
+                                logger.debug(f"Could not extract content from choice: {choice}")
+
+                    # Try to get chat responses
+                    if hasattr(output, 'choices') and len(output.choices) > 0:
+                        choice = output.choices[0]
+                        if hasattr(choice, 'model_dump'):
+                            logger.debug("Using model_dump to extract chat responses")
+                            chat_responses = choice.model_dump()
+                        else:
+                            # Fall back to a simpler representation
+                            logger.debug("Falling back to simple representation for chat responses")
+                            chat_responses = {"content": model_output}
+
+                    # Try to get token usage
+                    if hasattr(output, 'model_extra') and 'usage' in output.model_extra:
+                        usage_data = output.model_extra['usage']
+                        logger.debug(f"Found usage data of type {type(usage_data)}")
+
+                        # Special debug for the test case
+                        if hasattr(usage_data, 'prompt_tokens'
+                                   ) and usage_data.prompt_tokens == 20 and usage_data.completion_tokens == 15:
+                            logger.debug("Found test case token usage object with 20/15/35 tokens")
+
+                        if hasattr(usage_data, 'model_dump'):
+                            logger.debug("Using model_dump to extract token usage")
+                            token_usage = TokenUsageBaseModel(**usage_data.model_dump())
+                        elif isinstance(usage_data, dict):
+                            logger.debug("Extracting token usage from dictionary")
+                            token_usage = TokenUsageBaseModel(prompt_tokens=usage_data.get('prompt_tokens', 0),
+                                                              completion_tokens=usage_data.get('completion_tokens', 0),
+                                                              total_tokens=usage_data.get('total_tokens', 0))
+                        elif isinstance(usage_data, TokenUsageBaseModel):
+                            # If it's already a TokenUsageBaseModel instance, use it directly
+                            logger.debug("Using TokenUsageBaseModel directly")
+                            token_usage = usage_data
+                        elif hasattr(usage_data, 'prompt_tokens') and hasattr(
+                                usage_data, 'completion_tokens') and hasattr(usage_data, 'total_tokens'):
+                            # For objects that have the needed properties but aren't TokenUsageBaseModel
+                            logger.debug("Using object with token properties")
+                            token_usage = TokenUsageBaseModel(prompt_tokens=usage_data.prompt_tokens,
+                                                              completion_tokens=usage_data.completion_tokens,
+                                                              total_tokens=usage_data.total_tokens)
+
+                        logger.debug(f"Final token usage: prompt={token_usage.prompt_tokens}, "
+                                     f"completion={token_usage.completion_tokens}, "
+                                     f"total={token_usage.total_tokens}")
+                except Exception as e:
+                    logger.exception("Error getting model output: %s", e)
+
+            now = time.time()
+            # Record the end event
+            output_stats = IntermediateStepPayload(event_type=IntermediateStepType.LLM_END,
+                                                   span_event_timestamp=now,
+                                                   framework=LLMFrameworkEnum.AGNO,
+                                                   name=model_name,
+                                                   UUID=uuid,
+                                                   data=StreamEventData(input=model_input, output=model_output),
+                                                   metadata=TraceMetadata(chat_responses=chat_responses),
+                                                   usage_info=UsageInfo(token_usage=token_usage,
+                                                                        num_llm_calls=1,
+                                                                        seconds_between_calls=seconds_between_calls))
+
+            self.step_manager.push_intermediate_step(output_stats)
+            return output
+
+        return wrapped_llm_call

aiq/profiler/callbacks/base_callback_class.py

@@ -0,0 +1,20 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC
+
+
+class BaseProfilerCallback(ABC):
+    pass

aiq/profiler/callbacks/langchain_callback_handler.py

@@ -0,0 +1,278 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import copy
+import logging
+import threading
+import time
+from typing import Any
+from uuid import UUID
+from uuid import uuid4
+
+from langchain_core.callbacks import AsyncCallbackHandler
+from langchain_core.messages import AIMessage
+from langchain_core.messages import BaseMessage
+from langchain_core.outputs import ChatGeneration
+from langchain_core.outputs import LLMResult
+
+from aiq.builder.context import AIQContext
+from aiq.builder.framework_enum import LLMFrameworkEnum
+from aiq.data_models.intermediate_step import IntermediateStepPayload
+from aiq.data_models.intermediate_step import IntermediateStepType
+from aiq.data_models.intermediate_step import StreamEventData
+from aiq.data_models.intermediate_step import TraceMetadata
+from aiq.data_models.intermediate_step import UsageInfo
+from aiq.profiler.callbacks.base_callback_class import BaseProfilerCallback
+from aiq.profiler.callbacks.token_usage_base_model import TokenUsageBaseModel
+
+logger = logging.getLogger(__name__)
+
+
+class LangchainProfilerHandler(AsyncCallbackHandler, BaseProfilerCallback):  # pylint: disable=R0901
+    """Callback Handler that tracks NIM info."""
+
+    total_tokens: int = 0
+    prompt_tokens: int = 0
+    completion_tokens: int = 0
+    successful_requests: int = 0
+    raise_error = True  # Override to raise error and run inline
+    run_inline = True
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._lock = threading.Lock()
+        self.last_call_ts = time.time()
+
+        self.step_manager = AIQContext.get().intermediate_step_manager
+        self._state = IntermediateStepType.LLM_END
+
+        self._run_id_to_model_name = {}
+        self._run_id_to_llm_input = {}
+        self._run_id_to_tool_input = {}
+        self._run_id_to_start_time = {}
+
+    def __repr__(self) -> str:
+        return (f"Tokens Used: {self.total_tokens}\n"
+                f"\tPrompt Tokens: {self.prompt_tokens}\n"
+                f"\tCompletion Tokens: {self.completion_tokens}\n"
+                f"Successful Requests: {self.successful_requests}\n")
+
+    @property
+    def always_verbose(self) -> bool:
+        """Whether to call verbose callbacks even if verbose is False."""
+        return True
+
+    def _extract_token_base_model(self, usage_metadata: dict[str, Any]) -> TokenUsageBaseModel:
+        if usage_metadata:
+            prompt_tokens = usage_metadata.get("input_tokens", 0)
+            completion_tokens = usage_metadata.get("output_tokens", 0)
+            total_tokens = usage_metadata.get("total_tokens", 0)
+
+            return TokenUsageBaseModel(
+                prompt_tokens=prompt_tokens,
+                completion_tokens=completion_tokens,
+                total_tokens=total_tokens,
+            )
+        return TokenUsageBaseModel()
+
+    async def on_llm_start(self, serialized: dict[str, Any], prompts: list[str], **kwargs: Any) -> None:
+
+        model_name = ""
+        try:
+            model_name = kwargs.get("metadata")["ls_model_name"]
+        except Exception as e:
+            logger.exception("Error getting model name: %s", e, exc_info=True)
+
+        run_id = str(kwargs.get("run_id", str(uuid4())))
+        self._run_id_to_model_name[run_id] = model_name
+
+        stats = IntermediateStepPayload(event_type=IntermediateStepType.LLM_START,
+                                        framework=LLMFrameworkEnum.LANGCHAIN,
+                                        name=model_name,
+                                        UUID=run_id,
+                                        data=StreamEventData(input=prompts[-1]),
+                                        metadata=TraceMetadata(chat_inputs=copy.deepcopy(prompts)),
+                                        usage_info=UsageInfo(token_usage=TokenUsageBaseModel(),
+                                                             num_llm_calls=1,
+                                                             seconds_between_calls=int(time.time() -
+                                                                                       self.last_call_ts)))
+
+        self.step_manager.push_intermediate_step(stats)
+        self._run_id_to_llm_input[run_id] = prompts[-1]
+        self._state = IntermediateStepType.LLM_START
+        self.last_call_ts = time.time()
+        self._run_id_to_start_time[run_id] = time.time()
+
+    async def on_chat_model_start(
+        self,
+        serialized: dict[str, Any],
+        messages: list[list[BaseMessage]],
+        *,
+        run_id: UUID,
+        parent_run_id: UUID | None = None,
+        tags: list[str] | None = None,
+        metadata: dict[str, Any] | None = None,
+        **kwargs: Any,
+    ) -> Any:
+
+        model_name = ""
+        try:
+            model_name = metadata["ls_model_name"] if metadata else kwargs.get("metadata")["ls_model_name"]
+        except Exception as e:
+            logger.exception("Error getting model name: %s", e, exc_info=True)
+
+        run_id = str(run_id)
+        self._run_id_to_model_name[run_id] = model_name
+
+        stats = IntermediateStepPayload(event_type=IntermediateStepType.LLM_START,
+                                        framework=LLMFrameworkEnum.LANGCHAIN,
+                                        name=model_name,
+                                        UUID=run_id,
+                                        data=StreamEventData(input=copy.deepcopy(messages[0])),
+                                        metadata=TraceMetadata(chat_inputs=copy.deepcopy(messages[0])),
+                                        usage_info=UsageInfo(token_usage=TokenUsageBaseModel(),
+                                                             num_llm_calls=1,
+                                                             seconds_between_calls=int(time.time() -
+                                                                                       self.last_call_ts)))
+
+        self.step_manager.push_intermediate_step(stats)
+        self._run_id_to_llm_input[run_id] = messages[0][-1].content
+        self._state = IntermediateStepType.LLM_START
+        self.last_call_ts = time.time()
+        self._run_id_to_start_time[run_id] = time.time()
+
+    async def on_llm_new_token(self, token: str, **kwargs: Any) -> None:
+        """Collect stats for just the token"""
+        model_name = ""
+        try:
+            model_name = self._run_id_to_model_name.get(str(kwargs.get("run_id", "")), "")
+        except Exception as e:
+            logger.exception("Error getting model name: %s", e, exc_info=True)
+
+        usage_metadata = {}
+        try:
+            usage_metadata = kwargs.get("chunk").message.usage_metadata if kwargs.get("chunk") else {}
+        except Exception as e:
+            logger.exception("Error getting usage metadata: %s", e, exc_info=True)
+
+        stats = IntermediateStepPayload(
+            event_type=IntermediateStepType.LLM_NEW_TOKEN,
+            framework=LLMFrameworkEnum.LANGCHAIN,
+            name=model_name,
+            UUID=str(kwargs.get("run_id", str(uuid4()))),
+            data=StreamEventData(input=self._run_id_to_llm_input.get(str(kwargs.get("run_id", "")), ""), chunk=token),
+            usage_info=UsageInfo(token_usage=self._extract_token_base_model(usage_metadata),
+                                 num_llm_calls=1,
+                                 seconds_between_calls=int(time.time() - self.last_call_ts)),
+            metadata=TraceMetadata(chat_responses=[kwargs.get("chunk")] if kwargs.get("chunk") else []))
+
+        self.step_manager.push_intermediate_step(stats)
+
+    async def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
+        """Collect token usage."""
+
+        usage_metadata = {}
+
+        model_name = ""
+        try:
+            model_name = response.llm_output["model_name"]
+        except Exception as e:
+            try:
+                model_name = self._run_id_to_model_name.get(str(kwargs.get("run_id", "")), "")
+            except Exception as e_inner:
+                logger.exception("Error getting model name: %s from outer error %s", e_inner, e, exc_info=True)
+
+        try:
+            generation = response.generations[0][0]
+        except IndexError:
+            generation = None
+
+        if isinstance(generation, ChatGeneration):
+            try:
+                message = generation.message
+                if isinstance(message, AIMessage):
+                    usage_metadata = message.usage_metadata
+                else:
+                    usage_metadata = {}
+            except AttributeError:
+                usage_metadata = {}
+
+        llm_text_output = generation.message.content if generation else ""
+
+        # update shared state behind lock
+        with self._lock:
+            usage_stat = IntermediateStepPayload(
+                span_event_timestamp=self._run_id_to_start_time.get(str(kwargs.get("run_id", "")), time.time()),
+                event_type=IntermediateStepType.LLM_END,
+                framework=LLMFrameworkEnum.LANGCHAIN,
+                name=model_name,
+                UUID=str(kwargs.get("run_id", str(uuid4()))),
+                data=StreamEventData(input=self._run_id_to_llm_input.get(str(kwargs.get("run_id", "")), ""),
+                                     output=llm_text_output),
+                usage_info=UsageInfo(token_usage=self._extract_token_base_model(usage_metadata)),
+                metadata=TraceMetadata(chat_responses=[generation] if generation else []))
+
+            self.step_manager.push_intermediate_step(usage_stat)
+
+        self._state = IntermediateStepType.LLM_END
+
+    async def on_tool_start(
+        self,
+        serialized: dict[str, Any],
+        input_str: str,
+        *,
+        run_id: UUID,
+        parent_run_id: UUID | None = None,
+        tags: list[str] | None = None,
+        metadata: dict[str, Any] | None = None,
+        inputs: dict[str, Any] | None = None,
+        **kwargs: Any,
+    ) -> Any:
+
+        stats = IntermediateStepPayload(event_type=IntermediateStepType.TOOL_START,
+                                        framework=LLMFrameworkEnum.LANGCHAIN,
+                                        name=serialized.get("name", ""),
+                                        UUID=str(run_id),
+                                        data=StreamEventData(input=input_str),
+                                        metadata=TraceMetadata(tool_inputs=copy.deepcopy(inputs),
+                                                               tool_info=copy.deepcopy(serialized)),
+                                        usage_info=UsageInfo(token_usage=TokenUsageBaseModel()))
+
+        self.step_manager.push_intermediate_step(stats)
+        self._run_id_to_tool_input[str(run_id)] = input_str
+        self._run_id_to_start_time[str(run_id)] = time.time()
+
+    async def on_tool_end(
+        self,
+        output: Any,
+        *,
+        run_id: UUID,
+        parent_run_id: UUID | None = None,
+        **kwargs: Any,
+    ) -> Any:
+
+        stats = IntermediateStepPayload(event_type=IntermediateStepType.TOOL_END,
+                                        span_event_timestamp=self._run_id_to_start_time.get(str(run_id), time.time()),
+                                        framework=LLMFrameworkEnum.LANGCHAIN,
+                                        name=kwargs.get("name", ""),
+                                        UUID=str(run_id),
+                                        metadata=TraceMetadata(tool_outputs=output),
+                                        usage_info=UsageInfo(token_usage=TokenUsageBaseModel()),
+                                        data=StreamEventData(input=self._run_id_to_tool_input.get(str(run_id), ""),
+                                                             output=output))
+
+        self.step_manager.push_intermediate_step(stats)