llama-stack 0.0.42__py3-none-any.whl → 0.3.4__py3-none-any.whl

llama_stack/providers/tests/agents/test_agents.py

@@ -1,210 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the terms described in the LICENSE file in
-# the root directory of this source tree.
-
-import os
-
-import pytest
-import pytest_asyncio
-
-from llama_stack.apis.agents import *  # noqa: F403
-from llama_stack.providers.tests.resolver import resolve_impls_for_test
-from llama_stack.providers.datatypes import *  # noqa: F403
-
-from dotenv import load_dotenv
-
-# How to run this test:
-#
-# 1. Ensure you have a conda environment with the right dependencies installed.
-#    This includes `pytest` and `pytest-asyncio`.
-#
-# 2. Copy and modify the provider_config_example.yaml depending on the provider you are testing.
-#
-# 3. Run:
-#
-# ```bash
-# PROVIDER_ID=<your_provider> \
-#   PROVIDER_CONFIG=provider_config.yaml \
-#   pytest -s llama_stack/providers/tests/agents/test_agents.py \
-#   --tb=short --disable-warnings
-# ```
-
-load_dotenv()
-
-
-@pytest_asyncio.fixture(scope="session")
-async def agents_settings():
-    impls = await resolve_impls_for_test(
-        Api.agents, deps=[Api.inference, Api.memory, Api.safety]
-    )
-
-    return {
-        "impl": impls[Api.agents],
-        "memory_impl": impls[Api.memory],
-        "common_params": {
-            "model": "Llama3.1-8B-Instruct",
-            "instructions": "You are a helpful assistant.",
-        },
-    }
-
-
-@pytest.fixture
-def sample_messages():
-    return [
-        UserMessage(content="What's the weather like today?"),
-    ]
-
-
-@pytest.fixture
-def search_query_messages():
-    return [
-        UserMessage(content="What are the latest developments in quantum computing?"),
-    ]
-
-
-@pytest.mark.asyncio
-async def test_create_agent_turn(agents_settings, sample_messages):
-    agents_impl = agents_settings["impl"]
-
-    # First, create an agent
-    agent_config = AgentConfig(
-        model=agents_settings["common_params"]["model"],
-        instructions=agents_settings["common_params"]["instructions"],
-        enable_session_persistence=True,
-        sampling_params=SamplingParams(temperature=0.7, top_p=0.95),
-        input_shields=[],
-        output_shields=[],
-        tools=[],
-        max_infer_iters=5,
-    )
-
-    create_response = await agents_impl.create_agent(agent_config)
-    agent_id = create_response.agent_id
-
-    # Create a session
-    session_create_response = await agents_impl.create_agent_session(
-        agent_id, "Test Session"
-    )
-    session_id = session_create_response.session_id
-
-    # Create and execute a turn
-    turn_request = dict(
-        agent_id=agent_id,
-        session_id=session_id,
-        messages=sample_messages,
-        stream=True,
-    )
-
-    turn_response = [
-        chunk async for chunk in agents_impl.create_agent_turn(**turn_request)
-    ]
-
-    assert len(turn_response) > 0
-    assert all(
-        isinstance(chunk, AgentTurnResponseStreamChunk) for chunk in turn_response
-    )
-
-    # Check for expected event types
-    event_types = [chunk.event.payload.event_type for chunk in turn_response]
-    assert AgentTurnResponseEventType.turn_start.value in event_types
-    assert AgentTurnResponseEventType.step_start.value in event_types
-    assert AgentTurnResponseEventType.step_complete.value in event_types
-    assert AgentTurnResponseEventType.turn_complete.value in event_types
-
-    # Check the final turn complete event
-    final_event = turn_response[-1].event.payload
-    assert isinstance(final_event, AgentTurnResponseTurnCompletePayload)
-    assert isinstance(final_event.turn, Turn)
-    assert final_event.turn.session_id == session_id
-    assert final_event.turn.input_messages == sample_messages
-    assert isinstance(final_event.turn.output_message, CompletionMessage)
-    assert len(final_event.turn.output_message.content) > 0
-
-
-@pytest.mark.asyncio
-async def test_create_agent_turn_with_brave_search(
-    agents_settings, search_query_messages
-):
-    agents_impl = agents_settings["impl"]
-
-    if "BRAVE_SEARCH_API_KEY" not in os.environ:
-        pytest.skip("BRAVE_SEARCH_API_KEY not set, skipping test")
-
-    # Create an agent with Brave search tool
-    agent_config = AgentConfig(
-        model=agents_settings["common_params"]["model"],
-        instructions=agents_settings["common_params"]["instructions"],
-        enable_session_persistence=True,
-        sampling_params=SamplingParams(temperature=0.7, top_p=0.95),
-        input_shields=[],
-        output_shields=[],
-        tools=[
-            SearchToolDefinition(
-                type=AgentTool.brave_search.value,
-                api_key=os.environ["BRAVE_SEARCH_API_KEY"],
-                engine=SearchEngineType.brave,
-            )
-        ],
-        tool_choice=ToolChoice.auto,
-        max_infer_iters=5,
-    )
-
-    create_response = await agents_impl.create_agent(agent_config)
-    agent_id = create_response.agent_id
-
-    # Create a session
-    session_create_response = await agents_impl.create_agent_session(
-        agent_id, "Test Session with Brave Search"
-    )
-    session_id = session_create_response.session_id
-
-    # Create and execute a turn
-    turn_request = dict(
-        agent_id=agent_id,
-        session_id=session_id,
-        messages=search_query_messages,
-        stream=True,
-    )
-
-    turn_response = [
-        chunk async for chunk in agents_impl.create_agent_turn(**turn_request)
-    ]
-
-    assert len(turn_response) > 0
-    assert all(
-        isinstance(chunk, AgentTurnResponseStreamChunk) for chunk in turn_response
-    )
-
-    # Check for expected event types
-    event_types = [chunk.event.payload.event_type for chunk in turn_response]
-    assert AgentTurnResponseEventType.turn_start.value in event_types
-    assert AgentTurnResponseEventType.step_start.value in event_types
-    assert AgentTurnResponseEventType.step_complete.value in event_types
-    assert AgentTurnResponseEventType.turn_complete.value in event_types
-
-    # Check for tool execution events
-    tool_execution_events = [
-        chunk
-        for chunk in turn_response
-        if isinstance(chunk.event.payload, AgentTurnResponseStepCompletePayload)
-        and chunk.event.payload.step_details.step_type == StepType.tool_execution.value
-    ]
-    assert len(tool_execution_events) > 0, "No tool execution events found"
-
-    # Check the tool execution details
-    tool_execution = tool_execution_events[0].event.payload.step_details
-    assert isinstance(tool_execution, ToolExecutionStep)
-    assert len(tool_execution.tool_calls) > 0
-    assert tool_execution.tool_calls[0].tool_name == BuiltinTool.brave_search
-    assert len(tool_execution.tool_responses) > 0
-
-    # Check the final turn complete event
-    final_event = turn_response[-1].event.payload
-    assert isinstance(final_event, AgentTurnResponseTurnCompletePayload)
-    assert isinstance(final_event.turn, Turn)
-    assert final_event.turn.session_id == session_id
-    assert final_event.turn.input_messages == search_query_messages
-    assert isinstance(final_event.turn.output_message, CompletionMessage)
-    assert len(final_event.turn.output_message.content) > 0

llama_stack/providers/tests/inference/test_inference.py

@@ -1,257 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the terms described in the LICENSE file in
-# the root directory of this source tree.
-
-import itertools
-
-import pytest
-import pytest_asyncio
-
-from llama_models.llama3.api.datatypes import *  # noqa: F403
-from llama_stack.apis.inference import *  # noqa: F403
-
-from llama_stack.distribution.datatypes import *  # noqa: F403
-from llama_stack.providers.tests.resolver import resolve_impls_for_test
-
-# How to run this test:
-#
-# 1. Ensure you have a conda with the right dependencies installed. This is a bit tricky
-#    since it depends on the provider you are testing. On top of that you need
-#    `pytest` and `pytest-asyncio` installed.
-#
-# 2. Copy and modify the provider_config_example.yaml depending on the provider you are testing.
-#
-# 3. Run:
-#
-# ```bash
-# PROVIDER_ID=<your_provider> \
-#   PROVIDER_CONFIG=provider_config.yaml \
-#   pytest -s llama_stack/providers/tests/inference/test_inference.py \
-#   --tb=short --disable-warnings
-# ```
-
-
-def group_chunks(response):
-    return {
-        event_type: list(group)
-        for event_type, group in itertools.groupby(
-            response, key=lambda chunk: chunk.event.event_type
-        )
-    }
-
-
-Llama_8B = "Llama3.1-8B-Instruct"
-Llama_3B = "Llama3.2-3B-Instruct"
-
-
-def get_expected_stop_reason(model: str):
-    return StopReason.end_of_message if "Llama3.1" in model else StopReason.end_of_turn
-
-
-# This is going to create multiple Stack impls without tearing down the previous one
-# Fix that!
-@pytest_asyncio.fixture(
-    scope="session",
-    params=[
-        {"model": Llama_8B},
-        {"model": Llama_3B},
-    ],
-    ids=lambda d: d["model"],
-)
-async def inference_settings(request):
-    model = request.param["model"]
-    impls = await resolve_impls_for_test(
-        Api.inference,
-    )
-
-    return {
-        "impl": impls[Api.inference],
-        "models_impl": impls[Api.models],
-        "common_params": {
-            "model": model,
-            "tool_choice": ToolChoice.auto,
-            "tool_prompt_format": (
-                ToolPromptFormat.json
-                if "Llama3.1" in model
-                else ToolPromptFormat.python_list
-            ),
-        },
-    }
-
-
-@pytest.fixture
-def sample_messages():
-    return [
-        SystemMessage(content="You are a helpful assistant."),
-        UserMessage(content="What's the weather like today?"),
-    ]
-
-
-@pytest.fixture
-def sample_tool_definition():
-    return ToolDefinition(
-        tool_name="get_weather",
-        description="Get the current weather",
-        parameters={
-            "location": ToolParamDefinition(
-                param_type="string",
-                description="The city and state, e.g. San Francisco, CA",
-            ),
-        },
-    )
-
-
-@pytest.mark.asyncio
-async def test_model_list(inference_settings):
-    params = inference_settings["common_params"]
-    models_impl = inference_settings["models_impl"]
-    response = await models_impl.list_models()
-    assert isinstance(response, list)
-    assert len(response) >= 1
-    assert all(isinstance(model, ModelDefWithProvider) for model in response)
-
-    model_def = None
-    for model in response:
-        if model.identifier == params["model"]:
-            model_def = model
-            break
-
-    assert model_def is not None
-    assert model_def.identifier == params["model"]
-
-
-@pytest.mark.asyncio
-async def test_chat_completion_non_streaming(inference_settings, sample_messages):
-    inference_impl = inference_settings["impl"]
-    response = await inference_impl.chat_completion(
-        messages=sample_messages,
-        stream=False,
-        **inference_settings["common_params"],
-    )
-
-    assert isinstance(response, ChatCompletionResponse)
-    assert response.completion_message.role == "assistant"
-    assert isinstance(response.completion_message.content, str)
-    assert len(response.completion_message.content) > 0
-
-
-@pytest.mark.asyncio
-async def test_chat_completion_streaming(inference_settings, sample_messages):
-    inference_impl = inference_settings["impl"]
-    response = [
-        r
-        async for r in inference_impl.chat_completion(
-            messages=sample_messages,
-            stream=True,
-            **inference_settings["common_params"],
-        )
-    ]
-
-    assert len(response) > 0
-    assert all(
-        isinstance(chunk, ChatCompletionResponseStreamChunk) for chunk in response
-    )
-    grouped = group_chunks(response)
-    assert len(grouped[ChatCompletionResponseEventType.start]) == 1
-    assert len(grouped[ChatCompletionResponseEventType.progress]) > 0
-    assert len(grouped[ChatCompletionResponseEventType.complete]) == 1
-
-    end = grouped[ChatCompletionResponseEventType.complete][0]
-    assert end.event.stop_reason == StopReason.end_of_turn
-
-
-@pytest.mark.asyncio
-async def test_chat_completion_with_tool_calling(
-    inference_settings,
-    sample_messages,
-    sample_tool_definition,
-):
-    inference_impl = inference_settings["impl"]
-    messages = sample_messages + [
-        UserMessage(
-            content="What's the weather like in San Francisco?",
-        )
-    ]
-
-    response = await inference_impl.chat_completion(
-        messages=messages,
-        tools=[sample_tool_definition],
-        stream=False,
-        **inference_settings["common_params"],
-    )
-
-    assert isinstance(response, ChatCompletionResponse)
-
-    message = response.completion_message
-
-    # This is not supported in most providers :/ they don't return eom_id / eot_id
-    # stop_reason = get_expected_stop_reason(inference_settings["common_params"]["model"])
-    # assert message.stop_reason == stop_reason
-    assert message.tool_calls is not None
-    assert len(message.tool_calls) > 0
-
-    call = message.tool_calls[0]
-    assert call.tool_name == "get_weather"
-    assert "location" in call.arguments
-    assert "San Francisco" in call.arguments["location"]
-
-
-@pytest.mark.asyncio
-async def test_chat_completion_with_tool_calling_streaming(
-    inference_settings,
-    sample_messages,
-    sample_tool_definition,
-):
-    inference_impl = inference_settings["impl"]
-    messages = sample_messages + [
-        UserMessage(
-            content="What's the weather like in San Francisco?",
-        )
-    ]
-
-    response = [
-        r
-        async for r in inference_impl.chat_completion(
-            messages=messages,
-            tools=[sample_tool_definition],
-            stream=True,
-            **inference_settings["common_params"],
-        )
-    ]
-
-    assert len(response) > 0
-    assert all(
-        isinstance(chunk, ChatCompletionResponseStreamChunk) for chunk in response
-    )
-    grouped = group_chunks(response)
-    assert len(grouped[ChatCompletionResponseEventType.start]) == 1
-    assert len(grouped[ChatCompletionResponseEventType.progress]) > 0
-    assert len(grouped[ChatCompletionResponseEventType.complete]) == 1
-
-    # This is not supported in most providers :/ they don't return eom_id / eot_id
-    # expected_stop_reason = get_expected_stop_reason(
-    #     inference_settings["common_params"]["model"]
-    # )
-    # end = grouped[ChatCompletionResponseEventType.complete][0]
-    # assert end.event.stop_reason == expected_stop_reason
-
-    model = inference_settings["common_params"]["model"]
-    if "Llama3.1" in model:
-        assert all(
-            isinstance(chunk.event.delta, ToolCallDelta)
-            for chunk in grouped[ChatCompletionResponseEventType.progress]
-        )
-        first = grouped[ChatCompletionResponseEventType.progress][0]
-        assert first.event.delta.parse_status == ToolCallParseStatus.started
-
-    last = grouped[ChatCompletionResponseEventType.progress][-1]
-    # assert last.event.stop_reason == expected_stop_reason
-    assert last.event.delta.parse_status == ToolCallParseStatus.success
-    assert isinstance(last.event.delta.content, ToolCall)
-
-    call = last.event.delta.content
-    assert call.tool_name == "get_weather"
-    assert "location" in call.arguments
-    assert "San Francisco" in call.arguments["location"]

llama_stack/providers/tests/inference/test_prompt_adapter.py

@@ -1,126 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the terms described in the LICENSE file in
-# the root directory of this source tree.
-
-import unittest
-
-from llama_models.llama3.api import *  # noqa: F403
-from llama_stack.inference.api import *  # noqa: F403
-from llama_stack.inference.prompt_adapter import chat_completion_request_to_messages
-
-MODEL = "Llama3.1-8B-Instruct"
-
-
-class PrepareMessagesTests(unittest.IsolatedAsyncioTestCase):
-    async def test_system_default(self):
-        content = "Hello !"
-        request = ChatCompletionRequest(
-            model=MODEL,
-            messages=[
-                UserMessage(content=content),
-            ],
-        )
-        messages = chat_completion_request_to_messages(request)
-        self.assertEqual(len(messages), 2)
-        self.assertEqual(messages[-1].content, content)
-        self.assertTrue("Cutting Knowledge Date: December 2023" in messages[0].content)
-
-    async def test_system_builtin_only(self):
-        content = "Hello !"
-        request = ChatCompletionRequest(
-            model=MODEL,
-            messages=[
-                UserMessage(content=content),
-            ],
-            tools=[
-                ToolDefinition(tool_name=BuiltinTool.code_interpreter),
-                ToolDefinition(tool_name=BuiltinTool.brave_search),
-            ],
-        )
-        messages = chat_completion_request_to_messages(request)
-        self.assertEqual(len(messages), 2)
-        self.assertEqual(messages[-1].content, content)
-        self.assertTrue("Cutting Knowledge Date: December 2023" in messages[0].content)
-        self.assertTrue("Tools: brave_search" in messages[0].content)
-
-    async def test_system_custom_only(self):
-        content = "Hello !"
-        request = ChatCompletionRequest(
-            model=MODEL,
-            messages=[
-                UserMessage(content=content),
-            ],
-            tools=[
-                ToolDefinition(
-                    tool_name="custom1",
-                    description="custom1 tool",
-                    parameters={
-                        "param1": ToolParamDefinition(
-                            param_type="str",
-                            description="param1 description",
-                            required=True,
-                        ),
-                    },
-                )
-            ],
-            tool_prompt_format=ToolPromptFormat.json,
-        )
-        messages = chat_completion_request_to_messages(request)
-        self.assertEqual(len(messages), 3)
-        self.assertTrue("Environment: ipython" in messages[0].content)
-
-        self.assertTrue("Return function calls in JSON format" in messages[1].content)
-        self.assertEqual(messages[-1].content, content)
-
-    async def test_system_custom_and_builtin(self):
-        content = "Hello !"
-        request = ChatCompletionRequest(
-            model=MODEL,
-            messages=[
-                UserMessage(content=content),
-            ],
-            tools=[
-                ToolDefinition(tool_name=BuiltinTool.code_interpreter),
-                ToolDefinition(tool_name=BuiltinTool.brave_search),
-                ToolDefinition(
-                    tool_name="custom1",
-                    description="custom1 tool",
-                    parameters={
-                        "param1": ToolParamDefinition(
-                            param_type="str",
-                            description="param1 description",
-                            required=True,
-                        ),
-                    },
-                ),
-            ],
-        )
-        messages = chat_completion_request_to_messages(request)
-        self.assertEqual(len(messages), 3)
-
-        self.assertTrue("Environment: ipython" in messages[0].content)
-        self.assertTrue("Tools: brave_search" in messages[0].content)
-
-        self.assertTrue("Return function calls in JSON format" in messages[1].content)
-        self.assertEqual(messages[-1].content, content)
-
-    async def test_user_provided_system_message(self):
-        content = "Hello !"
-        system_prompt = "You are a pirate"
-        request = ChatCompletionRequest(
-            model=MODEL,
-            messages=[
-                SystemMessage(content=system_prompt),
-                UserMessage(content=content),
-            ],
-            tools=[
-                ToolDefinition(tool_name=BuiltinTool.code_interpreter),
-            ],
-        )
-        messages = chat_completion_request_to_messages(request)
-        self.assertEqual(len(messages), 2, messages)
-        self.assertTrue(messages[0].content.endswith(system_prompt))
-
-        self.assertEqual(messages[-1].content, content)