codemie-test-harness 0.1.183__py3-none-any.whl → 0.1.197__py3-none-any.whl

codemie_test_harness/tests/test_data/integrations_test_data.py

@@ -1,7 +1,27 @@
 import pytest
 
-from codemie_sdk.models.integration import CredentialTypes
+from codemie_sdk.models.integration import CredentialTypes, CredentialValues
 from codemie_test_harness.tests.enums.integrations import DataBaseDialect
+from codemie_test_harness.tests.enums.tools import (
+    Toolkit,
+    VcsTool,
+    NotificationTool,
+    ProjectManagementTool,
+    ServiceNowTool,
+    OpenApiTool,
+)
+from codemie_test_harness.tests.test_data.notification_tools_test_data import (
+    EMAIL_TOOL_PROMPT,
+    TELEGRAM_TOOL_PROMPT,
+)
+from codemie_test_harness.tests.test_data.open_api_tools_test_data import (
+    OPEN_API_SPEC_TOOL_TASK,
+)
+from codemie_test_harness.tests.test_data.pm_tools_test_data import (
+    JIRA_TOOL_PROMPT,
+    CONFLUENCE_TOOL_PROMPT,
+)
+from codemie_test_harness.tests.test_data.servicenow_tools_test_data import PROMPT
 from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
 from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
 
@@ -362,7 +382,7 @@ invalid_integrations = [
     pytest.param(
         CredentialTypes.EMAIL,
         CredentialsManager.invalid_gmail_credentials(),
-        "SMTP Code: 535. SMTP error",
+        "SMTP Code: 535, Message: 5.7.8 Username and Password not accepted.",
         marks=[
             pytest.mark.email,
             pytest.mark.notification,
@@ -412,3 +432,144 @@ invalid_integrations = [
         id=CredentialTypes.REPORT_PORTAL,
     ),
 ]
+
+empty_credentials_integrations = [
+    pytest.param(
+        [
+            CredentialValues(key="url", value="https://gitlab.com"),
+        ],
+        CredentialTypes.GIT,
+        Toolkit.VCS,
+        VcsTool.GITLAB,
+        f"Using gitlab tool get info about MR №7014 for repo with id '{CredentialsManager.get_parameter('GITLAB_PROJECT_ID')}'",
+        """
+            It seems there is an issue with the GitLab configuration, as it hasn't been set up properly for this tool.
+            This requires setting up authentication and configuration to connect to a GitLab instance.
+    
+            Could you please check your GitLab tool configuration and provide the necessary access details so that I can assist you further?
+        """,
+        marks=[pytest.mark.gitlab],
+        id=f"{CredentialTypes.GIT}_gitlab_empty",
+    ),
+    pytest.param(
+        [
+            CredentialValues(key="url", value="https://github.com"),
+        ],
+        CredentialTypes.GIT,
+        Toolkit.VCS,
+        VcsTool.GITHUB,
+        f"Using github tool get info about issue №5 for the repo {CredentialsManager.get_parameter('GITHUB_PROJECT')}.",
+        """
+            It seems there is an issue with accessing the GitHub API due to a missing configuration.
+            Unfortunately, I can't fetch the information directly at the moment.
+            However, you can access information about an issue using the following steps on your own:
+            
+            1. Navigate to the Issues tab of the repository [wild47/final_task](https://github.com/wild47/final_task).
+            2. Look for issue number 5 or directly visit `https://github.com/wild47/final_task/issues/5`,
+            and it should display all the details regarding that specific issue.
+            
+            If there's anything else you'd like assistance with, please let me know!
+        """,
+        marks=[pytest.mark.github],
+        id=f"{CredentialTypes.GIT}_github_empty",
+    ),
+    pytest.param(
+        [
+            CredentialValues(key="url", value=CredentialsManager.AUTO_GENERATED),
+        ],
+        CredentialTypes.EMAIL,
+        Toolkit.NOTIFICATION,
+        NotificationTool.EMAIL,
+        EMAIL_TOOL_PROMPT,
+        """
+            It looks like there's an issue with the SMTP configuration; it's missing a valid SMTP URL to send the email.
+            Please ensure that the SMTP settings are properly configured in the system.
+            If you have access to the SMTP details, such as the server URL, username, and password, you could input them into the system to resolve this issue.
+        """,
+        marks=[pytest.mark.email, pytest.mark.notification],
+        id=f"{CredentialTypes.EMAIL}_empty",
+    ),
+    pytest.param(
+        [
+            CredentialValues(key="url", value=CredentialsManager.AUTO_GENERATED),
+        ],
+        CredentialTypes.TELEGRAM,
+        Toolkit.NOTIFICATION,
+        NotificationTool.TELEGRAM,
+        TELEGRAM_TOOL_PROMPT,
+        """
+            It seems I don't have access to the necessary Telegram bot token to send a message. 
+            
+            You should verify that the bot token is set up correctly or send the message directly through your Telegram bot using your bot's token. 
+            
+            If you have your bot token and need help with the API request, let me know!
+        """,
+        marks=[pytest.mark.telegram, pytest.mark.notification],
+        id=f"{CredentialTypes.TELEGRAM}_empty",
+    ),
+    pytest.param(
+        [
+            CredentialValues(
+                key="url", value=CredentialsManager.get_parameter("JIRA_URL")
+            )
+        ],
+        CredentialTypes.JIRA,
+        Toolkit.PROJECT_MANAGEMENT,
+        ProjectManagementTool.JIRA,
+        JIRA_TOOL_PROMPT,
+        """
+            It seems that there's an authorization issue when trying to access the JIRA ticket.
+            The error indicates that the request is unauthorized.
+            You may need to ensure that the correct credentials are being used, or check if you have the necessary permissions to access the ticket.
+            
+            If there's anything else you'd like to try or if you have further details, let me know!
+        """,
+        marks=[pytest.mark.jira, pytest.mark.project_management],
+        id=f"{CredentialTypes.JIRA}_empty",
+    ),
+    pytest.param(
+        [],
+        CredentialTypes.CONFLUENCE,
+        Toolkit.PROJECT_MANAGEMENT,
+        ProjectManagementTool.CONFLUENCE,
+        CONFLUENCE_TOOL_PROMPT,
+        """
+            It looks like I don't have the required Confluence URL and credentials set up to perform the requested action.
+            If you have access to Confluence, please provide these details or try to access the page directly through your Confluence account.
+        """,
+        marks=[pytest.mark.confluence, pytest.mark.project_management],
+        id=f"{CredentialTypes.CONFLUENCE}_empty",
+    ),
+    pytest.param(
+        [
+            CredentialValues(key="url", value=""),
+        ],
+        CredentialTypes.SERVICE_NOW,
+        Toolkit.SERVICENOW,
+        ServiceNowTool.SERVICE_NOW,
+        PROMPT,
+        """
+            It appears that the ServiceNow configuration is not set, which prevents me from executing the request.
+            Please ensure that the ServiceNow configuration, including necessary authentication details,
+            is properly set up before using the ServiceNow tool. If you have access to configure these details, please do so and try again.
+        """,
+        marks=[pytest.mark.servicenow],
+        id=f"{CredentialTypes.SERVICE_NOW}_empty",
+    ),
+    pytest.param(
+        [
+            CredentialValues(key="url", value=CredentialsManager.AUTO_GENERATED),
+        ],
+        CredentialTypes.OPENAPI,
+        Toolkit.OPEN_API,
+        OpenApiTool.GET_OPEN_API_SPEC,
+        OPEN_API_SPEC_TOOL_TASK,
+        """
+            It seems that I don't have access to the OpenAPI specification for the `/v1/assistants` endpoint right now.
+            To help you better, could you please provide more details about what you'd like to know regarding this endpoint?
+            Are you looking for information on request parameters, expected responses, authentication, or something else?
+        """,
+        marks=[pytest.mark.openapi],
+        id=f"{CredentialTypes.OPENAPI}_empty",
+    ),
+]

codemie_test_harness/tests/test_data/llm_test_data.py

@@ -55,6 +55,7 @@ MODEL_RESPONSES = [
     LlmResponseData(ModelTypes.CLAUDE_4_OPUS, AWS_ENVS),
     LlmResponseData(ModelTypes.CLAUDE_4_1_OPUS, AWS_ENVS),
     LlmResponseData(ModelTypes.CLAUDE_4_SONNET_1M, AWS_ENVS),
+    LlmResponseData(ModelTypes.CLAUDE_4_5_SONNET, AWS_ENVS),
     # Other LLMs test data
     LlmResponseData(ModelTypes.RLAB_QWQ_32B, OTHER_ENVS),
     LlmResponseData(ModelTypes.DEEPSEEK_R1, OTHER_ENVS),

codemie_test_harness/tests/test_data/open_api_tools_test_data.py

@@ -58,6 +58,22 @@ RESPONSE_FOR_OPEN_API_SPEC = """
     These endpoints relate to managing assistants, allowing you to list and create assistants in the system.
 """
 
+INVOKE_EXTERNAL_API_TOOL_TASK = (
+    "Invoke GET https://api.open-meteo.com/v1/forecast?latitude=51.5&longitude=-0.1"
+)
+
+RESPONSE_EXTERNAL_FOR_INVOKE_API = """
+    The response from the Open Meteo API for the latitude 51.5 and longitude -0.1 is as follows:
+
+    - **Latitude**: 51.5
+    - **Longitude**: -0.1
+    - **Generation Time (ms)**: 0.0019
+    - **UTC Offset (seconds)**: 0
+    - **Timezone**: GMT
+    - **Timezone Abbreviation**: GMT
+    - **Elevation**: 12 meters
+"""
+
 open_api_tools_test_data = [
     (
         (OpenApiTool.INVOKE_EXTERNAL_API, OpenApiTool.GET_OPEN_API_SPEC),
@@ -65,8 +81,13 @@ open_api_tools_test_data = [
         RESPONSE_FOR_INVOKE_API,
     ),
     (
-        OpenApiTool.INVOKE_EXTERNAL_API,
+        OpenApiTool.GET_OPEN_API_SPEC,
         OPEN_API_SPEC_TOOL_TASK,
         RESPONSE_FOR_OPEN_API_SPEC,
     ),
+    (
+        OpenApiTool.INVOKE_EXTERNAL_API,
+        INVOKE_EXTERNAL_API_TOOL_TASK,
+        RESPONSE_EXTERNAL_FOR_INVOKE_API,
+    ),
 ]

codemie_test_harness/tests/test_data/report_portal_tools_test_data.py

@@ -4,7 +4,7 @@ rp_test_data = [
     (
         Toolkit.REPORT_PORTAL,
         ReportPortalTool.GET_EXTENDED_LAUNCH_DATA_AS_RAW,
-        "Get details for the launch with ID 23 in a raw HTML format",
+        "Get details for the launch with ID 23 in a raw HTML format. If content exceeds the token limit for display show truncated version",
         """
         The launch with ID 23 has been successfully retrieved in HTML format. Here is the raw HTML content for your reference:
 
@@ -90,36 +90,68 @@ rp_test_data = [
     (
         Toolkit.REPORT_PORTAL,
         ReportPortalTool.FIND_TEST_ITEM_BY_ID,
-        "Get test item by ID 2884",
+        "Get test item by ID 111487",
         """
-        Test Item Details: ID 2884
-        Test Information
-        Name: test_workflow_with_assistant_with_ado_test_plan_tools
-        Status: FAILED
-        Type: STEP
-        Path: workflow/assistant_tools/ado
-        Launch ID: 23
-        Launch Number: 2
-        Location
-        File: codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_test_plan_tools.py
-        Timing
-        Start Time: September 8, 2025, 08:51:45
-        End Time: September 8, 2025, 08:52:54
-        Duration: 1 minute, 9 seconds
-        Test Statistics
-        Executions: Failed: 1
-        Defects: To Investigate: 1 (ti001)
-        Attributes
-        Type: Regression
-        Issue Analysis
-        This test attempts to create an assistant with Azure DevOps Test Plan tools and create a test plan in the "CodemieAnton" project.
-        The test appears to have failed, though the logs show that the assistant was created successfully and the test plan was created with ID 3.
-        Without seeing explicit error messages in the logs, the failure might be related to:
-        Assertion failures that weren't logged
-        Issues during test teardown
-        Expected vs. actual state differences
-        Timeout issues
-        Would you like me to retrieve the logs for this test item to investigate the failure further?
+            Here are the details for the test item with ID **111487**:
+            
+            - **Name**: `codemie_test_harness/tests/assistant/tools/plugin/test_assistant_with_plugin_and_mcp_servers.py::test_assistant_with_plugin_and_cli_mcp_server[ls]`
+            - **Code Reference**: `codemie_test_harness/tests/assistant/tools/plugin/test_assistant_with_plugin_and_mcp_servers.py:test_assistant_with_plugin_and_cli_mcp_server`
+            
+            ### Parameters
+            - **prompt**: `ls`
+            - **expected_response**: 
+              ```
+              Here is a list of files and directories in `/apps/codemie-sdk/test-harness/codemie_test_harness/tests`:
+            
+              - Files:
+                - `.DS_Store`
+                - `__init__.py`
+                - `conftest.py`
+            
+              - Directories:
+                - `__pycache__`
+                - `assistant`
+                - `e2e`
+                - `enums`
+                - `integrations`
+                - `llm`
+                - `providers`
+                - `search`
+                - `service`
+                - `test_data`
+                - `ui`
+                - `utils`
+                - `workflow`
+              ```
+            
+            ### Attributes
+            - **plugin**
+            - **assistant**
+            - **api**
+            - **mcp**
+            
+            ### Status
+            - **Start Time**: October 8, 2025, 10:02:41 AM UTC
+            - **End Time**: October 8, 2025, 10:05:09 AM UTC
+            - **Status**: **FAILED**
+            
+            ### Execution Statistics
+            - **Total Executions**: 1
+            - **Failed Executions**: 1
+            - **Defects**:
+              - **To Investigate**: `ti001` (1 instance)
+            
+            ### Issue Details
+            - **Issue Type**: `ti001`
+            - **Auto Analyzed**: No
+            - **Ignore Analyzer**: No
+            
+            ### Launch Information
+            - **Launch Name**: Pytest Regression
+            - **Launch Number**: 121
+            - **Launch ID**: 321
+            
+            This test appears to have failed due to a defect marked for investigation. If you need further insights or actions, please let me know!
         """,
     ),
     (
@@ -168,89 +200,34 @@ rp_test_data = [
     (
         Toolkit.REPORT_PORTAL,
         ReportPortalTool.GET_LOGS_FOR_TEST_ITEM,
-        "Get logs for test item ID 2884",
+        "Get first page of the logs for test item ID 111487",
         """
-        Here are the logs for test item ID 2884:
-       
-       1. **Time:** 2025-09-08T08:51:45.164Z  
-          **Level:** DEBUG  
-          **Message:** Making /v1/assistants request to https://codemie-azure.eks-sandbox.aws.main.edp.projects.epam.com/code-assistant-api<class 'dict'>
-          
-       2. **Time:** 2025-09-08T08:51:45.164Z  
-          **Level:** DEBUG  
-          **Message:** Request body: {'name': 'sdp_xjwyipzcypfocwq', 'description': 'Integration test assistant', 'system_prompt': 'You are a helpful integration test assistant', 'project': 'codemie', 'context': [], 'llm_model_type': 'gpt-4o-2024-08-06', 'toolkits': [{'toolkit': 'Azure DevOps Test Plan', 'tools': [{'name': 'create_test_plan', ...}]  
-       
-       3. **Time:** 2025-09-08T08:51:45.165Z  
-          **Level:** DEBUG  
-          **Message:** Starting new HTTPS connection (1): codemie-azure.eks-sandbox.aws.main.edp.projects.epam.com:443
-          
-       4. **Time:** 2025-09-08T08:51:45.226Z  
-          **Level:** DEBUG  
-          **Message:** https://codemie-azure.eks-sandbox.aws.main.edp.projects.epam.com:443 "POST /code-assistant-api/v1/assistants HTTP/1.1" 200 39
-          
-       5. **Time:** 2025-09-08T08:51:45.226Z  
-          **Level:** DEBUG  
-          **Message:** Received response with status 200
-          
-       6. **Time:** 2025-09-08T08:51:45.226Z  
-          **Level:** DEBUG  
-          **Message:** Response datasource_type: <class 'dict'>
-          
-       7. **Time:** 2025-09-08T08:51:45.228Z  
-          **Level:** INFO  
-          **Message:** Successfully processed /v1/assistants request to <class 'dict'>
-          
-       8. **Time:** 2025-09-08T08:51:45.228Z  
-          **Level:** DEBUG  
-          **Message:** Making /v1/assistants request to https://codemie-azure.eks-sandbox.aws.main.edp.projects.epam.com/code-assistant-apityping.List[codemie_sdk.models.assistant.AssistantBase]
-          
-       9. **Time:** 2025-09-08T08:51:45.228Z  
-          **Level:** DEBUG  
-          **Message:** Request params: {'page': 0, 'per_page': 200, 'scope': 'visible_to_user', 'minimal_response': True}
-          
-       10. **Time:** 2025-09-08T08:51:45.229Z  
-           **Level:** DEBUG  
-           **Message:** Starting new HTTPS connection (1): codemie-azure.eks-sandbox.aws.main.edp.projects.epam.com:443
-          
-       11. **Time:** 2025-09-08T08:51:45.299Z  
-           **Level:** DEBUG  
-           **Message:** https://codemie-azure.eks-sandbox.aws.main.edp.projects.epam.com:443 "GET /code-assistant-api/v1/assistants?page=0&per_page=200&scope=visible_to_user&minimal_response=True HTTP/1.1" 200 69989
-          
-       12. **Time:** 2025-09-08T08:51:45.310Z  
-           **Level:** DEBUG  
-           **Message:** Received response with status 200
-          
-       13. **Time:** 2025-09-08T08:51:45.311Z  
-           **Level:** DEBUG  
-           **Message:** Response datasource_type: <class 'dict'>
-          
-       14. **Time:** 2025-09-08T08:51:45.312Z  
-           **Level:** INFO  
-           **Message:** Successfully processed /v1/assistants request to typing.List[codemie_sdk.models.assistant.AssistantBase]
-          
-       15. **Time:** 2025-09-08T08:51:45.314Z  
-           **Level:** DEBUG  
-           **Message:** Making /v1/workflows request to https://codemie-azure.eks-sandbox.aws.main.edp.projects.epam.com/code-assistant-api<class 'dict'>
-          
-       16. **Time:** 2025-09-08T08:51:45.314Z  
-           **Level:** DEBUG  
-           **Message:** Request body: {'project': 'codemie', 'name': 'sdp_xpgivblpavtdeza', 'description': 'Test Workflow', 'yaml_config': 'enable_summarization_node: false\\ntools: []\\nassistants:\\n- id: sdp_xjwyipzcypfocwq\\n  model: gpt-4o-2024-08-06\\n  assistant_id: 6ef6a01d-70ef-4211-a385-8c99b87eee72\\nstates:\\n- id: ...
-          
-       17. **Time:** 2025-09-08T08:51:45.315Z  
-           **Level:** DEBUG  
-           **Message:** Starting new HTTPS connection (1): codemie-azure.eks-sandbox.aws.main.edp.projects.epam.com:443
-          
-       18. **Time:** 2025-09-08T08:51:45.394Z  
-           **Level:** DEBUG  
-           **Message:** https://codemie-azure.eks-sandbox.aws.main.edp.projects.epam.com:443 "POST /code-assistant-api/v1/workflows HTTP/1.1" 200 2119
-          
-       19. **Time:** 2025-09-08T08:51:45.394Z  
-           **Level:** DEBUG  
-           **Message:** Received response with status 200
-          
-       20. **Time:** 2025-09-08T08:51:45.394Z  
-           **Level:** DEBUG  
-           **Message:** Response datasource_type: <class 'dict'>
+            Here are the logs from the first page for test item ID 111487:
+            
+            | Time                          | Level  | Message                                                                                                                   |
+            |-------------------------------|--------|---------------------------------------------------------------------------------------------------------------------------|
+            | 2025-10-08T10:02:41.368Z      | DEBUG  | Starting new HTTPS connection (1): keycloak.eks-core.aws.main.edp.projects.epam.com:443                                    |
+            | 2025-10-08T10:02:41.647Z      | DEBUG  | https://keycloak.eks-core.aws.main.edp.projects.epam.com:443 "POST /auth/realms/codemie-prod/protocol/openid-connect/token HTTP/1.1" 200 2500 |
+            | 2025-10-08T10:02:41.651Z      | INFO   | Plugin key: f6455cc5-af78-4420-86a9-965ac6f0425e                                                                           |
+            | 2025-10-08T10:02:41.652Z      | DEBUG  | Making /v1/settings/project request to https://codemie-preview.lab.epam.com/code-assistant-api<class 'dict'>                  |
+            | 2025-10-08T10:02:41.652Z      | DEBUG  | Request body: {'project_name': 'codemie', 'alias': 'cfs_qlbpwnkmgoqgkcc', 'default': False, 'is_global': False, 'credential_type': <CredentialTypes.PLUGIN: 'Plugin'>, 'credential_values': [{'key': 'url', 'value': 'AUTO_GENERATED'}, {'key': 'plugin_key', 'value': 'f6455cc5-af78-4420-86a9-965ac6f0425e'}], 'setting_type': <IntegrationType.PROJECT: 'project'>} |
+            | 2025-10-08T10:02:41.653Z      | DEBUG  | Starting new HTTPS connection (1): codemie-preview.lab.epam.com:443                                                        |
+            | 2025-10-08T10:02:42.051Z      | DEBUG  | https://codemie-preview.lab.epam.com:443 "POST /code-assistant-api/v1/settings/project HTTP/1.1" 200 41                     |
+            | 2025-10-08T10:02:42.052Z      | DEBUG  | Received response with status 200                                                                                           |
+            | 2025-10-08T10:02:42.052Z      | DEBUG  | Response datasource_type: <class 'dict'>                                                                                    |
+            | 2025-10-08T10:02:42.060Z      | INFO   | Successfully processed /v1/settings/project request to <class 'dict'>                                                       |
+            | 2025-10-08T10:02:42.060Z      | DEBUG  | Making /v1/settings/project request to https://codemie-preview.lab.epam.com/code-assistant-apityping.List[codemie_sdk.models.integration.Integration]            |
+            | 2025-10-08T10:02:42.061Z      | DEBUG  | Request params: {'page': 0, 'per_page': 200}                                                                               |
+            | 2025-10-08T10:02:42.062Z      | DEBUG  | Starting new HTTPS connection (1): codemie-preview.lab.epam.com:443                                                        |
+            | 2025-10-08T10:02:42.562Z      | DEBUG  | https://codemie-preview.lab.epam.com:443 "GET /code-assistant-api/v1/settings/project?page=0&per_page=200 HTTP/1.1" 200 93345 |
+            | 2025-10-08T10:02:42.573Z      | DEBUG  | Received response with status 200                                                                                           |
+            | 2025-10-08T10:02:42.573Z      | DEBUG  | Response datasource_type: <class 'dict'>                                                                                    |
+            | 2025-10-08T10:02:42.585Z      | INFO   | Successfully processed /v1/settings/project request to typing.List[codemie_sdk.models.integration.Integration]                |
+            | 2025-10-08T10:02:42.647Z      | DEBUG  | Making /v1/settings/project request to https://codemie-preview.lab.epam.com/code-assistant-apityping.List[codemie_sdk.models.integration.Integration]            |
+            | 2025-10-08T10:02:42.647Z      | DEBUG  | Request params: {'page': 0, 'per_page': 200}                                                                               |
+            | 2025-10-08T10:02:42.649Z      | DEBUG  | Starting new HTTPS connection (1): codemie-preview.lab.epam.com:443                                                        |
+            
+            If you need additional pages of logs, please let me know.
         """,
     ),
     (

codemie_test_harness/tests/test_data/research_tools_test_data.py

@@ -26,7 +26,7 @@ search_tools_test_data = [
 interactions_tools_test_data = [
     (
         ResearchToolName.GOOGLE_PLACES,
-        "Find McDonald in the Kyiv on the Khreschatyk in radius 2 km",
+        "Find McDonald in the Kyiv on the Khreschatyk, 19-a",
         """
             I found a McDonald's location on Khreschatyk Street in Kyiv:
             McDonald's
@@ -40,12 +40,34 @@ interactions_tools_test_data = [
         ResearchToolName.GOOGLE_PLACES_FIND_NEAR,
         "Find McDonalds in the Kyiv near Khreschatyk in radius 2 km",
         """
-            I found a McDonald's location on Khreschatyk Street in Kyiv:
-            McDonald's
-            Address: Khreschatyk St, 19-a, Kyiv, Ukraine
-            Rating: 4.4 (based on 21,802 user ratings)
-            Open Now: Yes
-            Price Level: 2
+            I found several McDonald's locations near Khreschatyk in Kyiv, within a 2 km radius. However, all these locations are temporarily closed. Here are the details:
+            
+            1. **McDonald's**
+               - **Address:** Sofiivs'ka St, 1/2, Kyiv
+               - **Rating:** 4.3 based on 11,090 reviews
+               - **[View Photo](https://maps.googleapis.com/maps/api/place/photo?maxwidth=400&photoreference=AciIO2fMNnp003CQSug6AmJuJXZ0myKILfuo4iNiSDpi6mTJm2zgnwFbWazL-5UC9MGU6e8TZ5jzorgsFj3QfmeEuICUKkKv9vvgKWXItQo4h2TdXgNxV2U-WHuz-cLDT0d56YoVpRFxjitp1Fmly-XEwhZj_UqgVW2d2qAI5lwjk4_aLdbZznWWZ212blOjpF4VY987Q8D9wVEUHnl-y72Ev6A7ywNvV3VJKGuE5szP6JRj7GI9r-0ToIuC8dub3-lkgyQUuF56Sfki6M46ePndL5mk-KIuJOfH54EINyBqp-z3Q9xA85T93pCcW6bMxDk5XqfG7nQfpZr5yO8-zfJj51NNJzjF8vsevvfXDIoUVDGXNJfllLcXCxA6uaoYHK0ynmMaX4_6X_R2FtM_UoimzUuw9cqQPBK_FryGvD9qO5Bx7zW_HMb3D6TMax2iNtSrkwd59Bz7XC4ud6MqdoWn7rXGR_Z52XR1KaGFnFMW_4NI0MI6_hFVjvUBp7xnzvfABaaG5o2imuVRzf4khUthFRyXTil8ht36C-ijIVvg8HdODerT8LN6IoV4pCg5XekDNpIqxhJf)**
+              
+            2. **McDonald's**
+               - **Address:** Khreschatyk St, 19-a, Kyiv
+               - **Rating:** 4.4 based on 21,848 reviews
+               - **[View Photo](https://maps.googleapis.com/maps/api/place/photo?maxwidth=400&photoreference=AciIO2fLPseD2rR9t8necj88rGG-3xzeNQGmho05h5SQ7i8tjQ8zG0wPEbLXz0m0ZzKxR6inczyc5GW_8WkJ5PmuDJLXYuEB0-pwow4tdnXkW5GynclZsRklxzEAV49X7IBsLXSaw1-WhISnkEWYMzb8PM6WkORf_IKBw8cD6AsQ93rLMusReVT37Cwo9ZiCqaOeMTDyc7MSN9Md7MhN0ezi-cphTe6VfFT9zMPSVyM-_yjb9bjzHsFjnK343C1QjMIZeBdRwz-Raw68xvecUu9FF6JLx8vPlW8kkbgdS9p9DX7AAS3NccBwFEBEGPryDYCzNKHew_0NRqcaQW9KQfWPuBeApArqbEXyeVCRdCPe0MwOy4wQER9_PtnaltlfKGBHD8fAWH3hS35erkkQE6GQ7_5KQxExVsXSq7hiVxUd_Mh0geOT2KSUl7A8Gkcq3AbqeTXTp5V6454vX-yq55WEpCUvdB6DJNGGxdCO8q4YJm3SuEyyN3B4f3QIg2HsiOVv9KPK3A29hYsX_IGgDcwsD720zltJEYbIevGZ3OtFrv_Yp-kGDBaYPmZD-M6i8qy6zS7aTfkWNLjcjQ)**
+              
+            3. **McDonald's**
+               - **Address:** Velyka Vasylkivska St, 22, Kyiv
+               - **Rating:** 4.3 based on 10,710 reviews
+               - **[View Photo](https://maps.googleapis.com/maps/api/place/photo?maxwidth=400&photoreference=AciIO2dOxDjqq7UHyeqKwh9VO8o9uq3iqfWDPLgVKMwwwjBhmb3cG2rAohIjtemD29g67NcsKzx-dWVqsmkRkqz8XhU6HlbrJEtq_TL7xVYQTImFHXq9kElkR9I0gnpA-rDZoD_RYlcmuRKURTNom8VC6sjzszcozz6QNwjOnl7f55MJq6O9eoJQ0g3iu-ecXGt6bELpx2oa9urOqqjrapGH4yKOQKPp_36MEJVkuD6IohzuLMqpQdlIlNp8WP7SiEZb9CD1DalNuk5H__jsZ35zMKVCpWYMnkZGSN0_-DOFrfXnhC-b6MKYrB_9D6pUjpsfIPvi2qHHf4LRho-P2OR71g1P7WlRf25h1g9DdX3uMe7kMtU0sxRvUMSiAe6Sy9D8s7qrIEb8DcfFQxkFgoQs6fTb-RCEKAwqQtMlJ96nd2kmRU100MDSWsiUy7ifLn6LK343O9_e-RJyobXSo8WovvUs1QxA2kh2z6UMHVxorNPV5wclkE3J3cLYzUTvL5P7t3OBDVGiTdcPzE-5Z7P63VF-jUHU9Hv2l6-KHjwa4r8wIDgB7shW_iJEuHgv04NxD2p95onx)**
+              
+            4. **McDonald's**
+               - **Address:** Borychiv Descent, 10А, Kyiv
+               - **Rating:** 4.5 based on 14,818 reviews
+               - **[View Photo](https://maps.googleapis.com/maps/api/place/photo?maxwidth=400&photoreference=AciIO2efC3DMYMgEycgpEkR5IhrJIwufUliminn1HZmICsch982EeMsgXS9kKkZRvOkvWnnI-os45avQFWAUFR_bWy8R2BwGbW1V37En8BwrFVGznWkIEFZeCVTVPI_s8K4Hj77_l5WLzYnh5wCMCsPZQtTjk26CdvM3Kg1MdJQdZAIFcc26jErp3h6bw7MRY245XbVl2w-41jSOlayQ8voyUUtVVzeGgKVN1kUh-cc2An72nvGjdBQP2S-_8pwGJ5Sz0CcakvCT2VnAdTKz4ywZ5pNxdpKRz17wPvQAkj9jdKrHN5J5jmqjUu_QQFMFfkfNR2P6OEaNxMT5mPizohzU7e1_A3dWHRZxB9bAmxZ9EpQkyeZb3ppwtM24pCkSQ8Cz_9b8qBLleIWolLtXBLyTM54z4XtCSlc-X1wmjyIkca6Cd5D_8jafJbd10uj6wiA5VdhWcv1l_yJSnq0-0t1STfqI0VUVITpAEEU0ZB0TTheuQUuLa6quMjzU8fnYiMzh9YyY8CrLt8xXEMuOwPJrh94iA4-5AKOeJO4LPVehpNdKVWLKXYEwfilMaouuEHheVBS6ANRn)**
+              
+            5. **McDonald's**
+               - **Address:** Bohdana Khmel'nyts'koho St, 40/25, Kyiv
+               - **Rating:** 4.3 based on 6,187 reviews
+               - **[View Photo](https://maps.googleapis.com/maps/api/place/photo?maxwidth=400&photoreference=AciIO2eeKwvPbZ89_onqK5_s5Zv8kV5TxY5ZIM5ZwR-q8vPuyxAhXp-X6A4LDdLQEtBc3WhH3IJtIiwf6wSqGqLQeuoxIzUqqTR1VM2bYp1FfQwVhVrTVf2bHixPt1z847PYQMCO4ibGE1c0EUQI8S6K_2SnCvmb0PO8AC5GMY-ociHR6rbR8QEiKnYFSd04w349_u-f_q8kntKLS1TnOphOhOFatS_TJFS8AO8JQbV9jILByYncmH6dkQxw7eqqpY--bFjk5y3n1bsP0haKdyLGRwN3d0aqFkRUvtqNPg7KU1Hx0_5uwBb4C_o4XXZXvWhi_fwanNTF_XpyTTcrmw_7IerLVMhoZmM89j3Un8qQi7_prhJBT3Un3zwLKvXs-cr4R9StGn_dT9D73RoOGHP16wjSrfCZ8BMNBM6JgqTxKzLjvf8yGKWeYs5pTctcjcsqw_vc9cOhTHewN2wcRc_AdQ6Snj191P6UiVUGnytXyhKCM6yjOXvVjNLjbDNfZz5CeJZjsNJDuL7kH64vG_EQgR5eHqQUTVWD2Fm7wJE-YtP2MtOGEptncDVQtRSt6ALBj0kUi9t3)**
+              
+            Please note that the status may change, so it's advisable to check the latest updates before making plans.
         """,
     ),
     (

codemie_test_harness/tests/utils/assistant_utils.py

@@ -109,6 +109,7 @@ class AssistantUtils(BaseUtils):
         conversation_id=None,
         history=(),
         output_schema=None,
+        extract_failed_tools=False,
     ):
         chat_request = AssistantChatRequest(
             text=user_prompt,
@@ -129,18 +130,21 @@ class AssistantUtils(BaseUtils):
             return response.generated
         else:
             # Extract triggered tools from response thoughts
-            triggered_tools = self._extract_triggered_tools(response)
+            triggered_tools = self._extract_triggered_tools(
+                response, extract_failed_tools
+            )
             return response.generated, triggered_tools
 
-    def _extract_triggered_tools(self, response):
+    def _extract_triggered_tools(self, response, extract_failed_tools):
         """
         Extract triggered tools from response thoughts.
 
-        Filters out 'Codemie Thoughts' entries and error entries,
+        Filters out 'Codemie Thoughts' entries and optionally error entries,
         returning a list of tool names in lowercase.
 
         Args:
             response: The assistant response containing thoughts
+            extract_failed_tools: If True, include tools that failed with errors
 
         Returns:
             list: List of triggered tool names in lowercase
@@ -148,15 +152,21 @@ class AssistantUtils(BaseUtils):
         triggered_tools = []
 
         # Check if response has thoughts attribute
-        if hasattr(response, "thoughts") and response.thoughts:
-            for thought in response.thoughts:
-                # Filter out 'Codemie Thoughts' and error entries
-                if thought.get("author_name") != "Codemie Thoughts" and not thought.get(
-                    "error", False
-                ):
-                    author_name = thought.get("author_name", "")
-                    if author_name:
-                        triggered_tools.append(author_name.lower())
+        if not (hasattr(response, "thoughts") and response.thoughts):
+            return triggered_tools
+
+        for thought in response.thoughts:
+            author_name = thought.get("author_name", "")
+
+            # Skip if no author name or if it's 'Codemie Thoughts'
+            if not author_name or author_name == "Codemie Thoughts":
+                continue
+
+            # If not extracting failed tools, skip error entries
+            if not extract_failed_tools and thought.get("error", False):
+                continue
+
+            triggered_tools.append(author_name.lower())
 
         return triggered_tools