codemie-test-harness 0.1.213__tar.gz → 0.1.215__tar.gz

{codemie_test_harness-0.1.213 → codemie_test_harness-0.1.215}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: codemie-test-harness
-Version: 0.1.213
+Version: 0.1.215
 Summary: Autotest for CodeMie backend and UI
 Author: Anton Yeromin
 Author-email: anton_yeromin@epam.com
@@ -13,7 +13,7 @@ Requires-Dist: aws-assume-role-lib (>=2.10.0,<3.0.0)
 Requires-Dist: boto3 (>=1.39.8,<2.0.0)
 Requires-Dist: click (>=8.1.7,<9.0.0)
 Requires-Dist: codemie-plugins (>=0.1.123,<0.2.0)
-Requires-Dist: codemie-sdk-python (==0.1.213)
+Requires-Dist: codemie-sdk-python (==0.1.215)
 Requires-Dist: pytest (>=8.4.1,<9.0.0)
 Requires-Dist: pytest-playwright (>=0.7.0,<0.8.0)
 Requires-Dist: pytest-repeat (>=0.9.3,<0.10.0)

{codemie_test_harness-0.1.213 → codemie_test_harness-0.1.215}/codemie_test_harness/tests/assistant/tools/project_management/test_assistant_pm_tools.py

@@ -16,6 +16,7 @@ from codemie_test_harness.tests.test_data.project_management_test_data import (
 from codemie_test_harness.tests.utils.base_utils import (
     credentials_to_dict,
     assert_tool_triggered,
+    get_random_name,
 )
 from codemie_test_harness.tests.utils.constants import (
     project_management_integrations,
@@ -45,6 +46,9 @@ def test_assistant_with_project_management_tools(
         project_management_integrations[integration_type]
     )
 
+    if "%s" in prompt:
+        prompt = prompt % get_random_name()
+
     assistant = assistant(
         Toolkit.PROJECT_MANAGEMENT,
         tool_name,

{codemie_test_harness-0.1.213 → codemie_test_harness-0.1.215}/codemie_test_harness/tests/test_data/assistant_test_data.py

@@ -803,36 +803,101 @@ DOCX_TOOL_TEST_DATA = [
         id="image_extraction",
     ),
     pytest.param(
-        "From test_extended.docx extract and show the text with OCR from images",
+        "From test_extended.docx extract and show the OCR from images and text from the first page",
         """
-            ### Extracted Text with OCR from Images in *test_extended.docx*
-
-            #### **Image 1: Capability Configuration**
-            The image contains text related to client authentication, authorization settings, and available flows. Below is the OCR-extracted text:
-
+            Here is the extracted content from the first page of the `test_extended.docx` document including text and OCR results from images:
+            
+            ### Text Content
             ```
-            **Capability config**
-
-            - **Client authentication**
-              - 🟦 On
-
-            - **Authorization**
-              - ⚪ Off
-
+            1. Getting Started with AI/RunTM CodeMie
+            
+            This section provides a brief description of what AI/RunTM CodeMie is and describes how to start using the AI/RunTM CodeMie tool as soon as possible.
+            
+            1.1 What is AI/RunTM CodeMie?
+            
+            AI/RunTM CodeMie is a sophisticated AI assistant tool tailored specifically for developers. AI/RunTM CodeMie is an innovative LLM-powered platform designed to help users to address specific challenges and find solutions for their needs. Unlike conventional large language models (LLMs) such as ChatGPT, Bard, and Copilot, AI/RunTM CodeMie is engineered to support development processes more comprehensively. It integrates seamlessly with essential development tools including Git, Jira, Confluence, and various search engines. This integration empowers AI/RunTM CodeMie to go beyond the capabilities of a standard chatbot, enabling it to manage Jira issues, devise appropriate implementations, and generate pull requests. 
+            
+            AI/RunTM CodeMie leverages AI to automate coding tasks, reduce technical debt, streamline code reviews and legacy application migrations, enhance onboarding, improve information access and analysis, optimize call center operations, ensure consistent customer support, maintain data security, and analyze market sentiment, ultimately enhancing productivity and reducing costs. The development of such a powerful assistant necessitates extensive customization and fine-tuning. The following sections will delve into the intricate adjustments and configurations required to harness the full potential of AI/RunTM CodeMie.
+            
+            1.2 What Are AI/RunTM CodeMie Capabilities?
+            
+            AI/RunTM CodeMie efficiently addresses numerous tasks across varying difficulty levels. Below is an overview of AI/RunTM CodeMie’s key capabilities:
+            
+            Comprehensive SDLC Smart Assistance and Assistants Library: AI/RunTM CodeMie offers robust smart assistance across all phases of the SDLC process by leveraging a variety of AI assistant roles, such as Business Analyst (BA), Developer, Quality Assurance (QA), Project Manager (PM), and more. These pre-built AI assistants enhance performance and productivity, and automate routine work, significantly reducing process costs and accelerating the software development cycle. The platform comes with a comprehensive library of pre-built AI assistants tailored to various roles within the SDLC to suit the diverse needs within a project.
+            
+            Assistants Constructor: Provides the flexibility to create personalized assistants equipped with specific tools and abilities tailored to your project's needs.
+            
+            Indexing and Data Management: AI/RunTM CodeMie provides options for data indexing, including the ability to monitor the current progress and status of the indexing process, perform incremental or full reindexing, and manage indexed data sources effectively. Supported data sources include Jira, Confluence, various file formats (PDF, PPT, Excel, etc.), and Git.
+            
+            Support for Multi-Agent Workflows: AI/RunTM аCodeMie supports multi-agent workflows, allowing multiple AI assistants to collaborate seamlessly on complex tasks and workflows. This capability covers use cases where different agents need to interact and share information to achieve a common goal, enhancing coordination and efficiency across various phases of the SDLC.
+            
+            Ease of Use for Beginners: Simple use cases for newcomers include code review, newcomer training, and user story creation. These require minimal setup, such as configuring your Git token for code-related tasks or your Jira token for project management tasks.
+            
+            Extensive Library of Tools: AI/RunTM CodeMie includes a wide array of tools to support various aspects of software development and project management:
+            
+            Version Control Systems (VCS): Tools for managing and tracking changes in the codebase, such as Git.
+            
+            Codebase Tools: Tools for code review, static code analysis, and automated code formatting.
+            
+            Research Tools: Tools to assist in gathering and organizing research data and documentation.
+            
+            Cloud Tools: Integration with major cloud providers for deployment, monitoring, and management of cloud resources.
+            
+            Project Management Tool (Jira: For project management, task tracking, and issue tracking, 
+            
+            Confluence: For documentation, knowledge sharing, and collaboration.)
+            
+            Open API: Integration with various open APIs to extend functionality and connect with other services.
+            
+            Notification Tools: Tools for sending notifications and alerts via email, chat, or other communication channels.
+            
+            Data Management Tools: Tools for querying Elasticsearch indexes.
+            
+            Git
+            
+            Access Management: Keycloak
+            
+            Plugin
+            
+            Open API
+            
+            Notification
+            
+            File Management
+            
+            Data Management
+            
+            Quality Assurance
+            
+            There is no priority or sequential system in place. Everything depends on the given instructions. It is possible to instruct the model to use a specific data source for a particular use case or provide a description for the data source when it is created. Data source descriptions are provided to the model so it can understand better use cases for it.
+            
+            The data source does not have rules, only description. System Instructions(System prompt) extends based on data source description. If contradictions arise, the model will use its creative problem-solving abilities to address them. 
+            
+            The model has a context window, but it is irrelevant to data source size. The model answer will depend on query quality. There is a use case when the model has thousands of data sources, however with queries that are specific and on point there are no problems. If a poor query provided e.g., “tell me about something” answer would be vague.
+            
+            Other than the context window there is no limit, however there is a simple rule: the smaller and clearer the instructions, the better. Also, the instructions must be uncontradictory, as this will reduce the risk of confusing LLM.
+            
+            1.3 Meet the AI/Run FAQ Assistant
+            ```
+            
+            ### OCR from Image Text
+            ```
+            Capability config
+            
+            - **Client authentication** 
+              - On
+            
+            - **Authorization** 
+              - Off
+            
             - **Authentication flow**
-              - ☑️ Standard flow
-              - ☑️ Direct access grants
-              - ☐ Implicit flow
-              - ☑️ Service accounts roles
-              - ☐ OAuth 2.0 Device Authorization Grant
-              - ☐ OIDC CIBA Grant
+              - [x] Standard flow
+              - [x] Direct access grants
+              - [ ] Implicit flow
+              - [x] Service accounts roles
+              - [ ] OAuth 2.0 Device Authorization Grant
+              - [ ] OIDC CIBA Grant
             ```
-
-            This image primarily highlights authentication configurations, flow types, and the enabling/disabling of various features.
-
-            ---
-
-            Let me know if you'd like to perform further analysis or extract more details!
         """,
         id="text_with_images",
     ),

{codemie_test_harness-0.1.213 → codemie_test_harness-0.1.215}/codemie_test_harness/tests/test_data/pm_tools_test_data.py

@@ -1,4 +1,3 @@
-from codemie_test_harness.tests.utils.base_utils import get_random_name
 from codemie_test_harness.tests.utils.confluence_utils import CONFLUENCE_SPACE_KEYS
 from codemie_test_harness.tests.utils.jira_utils import JIRA_PROJECT_KEYS
 
@@ -101,7 +100,7 @@ def get_jira_tool_create_prompt(jira_type: str) -> str:
         "For generic jira tool use exactly the same parameters: "
         "relative_url=/rest/api/2/issue "
         "method=POST "
-        f"params={{'fields': {{'project': {{'key': '{JIRA_PROJECT_KEYS[jira_type]}'}}, 'summary': '{get_random_name()} Test issue created by automation', "
+        f"params={{'fields': {{'project': {{'key': '{JIRA_PROJECT_KEYS[jira_type]}'}}, 'summary': '%s Test issue created by automation', "
         "'description': 'This is a test issue created for testing purposes', 'issuetype': {'name': 'Task'}}}}}"
     )
 
@@ -121,7 +120,7 @@ def get_confluence_tool_create_prompt(confluence_type: str) -> str:
     Create a new page in Confluence with the following details using Generic Confluence Tool with the params:
     relative_url='/rest/api/content'
     method='POST'
-    params={{'type': 'page', 'title': '{get_random_name()} Test Page Created by Automation', 'space': {{'key': '{CONFLUENCE_SPACE_KEYS[confluence_type]}'}},
+    params={{'type': 'page', 'title': '%s Test Page Created by Automation', 'space': {{'key': '{CONFLUENCE_SPACE_KEYS[confluence_type]}'}},
     'body': {{'storage': {{'value': '<p>This is a test page created for testing purposes.</p>', 'representation': 'storage'}}}}}}
 """
 

{codemie_test_harness-0.1.213 → codemie_test_harness-0.1.215}/codemie_test_harness/tests/utils/base_utils.py

@@ -4,21 +4,11 @@ import re
 import string
 import time
 import unicodedata
-from time import sleep
 
 from hamcrest import assert_that, equal_to, contains_string
 
 from codemie_sdk import CodeMieClient
-from codemie_sdk.exceptions import ApiError
-from codemie_sdk.models.workflow_state import (
-    WorkflowExecutionState,
-    WorkflowExecutionStatusEnum,
-)
-from codemie_sdk.services.workflow_execution_state import (
-    WorkflowExecutionStateService,
-)
 from codemie_test_harness.tests import autotest_entity_prefix
-from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
 
 
 class BaseUtils:
@@ -47,29 +37,6 @@ def to_camel_case(input_string):
     return camel_case
 
 
-def wait_for_completion(
-    execution_state_service: WorkflowExecutionStateService,
-    state_id: str,
-    timeout: int = int(CredentialsManager.get_parameter("DEFAULT_TIMEOUT", "120")),
-    pool_interval: int = 3,
-) -> WorkflowExecutionState:
-    start_time = time.time()
-    while time.time() - start_time < timeout:
-        states = [row for row in execution_state_service.list() if row.id == state_id]
-        if len(states) == 0:
-            sleep(pool_interval)
-            continue
-        state = states[0]
-        if state.status == WorkflowExecutionStatusEnum.SUCCEEDED:
-            return state
-        elif state.status == WorkflowExecutionStatusEnum.FAILED:
-            raise ApiError(
-                f"State execution failed: {execution_state_service.get_output(state_id)}"
-            )
-        sleep(pool_interval)
-    raise TimeoutError("State was not executed within the timeout period.")
-
-
 def clean_json(json_str):
     try:
         # Attempt to parse the JSON string directly

{codemie_test_harness-0.1.213 → codemie_test_harness-0.1.215}/codemie_test_harness/tests/utils/similarity_check.py

@@ -3,9 +3,9 @@ from hamcrest import assert_that, greater_than_or_equal_to
 
 from codemie_test_harness.tests.utils.base_utils import (
     wait_for_entity,
-    wait_for_completion,
     clean_json,
 )
+from codemie_test_harness.tests.utils.workflow_utils import WorkflowUtils
 
 
 class SimilarityCheck:
@@ -27,7 +27,7 @@ class SimilarityCheck:
             lambda: states_service.list(),
             entity_name="similarity_analysis",
         )
-        wait_for_completion(execution_state_service=states_service, state_id=state.id)
+        WorkflowUtils.wait_for_completion(service=states_service, _id=state.id)
         output = states_service.get_output(state_id=state.id).output
         output = int(clean_json(output).get("similarity_rank"))
         assert_that(

{codemie_test_harness-0.1.213 → codemie_test_harness-0.1.215}/codemie_test_harness/tests/utils/workflow_utils.py

@@ -1,21 +1,37 @@
 import os
 from pathlib import Path
 from typing import Optional
+import time
 
+
+from codemie_sdk.exceptions import ApiError
 from codemie_sdk.models.workflow import (
     WorkflowCreateRequest,
     WorkflowUpdateRequest,
     WorkflowMode,
+    WorkflowExecution,
+)
+from codemie_sdk.models.workflow_state import (
+    WorkflowExecutionState,
+    WorkflowExecutionStatusEnum,
+)
+from codemie_sdk.services.workflow_execution import WorkflowExecutionService
+from codemie_sdk.services.workflow_execution_state import WorkflowExecutionStateService
+
+from codemie_test_harness.tests import (
+    PROJECT,
+    VERIFY_SSL,
+    API_DOMEN,
+    CredentialsManager,
 )
-
-from codemie_test_harness.tests import PROJECT, VERIFY_SSL, API_DOMEN
 from codemie_test_harness.tests.utils.base_utils import (
     BaseUtils,
     get_random_name,
     wait_for_entity,
-    wait_for_completion,
 )
 from codemie_test_harness.tests.utils.http_utils import RequestHandler
+from time import sleep
+
 
 workflow_endpoint = "/v1/workflows"
 
@@ -106,17 +122,21 @@ class WorkflowUtils(BaseUtils):
         file_name: Optional[str] = None,
     ):
         self.client.workflows.run(workflow, user_input=user_input, file_name=file_name)
-        executions = self.client.workflows.executions(workflow)
+        executions_service = self.client.workflows.executions(workflow)
         execution_id = next(
-            row.execution_id for row in executions.list() if row.prompt == user_input
+            row.execution_id
+            for row in executions_service.list()
+            if row.prompt == user_input
         )
-        states_service = executions.states(execution_id)
+        states_service = executions_service.states(execution_id)
         state = wait_for_entity(
             lambda: states_service.list(),
             entity_name=execution_name,
         )
 
-        wait_for_completion(execution_state_service=states_service, state_id=state.id)
+        self.wait_for_completion(service=states_service, _id=state.id)
+        self.wait_for_completion(service=executions_service, _id=execution_id)
+
         return states_service.get_output(state_id=state.id).output
 
     def run_workflow(self, workflow_id, user_input=""):
@@ -279,3 +299,39 @@ class WorkflowUtils(BaseUtils):
                 process_thought(thought)
 
         return triggered_tools
+
+    @staticmethod
+    def wait_for_completion(
+        service: WorkflowExecutionStateService | WorkflowExecutionService,
+        _id: str,
+        timeout: int = int(CredentialsManager.get_parameter("DEFAULT_TIMEOUT", "120")),
+        pool_interval: int = 3,
+    ) -> WorkflowExecutionState | WorkflowExecution:
+        is_workflow_execution_state_state = isinstance(
+            service, WorkflowExecutionStateService
+        )
+        start_time = time.time()
+        while time.time() - start_time < timeout:
+            executions_or_states = [
+                row
+                for row in service.list()
+                if row.id == _id or row.execution_id == _id
+            ]
+            if len(executions_or_states) == 0:
+                sleep(pool_interval)
+                continue
+            execution_or_state = executions_or_states[0]
+            if execution_or_state.status == WorkflowExecutionStatusEnum.SUCCEEDED:
+                return execution_or_state
+            elif execution_or_state.status == WorkflowExecutionStatusEnum.FAILED:
+                if is_workflow_execution_state_state:
+                    raise ApiError(f"State execution failed: {service.get_output(_id)}")
+                else:
+                    raise ApiError(
+                        f"Execution failed. Execution id: {execution_or_state.execution_id}"
+                    )
+            sleep(pool_interval)
+        if is_workflow_execution_state_state:
+            raise TimeoutError("State was not executed within the timeout period.")
+        else:
+            raise TimeoutError("Workflow was not executed within the timeout period.")

{codemie_test_harness-0.1.213 → codemie_test_harness-0.1.215}/codemie_test_harness/tests/utils/yaml_utils.py

@@ -54,7 +54,6 @@ class StateModel(BaseModel):
 
 
 class WorkflowYamlModel(BaseModel):
-    enable_summarization_node: bool = False
     tools: List[ToolModel] = Field(default_factory=list)
     assistants: List[AssistantModel] = Field(default_factory=list)
     states: List[StateModel] = Field(default_factory=list)

{codemie_test_harness-0.1.213 → codemie_test_harness-0.1.215}/codemie_test_harness/tests/workflow/assistant_tools/project_management/test_workflow_with_assistant_pm_tools.py

@@ -4,7 +4,10 @@ from codemie_test_harness.tests.enums.tools import Toolkit
 from codemie_test_harness.tests.test_data.project_management_test_data import (
     pm_tools_test_data,
 )
-from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
+from codemie_test_harness.tests.utils.base_utils import (
+    assert_tool_triggered,
+    get_random_name,
+)
 from codemie_test_harness.tests.utils.constants import (
     project_management_integrations,
 )
@@ -36,6 +39,9 @@ def test_workflow_with_assistant_with_project_management_tools(
         project_management_integrations[integration_type]
     )
 
+    if "%s" in prompt:
+        prompt = prompt % get_random_name()
+
     assistant = assistant(
         Toolkit.PROJECT_MANAGEMENT,
         tool_name,

{codemie_test_harness-0.1.213 → codemie_test_harness-0.1.215}/codemie_test_harness/tests/workflow/test_workflows.py

@@ -3,6 +3,9 @@ import json
 import pytest
 from hamcrest import assert_that, equal_to
 
+from codemie_test_harness.tests.assistant.test_assistants import (
+    system_prompt_for_chatting_with_files,
+)
 from codemie_test_harness.tests.enums.tools import Default
 from codemie_test_harness.tests.test_data.assistant_test_data import (
     EXCEL_TOOL_TEST_DATA,
@@ -84,15 +87,17 @@ def test_workflow_with_user_input_and_file_attachment(
 
     # Create workflow with virtual assistant that has file analysis capabilities
     # Note: File analysis tools are automatically available when files are processed
-    system_prompt = "You are a helpful assistant that can analyze and process files. "
+    task = (
+        f"What is the content/text of the {file_name}. Show information from ALL pages. "
+        "For PDF use 'Text' for query. If provided file is CSV then run python_repl_ast tool and show first 5 rows."
+        "For images explain what you see on it."
+        "For excel show data from the sheet with name 'Sheet1'"
+    )
 
     workflow_instance = workflow_with_virtual_assistant(
         assistant_and_state_name=assistant_and_state_name,
-        system_prompt=system_prompt,
-        task=(
-            "Analyze the uploaded file from the provided file URL and give a detailed summary. "
-            "Use the appropriate file analysis tools based on the file type to extract and process the content."
-        ),
+        system_prompt=system_prompt_for_chatting_with_files,
+        task=task,
     )
 
     # Prepare user input that includes file reference

{codemie_test_harness-0.1.213 → codemie_test_harness-0.1.215}/codemie_test_harness/tests/workflow/virtual_assistant_tools/project_management/test_workflow_with_project_management_tools.py

@@ -36,6 +36,10 @@ def test_workflow_with_virtual_assistant_with_project_management_tools(
         project_management_integrations[integration_type]
     )
     assistant_and_state_name = get_random_name()
+
+    if "%s" in prompt:
+        prompt = prompt % get_random_name()
+
     test_workflow = workflow_with_virtual_assistant(
         assistant_and_state_name, tool_name, integration=integration
     )

{codemie_test_harness-0.1.213 → codemie_test_harness-0.1.215}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "codemie-test-harness"
-version = "0.1.213"
+version = "0.1.215"
 description = "Autotest for CodeMie backend and UI"
 authors = [
     "Anton Yeromin <anton_yeromin@epam.com>",
@@ -12,7 +12,7 @@ packages = [
 
 [tool.poetry.dependencies]
 python = "^3.12"
-codemie-sdk-python = "0.1.213"
+codemie-sdk-python = "0.1.215"
 pytest = "^8.4.1"
 python-dotenv = "^1.1.0"
 PyHamcrest = "^2.1.0"