codemie-test-harness 0.1.200__py3-none-any.whl → 0.1.202__py3-none-any.whl

codemie_test_harness/tests/utils/workflow_utils.py

@@ -1,10 +1,13 @@
 import os
+from pathlib import Path
+from typing import Optional
 
 from codemie_sdk.models.workflow import (
     WorkflowCreateRequest,
     WorkflowUpdateRequest,
     WorkflowMode,
 )
+
 from codemie_test_harness.tests import PROJECT
 from codemie_test_harness.tests.utils.base_utils import (
     BaseUtils,
@@ -97,8 +100,14 @@ class WorkflowUtils(BaseUtils):
             entity_name=response[1],
         )
 
-    def execute_workflow(self, workflow, execution_name, user_input=""):
-        self.client.workflows.run(workflow, user_input=user_input)
+    def execute_workflow(
+        self,
+        workflow,
+        execution_name,
+        user_input="",
+        file_name: Optional[str] = None,
+    ):
+        self.client.workflows.run(workflow, user_input=user_input, file_name=file_name)
         executions = self.client.workflows.executions(workflow)
         execution_id = next(
             row.execution_id for row in executions.list() if row.prompt == user_input
@@ -232,6 +241,9 @@ class WorkflowUtils(BaseUtils):
 
         return triggered_tools
 
+    def upload_file(self, file_path: Path):
+        return self.client.files.bulk_upload([file_path])
+
     @staticmethod
     def _extract_tools_from_thoughts(thoughts_data):
         """

codemie_test_harness/tests/workflow/test_workflows.py

@@ -3,8 +3,18 @@ import json
 import pytest
 from hamcrest import assert_that, equal_to
 
+from codemie_test_harness.tests.enums.tools import Default
+from codemie_test_harness.tests.test_data.assistant_test_data import (
+    EXCEL_TOOL_TEST_DATA,
+    DOCX_TOOL_TEST_DATA,
+)
+from codemie_test_harness.tests.test_data.file_test_data import file_test_data
 from codemie_test_harness.tests.test_data.output_schema_test_data import output_schema
-from codemie_test_harness.tests.utils.base_utils import get_random_name
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_tool_triggered,
+)
+from codemie_test_harness.tests.utils.constants import FILES_PATH
 from codemie_test_harness.tests.utils.yaml_utils import AssistantModel, StateModel
 
 
@@ -37,3 +47,191 @@ def test_workflow_with_json_output_schema(default_llm, workflow, workflow_utils)
     )
 
     assert_that(json.loads(response)["results"][0], equal_to(2))
+
+
+@pytest.mark.workflow
+@pytest.mark.api
+@pytest.mark.smoke
+@pytest.mark.file
+@pytest.mark.parametrize(
+    "file_name, expected_response, expected_tool",
+    file_test_data,
+    ids=[f"{row[0]}" for row in file_test_data],
+)
+def test_workflow_with_user_input_and_file_attachment(
+    workflow_with_virtual_assistant,
+    workflow_utils,
+    assistant_utils,
+    file_name,
+    expected_response,
+    expected_tool,
+    similarity_check,
+):
+    """
+    Test workflow execution with user input that includes file attachment.
+
+    This test demonstrates how workflows can handle file attachments by:
+    1. Uploading a file
+    2. Creating a workflow with file processing capabilities
+    3. Executing the workflow with user input that references the uploaded file
+    4. Verifying that file processing tools are triggered
+    """
+    assistant_and_state_name = get_random_name()
+
+    # Upload file to get file URL
+    upload_response = workflow_utils.upload_file(FILES_PATH / file_name)
+    file_url = upload_response.files[0].file_url
+
+    # Create workflow with virtual assistant that has file analysis capabilities
+    # Note: File analysis tools are automatically available when files are processed
+    system_prompt = "You are a helpful assistant that can analyze and process files. "
+
+    workflow_instance = workflow_with_virtual_assistant(
+        assistant_and_state_name=assistant_and_state_name,
+        system_prompt=system_prompt,
+        task=(
+            "Analyze the uploaded file from the provided file URL and give a detailed summary. "
+            "Use the appropriate file analysis tools based on the file type to extract and process the content."
+        ),
+    )
+
+    # Prepare user input that includes file reference
+    user_input = "Please provide a summary about file content."
+
+    # Execute workflow
+    response = workflow_utils.execute_workflow(
+        workflow_instance.id,
+        assistant_and_state_name,
+        user_input=user_input,
+        file_name=file_url,
+    )
+
+    # Extract triggered tools from execution
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        workflow_instance
+    )
+
+    assert_tool_triggered(expected_tool, triggered_tools)
+    similarity_check.check_similarity(response, expected_response)
+
+
+@pytest.mark.workflow
+@pytest.mark.file
+@pytest.mark.api
+@pytest.mark.smoke
+@pytest.mark.parametrize("prompt, expected_response", EXCEL_TOOL_TEST_DATA)
+def test_workflow_excel_tool_extended_functionality(
+    workflow_with_virtual_assistant,
+    workflow_utils,
+    similarity_check,
+    prompt,
+    expected_response,
+):
+    """
+    Test extended Excel tool functionality with various scenarios in workflow.
+
+    This test covers:
+    - Data extraction from visible sheets only
+    - All data including hidden sheets
+    - Sheet name listing functionality
+    - File statistics and structure analysis
+    - Single sheet extraction by index and name
+    - Data cleaning and normalization
+    - Hidden sheet visibility control
+    - Column structure and data type analysis
+    - Tabular structure normalization
+    - Multi-sheet comprehensive analysis
+
+    """
+    assistant_and_state_name = get_random_name()
+
+    # Upload file to get file URL
+    upload_response = workflow_utils.upload_file(FILES_PATH / "test_extended.xlsx")
+    file_url = upload_response.files[0].file_url
+
+    # Create workflow with virtual assistant that has Excel file processing capabilities
+    system_prompt = "You have all required information in initial prompt. Do not ask additional questions and proceed with request."
+
+    workflow_instance = workflow_with_virtual_assistant(
+        assistant_and_state_name=assistant_and_state_name,
+        system_prompt=system_prompt,
+        task=prompt,
+    )
+
+    # Execute workflow with the file URL
+    response = workflow_utils.execute_workflow(
+        workflow_instance.id,
+        assistant_and_state_name,
+        user_input="Process the uploaded Excel file as requested.",
+        file_name=file_url,
+    )
+
+    # Extract triggered tools from execution
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        workflow_instance
+    )
+
+    assert_tool_triggered(Default.EXCEL_TOOL, triggered_tools)
+    similarity_check.check_similarity(response, expected_response)
+
+
+@pytest.mark.workflow
+@pytest.mark.file
+@pytest.mark.api
+@pytest.mark.smoke
+@pytest.mark.parametrize("prompt, expected_response", DOCX_TOOL_TEST_DATA)
+def test_workflow_docx_tool_extended_functionality(
+    workflow_with_virtual_assistant,
+    workflow_utils,
+    similarity_check,
+    prompt,
+    expected_response,
+):
+    """
+    Test extended Docx tool functionality with various scenarios in workflow.
+
+    This test covers:
+    - Extract plain text using 'text' query
+    - Extract text with metadata using 'text_with_metadata' query
+    - Extract document structure using 'structure_only' query
+    - Extract tables using 'table_extraction' query
+    - Generate summary using 'summary' query
+    - Perform analysis with custom instructions using 'analyze' query
+    - Process specific pages '1-3' using pages parameter
+    - Process specific pages '1,5,10' using pages parameter
+    - Extract images using 'image_extraction' query
+    - Extract text with OCR from images using 'text_with_images' query
+
+    """
+    assistant_and_state_name = get_random_name()
+
+    # Upload file to get file URL
+    upload_response = workflow_utils.upload_file(FILES_PATH / "test_extended.docx")
+    file_url = upload_response.files[0].file_url
+
+    # Create workflow with virtual assistant that has DOCX file processing capabilities
+    system_prompt = """You are a helpful assistant that can analyze and process DOCX files.
+                       You have all required information in initial prompt.
+                       Do not ask additional questions and proceed with request."""
+
+    workflow_instance = workflow_with_virtual_assistant(
+        assistant_and_state_name=assistant_and_state_name,
+        system_prompt=system_prompt,
+        task=prompt,
+    )
+
+    # Execute workflow with the file URL
+    response = workflow_utils.execute_workflow(
+        workflow_instance.id,
+        assistant_and_state_name,
+        user_input="Process the uploaded DOCX file as requested.",
+        file_name=file_url,
+    )
+
+    # Extract triggered tools from execution
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        workflow_instance
+    )
+
+    assert_tool_triggered(Default.DOCX_TOOL, triggered_tools)
+    similarity_check.check_similarity(response, expected_response)

{codemie_test_harness-0.1.200.dist-info → codemie_test_harness-0.1.202.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: codemie-test-harness
-Version: 0.1.200
+Version: 0.1.202
 Summary: Autotest for CodeMie backend and UI
 Author: Anton Yeromin
 Author-email: anton_yeromin@epam.com
@@ -13,7 +13,7 @@ Requires-Dist: aws-assume-role-lib (>=2.10.0,<3.0.0)
 Requires-Dist: boto3 (>=1.39.8,<2.0.0)
 Requires-Dist: click (>=8.1.7,<9.0.0)
 Requires-Dist: codemie-plugins (>=0.1.123,<0.2.0)
-Requires-Dist: codemie-sdk-python (==0.1.200)
+Requires-Dist: codemie-sdk-python (==0.1.202)
 Requires-Dist: pytest (>=8.4.1,<9.0.0)
 Requires-Dist: pytest-playwright (>=0.7.0,<0.8.0)
 Requires-Dist: pytest-repeat (>=0.9.3,<0.10.0)
@@ -59,6 +59,7 @@ The CLI now provides four main command groups:
 2. **`run`** - Enhanced test execution with flexible parameters
 3. **`assistant`** - Direct assistant interaction and chat capabilities
 4. **`workflow`** - Workflow execution
+5. **`marks`** - List all available pytest marks in the test suite.
 
 Each command group includes extensive help and validation features.
 
@@ -101,6 +102,49 @@ codemie-test-harness config set PYTEST_RERUNS 2
 codemie-test-harness config set PYTEST_COUNT 10  # For performance testing (optional)
 ```
 
+#### Running Tests Locally (Minimal Configuration)
+
+If you're running tests against a **local CodeMie instance** and want to use integration settings already stored in AWS Parameter Store (instead of configuring each integration manually), you only need to set these **4 minimal keys**:
+
+```shell
+# Minimal local setup - uses AWS Parameter Store for all integrations
+codemie-test-harness config set AWS_ACCESS_KEY <your_aws_access_key>
+codemie-test-harness config set AWS_SECRET_KEY <your_aws_secret_key>
+codemie-test-harness config set CODEMIE_API_DOMAIN http://localhost:8080
+codemie-test-harness config set TEST_USER_FULL_NAME "dev-codemie-user"
+```
+
+Then run your tests:
+
+```shell
+# Run all tests locally
+codemie-test-harness run
+
+# Run specific test categories
+codemie-test-harness run --marks smoke
+codemie-test-harness run --marks "api and not not_for_parallel_run" -n 8
+```
+
+**How it works:**
+- The AWS credentials allow the test harness to automatically fetch integration credentials (GitLab, JIRA, Confluence, etc.) from AWS Parameter Store
+- You don't need to manually configure individual integrations unless you want to override specific values
+- All 86+ integration variables are automatically pulled from Parameter Store as needed
+- This is ideal for local development and testing against your local CodeMie backend
+
+**To override specific integrations locally:**
+
+If you need to use your own tokens instead of shared Parameter Store values:
+
+```shell
+# Override with personal GitLab token
+codemie-test-harness config set GITLAB_TOKEN <your_personal_token>
+
+# Override with personal JIRA credentials
+codemie-test-harness config set JIRA_TOKEN <your_personal_token>
+```
+
+Values set explicitly in the config take priority over AWS Parameter Store values.
+
 #### Integration Categories & Management
 
 The CLI supports **10 major integration categories** with comprehensive credential management:
@@ -296,6 +340,35 @@ codemie-test-harness --git-env github run --marks git
 codemie-test-harness run --marks ui --headless
 ```
 
+#### Advanced Marks Usage (Logical Operators)
+
+Combine multiple markers using `and`, `or`, and `not` keywords for fine-grained test selection:
+
+```shell
+# OR operator - run tests with either marker
+codemie-test-harness run --marks "smoke or gitlab" -n 8
+codemie-test-harness run --marks "jira_kb or confluence_kb" -n 6
+
+# AND operator - run tests with both markers
+codemie-test-harness run --marks "api and smoke" -n 10
+codemie-test-harness run --marks "gitlab and code_kb" -n 4
+
+# NOT operator - exclude specific markers
+codemie-test-harness run --marks "api and not ui" -n 10
+codemie-test-harness run --marks "not not_for_parallel_run" -n 12
+
+# Complex combinations with parentheses
+codemie-test-harness run --marks "(smoke or api) and not ui" -n 8
+codemie-test-harness run --marks "(gitlab or github) and not not_for_parallel_run" -n 10
+codemie-test-harness run --marks "smoke and (jira_kb or confluence_kb)" -n 6
+
+# Exclude multiple markers
+codemie-test-harness run --marks "api and not (ui or not_for_parallel_run)" -n 10
+
+# Run all knowledge base tests except code
+codemie-test-harness run --marks "(jira_kb or confluence_kb) and not code_kb" -n 8
+```
+
 #### Performance and Load Testing
 
 Run tests multiple times in parallel to simulate load and measure performance:
@@ -542,7 +615,7 @@ The credentials manager supports **86+ environment variables** across **10 categ
 2) Create a .env file in the project root. If you provide AWS credentials, the suite will fetch additional values from AWS Systems Manager Parameter Store and recreate .env accordingly.
 
 ```properties
-ENV=local
+CODEMIE_API_DOMAIN=http://localhost:8080
 
 AWS_ACCESS_KEY=<aws_access_token>
 AWS_SECRET_KEY=<aws_secret_key>
@@ -594,9 +667,42 @@ pytest -n 10 --count 50 -m excel_generation  # Run 50 times with 10 workers
 pytest -n 20 --count 100 -m smoke --reruns 2  # Heavy load with retries
 ```
 
+**Advanced Marks Usage with Logical Operators:**
+
+Combine markers using `and`, `or`, and `not` for precise test selection:
+
+```shell
+# OR operator - run tests with either marker
+pytest -n 8 -m "smoke or gitlab" --reruns 2
+pytest -n 6 -m "jira_kb or confluence_kb" --reruns 2
+
+# AND operator - run tests with both markers
+pytest -n 10 -m "api and smoke" --reruns 2
+pytest -n 4 -m "gitlab and code_kb" --reruns 2
+
+# NOT operator - exclude specific markers
+pytest -n 10 -m "api and not ui" --reruns 2
+pytest -n 12 -m "not not_for_parallel_run" --reruns 2
+
+# Complex combinations with parentheses
+pytest -n 8 -m "(smoke or api) and not ui" --reruns 2
+pytest -n 10 -m "(gitlab or github) and not not_for_parallel_run" --reruns 2
+pytest -n 6 -m "smoke and (jira_kb or confluence_kb)" --reruns 2
+
+# Exclude multiple markers
+pytest -n 10 -m "api and not (ui or not_for_parallel_run)" --reruns 2
+
+# Run all knowledge base tests except code KB
+pytest -n 8 -m "(jira_kb or confluence_kb) and not code_kb" --reruns 2
+
+# Run all Git-related tests (GitLab or GitHub)
+pytest -n 8 -m "gitlab or github or git" --reruns 2
+```
+
 **Notes:**
 - `--reruns 2` uses pytest-rerunfailures to improve resiliency in flaky environments
 - `--count N` uses pytest-repeat to run each test N times (useful for performance/load testing)
+- Use quotes around marker expressions containing spaces or special characters
 
 #### Test Timeout Configuration
 

{codemie_test_harness-0.1.200.dist-info → codemie_test_harness-0.1.202.dist-info}/RECORD

@@ -284,7 +284,7 @@ codemie_test_harness/tests/utils/pytest_utils.py,sha256=k-mEjX2qpnh37sqKpJqYhZT6
 codemie_test_harness/tests/utils/search_utils.py,sha256=SrXiB2d9wiI5ka9bgg0CD73GOX_1mqi2Hz5FBm5DsEU,1435
 codemie_test_harness/tests/utils/similarity_check.py,sha256=1U66NGh6esISKABodtVobE2WnuFt0f6vcK3qUri6ZqU,1485
 codemie_test_harness/tests/utils/user_utils.py,sha256=zJNrmL3Fb7iGuaVRobUMwJ2Og6NqEPcM_9lw60m18T8,242
-codemie_test_harness/tests/utils/workflow_utils.py,sha256=awEoO0U23_JkHzg_DAZyblM9GW-rQ6hYjdsqRBdyeOw,9101
+codemie_test_harness/tests/utils/workflow_utils.py,sha256=Kz4F4Xzdx42PNmye--ENpjSakdB1Y-QL_nwSfKhMAXE,9359
 codemie_test_harness/tests/utils/yaml_utils.py,sha256=y9fUf4u4G4SoCktPOwaC5x71iaDKhktbz_XUfI9kNis,1661
 codemie_test_harness/tests/workflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/workflow/assistant_tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -350,7 +350,7 @@ codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_repo
 codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_research_tools.py,sha256=lQzzWV6mcKH1_8k4-OxXoI9_hb48JJbYAmDT9bj-WQ0,2558
 codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_servicenow_tools.py,sha256=meR9c8BgMTgM6LpwqE0n7ygTCIHGVooMvYfb6kGF7UE,2377
 codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_vcs_tools.py,sha256=dbfjHnIhjneW3_JZyElvmswE7omOp0A0EgzoztZ7vow,3098
-codemie_test_harness/tests/workflow/test_workflows.py,sha256=9AJbZ3URcJH3M9yu6NMJmO7BXv8-CibiznN-qk38DmA,1156
+codemie_test_harness/tests/workflow/test_workflows.py,sha256=HxwY2VKGLvLW2sl3CwR_0xFOmXrLwj_nfGdqnC39CSw,8129
 codemie_test_harness/tests/workflow/virtual_assistant_tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/workflow/virtual_assistant_tools/access_management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/workflow/virtual_assistant_tools/access_management/test_workflow_with_keycloak_tool.py,sha256=eLM804LRG-Rd5ZIgrO_W53FGoSJ_TxWPKQPtWG598CY,1203
@@ -391,7 +391,7 @@ codemie_test_harness/tests/workflow/virtual_assistant_tools/servicenow/__init__.
 codemie_test_harness/tests/workflow/virtual_assistant_tools/servicenow/test_workflow_with_servicenow_tools.py,sha256=D835gaRbCnB4va5mi9TdA_u9StSpGXQ_fgzwW0S2pwo,1173
 codemie_test_harness/tests/workflow/virtual_assistant_tools/vcs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/workflow/virtual_assistant_tools/vcs/test_workflow_with_vcs_tools.py,sha256=Se9imIiBYuJU78m1pLu0g4ZmHygKZjr6JjIWkGXTy1Q,1364
-codemie_test_harness-0.1.200.dist-info/METADATA,sha256=s-ASsAfnRD0a5t_Yiy30KzzDXtV-O5P1xICb8YTWEeE,22968
-codemie_test_harness-0.1.200.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-codemie_test_harness-0.1.200.dist-info/entry_points.txt,sha256=n98t-EOM5M1mnMl_j2X4siyeO9zr0WD9a5LF7JyElIM,73
-codemie_test_harness-0.1.200.dist-info/RECORD,,
+codemie_test_harness-0.1.202.dist-info/METADATA,sha256=9xYAqqEciTJKycdxV0HID1vYNmZFI-qLDaRKtFkC1lE,27184
+codemie_test_harness-0.1.202.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+codemie_test_harness-0.1.202.dist-info/entry_points.txt,sha256=n98t-EOM5M1mnMl_j2X4siyeO9zr0WD9a5LF7JyElIM,73
+codemie_test_harness-0.1.202.dist-info/RECORD,,