codemie-test-harness 0.1.159__py3-none-any.whl → 0.1.160__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of codemie-test-harness might be problematic. Click here for more details.

Files changed (59) hide show
  1. codemie_test_harness/tests/assistant/datasource/test_confluence_datasource.py +2 -1
  2. codemie_test_harness/tests/assistant/datasource/test_jira_datasource.py +2 -1
  3. codemie_test_harness/tests/assistant/tools/cloud/test_cloud_tools.py +0 -7
  4. codemie_test_harness/tests/assistant/tools/codebase/test_codebase_tools.py +0 -1
  5. codemie_test_harness/tests/assistant/tools/datamanagement/test_assistant_with_data_management_tools.py +4 -5
  6. codemie_test_harness/tests/assistant/tools/filemanagement/test_assistant_with_file_management_tools.py +2 -9
  7. codemie_test_harness/tests/assistant/tools/mcp/test_cli_mcp_server.py +5 -7
  8. codemie_test_harness/tests/assistant/tools/mcp/test_mcp_servers.py +5 -7
  9. codemie_test_harness/tests/assistant/tools/notification/test_assistant_notification_tools.py +3 -3
  10. codemie_test_harness/tests/assistant/tools/openapi/test_assistant_with_open_api_tools.py +3 -2
  11. codemie_test_harness/tests/conftest.py +6 -2
  12. codemie_test_harness/tests/enums/environment.py +102 -0
  13. codemie_test_harness/tests/enums/model_types.py +1 -0
  14. codemie_test_harness/tests/integrations/project/test_default_integrations.py +3 -11
  15. codemie_test_harness/tests/integrations/project/test_project_integrations.py +0 -132
  16. codemie_test_harness/tests/integrations/user/test_default_integrations.py +3 -11
  17. codemie_test_harness/tests/integrations/user/test_user_integrations.py +0 -132
  18. codemie_test_harness/tests/llm/assistants/test_llm.py +3 -3
  19. codemie_test_harness/tests/service/test_assistant_service.py +2 -2
  20. codemie_test_harness/tests/test_data/cloud_tools_test_data.py +32 -11
  21. codemie_test_harness/tests/test_data/codebase_tools_test_data.py +2 -0
  22. codemie_test_harness/tests/test_data/data_management_tools_test_data.py +3 -3
  23. codemie_test_harness/tests/test_data/direct_tools/cloud_tools_test_data.py +7 -4
  24. codemie_test_harness/tests/test_data/direct_tools/codebase_tools_test_data.py +2 -0
  25. codemie_test_harness/tests/test_data/direct_tools/data_management_tools_test_data.py +4 -5
  26. codemie_test_harness/tests/test_data/direct_tools/file_management_tools_test_data.py +2 -2
  27. codemie_test_harness/tests/test_data/direct_tools/notification_tools_test_data.py +5 -2
  28. codemie_test_harness/tests/test_data/direct_tools/project_management_tools_test_data.py +2 -0
  29. codemie_test_harness/tests/test_data/direct_tools/research_tools_test_data.py +1 -0
  30. codemie_test_harness/tests/test_data/direct_tools/vcs_tools_test_data.py +3 -0
  31. codemie_test_harness/tests/test_data/file_management_tools_test_data.py +9 -5
  32. codemie_test_harness/tests/test_data/index_test_data.py +9 -11
  33. codemie_test_harness/tests/test_data/integrations_test_data.py +55 -9
  34. codemie_test_harness/tests/test_data/llm_test_data.py +8 -6
  35. codemie_test_harness/tests/test_data/project_management_test_data.py +4 -0
  36. codemie_test_harness/tests/test_data/vcs_tools_test_data.py +11 -2
  37. codemie_test_harness/tests/utils/aws_parameters_store.py +5 -2
  38. codemie_test_harness/tests/utils/constants.py +1 -1
  39. codemie_test_harness/tests/utils/env_resolver.py +119 -0
  40. codemie_test_harness/tests/workflow/assistant_tools/cloud/test_workflow_with_assistant_cloud_tools.py +0 -7
  41. codemie_test_harness/tests/workflow/assistant_tools/codebase/test_worfklow_with_assistant_codebase_tools.py +0 -1
  42. codemie_test_harness/tests/workflow/assistant_tools/data_management/test_workflow_with_assistant_with_data_management_tools.py +3 -5
  43. codemie_test_harness/tests/workflow/assistant_tools/file_management/test_workflow_with_assistant_with_file_management_tools.py +2 -9
  44. codemie_test_harness/tests/workflow/assistant_tools/mcp/test_workflow_with_assistant_with_mcp_server.py +5 -10
  45. codemie_test_harness/tests/workflow/assistant_tools/notification/test_workflow_with_assistant_notification_tools.py +3 -2
  46. codemie_test_harness/tests/workflow/assistant_tools/open_api/test_workflow_with_assistant_with_open_api_tools.py +3 -2
  47. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_data_management_tools_sql.py +3 -2
  48. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_open_api_tools.py +3 -2
  49. codemie_test_harness/tests/workflow/virtual_assistant_tools/cloud/test_workflow_with_cloud_tools.py +0 -7
  50. codemie_test_harness/tests/workflow/virtual_assistant_tools/codebase/test_workflow_with_codebase_tools.py +0 -1
  51. codemie_test_harness/tests/workflow/virtual_assistant_tools/data_management/test_workflow_with_data_management_tools.py +3 -5
  52. codemie_test_harness/tests/workflow/virtual_assistant_tools/file_management/test_workflow_with_file_management_tools.py +2 -9
  53. codemie_test_harness/tests/workflow/virtual_assistant_tools/mcp/test_workflow_with_mcp_server.py +5 -11
  54. codemie_test_harness/tests/workflow/virtual_assistant_tools/notification/test_workflow_with_notification_tools.py +3 -3
  55. codemie_test_harness/tests/workflow/virtual_assistant_tools/open_api/test_workflow_with_open_api_tools.py +3 -3
  56. {codemie_test_harness-0.1.159.dist-info → codemie_test_harness-0.1.160.dist-info}/METADATA +2 -2
  57. {codemie_test_harness-0.1.159.dist-info → codemie_test_harness-0.1.160.dist-info}/RECORD +59 -57
  58. {codemie_test_harness-0.1.159.dist-info → codemie_test_harness-0.1.160.dist-info}/WHEEL +0 -0
  59. {codemie_test_harness-0.1.159.dist-info → codemie_test_harness-0.1.160.dist-info}/entry_points.txt +0 -0
@@ -1,5 +1,3 @@
1
- import os
2
-
3
1
  import pytest
4
2
  from hamcrest import assert_that, contains_string, is_not, all_of
5
3
 
@@ -18,6 +16,7 @@ from codemie_test_harness.tests.test_data.file_management_tools_test_data import
18
16
  RESPONSE_FOR_FILE_EDITOR,
19
17
  )
20
18
  from codemie_test_harness.tests.utils.base_utils import get_random_name
19
+ from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
21
20
 
22
21
 
23
22
  @pytest.mark.workflow
@@ -27,12 +26,6 @@ from codemie_test_harness.tests.utils.base_utils import get_random_name
27
26
  @pytest.mark.parametrize(
28
27
  "tool_name, prompt, expected_response",
29
28
  file_management_tools_test_data,
30
- ids=[
31
- FileManagementTool.PYTHON_CODE_INTERPRETER,
32
- FileManagementTool.LIST_DIRECTORY,
33
- FileManagementTool.WRITE_FILE,
34
- FileManagementTool.RUN_COMMAND_LINE,
35
- ],
36
29
  )
37
30
  def test_workflow_with_assistant_with_file_management_tools(
38
31
  assistant,
@@ -58,7 +51,7 @@ def test_workflow_with_assistant_with_file_management_tools(
58
51
  @pytest.mark.file_management
59
52
  @pytest.mark.regression
60
53
  @pytest.mark.skipif(
61
- os.getenv("ENV") == "local", reason="Skipping this test on local environment"
54
+ EnvironmentResolver.is_localhost(), reason="Skipping this test on local environment"
62
55
  )
63
56
  def test_workflow_with_assistant_with_generate_image_tool(
64
57
  assistant,
@@ -1,4 +1,4 @@
1
- import os
1
+ from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
2
2
 
3
3
  import pytest
4
4
 
@@ -15,14 +15,15 @@ from codemie_test_harness.tests.test_data.mcp_server_test_data import (
15
15
  CLI_MCP_SERVER,
16
16
  )
17
17
 
18
+ pytestmark = pytest.mark.skipif(
19
+ EnvironmentResolver.is_localhost(), reason="Skipping this test on local environment"
20
+ )
21
+
18
22
 
19
23
  @pytest.mark.workflow
20
24
  @pytest.mark.workflow_with_assistant
21
25
  @pytest.mark.mcp
22
26
  @pytest.mark.regression
23
- @pytest.mark.skipif(
24
- os.getenv("ENV") == "local", reason="Skipping this test on local environment"
25
- )
26
27
  def test_workflow_with_assistant_with_time_mcp_server(
27
28
  assistant,
28
29
  workflow_with_assistant,
@@ -49,9 +50,6 @@ def test_workflow_with_assistant_with_time_mcp_server(
49
50
  cli_mcp_server_test_data,
50
51
  ids=[f"{row[0]}" for row in cli_mcp_server_test_data],
51
52
  )
52
- @pytest.mark.skipif(
53
- os.getenv("ENV") == "local", reason="Skipping this test on local environment"
54
- )
55
53
  def test_workflow_with_assistant_with_cli_mcp_server(
56
54
  assistant,
57
55
  workflow_with_assistant,
@@ -77,9 +75,6 @@ def test_workflow_with_assistant_with_cli_mcp_server(
77
75
  @pytest.mark.workflow_with_assistant
78
76
  @pytest.mark.mcp
79
77
  @pytest.mark.regression
80
- @pytest.mark.skipif(
81
- os.getenv("ENV") == "local", reason="Skipping this test on local environment"
82
- )
83
78
  def test_workflow_with_assistant_with_fetch_mcp_server(
84
79
  assistant,
85
80
  workflow_with_assistant,
@@ -1,4 +1,5 @@
1
- import os
1
+ from codemie_test_harness.tests.enums.environment import Environment
2
+ from codemie_test_harness.tests.utils.env_resolver import get_environment
2
3
 
3
4
  import pytest
4
5
  from hamcrest import assert_that, equal_to
@@ -20,7 +21,7 @@ from codemie_test_harness.tests.test_data.notification_tools_test_data import (
20
21
  @pytest.mark.email
21
22
  @pytest.mark.regression
22
23
  @pytest.mark.skipif(
23
- os.getenv("ENV") in ["local", "gcp"],
24
+ get_environment() in [Environment.LOCALHOST, Environment.GCP],
24
25
  reason="Skipping this test on local environment",
25
26
  )
26
27
  def test_workflow_with_email_tool(
@@ -1,4 +1,4 @@
1
- import os
1
+ from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
2
2
 
3
3
  import pytest
4
4
 
@@ -18,7 +18,8 @@ from codemie_test_harness.tests.test_data.open_api_tools_test_data import (
18
18
  ids=[f"{row[0]}" for row in open_api_tools_test_data],
19
19
  )
20
20
  @pytest.mark.skipif(
21
- os.getenv("ENV") == "azure", reason="Still have an issue with encoding long strings"
21
+ EnvironmentResolver.is_azure(),
22
+ reason="Still have an issue with encoding long strings",
22
23
  )
23
24
  def test_workflow_with_assistant_with_open_api_tools(
24
25
  assistant,
@@ -1,6 +1,5 @@
1
1
  import copy
2
2
  import json
3
- import os
4
3
  import random
5
4
 
6
5
  import pytest
@@ -11,10 +10,12 @@ from codemie_test_harness.tests.test_data.direct_tools.data_management_tools_tes
11
10
  sql_tools_test_data,
12
11
  )
13
12
  from codemie_test_harness.tests.utils.aws_parameters_store import CredentialsUtil
13
+ from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
14
14
  from codemie_test_harness.tests.utils.base_utils import get_random_name
15
15
 
16
16
  pytestmark = pytest.mark.skipif(
17
- os.getenv("ENV") == "local", reason="Skipping this tests on local environment"
17
+ EnvironmentResolver.is_localhost(),
18
+ reason="Skipping this tests on local environment",
18
19
  )
19
20
 
20
21
 
@@ -1,6 +1,5 @@
1
1
  import copy
2
2
  import json
3
- import os
4
3
  import random
5
4
 
6
5
  import pytest
@@ -9,9 +8,11 @@ from codemie_test_harness.tests.test_data.direct_tools.open_api_tools_test_data
9
8
  open_api_tools_test_data,
10
9
  )
11
10
  from codemie_test_harness.tests.utils.base_utils import get_random_name
11
+ from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
12
12
 
13
13
  pytestmark = pytest.mark.skipif(
14
- os.getenv("ENV") == "azure", reason="Still have an issue with encoding long strings"
14
+ EnvironmentResolver.is_azure(),
15
+ reason="Still have an issue with encoding long strings",
15
16
  )
16
17
 
17
18
 
@@ -1,6 +1,5 @@
1
1
  import pytest
2
2
 
3
- from codemie_test_harness.tests.enums.tools import Toolkit, CloudTool
4
3
  from codemie_test_harness.tests.test_data.cloud_tools_test_data import cloud_test_data
5
4
  from codemie_test_harness.tests.utils.base_utils import get_random_name
6
5
 
@@ -15,12 +14,6 @@ from codemie_test_harness.tests.utils.base_utils import get_random_name
15
14
  @pytest.mark.parametrize(
16
15
  "toolkit, tool_name, credential_type, credentials, prompt, expected_response",
17
16
  cloud_test_data,
18
- ids=[
19
- f"{Toolkit.CLOUD}_{CloudTool.AWS}",
20
- f"{Toolkit.CLOUD}_{CloudTool.AZURE}",
21
- f"{Toolkit.CLOUD}_{CloudTool.GCP}",
22
- f"{Toolkit.CLOUD}_{CloudTool.KUBERNETES}",
23
- ],
24
17
  )
25
18
  def test_workflow_with_cloud_tools(
26
19
  workflow_with_virtual_assistant,
@@ -70,7 +70,6 @@ def test_workflow_with_codebase_tools(
70
70
  @pytest.mark.parametrize(
71
71
  "toolkit, tool_name, credentials, prompt, expected_response",
72
72
  sonar_tools_test_data,
73
- ids=[f"{row[1]}" for row in sonar_tools_test_data],
74
73
  )
75
74
  def test_workflow_with_sonar_tools(
76
75
  workflow_with_virtual_assistant,
@@ -1,9 +1,6 @@
1
- import os
2
-
3
1
  import pytest
4
2
  from codemie_sdk.models.integration import CredentialTypes
5
3
 
6
- from codemie_test_harness.tests.enums.integrations import DataBaseDialect
7
4
  from codemie_test_harness.tests.enums.tools import DataManagementTool
8
5
  from codemie_test_harness.tests.test_data.data_management_tools_test_data import (
9
6
  ELASTIC_TOOL_TASK,
@@ -16,10 +13,12 @@ from codemie_test_harness.tests.test_data.data_management_tools_test_data import
16
13
  RESPONSE_FOR_SQL,
17
14
  )
18
15
  from codemie_test_harness.tests.utils.aws_parameters_store import CredentialsUtil
16
+ from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
19
17
  from codemie_test_harness.tests.utils.base_utils import get_random_name
20
18
 
21
19
  pytestmark = pytest.mark.skipif(
22
- os.getenv("ENV") == "local", reason="Skipping this tests on local environment"
20
+ EnvironmentResolver.is_localhost(),
21
+ reason="Skipping this tests on local environment",
23
22
  )
24
23
 
25
24
 
@@ -61,7 +60,6 @@ def test_workflow_with_elastic_tools(
61
60
  @pytest.mark.parametrize(
62
61
  "db_dialect",
63
62
  sql_tools_test_data,
64
- ids=[DataBaseDialect.MY_SQL, DataBaseDialect.POSTGRES, DataBaseDialect.MS_SQL],
65
63
  )
66
64
  def test_workflow_with_sql_tools(
67
65
  workflow_with_virtual_assistant,
@@ -1,5 +1,3 @@
1
- import os
2
-
3
1
  import pytest
4
2
  from hamcrest import assert_that, contains_string, is_not, all_of
5
3
 
@@ -18,6 +16,7 @@ from codemie_test_harness.tests.test_data.file_management_tools_test_data import
18
16
  RESPONSE_FOR_FILE_EDITOR,
19
17
  )
20
18
  from codemie_test_harness.tests.utils.base_utils import get_random_name
19
+ from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
21
20
 
22
21
 
23
22
  @pytest.mark.workflow
@@ -28,12 +27,6 @@ from codemie_test_harness.tests.utils.base_utils import get_random_name
28
27
  @pytest.mark.parametrize(
29
28
  "tool_name, prompt, expected_response",
30
29
  file_management_tools_test_data,
31
- ids=[
32
- FileManagementTool.PYTHON_CODE_INTERPRETER,
33
- FileManagementTool.LIST_DIRECTORY,
34
- FileManagementTool.WRITE_FILE,
35
- FileManagementTool.RUN_COMMAND_LINE,
36
- ],
37
30
  )
38
31
  def test_workflow_with_file_management_tools(
39
32
  workflow_with_virtual_assistant,
@@ -65,7 +58,7 @@ def test_workflow_with_file_management_tools(
65
58
  @pytest.mark.regression
66
59
  @pytest.mark.testcase("EPMCDME-6561")
67
60
  @pytest.mark.skipif(
68
- os.getenv("ENV") == "local", reason="Skipping this test on local environment"
61
+ EnvironmentResolver.is_localhost(), reason="Skipping this test on local environment"
69
62
  )
70
63
  def test_workflow_with_generate_image_tool(
71
64
  workflow_with_virtual_assistant, filesystem_integration, workflow_utils
@@ -1,5 +1,3 @@
1
- import os
2
-
3
1
  import pytest
4
2
 
5
3
  from codemie_test_harness.tests.test_data.mcp_server_test_data import (
@@ -15,6 +13,11 @@ from codemie_test_harness.tests.test_data.mcp_server_test_data import (
15
13
  CLI_MCP_SERVER,
16
14
  )
17
15
  from codemie_test_harness.tests.utils.base_utils import get_random_name
16
+ from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
17
+
18
+ pytestmark = pytest.mark.skipif(
19
+ EnvironmentResolver.is_localhost(), reason="Skipping this test on local environment"
20
+ )
18
21
 
19
22
 
20
23
  @pytest.mark.workflow
@@ -22,9 +25,6 @@ from codemie_test_harness.tests.utils.base_utils import get_random_name
22
25
  @pytest.mark.mcp
23
26
  @pytest.mark.regression
24
27
  @pytest.mark.testcase("EPMCDME-6419")
25
- @pytest.mark.skipif(
26
- os.getenv("ENV") == "local", reason="Skipping this test on local environment"
27
- )
28
28
  def test_workflow_with_time_mcp_server(
29
29
  workflow_with_virtual_assistant,
30
30
  workflow_utils,
@@ -51,9 +51,6 @@ def test_workflow_with_time_mcp_server(
51
51
  @pytest.mark.mcp
52
52
  @pytest.mark.regression
53
53
  @pytest.mark.testcase("EPMCDME-6419")
54
- @pytest.mark.skipif(
55
- os.getenv("ENV") == "local", reason="Skipping this test on local environment"
56
- )
57
54
  @pytest.mark.parametrize(
58
55
  "command, expected_answer",
59
56
  cli_mcp_server_test_data,
@@ -86,9 +83,6 @@ def test_workflow_with_cli_mcp_server(
86
83
  @pytest.mark.mcp
87
84
  @pytest.mark.regression
88
85
  @pytest.mark.testcase("EPMCDME-6419")
89
- @pytest.mark.skipif(
90
- os.getenv("ENV") == "local", reason="Skipping this test on local environment"
91
- )
92
86
  def test_workflow_with_fetch_mcp_server(
93
87
  workflow_with_virtual_assistant,
94
88
  workflow_utils,
@@ -1,5 +1,3 @@
1
- import os
2
-
3
1
  import pytest
4
2
  from hamcrest import assert_that, equal_to
5
3
 
@@ -13,6 +11,8 @@ from codemie_test_harness.tests.test_data.notification_tools_test_data import (
13
11
  TELEGRAM_RESPONSE,
14
12
  )
15
13
  from codemie_test_harness.tests.utils.base_utils import get_random_name
14
+ from codemie_test_harness.tests.enums.environment import Environment
15
+ from codemie_test_harness.tests.utils.env_resolver import get_environment
16
16
 
17
17
 
18
18
  @pytest.mark.workflow
@@ -22,7 +22,7 @@ from codemie_test_harness.tests.utils.base_utils import get_random_name
22
22
  @pytest.mark.regression
23
23
  @pytest.mark.tescase("EPMCDME-6652")
24
24
  @pytest.mark.skipif(
25
- os.getenv("ENV") in ["local", "gcp"],
25
+ get_environment() in [Environment.LOCALHOST, Environment.GCP],
26
26
  reason="Skipping this test on local environment",
27
27
  )
28
28
  def test_workflow_with_notification_email_tool(
@@ -1,11 +1,10 @@
1
- import os
2
-
3
1
  import pytest
4
2
 
5
3
  from codemie_test_harness.tests.test_data.open_api_tools_test_data import (
6
4
  open_api_tools_test_data,
7
5
  )
8
6
  from codemie_test_harness.tests.utils.base_utils import get_random_name
7
+ from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
9
8
 
10
9
 
11
10
  @pytest.mark.workflow
@@ -14,7 +13,8 @@ from codemie_test_harness.tests.utils.base_utils import get_random_name
14
13
  @pytest.mark.regression
15
14
  @pytest.mark.tescase("EPMCDME-6605")
16
15
  @pytest.mark.skipif(
17
- os.getenv("ENV") == "azure", reason="Still have an issue with encoding long strings"
16
+ EnvironmentResolver.is_azure(),
17
+ reason="Still have an issue with encoding long strings",
18
18
  )
19
19
  @pytest.mark.parametrize(
20
20
  "tool_name, prompt, expected_response",
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: codemie-test-harness
3
- Version: 0.1.159
3
+ Version: 0.1.160
4
4
  Summary: Autotest for CodeMie backend and UI
5
5
  Author: Anton Yeromin
6
6
  Author-email: anton_yeromin@epam.com
@@ -13,7 +13,7 @@ Requires-Dist: aws-assume-role-lib (>=2.10.0,<3.0.0)
13
13
  Requires-Dist: boto3 (>=1.39.8,<2.0.0)
14
14
  Requires-Dist: click (>=8.1.7,<9.0.0)
15
15
  Requires-Dist: codemie-plugins (>=0.1.123,<0.2.0)
16
- Requires-Dist: codemie-sdk-python (==0.1.159)
16
+ Requires-Dist: codemie-sdk-python (==0.1.160)
17
17
  Requires-Dist: pytest (>=8.4.1,<9.0.0)
18
18
  Requires-Dist: pytest-playwright (>=0.7.0,<0.8.0)
19
19
  Requires-Dist: pytest-reportportal (>=5.5.2,<6.0.0)