codemie-test-harness 0.1.172__py3-none-any.whl → 0.1.174__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codemie_test_harness/tests/ui/_test_data/__init__.py +0 -0
- codemie_test_harness/tests/ui/_test_data/integration_test_data.py +121 -0
- codemie_test_harness/tests/ui/assistants/test_create_assistant.py +1 -1
- codemie_test_harness/tests/ui/conftest.py +25 -0
- codemie_test_harness/tests/ui/integrations/__init__.py +0 -0
- codemie_test_harness/tests/ui/integrations/test_create_integration.py +320 -0
- codemie_test_harness/tests/ui/pageobject/assistants/create_assistant_page.py +0 -20
- codemie_test_harness/tests/ui/pageobject/base_page.py +19 -6
- codemie_test_harness/tests/ui/pageobject/components/integration_row.py +299 -0
- codemie_test_harness/tests/ui/pageobject/integrations/create_integration_page.py +772 -0
- codemie_test_harness/tests/ui/pageobject/integrations/integrations_page.py +434 -0
- codemie_test_harness-0.1.174.dist-info/METADATA +567 -0
- {codemie_test_harness-0.1.172.dist-info → codemie_test_harness-0.1.174.dist-info}/RECORD +16 -9
- codemie_test_harness-0.1.172.dist-info/METADATA +0 -306
- /codemie_test_harness/tests/{test_data → ui/_test_data}/assistant_test_data.py +0 -0
- {codemie_test_harness-0.1.172.dist-info → codemie_test_harness-0.1.174.dist-info}/WHEEL +0 -0
- {codemie_test_harness-0.1.172.dist-info → codemie_test_harness-0.1.174.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,567 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: codemie-test-harness
|
|
3
|
+
Version: 0.1.174
|
|
4
|
+
Summary: Autotest for CodeMie backend and UI
|
|
5
|
+
Author: Anton Yeromin
|
|
6
|
+
Author-email: anton_yeromin@epam.com
|
|
7
|
+
Requires-Python: >=3.12,<4.0
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
11
|
+
Requires-Dist: PyHamcrest (>=2.1.0,<3.0.0)
|
|
12
|
+
Requires-Dist: aws-assume-role-lib (>=2.10.0,<3.0.0)
|
|
13
|
+
Requires-Dist: boto3 (>=1.39.8,<2.0.0)
|
|
14
|
+
Requires-Dist: click (>=8.1.7,<9.0.0)
|
|
15
|
+
Requires-Dist: codemie-plugins (>=0.1.123,<0.2.0)
|
|
16
|
+
Requires-Dist: codemie-sdk-python (==0.1.174)
|
|
17
|
+
Requires-Dist: pytest (>=8.4.1,<9.0.0)
|
|
18
|
+
Requires-Dist: pytest-playwright (>=0.7.0,<0.8.0)
|
|
19
|
+
Requires-Dist: pytest-reportportal (>=5.5.2,<6.0.0)
|
|
20
|
+
Requires-Dist: pytest-rerunfailures (>=15.1,<16.0)
|
|
21
|
+
Requires-Dist: pytest-xdist (>=3.6.1,<4.0.0)
|
|
22
|
+
Requires-Dist: python-dotenv (>=1.1.0,<2.0.0)
|
|
23
|
+
Requires-Dist: python-gitlab (>=5.6.0,<6.0.0)
|
|
24
|
+
Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
|
|
25
|
+
Requires-Dist: tzlocal (>=5.3.1,<6.0.0)
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
28
|
+
# CodeMie Test Harness
|
|
29
|
+
|
|
30
|
+
End-to-end, integration, and UI test suite for CodeMie services. This repository exercises CodeMie APIs (LLM, assistants, workflows, tools) and common integrations.
|
|
31
|
+
|
|
32
|
+
The suite is designed for high-parallel execution (pytest-xdist), resilient runs (pytest-rerunfailures), and optional reporting to ReportPortal.
|
|
33
|
+
|
|
34
|
+
## Table of Contents
|
|
35
|
+
|
|
36
|
+
- Part 1: codemie-test-harness command line (recommended)
|
|
37
|
+
- Installation
|
|
38
|
+
- Configuration (CLI)
|
|
39
|
+
- Run with command line
|
|
40
|
+
- Useful CLI commands and common markers
|
|
41
|
+
- Part 2: Contributors (pytest from repo)
|
|
42
|
+
- Install and configure with .env (PREVIEW/AZURE/GCP/AWS/PROD/LOCAL)
|
|
43
|
+
- Local with custom GitLab, GitHub, Jira and Confluence tokens
|
|
44
|
+
- UI tests (Playwright)
|
|
45
|
+
- ReportPortal integration
|
|
46
|
+
- Makefile targets
|
|
47
|
+
- Troubleshooting
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
## Part 1: codemie-test-harness command line (recommended)
|
|
51
|
+
|
|
52
|
+
### New Command Structure Overview
|
|
53
|
+
|
|
54
|
+
The CLI now provides four main command groups:
|
|
55
|
+
|
|
56
|
+
1. **`config`** - Comprehensive configuration management (10 categories, 86+ variables)
|
|
57
|
+
2. **`run`** - Enhanced test execution with flexible parameters
|
|
58
|
+
3. **`assistant`** - Direct assistant interaction and chat capabilities
|
|
59
|
+
4. **`workflow`** - Workflow execution
|
|
60
|
+
|
|
61
|
+
Each command group includes extensive help and validation features.
|
|
62
|
+
|
|
63
|
+
---
|
|
64
|
+
|
|
65
|
+
Use the CLI to install, configure, and run tests against your custom environment. No .env file is used in this flow. Values are stored in ~/.codemie/test-harness.json.
|
|
66
|
+
|
|
67
|
+
### Installation
|
|
68
|
+
|
|
69
|
+
Install from PyPI:
|
|
70
|
+
|
|
71
|
+
```shell
|
|
72
|
+
pip install codemie-test-harness
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Tip: Use a virtual environment (e.g., python -m venv .venv && source .venv/bin/activate).
|
|
76
|
+
|
|
77
|
+
### Configuration (CLI)
|
|
78
|
+
|
|
79
|
+
The CLI provides comprehensive configuration management with **86+ environment variables** across **10 integration categories**. Configuration values are securely stored and support interactive setup, validation, and easy management.
|
|
80
|
+
|
|
81
|
+
#### Quick Setup
|
|
82
|
+
|
|
83
|
+
Set required Auth/API values once (saved under ~/.codemie/test-harness.json):
|
|
84
|
+
|
|
85
|
+
```shell
|
|
86
|
+
codemie-test-harness config set AUTH_SERVER_URL <auth_server_url>
|
|
87
|
+
codemie-test-harness config set AUTH_CLIENT_ID <client_id>
|
|
88
|
+
codemie-test-harness config set AUTH_CLIENT_SECRET <client_secret>
|
|
89
|
+
codemie-test-harness config set AUTH_REALM_NAME <realm_name>
|
|
90
|
+
codemie-test-harness config set CODEMIE_API_DOMAIN <codemie_api_domain_url>
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Optional defaults for pytest:
|
|
94
|
+
|
|
95
|
+
```shell
|
|
96
|
+
codemie-test-harness config set PYTEST_MARKS "smoke"
|
|
97
|
+
codemie-test-harness config set PYTEST_N 8
|
|
98
|
+
codemie-test-harness config set PYTEST_RERUNS 2
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
#### Integration Categories & Management
|
|
102
|
+
|
|
103
|
+
The CLI supports **10 major integration categories** with comprehensive credential management:
|
|
104
|
+
|
|
105
|
+
1. **Version Control**: GitLab, GitHub
|
|
106
|
+
2. **Code Base**: SonarQube, SonarCloud
|
|
107
|
+
3. **Project Management**: JIRA (Server & Cloud), Confluence (Server & Cloud)
|
|
108
|
+
4. **Cloud Providers**: AWS, GCP, Azure Cloud, Kubernetes
|
|
109
|
+
5. **Azure DevOps**: Azure DevOps services
|
|
110
|
+
6. **Access Management**: Keycloak
|
|
111
|
+
7. **Notification Systems**: Email/Gmail, OAuth, Telegram
|
|
112
|
+
8. **Data Management**: SQL databases (MySQL, PostgreSQL, MSSQL), LiteLLM, Elasticsearch
|
|
113
|
+
9. **IT Service Management**: ServiceNow
|
|
114
|
+
10. **Quality Assurance**: Report Portal
|
|
115
|
+
|
|
116
|
+
#### Configuration Commands
|
|
117
|
+
|
|
118
|
+
**List and View Configurations:**
|
|
119
|
+
```shell
|
|
120
|
+
# List all available categories
|
|
121
|
+
codemie-test-harness config categories
|
|
122
|
+
|
|
123
|
+
# List variables for specific category
|
|
124
|
+
codemie-test-harness config vars version-control
|
|
125
|
+
|
|
126
|
+
# List all configurations (masked by default)
|
|
127
|
+
codemie-test-harness config list
|
|
128
|
+
|
|
129
|
+
# Show actual values (use with caution)
|
|
130
|
+
codemie-test-harness config list --show-values
|
|
131
|
+
|
|
132
|
+
# Get specific value
|
|
133
|
+
codemie-test-harness config get AUTH_SERVER_URL
|
|
134
|
+
|
|
135
|
+
# Show integration credentials by category
|
|
136
|
+
codemie-test-harness config integrations --category project-management
|
|
137
|
+
codemie-test-harness config integrations --show-values
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
**Interactive Setup:**
|
|
141
|
+
```shell
|
|
142
|
+
# Interactive setup for specific category
|
|
143
|
+
codemie-test-harness config setup --category version-control
|
|
144
|
+
codemie-test-harness config setup --category project-management
|
|
145
|
+
|
|
146
|
+
# Setup all categories interactively
|
|
147
|
+
codemie-test-harness config setup --all
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
**Validation:**
|
|
151
|
+
```shell
|
|
152
|
+
# Validate all configured credentials
|
|
153
|
+
codemie-test-harness config validate
|
|
154
|
+
|
|
155
|
+
# Validate specific category
|
|
156
|
+
codemie-test-harness config validate --category cloud
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
**Management:**
|
|
160
|
+
```shell
|
|
161
|
+
# Set individual values
|
|
162
|
+
codemie-test-harness config set KEY VALUE
|
|
163
|
+
|
|
164
|
+
# Remove specific keys
|
|
165
|
+
codemie-test-harness config unset --keys GITLAB_TOKEN,GITHUB_TOKEN
|
|
166
|
+
|
|
167
|
+
# Remove entire category
|
|
168
|
+
codemie-test-harness config unset --category version-control
|
|
169
|
+
|
|
170
|
+
# Clear all configuration (with confirmation)
|
|
171
|
+
codemie-test-harness config clear
|
|
172
|
+
codemie-test-harness config clear --force
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
#### Sample Integration Configurations
|
|
176
|
+
|
|
177
|
+
**Version Control:**
|
|
178
|
+
```shell
|
|
179
|
+
# Git provider selection
|
|
180
|
+
codemie-test-harness config set GIT_ENV gitlab # or github
|
|
181
|
+
|
|
182
|
+
# GitLab
|
|
183
|
+
codemie-test-harness config set GITLAB_URL https://gitlab.example.com
|
|
184
|
+
codemie-test-harness config set GITLAB_TOKEN <gitlab_token>
|
|
185
|
+
codemie-test-harness config set GITLAB_PROJECT https://gitlab.example.com/group/project
|
|
186
|
+
codemie-test-harness config set GITLAB_PROJECT_ID 12345
|
|
187
|
+
|
|
188
|
+
# GitHub
|
|
189
|
+
codemie-test-harness config set GITHUB_URL https://github.com
|
|
190
|
+
codemie-test-harness config set GITHUB_TOKEN <github_token>
|
|
191
|
+
codemie-test-harness config set GITHUB_PROJECT https://github.com/org/repo
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
**Project Management:**
|
|
195
|
+
```shell
|
|
196
|
+
# JIRA Server
|
|
197
|
+
codemie-test-harness config set JIRA_URL https://jira.example.com
|
|
198
|
+
codemie-test-harness config set JIRA_TOKEN <jira_token>
|
|
199
|
+
codemie-test-harness config set JIRA_JQL "project = 'EPMCDME' and issuetype = 'Epic'"
|
|
200
|
+
|
|
201
|
+
# JIRA Cloud
|
|
202
|
+
codemie-test-harness config set JIRA_CLOUD_URL https://company.atlassian.net
|
|
203
|
+
codemie-test-harness config set JIRA_CLOUD_EMAIL user@company.com
|
|
204
|
+
codemie-test-harness config set JIRA_CLOUD_TOKEN <api_token>
|
|
205
|
+
|
|
206
|
+
# Confluence Server
|
|
207
|
+
codemie-test-harness config set CONFLUENCE_URL https://confluence.example.com
|
|
208
|
+
codemie-test-harness config set CONFLUENCE_TOKEN <confluence_token>
|
|
209
|
+
codemie-test-harness config set CONFLUENCE_CQL "space = EPMCDME and type = page"
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
**Cloud Providers:**
|
|
213
|
+
```shell
|
|
214
|
+
# AWS
|
|
215
|
+
codemie-test-harness config set AWS_ACCESS_KEY_ID <access_key>
|
|
216
|
+
codemie-test-harness config set AWS_SECRET_ACCESS_KEY <secret_key>
|
|
217
|
+
codemie-test-harness config set AWS_REGION us-east-1
|
|
218
|
+
|
|
219
|
+
# Azure
|
|
220
|
+
codemie-test-harness config set AZURE_CLIENT_ID <client_id>
|
|
221
|
+
codemie-test-harness config set AZURE_CLIENT_SECRET <client_secret>
|
|
222
|
+
codemie-test-harness config set AZURE_TENANT_ID <tenant_id>
|
|
223
|
+
|
|
224
|
+
# GCP
|
|
225
|
+
codemie-test-harness config set GCP_SA_KEY_BASE64 <base64_encoded_service_account>
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
**Quality Assurance:**
|
|
229
|
+
```shell
|
|
230
|
+
# Report Portal
|
|
231
|
+
codemie-test-harness config set RP_ENDPOINT https://rp.example.com
|
|
232
|
+
codemie-test-harness config set RP_PROJECT codemie_tests
|
|
233
|
+
codemie-test-harness config set RP_API_KEY <api_key>
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
#### Security Features
|
|
237
|
+
|
|
238
|
+
- **Credential Masking**: Sensitive values are masked by default in all displays
|
|
239
|
+
- **Show Values Flag**: Use `--show-values` only when needed to view actual credentials
|
|
240
|
+
- **Case Insensitive**: Configuration keys are case-insensitive for ease of use
|
|
241
|
+
|
|
242
|
+
**Notes:**
|
|
243
|
+
- Quoting is required for values with spaces (e.g., JQL/CQL)
|
|
244
|
+
- Resolution precedence when running: CLI flags > environment variables > saved config > defaults
|
|
245
|
+
- Config file path: ~/.codemie/test-harness.json
|
|
246
|
+
- All sensitive values are automatically masked in output unless `--show-values` is used
|
|
247
|
+
|
|
248
|
+
### Run with command line
|
|
249
|
+
|
|
250
|
+
#### Test Execution
|
|
251
|
+
|
|
252
|
+
Default run (uses saved config or defaults):
|
|
253
|
+
|
|
254
|
+
```shell
|
|
255
|
+
codemie-test-harness run
|
|
256
|
+
```
|
|
257
|
+
|
|
258
|
+
Override at runtime:
|
|
259
|
+
|
|
260
|
+
```shell
|
|
261
|
+
# Change marks, workers, reruns just for this run
|
|
262
|
+
codemie-test-harness run --marks "smoke or gitlab or jira_kb" -n 8 --reruns 2
|
|
263
|
+
|
|
264
|
+
# Run with specific environment settings
|
|
265
|
+
codemie-test-harness run --codemie-api-domain https://api.example.com --marks smoke
|
|
266
|
+
|
|
267
|
+
# Run with custom test patterns
|
|
268
|
+
codemie-test-harness run --test-path "tests/integration/" --marks regression
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
Provider-specific examples:
|
|
272
|
+
|
|
273
|
+
```shell
|
|
274
|
+
# Only GitLab
|
|
275
|
+
codemie-test-harness run --marks gitlab
|
|
276
|
+
|
|
277
|
+
# Only GitHub
|
|
278
|
+
codemie-test-harness run --marks github
|
|
279
|
+
|
|
280
|
+
# Jira knowledge base
|
|
281
|
+
codemie-test-harness run --marks jira_kb
|
|
282
|
+
|
|
283
|
+
# Confluence knowledge base
|
|
284
|
+
codemie-test-harness run --marks confluence_kb
|
|
285
|
+
|
|
286
|
+
# Code knowledge base
|
|
287
|
+
codemie-test-harness --git-env gitlab run --marks code_kb
|
|
288
|
+
|
|
289
|
+
# Git tool
|
|
290
|
+
codemie-test-harness --git-env github run --marks git
|
|
291
|
+
|
|
292
|
+
# UI tests with specific browser
|
|
293
|
+
codemie-test-harness run --marks ui --headless
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
#### Assistant Chat Interface
|
|
297
|
+
|
|
298
|
+
Interact directly with CodeMie assistants through the CLI:
|
|
299
|
+
|
|
300
|
+
```shell
|
|
301
|
+
# Start new conversation with assistant
|
|
302
|
+
codemie-test-harness assistant chat --assistant-id "asst_123" -m "Hello, help me with testing"
|
|
303
|
+
|
|
304
|
+
# Continue existing conversation
|
|
305
|
+
codemie-test-harness assistant chat --assistant-id "asst_123" --conversation-id "conv_456" -m "What's next?"
|
|
306
|
+
|
|
307
|
+
# Chat with streaming enabled
|
|
308
|
+
codemie-test-harness assistant chat --assistant-id "asst_123" --stream -m "Generate test cases"
|
|
309
|
+
|
|
310
|
+
# Chat with Langfuse tracing
|
|
311
|
+
codemie-test-harness assistant chat --assistant-id "asst_123" --langfuse-traces-enabled -m "Analyze logs"
|
|
312
|
+
```
|
|
313
|
+
|
|
314
|
+
#### Workflow Execution
|
|
315
|
+
|
|
316
|
+
Execute CodeMie workflows directly from the command line:
|
|
317
|
+
|
|
318
|
+
```shell
|
|
319
|
+
# Execute workflow
|
|
320
|
+
codemie-test-harness workflow execute --workflow-id "wf_123"
|
|
321
|
+
|
|
322
|
+
# Execute workflow with user input
|
|
323
|
+
codemie-test-harness workflow execute --workflow-id "wf_456" --user-input "process test data"
|
|
324
|
+
|
|
325
|
+
# Execute workflow with custom execution ID
|
|
326
|
+
codemie-test-harness workflow execute --workflow-id "wf_789" --execution-id "exec_custom_001"
|
|
327
|
+
```
|
|
328
|
+
|
|
329
|
+
### Useful CLI commands and common markers
|
|
330
|
+
|
|
331
|
+
#### CLI Basics
|
|
332
|
+
|
|
333
|
+
```shell
|
|
334
|
+
# General help
|
|
335
|
+
codemie-test-harness --help
|
|
336
|
+
|
|
337
|
+
# Command-specific help
|
|
338
|
+
codemie-test-harness config --help
|
|
339
|
+
codemie-test-harness run --help
|
|
340
|
+
codemie-test-harness assistant --help
|
|
341
|
+
codemie-test-harness workflow --help
|
|
342
|
+
|
|
343
|
+
# Configuration management
|
|
344
|
+
codemie-test-harness config list
|
|
345
|
+
codemie-test-harness config get AUTH_SERVER_URL
|
|
346
|
+
codemie-test-harness config set PYTEST_N 12
|
|
347
|
+
codemie-test-harness config categories
|
|
348
|
+
codemie-test-harness config validate
|
|
349
|
+
|
|
350
|
+
# Quick configuration check
|
|
351
|
+
codemie-test-harness config integrations --category version-control
|
|
352
|
+
```
|
|
353
|
+
|
|
354
|
+
#### Advanced CLI Features
|
|
355
|
+
|
|
356
|
+
```shell
|
|
357
|
+
# Interactive category setup
|
|
358
|
+
codemie-test-harness config setup --category project-management
|
|
359
|
+
|
|
360
|
+
# Bulk configuration removal
|
|
361
|
+
codemie-test-harness config unset --keys "GITLAB_TOKEN,GITHUB_TOKEN"
|
|
362
|
+
|
|
363
|
+
# Validate specific integration
|
|
364
|
+
codemie-test-harness config validate --category cloud
|
|
365
|
+
|
|
366
|
+
# Show all variables for a category
|
|
367
|
+
codemie-test-harness config vars data-management
|
|
368
|
+
|
|
369
|
+
# Test execution with multiple overrides
|
|
370
|
+
codemie-test-harness run --marks "smoke and not ui" -n 10 --reruns 3 --headless
|
|
371
|
+
```
|
|
372
|
+
|
|
373
|
+
#### Common Test Markers
|
|
374
|
+
|
|
375
|
+
Common markers in this repo include:
|
|
376
|
+
- **smoke** - Quick smoke tests
|
|
377
|
+
- **mcp** - Model Context Protocol tests
|
|
378
|
+
- **plugin** - Plugin functionality tests
|
|
379
|
+
- **regression** - Comprehensive regression tests
|
|
380
|
+
- **ui** - User interface tests (Playwright)
|
|
381
|
+
- **jira_kb, confluence_kb, code_kb** - Knowledge base tests
|
|
382
|
+
- **gitlab, github, git** - Version control integration tests
|
|
383
|
+
- **not_for_parallel_run** - Tests that must run sequentially
|
|
384
|
+
|
|
385
|
+
#### Environment-Specific Execution
|
|
386
|
+
|
|
387
|
+
```shell
|
|
388
|
+
# Target specific environments
|
|
389
|
+
codemie-test-harness run --codemie-api-domain https://preview.codemie.ai --marks smoke
|
|
390
|
+
codemie-test-harness run --codemie-api-domain https://prod.codemie.ai --marks regression
|
|
391
|
+
|
|
392
|
+
# Local development
|
|
393
|
+
codemie-test-harness run --codemie-api-domain http://localhost:8080 --marks "smoke and not ui"
|
|
394
|
+
```
|
|
395
|
+
|
|
396
|
+
---
|
|
397
|
+
|
|
398
|
+
## Part 2: Contributors (pytest from repo)
|
|
399
|
+
|
|
400
|
+
This section is for contributors who run tests from a cloned codemie-sdk repository (test-harness package). This flow uses a .env file and may pull values from AWS SSM Parameter Store.
|
|
401
|
+
|
|
402
|
+
### Credentials Management System
|
|
403
|
+
|
|
404
|
+
The test harness uses a sophisticated **unified credentials management system** that handles integration credentials through a **priority-based resolution approach**:
|
|
405
|
+
|
|
406
|
+
#### **Resolution Priority (Highest to Lowest):**
|
|
407
|
+
|
|
408
|
+
1. **Environment Variables (.env file)** - Highest priority
|
|
409
|
+
2. **AWS Parameter Store** - Fallback with JSON path navigation
|
|
410
|
+
3. **Default Values** - Final fallback
|
|
411
|
+
|
|
412
|
+
#### **AWS Parameter Store Integration**
|
|
413
|
+
|
|
414
|
+
The credentials manager automatically integrates with AWS Parameter Store using **JSON path navigation** for structured credential storage:
|
|
415
|
+
|
|
416
|
+
**Parameter Structure:**
|
|
417
|
+
- **Base Path**: `/codemie/autotests/integrations/`
|
|
418
|
+
- **Service Groupings**: `jira`, `confluence`, `git`, `aws`, `azure`, `gcp`, `sonar`, etc.
|
|
419
|
+
- **JSON Path Navigation**: Uses dot notation (e.g., `jira_server.url`, `jira_cloud.token`)
|
|
420
|
+
|
|
421
|
+
**Example AWS Parameter Store Structure:**
|
|
422
|
+
```json
|
|
423
|
+
{
|
|
424
|
+
"jira_server": {
|
|
425
|
+
"url": "https://jira.example.com",
|
|
426
|
+
"token": "server_token_here",
|
|
427
|
+
"jql": "project = 'EPMCDME'"
|
|
428
|
+
},
|
|
429
|
+
"jira_cloud": {
|
|
430
|
+
"url": "https://company.atlassian.net",
|
|
431
|
+
"email": "user@company.com",
|
|
432
|
+
"token": "cloud_token_here",
|
|
433
|
+
"jql": "project = 'CLOUD' and status = 'Open'"
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
```
|
|
437
|
+
|
|
438
|
+
**Environment-Aware Credential Resolution:**
|
|
439
|
+
- **Preview Environment**: Uses standard paths (e.g., `elasticsearch.*`, `preview.mysql.*`)
|
|
440
|
+
- **Sandbox Environments** (Azure/GCP/AWS): Uses sandbox-specific paths (e.g., `sandbox.elasticsearch.*`, `sandbox.mysql.*`)
|
|
441
|
+
- **Automatic Detection**: Environment resolver determines the appropriate credential set
|
|
442
|
+
|
|
443
|
+
#### **Supported Integration Categories**
|
|
444
|
+
|
|
445
|
+
The credentials manager supports **86+ environment variables** across **10 categories**:
|
|
446
|
+
|
|
447
|
+
1. **Version Control**: GitLab, GitHub (tokens, project IDs, URLs)
|
|
448
|
+
2. **Project Management**: JIRA Server/Cloud, Confluence Server/Cloud (tokens, JQL, CQL)
|
|
449
|
+
3. **Cloud Providers**: AWS (keys, regions), Azure (client credentials), GCP (service accounts)
|
|
450
|
+
4. **Code Quality**: SonarQube Server/Cloud (tokens, project keys)
|
|
451
|
+
5. **DevOps**: Azure DevOps (PATs, organization/project names)
|
|
452
|
+
6. **Access Management**: Keycloak (client credentials, realms)
|
|
453
|
+
7. **Notifications**: Email/Gmail (SMTP), OAuth (refresh tokens), Telegram (bot tokens)
|
|
454
|
+
8. **Data Management**: SQL databases (MySQL, PostgreSQL, MSSQL), Elasticsearch, LiteLLM
|
|
455
|
+
9. **IT Service**: ServiceNow (API keys)
|
|
456
|
+
10. **Quality Assurance**: Report Portal (API keys, projects), Kubernetes (bearer tokens)
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
### Install and configure with .env (PREVIEW/AZURE/GCP/AWS/PROD/LOCAL)
|
|
460
|
+
|
|
461
|
+
1) Clone the codemie-sdk repository and navigate to the test-harness folder.
|
|
462
|
+
2) Create a .env file in the project root. If you provide AWS credentials, the suite will fetch additional values from AWS Systems Manager Parameter Store and recreate .env accordingly.
|
|
463
|
+
|
|
464
|
+
```properties
|
|
465
|
+
ENV=local
|
|
466
|
+
|
|
467
|
+
AWS_ACCESS_KEY=<aws_access_token>
|
|
468
|
+
AWS_SECRET_KEY=<aws_secret_key>
|
|
469
|
+
```
|
|
470
|
+
|
|
471
|
+
### Local with custom GitLab, GitHub, Jira and Confluence tokens
|
|
472
|
+
|
|
473
|
+
1) Start from a .env populated via AWS (optional)
|
|
474
|
+
2) Replace the tokens below with your personal values
|
|
475
|
+
3) Add variables to integrations with your personal values (optional)
|
|
476
|
+
4) Important: After replacing tokens, remove AWS_ACCESS_KEY and AWS_SECRET_KEY from .env — otherwise they will overwrite your changes next time .env is regenerated
|
|
477
|
+
|
|
478
|
+
Full .env example:
|
|
479
|
+
|
|
480
|
+
```properties
|
|
481
|
+
PROJECT_NAME=codemie
|
|
482
|
+
GIT_ENV=gitlab # required for e2e tests only
|
|
483
|
+
DEFAULT_TIMEOUT=60
|
|
484
|
+
CLEANUP_DATA=True
|
|
485
|
+
LANGFUSE_TRACES_ENABLED=False
|
|
486
|
+
|
|
487
|
+
CODEMIE_API_DOMAIN=http://localhost:8080
|
|
488
|
+
|
|
489
|
+
FRONTEND_URL=https://localhost:5173/
|
|
490
|
+
HEADLESS=False
|
|
491
|
+
|
|
492
|
+
NATS_URL=nats://localhost:4222
|
|
493
|
+
|
|
494
|
+
TEST_USER_FULL_NAME=dev-codemie-user
|
|
495
|
+
|
|
496
|
+
RP_API_KEY=<report_portal_api_key>
|
|
497
|
+
```
|
|
498
|
+
|
|
499
|
+
Now you can run full or subset packs. Examples:
|
|
500
|
+
|
|
501
|
+
```shell
|
|
502
|
+
# All tests except tests that cannot be run in parallel (-n controls the number of workers)
|
|
503
|
+
pytest -n 10 -m "not not_for_parallel_run" --reruns 2
|
|
504
|
+
|
|
505
|
+
# Tests that cannot be run in parallel
|
|
506
|
+
pytest -m not_for_parallel_run --reruns 2
|
|
507
|
+
|
|
508
|
+
# Regression tests
|
|
509
|
+
pytest -n 10 -m "regression and not not_for_parallel_run" --reruns 2
|
|
510
|
+
pytest -m not_for_parallel_run --reruns 3
|
|
511
|
+
```
|
|
512
|
+
|
|
513
|
+
Note: "--reruns 2" uses pytest-rerunfailures to improve resiliency in flaky environments.
|
|
514
|
+
|
|
515
|
+
### UI tests (Playwright)
|
|
516
|
+
|
|
517
|
+
Install browsers once:
|
|
518
|
+
|
|
519
|
+
```shell
|
|
520
|
+
playwright install
|
|
521
|
+
```
|
|
522
|
+
|
|
523
|
+
Then run UI pack:
|
|
524
|
+
|
|
525
|
+
```shell
|
|
526
|
+
pytest -n 4 -m ui --reruns 2
|
|
527
|
+
```
|
|
528
|
+
|
|
529
|
+
Playwright docs: https://playwright.dev/python/docs/intro
|
|
530
|
+
|
|
531
|
+
### ReportPortal integration
|
|
532
|
+
|
|
533
|
+
pytest.ini is preconfigured with rp_endpoint, rp_project, and a default rp_launch. To publish results:
|
|
534
|
+
|
|
535
|
+
1) Set RP_API_KEY in .env
|
|
536
|
+
2) Add the flag:
|
|
537
|
+
|
|
538
|
+
```shell
|
|
539
|
+
pytest -n 10 -m "regression and not not_for_parallel_run" --reruns 2 --reportportal
|
|
540
|
+
```
|
|
541
|
+
|
|
542
|
+
If you need access to the ReportPortal project, contact: Anton Yeromin (anton_yeromin@epam.com).
|
|
543
|
+
|
|
544
|
+
### Makefile targets
|
|
545
|
+
|
|
546
|
+
- install — poetry install
|
|
547
|
+
- ruff — lint and format with Ruff
|
|
548
|
+
- ruff-format — format only
|
|
549
|
+
- ruff-fix — apply autofixes
|
|
550
|
+
- build — poetry build
|
|
551
|
+
- publish — poetry publish
|
|
552
|
+
|
|
553
|
+
Example:
|
|
554
|
+
|
|
555
|
+
```shell
|
|
556
|
+
make install
|
|
557
|
+
make ruff
|
|
558
|
+
```
|
|
559
|
+
|
|
560
|
+
### Troubleshooting
|
|
561
|
+
|
|
562
|
+
- Playwright not installed: Run playwright install.
|
|
563
|
+
- Headless issues locally: Set HEADLESS=True in .env for CI or False for local debugging.
|
|
564
|
+
- Env values keep reverting: Ensure AWS_ACCESS_KEY and AWS_SECRET_KEY are removed after manual edits to .env.
|
|
565
|
+
- Authentication failures: Verify AUTH_* variables and CODEMIE_API_DOMAIN are correct for the target environment.
|
|
566
|
+
- Slow or flaky runs: Reduce -n, increase timeouts, and/or use --reruns.
|
|
567
|
+
|
|
@@ -103,7 +103,6 @@ codemie_test_harness/tests/test_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeu
|
|
|
103
103
|
codemie_test_harness/tests/test_data/ado_test_plan_tools_test_data.py,sha256=Al5u4HNfrcoj-b072uEGsqUqjKqwXLGJXKQ0QeJT3PI,5778
|
|
104
104
|
codemie_test_harness/tests/test_data/ado_wiki_tools_test_data.py,sha256=xvgEja5vE0l41sP4fE0stdFLQ0M201FWynOCEcRYufE,3188
|
|
105
105
|
codemie_test_harness/tests/test_data/ado_work_item_tools_test_data.py,sha256=MHou6QGjufyX5t1oexnq2Y4UCjlE17eeyMuIz6Ml62s,5693
|
|
106
|
-
codemie_test_harness/tests/test_data/assistant_test_data.py,sha256=XYB0GN6_T6OSIWLDQBGFVbeFDOTxxD6kOae88VIxTPE,53505
|
|
107
106
|
codemie_test_harness/tests/test_data/cloud_tools_test_data.py,sha256=SWz-VTNcWteCvVupl2xksv4eEFkmtWMdRIqrZxjpFYk,6200
|
|
108
107
|
codemie_test_harness/tests/test_data/codebase_tools_test_data.py,sha256=xbnIlDbiZTibGekrodmhO7bOg7kilsoKSlfHAhcmyIQ,3312
|
|
109
108
|
codemie_test_harness/tests/test_data/data_management_tools_test_data.py,sha256=e5Cfqza3GUE3hRElm1bxgQO4PaN-jOiNd38OX9299Kc,2793
|
|
@@ -195,17 +194,23 @@ codemie_test_harness/tests/test_data/workflow/invalid_config/missing_required_to
|
|
|
195
194
|
codemie_test_harness/tests/test_data/workflow/invalid_config/missing_required_tools_name.yaml,sha256=Xh6TKSAGZyD2-gCxaW7BRW_9-_7-5EQA75djCc3FwLI,263
|
|
196
195
|
codemie_test_harness/tests/test_data/workflow_validation_messages.py,sha256=zg5BhMJ_tbzEeLSYJEnspHTuWar1qgoxqTfIXltlSPg,3282
|
|
197
196
|
codemie_test_harness/tests/ui/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
197
|
+
codemie_test_harness/tests/ui/_test_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
198
|
+
codemie_test_harness/tests/ui/_test_data/assistant_test_data.py,sha256=XYB0GN6_T6OSIWLDQBGFVbeFDOTxxD6kOae88VIxTPE,53505
|
|
199
|
+
codemie_test_harness/tests/ui/_test_data/integration_test_data.py,sha256=6YnDIWhOSTSmzIziBCfpqYikVT0kXw_vMrMJBmhnSnQ,4212
|
|
198
200
|
codemie_test_harness/tests/ui/assistants/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
199
|
-
codemie_test_harness/tests/ui/assistants/test_create_assistant.py,sha256=
|
|
200
|
-
codemie_test_harness/tests/ui/conftest.py,sha256=
|
|
201
|
+
codemie_test_harness/tests/ui/assistants/test_create_assistant.py,sha256=YWhz5--YgXBi10K8jPuhG9G2cK5GY9_xAnxDdN6-NJ8,14560
|
|
202
|
+
codemie_test_harness/tests/ui/conftest.py,sha256=KBM3g-lxPQEtbNh6i-1bAYYRlVycbEXg7ztn3x7VobI,4361
|
|
203
|
+
codemie_test_harness/tests/ui/integrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
204
|
+
codemie_test_harness/tests/ui/integrations/test_create_integration.py,sha256=ASe1i7Zv3x7RynVkT5rQz79z4lsah6MEPfew6aoTsy0,11962
|
|
201
205
|
codemie_test_harness/tests/ui/pageobject/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
202
206
|
codemie_test_harness/tests/ui/pageobject/assistants/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
203
207
|
codemie_test_harness/tests/ui/pageobject/assistants/assistants_page.py,sha256=pYUhVRGZhCVX5EILCTkqjvOBPfD9u2qc5Rsa54up5IA,6036
|
|
204
|
-
codemie_test_harness/tests/ui/pageobject/assistants/create_assistant_page.py,sha256=
|
|
208
|
+
codemie_test_harness/tests/ui/pageobject/assistants/create_assistant_page.py,sha256=xGl5nZAwJJtZ_ej2ycYgywK9EBGj_-AXCW4Z_eEWgYA,21791
|
|
205
209
|
codemie_test_harness/tests/ui/pageobject/assistants/generate_with_ai_modal.py,sha256=rsavzWxmawKzI4wHxixp49IN6m_ZUZNFTJTSnE8jBr8,13732
|
|
206
|
-
codemie_test_harness/tests/ui/pageobject/base_page.py,sha256=
|
|
210
|
+
codemie_test_harness/tests/ui/pageobject/base_page.py,sha256=pKi_Qv81W5XESdX-xFA3de98gyM4iJB1rkXNPyFnFx8,7960
|
|
207
211
|
codemie_test_harness/tests/ui/pageobject/components/__init__.py,sha256=6scUFCL2StHbKIoNgGGZdpeDZUwbCrKIH7hwaskAB4E,577
|
|
208
212
|
codemie_test_harness/tests/ui/pageobject/components/execution_history_row.py,sha256=aGHc5AOpGR0tlfmLQfk8272TN6TWiuiXHUcg6PtB1Iw,6499
|
|
213
|
+
codemie_test_harness/tests/ui/pageobject/components/integration_row.py,sha256=TCS6Si1Di6nrU9TTstcc4bxPztIqaWo11JxFzdqPX_c,8492
|
|
209
214
|
codemie_test_harness/tests/ui/pageobject/components/menu.py,sha256=llTWAbJHldo1-wY86k_2Dvs8iSEmMWzrw26AhQJ1kis,10501
|
|
210
215
|
codemie_test_harness/tests/ui/pageobject/components/pop_up.py,sha256=DfdQfLZp_We8K9rNyLXtJqy3vk8BxEnp32h9dTThJUM,3466
|
|
211
216
|
codemie_test_harness/tests/ui/pageobject/components/workflow_card.py,sha256=8M4JKYCI46_-0EosFBgWfFuD3-_AH8jVIhSRJCQqamQ,6712
|
|
@@ -213,6 +218,8 @@ codemie_test_harness/tests/ui/pageobject/components/workflow_execution_history_i
|
|
|
213
218
|
codemie_test_harness/tests/ui/pageobject/components/workflow_execution_state.py,sha256=IWir8fP-7oc9t17Rjs6mHCNCxfQfmdfbFY2SsNxZaE4,13837
|
|
214
219
|
codemie_test_harness/tests/ui/pageobject/components/workflow_sidebar.py,sha256=9rGoopJShDtASejDA1L8yL6TfRvGHD_5yNXtDKcKzoQ,17158
|
|
215
220
|
codemie_test_harness/tests/ui/pageobject/components/workflow_state_card.py,sha256=V7V1p3FNPrwVFM0CjZeg0MUkW9bGoF8UP5DuuH-r4gU,11548
|
|
221
|
+
codemie_test_harness/tests/ui/pageobject/integrations/create_integration_page.py,sha256=3-iI3WntzJRxkdjw6HwQkaz3TI3pAWWRkfJvBcb6klk,24911
|
|
222
|
+
codemie_test_harness/tests/ui/pageobject/integrations/integrations_page.py,sha256=Dsik34qzL8uK2KyIlKU--OFm61WyAWZTPBD__nwsBTM,13658
|
|
216
223
|
codemie_test_harness/tests/ui/pageobject/login_page.py,sha256=cs0nYtvYykXfli9vYKWPpIWOEQbksUDUGgq03hulfSg,3062
|
|
217
224
|
codemie_test_harness/tests/ui/pageobject/workflows/__init__.py,sha256=UaVz3N_M1qG3EetktyQVM1j03_6XNEmGgyO2W5clLoo,519
|
|
218
225
|
codemie_test_harness/tests/ui/pageobject/workflows/base_workflow_form_page.py,sha256=zkHtUIqRq6By5vKrLMoOmCCFwDlmhUUZsPruummftSw,14912
|
|
@@ -362,7 +369,7 @@ codemie_test_harness/tests/workflow/virtual_assistant_tools/servicenow/__init__.
|
|
|
362
369
|
codemie_test_harness/tests/workflow/virtual_assistant_tools/servicenow/test_workflow_with_servicenow_tools.py,sha256=vq6tucNBxiNIQSmIj_pYiiPm0lipU9X3kzeCd6xEbRM,966
|
|
363
370
|
codemie_test_harness/tests/workflow/virtual_assistant_tools/vcs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
364
371
|
codemie_test_harness/tests/workflow/virtual_assistant_tools/vcs/test_workflow_with_vcs_tools.py,sha256=uD2qs361j6Egp4UumfoQ4gC24-NioXfiW0IF53N9hVA,1175
|
|
365
|
-
codemie_test_harness-0.1.
|
|
366
|
-
codemie_test_harness-0.1.
|
|
367
|
-
codemie_test_harness-0.1.
|
|
368
|
-
codemie_test_harness-0.1.
|
|
372
|
+
codemie_test_harness-0.1.174.dist-info/METADATA,sha256=2OXIEt7z2LZihZ3M9tr9JAjGn3-UemD_QY_OQX9z-sE,18310
|
|
373
|
+
codemie_test_harness-0.1.174.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
|
|
374
|
+
codemie_test_harness-0.1.174.dist-info/entry_points.txt,sha256=n98t-EOM5M1mnMl_j2X4siyeO9zr0WD9a5LF7JyElIM,73
|
|
375
|
+
codemie_test_harness-0.1.174.dist-info/RECORD,,
|