robotframework-testselection 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/PKG-INFO +106 -4
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/README.md +102 -1
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/pyproject.toml +12 -1
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/__init__.py +1 -1
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/cli.py +116 -14
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/pipeline/cache.py +25 -7
- robotframework_testselection-0.3.0/src/TestSelection/pipeline/vectorize_pytest.py +129 -0
- robotframework_testselection-0.3.0/src/TestSelection/pytest/__init__.py +1 -0
- robotframework_testselection-0.3.0/src/TestSelection/pytest/collector.py +120 -0
- robotframework_testselection-0.3.0/src/TestSelection/pytest/plugin.py +238 -0
- robotframework_testselection-0.3.0/src/TestSelection/pytest/runner.py +118 -0
- robotframework_testselection-0.3.0/src/TestSelection/pytest/text_builder.py +220 -0
- robotframework_testselection-0.3.0/src/TestSelection/selection/dpp.py +59 -0
- robotframework_testselection-0.2.0/src/TestSelection/selection/dpp.py +0 -31
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/.gitignore +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/LICENSE +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/embedding/__init__.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/embedding/embedder.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/embedding/models.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/embedding/ports.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/execution/__init__.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/execution/listener.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/execution/prerun_modifier.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/execution/runner.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/parsing/__init__.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/parsing/datadriver_reader.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/parsing/keyword_resolver.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/parsing/suite_collector.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/parsing/text_builder.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/pipeline/__init__.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/pipeline/artifacts.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/pipeline/errors.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/pipeline/execute.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/pipeline/select.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/pipeline/vectorize.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/py.typed +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/selection/__init__.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/selection/facility.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/selection/filtering.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/selection/fps.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/selection/kmedoids.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/selection/registry.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/selection/strategy.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/shared/__init__.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/shared/config.py +0 -0
- {robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/shared/types.py +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: robotframework-testselection
|
|
3
|
-
Version: 0.
|
|
4
|
-
Summary: Vector-based diverse test case selection for Robot Framework
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Vector-based diverse test case selection for Robot Framework and pytest
|
|
5
5
|
Project-URL: Homepage, https://github.com/manykarim/robotframework-testselection
|
|
6
6
|
Project-URL: Documentation, https://github.com/manykarim/robotframework-testselection#readme
|
|
7
7
|
Project-URL: Repository, https://github.com/manykarim/robotframework-testselection
|
|
@@ -10,8 +10,9 @@ Project-URL: Changelog, https://github.com/manykarim/robotframework-testselectio
|
|
|
10
10
|
Author: Many Kasiriha
|
|
11
11
|
License: Apache-2.0
|
|
12
12
|
License-File: LICENSE
|
|
13
|
-
Keywords: diversity,embeddings,machine-learning,nlp,robotframework,test-selection,testing
|
|
13
|
+
Keywords: diversity,embeddings,machine-learning,nlp,pytest,robotframework,test-selection,testing
|
|
14
14
|
Classifier: Development Status :: 3 - Alpha
|
|
15
|
+
Classifier: Framework :: Pytest
|
|
15
16
|
Classifier: Framework :: Robot Framework
|
|
16
17
|
Classifier: Framework :: Robot Framework :: Library
|
|
17
18
|
Classifier: Framework :: Robot Framework :: Tool
|
|
@@ -58,7 +59,7 @@ Description-Content-Type: text/markdown
|
|
|
58
59
|
|
|
59
60
|
# robotframework-testselection
|
|
60
61
|
|
|
61
|
-
Vector-based diverse test case selection for Robot Framework. Embeds test cases as semantic vectors and selects maximally diverse subsets to reduce test suite execution time while preserving coverage breadth.
|
|
62
|
+
Vector-based diverse test case selection for Robot Framework and pytest. Embeds test cases as semantic vectors and selects maximally diverse subsets to reduce test suite execution time while preserving coverage breadth.
|
|
62
63
|
|
|
63
64
|
## How It Works
|
|
64
65
|
|
|
@@ -101,6 +102,8 @@ uv sync --extra vectorize
|
|
|
101
102
|
uv sync --extra all
|
|
102
103
|
```
|
|
103
104
|
|
|
105
|
+
The pytest plugin is included in the base install and activates when you pass `--diverse-k` to pytest.
|
|
106
|
+
|
|
104
107
|
### Dependency Groups
|
|
105
108
|
|
|
106
109
|
| Group | Packages | Purpose |
|
|
@@ -182,6 +185,38 @@ testcase-select run --suite tests/ --k 50 \
|
|
|
182
185
|
|
|
183
186
|
This works with both `run` and `execute` subcommands, including during graceful fallback (when selection fails and all tests are run).
|
|
184
187
|
|
|
188
|
+
### pytest Support
|
|
189
|
+
|
|
190
|
+
The diversity selection algorithm also works with pytest test suites, via a pytest plugin:
|
|
191
|
+
|
|
192
|
+
```bash
|
|
193
|
+
# Select 20 most diverse tests
|
|
194
|
+
pytest --diverse-k=20 tests/
|
|
195
|
+
|
|
196
|
+
# With custom strategy and seed
|
|
197
|
+
pytest --diverse-k=30 --diverse-strategy=fps_multi --diverse-seed=123 tests/
|
|
198
|
+
|
|
199
|
+
# Filter by markers before selection
|
|
200
|
+
pytest --diverse-k=20 --diverse-include-markers slow integration tests/
|
|
201
|
+
|
|
202
|
+
# Group parametrized tests (select at group level)
|
|
203
|
+
pytest --diverse-k=20 --diverse-group-parametrize tests/
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
Or via the `testcase-select` CLI:
|
|
207
|
+
|
|
208
|
+
```bash
|
|
209
|
+
# Full pipeline with pytest
|
|
210
|
+
testcase-select run --framework pytest --suite tests/ --k 20 --strategy fps
|
|
211
|
+
|
|
212
|
+
# Stage-by-stage
|
|
213
|
+
testcase-select vectorize --framework pytest --suite tests/ --output ./artifacts/
|
|
214
|
+
testcase-select select --artifacts ./artifacts/ --k 20
|
|
215
|
+
testcase-select execute --framework pytest --suite tests/ --selection selected_tests.json
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
The pytest plugin is installed automatically and activated by `--diverse-k`. It uses AST-based text representation with sentence embeddings for test diversity analysis.
|
|
219
|
+
|
|
185
220
|
### Direct Robot Framework Integration
|
|
186
221
|
|
|
187
222
|
You can also use the components directly with `robot`:
|
|
@@ -347,6 +382,11 @@ src/TestSelection/
|
|
|
347
382
|
prerun_modifier.py # DiversePreRunModifier (SuiteVisitor)
|
|
348
383
|
listener.py # DiverseDataDriverListener (Listener v3)
|
|
349
384
|
runner.py # ExecutionRunner
|
|
385
|
+
pytest/ # pytest integration
|
|
386
|
+
plugin.py # pytest plugin (pytest11 entry point)
|
|
387
|
+
collector.py # Programmatic test collection
|
|
388
|
+
text_builder.py # AST-based text representation
|
|
389
|
+
runner.py # pytest execution
|
|
350
390
|
pipeline/ # Orchestration layer
|
|
351
391
|
vectorize.py # Stage 1 orchestrator
|
|
352
392
|
select.py # Stage 2 orchestrator
|
|
@@ -401,6 +441,68 @@ uv run pytest -m integration
|
|
|
401
441
|
uv run pytest -m benchmark
|
|
402
442
|
```
|
|
403
443
|
|
|
444
|
+
## Coverage Retention Analysis
|
|
445
|
+
|
|
446
|
+
Measured on [robotframework-doctestlibrary](https://github.com/manykarim/robotframework-doctestlibrary) (5,045 statements in the DocTest package). All strategies use `all-MiniLM-L6-v2` embeddings (384-dim), seed=42.
|
|
447
|
+
|
|
448
|
+
### pytest (594 tests, 70.1% full coverage)
|
|
449
|
+
|
|
450
|
+
| Strategy | Selection | Tests | Coverage | Retention | Speedup |
|
|
451
|
+
|---|---|---|---|---|---|
|
|
452
|
+
| *(full suite)* | 100% | 594 | 70.1% | baseline | 1.0x |
|
|
453
|
+
| **fps** | **50%** | **297** | **65.9%** | **93.9%** | **3.4x** |
|
|
454
|
+
| dpp | 50% | 297 | 65.9% | 93.9% | 3.5x |
|
|
455
|
+
| kmedoids | 50% | 297 | 65.8% | 93.8% | 3.1x |
|
|
456
|
+
| fps_multi | 50% | 297 | 65.7% | 93.7% | 3.0x |
|
|
457
|
+
| facility | 50% | 297 | 64.9% | 92.6% | 2.9x |
|
|
458
|
+
| random | 50% | 297 | 64.2% | 91.5% | 2.3x |
|
|
459
|
+
| **facility** | **20%** | **119** | **58.6%** | **83.6%** | **7.5x** |
|
|
460
|
+
| kmedoids | 20% | 119 | 58.3% | 83.1% | 6.7x |
|
|
461
|
+
| fps | 20% | 119 | 56.1% | 80.0% | 6.7x |
|
|
462
|
+
| fps_multi | 20% | 119 | 53.8% | 76.8% | 8.0x |
|
|
463
|
+
| random | 20% | 119 | 53.2% | 75.8% | 5.4x |
|
|
464
|
+
| dpp | 20% | 119 | 47.6% | 67.8% | 6.0x |
|
|
465
|
+
| **fps_multi** | **10%** | **60** | **47.7%** | **67.9%** | **18.5x** |
|
|
466
|
+
| kmedoids | 10% | 60 | 46.9% | 66.9% | 14.9x |
|
|
467
|
+
| facility | 10% | 60 | 46.8% | 66.8% | 11.5x |
|
|
468
|
+
| random | 10% | 60 | 44.3% | 63.1% | 10.3x |
|
|
469
|
+
| dpp | 10% | 60 | 42.9% | 61.1% | 13.7x |
|
|
470
|
+
| fps | 10% | 60 | 40.6% | 57.9% | 21.4x |
|
|
471
|
+
|
|
472
|
+
### Robot Framework (126 tests, 63.5% full coverage)
|
|
473
|
+
|
|
474
|
+
| Strategy | Selection | Tests | Coverage | Retention | Speedup |
|
|
475
|
+
|---|---|---|---|---|---|
|
|
476
|
+
| *(full suite)* | 100% | 126 | 63.5% | baseline | 1.0x |
|
|
477
|
+
| **fps_multi** | **50%** | **63** | **61.6%** | **97.0%** | **1.5x** |
|
|
478
|
+
| facility | 50% | 63 | 61.6% | 96.9% | 1.5x |
|
|
479
|
+
| fps | 50% | 63 | 61.2% | 96.4% | 1.6x |
|
|
480
|
+
| kmedoids | 50% | 63 | 60.5% | 95.3% | 1.5x |
|
|
481
|
+
| dpp | 50% | 63 | 58.5% | 92.0% | 1.7x |
|
|
482
|
+
| random | 50% | 63 | 57.2% | 90.1% | 2.3x |
|
|
483
|
+
| **facility** | **20%** | **26** | **51.8%** | **81.5%** | **4.4x** |
|
|
484
|
+
| kmedoids | 20% | 26 | 47.5% | 74.8% | 5.5x |
|
|
485
|
+
| random | 20% | 26 | 46.0% | 72.5% | 5.7x |
|
|
486
|
+
| fps | 20% | 26 | 45.3% | 71.4% | 3.1x |
|
|
487
|
+
| fps_multi | 20% | 26 | 45.3% | 71.4% | 2.9x |
|
|
488
|
+
| dpp | 20% | 26 | 43.9% | 69.0% | 4.6x |
|
|
489
|
+
| **kmedoids** | **10%** | **13** | **42.3%** | **66.6%** | **11.3x** |
|
|
490
|
+
| dpp | 10% | 13 | 40.9% | 64.3% | 10.7x |
|
|
491
|
+
| facility | 10% | 13 | 40.2% | 63.3% | 12.9x |
|
|
492
|
+
| random | 10% | 13 | 37.6% | 59.2% | 11.3x |
|
|
493
|
+
| fps_multi | 10% | 13 | 37.5% | 59.1% | 10.7x |
|
|
494
|
+
| fps | 10% | 13 | 33.2% | 52.3% | 8.3x |
|
|
495
|
+
|
|
496
|
+
### Key Findings
|
|
497
|
+
|
|
498
|
+
- **50% selection retains 92-97% of coverage** across all strategies — the diversity algorithms select tests maximizing behavioral breadth, with minimal redundancy loss
|
|
499
|
+
- **Facility location excels at 20% selection** — 83.6% retention on pytest, 81.5% on Robot Framework, making it the best choice for aggressive smoke-test suites
|
|
500
|
+
- **kmedoids matches facility at 20%** on pytest (83.1% retention) and leads at 10% on Robot Framework (66.6%)
|
|
501
|
+
- **fps_multi leads at 10% pytest** (67.9% retention) where initial-point sensitivity matters most
|
|
502
|
+
- **DPP performs well at 50%** (93.9% retention, matching FPS) but drops at 20% due to FPS fallback for large k values
|
|
503
|
+
- **All diversity strategies outperform random** at 50% by 2-7 percentage points of retention
|
|
504
|
+
- At 50% robot selection, fps_multi achieves **97% retention** — meaning half the tests contribute almost no incremental coverage
|
|
505
|
+
|
|
404
506
|
## Performance
|
|
405
507
|
|
|
406
508
|
Benchmarked on 384-dimensional normalized vectors:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# robotframework-testselection
|
|
2
2
|
|
|
3
|
-
Vector-based diverse test case selection for Robot Framework. Embeds test cases as semantic vectors and selects maximally diverse subsets to reduce test suite execution time while preserving coverage breadth.
|
|
3
|
+
Vector-based diverse test case selection for Robot Framework and pytest. Embeds test cases as semantic vectors and selects maximally diverse subsets to reduce test suite execution time while preserving coverage breadth.
|
|
4
4
|
|
|
5
5
|
## How It Works
|
|
6
6
|
|
|
@@ -43,6 +43,8 @@ uv sync --extra vectorize
|
|
|
43
43
|
uv sync --extra all
|
|
44
44
|
```
|
|
45
45
|
|
|
46
|
+
The pytest plugin is included in the base install and activates when you pass `--diverse-k` to pytest.
|
|
47
|
+
|
|
46
48
|
### Dependency Groups
|
|
47
49
|
|
|
48
50
|
| Group | Packages | Purpose |
|
|
@@ -124,6 +126,38 @@ testcase-select run --suite tests/ --k 50 \
|
|
|
124
126
|
|
|
125
127
|
This works with both `run` and `execute` subcommands, including during graceful fallback (when selection fails and all tests are run).
|
|
126
128
|
|
|
129
|
+
### pytest Support
|
|
130
|
+
|
|
131
|
+
The diversity selection algorithm also works with pytest test suites, via a pytest plugin:
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
# Select 20 most diverse tests
|
|
135
|
+
pytest --diverse-k=20 tests/
|
|
136
|
+
|
|
137
|
+
# With custom strategy and seed
|
|
138
|
+
pytest --diverse-k=30 --diverse-strategy=fps_multi --diverse-seed=123 tests/
|
|
139
|
+
|
|
140
|
+
# Filter by markers before selection
|
|
141
|
+
pytest --diverse-k=20 --diverse-include-markers slow integration tests/
|
|
142
|
+
|
|
143
|
+
# Group parametrized tests (select at group level)
|
|
144
|
+
pytest --diverse-k=20 --diverse-group-parametrize tests/
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
Or via the `testcase-select` CLI:
|
|
148
|
+
|
|
149
|
+
```bash
|
|
150
|
+
# Full pipeline with pytest
|
|
151
|
+
testcase-select run --framework pytest --suite tests/ --k 20 --strategy fps
|
|
152
|
+
|
|
153
|
+
# Stage-by-stage
|
|
154
|
+
testcase-select vectorize --framework pytest --suite tests/ --output ./artifacts/
|
|
155
|
+
testcase-select select --artifacts ./artifacts/ --k 20
|
|
156
|
+
testcase-select execute --framework pytest --suite tests/ --selection selected_tests.json
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
The pytest plugin is installed automatically and activated by `--diverse-k`. It uses AST-based text representation with sentence embeddings for test diversity analysis.
|
|
160
|
+
|
|
127
161
|
### Direct Robot Framework Integration
|
|
128
162
|
|
|
129
163
|
You can also use the components directly with `robot`:
|
|
@@ -289,6 +323,11 @@ src/TestSelection/
|
|
|
289
323
|
prerun_modifier.py # DiversePreRunModifier (SuiteVisitor)
|
|
290
324
|
listener.py # DiverseDataDriverListener (Listener v3)
|
|
291
325
|
runner.py # ExecutionRunner
|
|
326
|
+
pytest/ # pytest integration
|
|
327
|
+
plugin.py # pytest plugin (pytest11 entry point)
|
|
328
|
+
collector.py # Programmatic test collection
|
|
329
|
+
text_builder.py # AST-based text representation
|
|
330
|
+
runner.py # pytest execution
|
|
292
331
|
pipeline/ # Orchestration layer
|
|
293
332
|
vectorize.py # Stage 1 orchestrator
|
|
294
333
|
select.py # Stage 2 orchestrator
|
|
@@ -343,6 +382,68 @@ uv run pytest -m integration
|
|
|
343
382
|
uv run pytest -m benchmark
|
|
344
383
|
```
|
|
345
384
|
|
|
385
|
+
## Coverage Retention Analysis
|
|
386
|
+
|
|
387
|
+
Measured on [robotframework-doctestlibrary](https://github.com/manykarim/robotframework-doctestlibrary) (5,045 statements in the DocTest package). All strategies use `all-MiniLM-L6-v2` embeddings (384-dim), seed=42.
|
|
388
|
+
|
|
389
|
+
### pytest (594 tests, 70.1% full coverage)
|
|
390
|
+
|
|
391
|
+
| Strategy | Selection | Tests | Coverage | Retention | Speedup |
|
|
392
|
+
|---|---|---|---|---|---|
|
|
393
|
+
| *(full suite)* | 100% | 594 | 70.1% | baseline | 1.0x |
|
|
394
|
+
| **fps** | **50%** | **297** | **65.9%** | **93.9%** | **3.4x** |
|
|
395
|
+
| dpp | 50% | 297 | 65.9% | 93.9% | 3.5x |
|
|
396
|
+
| kmedoids | 50% | 297 | 65.8% | 93.8% | 3.1x |
|
|
397
|
+
| fps_multi | 50% | 297 | 65.7% | 93.7% | 3.0x |
|
|
398
|
+
| facility | 50% | 297 | 64.9% | 92.6% | 2.9x |
|
|
399
|
+
| random | 50% | 297 | 64.2% | 91.5% | 2.3x |
|
|
400
|
+
| **facility** | **20%** | **119** | **58.6%** | **83.6%** | **7.5x** |
|
|
401
|
+
| kmedoids | 20% | 119 | 58.3% | 83.1% | 6.7x |
|
|
402
|
+
| fps | 20% | 119 | 56.1% | 80.0% | 6.7x |
|
|
403
|
+
| fps_multi | 20% | 119 | 53.8% | 76.8% | 8.0x |
|
|
404
|
+
| random | 20% | 119 | 53.2% | 75.8% | 5.4x |
|
|
405
|
+
| dpp | 20% | 119 | 47.6% | 67.8% | 6.0x |
|
|
406
|
+
| **fps_multi** | **10%** | **60** | **47.7%** | **67.9%** | **18.5x** |
|
|
407
|
+
| kmedoids | 10% | 60 | 46.9% | 66.9% | 14.9x |
|
|
408
|
+
| facility | 10% | 60 | 46.8% | 66.8% | 11.5x |
|
|
409
|
+
| random | 10% | 60 | 44.3% | 63.1% | 10.3x |
|
|
410
|
+
| dpp | 10% | 60 | 42.9% | 61.1% | 13.7x |
|
|
411
|
+
| fps | 10% | 60 | 40.6% | 57.9% | 21.4x |
|
|
412
|
+
|
|
413
|
+
### Robot Framework (126 tests, 63.5% full coverage)
|
|
414
|
+
|
|
415
|
+
| Strategy | Selection | Tests | Coverage | Retention | Speedup |
|
|
416
|
+
|---|---|---|---|---|---|
|
|
417
|
+
| *(full suite)* | 100% | 126 | 63.5% | baseline | 1.0x |
|
|
418
|
+
| **fps_multi** | **50%** | **63** | **61.6%** | **97.0%** | **1.5x** |
|
|
419
|
+
| facility | 50% | 63 | 61.6% | 96.9% | 1.5x |
|
|
420
|
+
| fps | 50% | 63 | 61.2% | 96.4% | 1.6x |
|
|
421
|
+
| kmedoids | 50% | 63 | 60.5% | 95.3% | 1.5x |
|
|
422
|
+
| dpp | 50% | 63 | 58.5% | 92.0% | 1.7x |
|
|
423
|
+
| random | 50% | 63 | 57.2% | 90.1% | 2.3x |
|
|
424
|
+
| **facility** | **20%** | **26** | **51.8%** | **81.5%** | **4.4x** |
|
|
425
|
+
| kmedoids | 20% | 26 | 47.5% | 74.8% | 5.5x |
|
|
426
|
+
| random | 20% | 26 | 46.0% | 72.5% | 5.7x |
|
|
427
|
+
| fps | 20% | 26 | 45.3% | 71.4% | 3.1x |
|
|
428
|
+
| fps_multi | 20% | 26 | 45.3% | 71.4% | 2.9x |
|
|
429
|
+
| dpp | 20% | 26 | 43.9% | 69.0% | 4.6x |
|
|
430
|
+
| **kmedoids** | **10%** | **13** | **42.3%** | **66.6%** | **11.3x** |
|
|
431
|
+
| dpp | 10% | 13 | 40.9% | 64.3% | 10.7x |
|
|
432
|
+
| facility | 10% | 13 | 40.2% | 63.3% | 12.9x |
|
|
433
|
+
| random | 10% | 13 | 37.6% | 59.2% | 11.3x |
|
|
434
|
+
| fps_multi | 10% | 13 | 37.5% | 59.1% | 10.7x |
|
|
435
|
+
| fps | 10% | 13 | 33.2% | 52.3% | 8.3x |
|
|
436
|
+
|
|
437
|
+
### Key Findings
|
|
438
|
+
|
|
439
|
+
- **50% selection retains 92-97% of coverage** across all strategies — the diversity algorithms select tests maximizing behavioral breadth, with minimal redundancy loss
|
|
440
|
+
- **Facility location excels at 20% selection** — 83.6% retention on pytest, 81.5% on Robot Framework, making it the best choice for aggressive smoke-test suites
|
|
441
|
+
- **kmedoids matches facility at 20%** on pytest (83.1% retention) and leads at 10% on Robot Framework (66.6%)
|
|
442
|
+
- **fps_multi leads at 10% pytest** (67.9% retention) where initial-point sensitivity matters most
|
|
443
|
+
- **DPP performs well at 50%** (93.9% retention, matching FPS) but drops at 20% due to FPS fallback for large k values
|
|
444
|
+
- **All diversity strategies outperform random** at 50% by 2-7 percentage points of retention
|
|
445
|
+
- At 50% robot selection, fps_multi achieves **97% retention** — meaning half the tests contribute almost no incremental coverage
|
|
446
|
+
|
|
346
447
|
## Performance
|
|
347
448
|
|
|
348
449
|
Benchmarked on 384-dimensional normalized vectors:
|
|
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "robotframework-testselection"
|
|
7
7
|
dynamic = ["version"]
|
|
8
|
-
description = "Vector-based diverse test case selection for Robot Framework"
|
|
8
|
+
description = "Vector-based diverse test case selection for Robot Framework and pytest"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
11
11
|
license = { text = "Apache-2.0" }
|
|
@@ -14,6 +14,7 @@ authors = [
|
|
|
14
14
|
]
|
|
15
15
|
keywords = [
|
|
16
16
|
"robotframework",
|
|
17
|
+
"pytest",
|
|
17
18
|
"testing",
|
|
18
19
|
"test-selection",
|
|
19
20
|
"diversity",
|
|
@@ -26,6 +27,7 @@ classifiers = [
|
|
|
26
27
|
"Framework :: Robot Framework",
|
|
27
28
|
"Framework :: Robot Framework :: Library",
|
|
28
29
|
"Framework :: Robot Framework :: Tool",
|
|
30
|
+
"Framework :: Pytest",
|
|
29
31
|
"Intended Audience :: Developers",
|
|
30
32
|
"License :: OSI Approved :: Apache Software License",
|
|
31
33
|
"Operating System :: OS Independent",
|
|
@@ -76,6 +78,9 @@ all = [
|
|
|
76
78
|
[project.scripts]
|
|
77
79
|
testcase-select = "TestSelection.cli:main"
|
|
78
80
|
|
|
81
|
+
[project.entry-points.pytest11]
|
|
82
|
+
diverse-selection = "TestSelection.pytest.plugin"
|
|
83
|
+
|
|
79
84
|
[tool.hatch.version]
|
|
80
85
|
path = "src/TestSelection/__init__.py"
|
|
81
86
|
|
|
@@ -128,3 +133,9 @@ module = [
|
|
|
128
133
|
"robot.*",
|
|
129
134
|
]
|
|
130
135
|
ignore_missing_imports = true
|
|
136
|
+
|
|
137
|
+
[dependency-groups]
|
|
138
|
+
dev = [
|
|
139
|
+
"coverage>=7.13.4",
|
|
140
|
+
"pytest-cov>=7.0.0",
|
|
141
|
+
]
|
{robotframework_testselection-0.2.0 → robotframework_testselection-0.3.0}/src/TestSelection/cli.py
RENAMED
|
@@ -18,13 +18,24 @@ def _setup_logging(verbose: bool = False) -> None:
|
|
|
18
18
|
)
|
|
19
19
|
|
|
20
20
|
|
|
21
|
+
def _add_framework_arg(parser: argparse.ArgumentParser) -> None:
|
|
22
|
+
"""Add --framework option to a parser."""
|
|
23
|
+
parser.add_argument(
|
|
24
|
+
"--framework",
|
|
25
|
+
choices=["robot", "pytest"],
|
|
26
|
+
default="robot",
|
|
27
|
+
help="Test framework: robot (default) or pytest.",
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
21
31
|
def _add_vectorize_parser(subparsers: argparse._SubParsersAction) -> None:
|
|
22
32
|
p = subparsers.add_parser(
|
|
23
33
|
"vectorize", help="Stage 1: vectorize test suite",
|
|
24
34
|
)
|
|
35
|
+
_add_framework_arg(p)
|
|
25
36
|
p.add_argument(
|
|
26
37
|
"--suite", required=True, type=Path,
|
|
27
|
-
help="Path to
|
|
38
|
+
help="Path to test suite directory",
|
|
28
39
|
)
|
|
29
40
|
p.add_argument(
|
|
30
41
|
"--output", required=True, type=Path,
|
|
@@ -36,7 +47,7 @@ def _add_vectorize_parser(subparsers: argparse._SubParsersAction) -> None:
|
|
|
36
47
|
)
|
|
37
48
|
p.add_argument(
|
|
38
49
|
"--resolve-depth", type=int, default=0,
|
|
39
|
-
help="Keyword resolve depth",
|
|
50
|
+
help="Keyword resolve depth (Robot Framework only)",
|
|
40
51
|
)
|
|
41
52
|
p.add_argument(
|
|
42
53
|
"--force", action="store_true",
|
|
@@ -44,7 +55,7 @@ def _add_vectorize_parser(subparsers: argparse._SubParsersAction) -> None:
|
|
|
44
55
|
)
|
|
45
56
|
p.add_argument(
|
|
46
57
|
"--datadriver-csv", nargs="*", type=Path,
|
|
47
|
-
help="DataDriver CSV files",
|
|
58
|
+
help="DataDriver CSV files (Robot Framework only)",
|
|
48
59
|
)
|
|
49
60
|
p.set_defaults(func=_cmd_vectorize)
|
|
50
61
|
|
|
@@ -99,15 +110,16 @@ def _add_execute_parser(subparsers: argparse._SubParsersAction) -> None:
|
|
|
99
110
|
"execute",
|
|
100
111
|
help="Stage 3: execute selected tests",
|
|
101
112
|
epilog=(
|
|
102
|
-
"All arguments after -- are passed directly to
|
|
113
|
+
"All arguments after -- are passed directly to the test runner. "
|
|
103
114
|
"Example: testcase-select execute --suite tests/ "
|
|
104
115
|
"--selection sel.json "
|
|
105
116
|
"-- --variable ENV:staging --loglevel DEBUG"
|
|
106
117
|
),
|
|
107
118
|
)
|
|
119
|
+
_add_framework_arg(p)
|
|
108
120
|
p.add_argument(
|
|
109
121
|
"--suite", required=True, type=Path,
|
|
110
|
-
help="Path to
|
|
122
|
+
help="Path to test suite directory",
|
|
111
123
|
)
|
|
112
124
|
p.add_argument(
|
|
113
125
|
"--selection", required=True, type=Path,
|
|
@@ -129,14 +141,15 @@ def _add_run_parser(subparsers: argparse._SubParsersAction) -> None:
|
|
|
129
141
|
"run",
|
|
130
142
|
help="Full pipeline: vectorize + select + execute",
|
|
131
143
|
epilog=(
|
|
132
|
-
"All arguments after -- are passed directly to
|
|
144
|
+
"All arguments after -- are passed directly to the test runner. "
|
|
133
145
|
"Example: testcase-select run --suite tests/ --k 20 "
|
|
134
146
|
"-- --include smoke --variable ENV:staging"
|
|
135
147
|
),
|
|
136
148
|
)
|
|
149
|
+
_add_framework_arg(p)
|
|
137
150
|
p.add_argument(
|
|
138
151
|
"--suite", required=True, type=Path,
|
|
139
|
-
help="Path to
|
|
152
|
+
help="Path to test suite directory",
|
|
140
153
|
)
|
|
141
154
|
p.add_argument(
|
|
142
155
|
"--k", type=int, default=default_k,
|
|
@@ -156,7 +169,7 @@ def _add_run_parser(subparsers: argparse._SubParsersAction) -> None:
|
|
|
156
169
|
)
|
|
157
170
|
p.add_argument(
|
|
158
171
|
"--resolve-depth", type=int, default=0,
|
|
159
|
-
help="Keyword resolve depth",
|
|
172
|
+
help="Keyword resolve depth (Robot Framework only)",
|
|
160
173
|
)
|
|
161
174
|
p.add_argument(
|
|
162
175
|
"--force", action="store_true",
|
|
@@ -170,6 +183,11 @@ def _add_run_parser(subparsers: argparse._SubParsersAction) -> None:
|
|
|
170
183
|
|
|
171
184
|
|
|
172
185
|
def _cmd_vectorize(args: argparse.Namespace) -> int:
|
|
186
|
+
framework = getattr(args, "framework", "robot")
|
|
187
|
+
|
|
188
|
+
if framework == "pytest":
|
|
189
|
+
return _cmd_vectorize_pytest(args)
|
|
190
|
+
|
|
173
191
|
from TestSelection.pipeline.vectorize import run_vectorize
|
|
174
192
|
|
|
175
193
|
try:
|
|
@@ -193,6 +211,29 @@ def _cmd_vectorize(args: argparse.Namespace) -> int:
|
|
|
193
211
|
return 2
|
|
194
212
|
|
|
195
213
|
|
|
214
|
+
def _cmd_vectorize_pytest(args: argparse.Namespace) -> int:
|
|
215
|
+
"""Vectorize pytest tests."""
|
|
216
|
+
from TestSelection.pipeline.vectorize_pytest import run_vectorize_pytest
|
|
217
|
+
|
|
218
|
+
try:
|
|
219
|
+
indexed = run_vectorize_pytest(
|
|
220
|
+
suite_path=args.suite,
|
|
221
|
+
artifact_dir=args.output,
|
|
222
|
+
model_name=args.model,
|
|
223
|
+
force=args.force,
|
|
224
|
+
)
|
|
225
|
+
if indexed:
|
|
226
|
+
logger.info("[DIVERSE-SELECT] Vectorization complete (pytest)")
|
|
227
|
+
else:
|
|
228
|
+
logger.info(
|
|
229
|
+
"[DIVERSE-SELECT] Vectorization skipped (no changes)"
|
|
230
|
+
)
|
|
231
|
+
return 0
|
|
232
|
+
except Exception as exc:
|
|
233
|
+
logger.error("[DIVERSE-SELECT] Vectorization failed: %s", exc)
|
|
234
|
+
return 2
|
|
235
|
+
|
|
236
|
+
|
|
196
237
|
def _cmd_select(args: argparse.Namespace) -> int:
|
|
197
238
|
from TestSelection.pipeline.select import run_select
|
|
198
239
|
|
|
@@ -219,6 +260,11 @@ def _cmd_select(args: argparse.Namespace) -> int:
|
|
|
219
260
|
|
|
220
261
|
|
|
221
262
|
def _cmd_execute(args: argparse.Namespace) -> int:
|
|
263
|
+
framework = getattr(args, "framework", "robot")
|
|
264
|
+
|
|
265
|
+
if framework == "pytest":
|
|
266
|
+
return _cmd_execute_pytest(args)
|
|
267
|
+
|
|
222
268
|
from TestSelection.pipeline.execute import run_execute
|
|
223
269
|
|
|
224
270
|
return run_execute(
|
|
@@ -229,7 +275,24 @@ def _cmd_execute(args: argparse.Namespace) -> int:
|
|
|
229
275
|
)
|
|
230
276
|
|
|
231
277
|
|
|
278
|
+
def _cmd_execute_pytest(args: argparse.Namespace) -> int:
|
|
279
|
+
"""Execute pytest with selection."""
|
|
280
|
+
from TestSelection.pytest.runner import run_pytest_with_selection
|
|
281
|
+
|
|
282
|
+
return run_pytest_with_selection(
|
|
283
|
+
suite_path=args.suite,
|
|
284
|
+
selection_file=args.selection,
|
|
285
|
+
output_dir=str(args.output_dir),
|
|
286
|
+
extra_args=args.robot_passthrough,
|
|
287
|
+
)
|
|
288
|
+
|
|
289
|
+
|
|
232
290
|
def _cmd_run(args: argparse.Namespace) -> int:
|
|
291
|
+
framework = getattr(args, "framework", "robot")
|
|
292
|
+
|
|
293
|
+
if framework == "pytest":
|
|
294
|
+
return _cmd_run_pytest(args)
|
|
295
|
+
|
|
233
296
|
from TestSelection.pipeline.execute import run_execute
|
|
234
297
|
from TestSelection.pipeline.select import run_select
|
|
235
298
|
from TestSelection.pipeline.vectorize import run_vectorize
|
|
@@ -280,6 +343,29 @@ def _cmd_run(args: argparse.Namespace) -> int:
|
|
|
280
343
|
)
|
|
281
344
|
|
|
282
345
|
|
|
346
|
+
def _cmd_run_pytest(args: argparse.Namespace) -> int:
|
|
347
|
+
"""Run full pipeline for pytest."""
|
|
348
|
+
from TestSelection.pytest.runner import run_pytest_pipeline
|
|
349
|
+
|
|
350
|
+
try:
|
|
351
|
+
return run_pytest_pipeline(
|
|
352
|
+
suite_path=args.suite,
|
|
353
|
+
k=args.k,
|
|
354
|
+
strategy=args.strategy,
|
|
355
|
+
seed=args.seed,
|
|
356
|
+
model_name=args.model,
|
|
357
|
+
output_dir=str(args.output_dir),
|
|
358
|
+
extra_args=args.robot_passthrough,
|
|
359
|
+
)
|
|
360
|
+
except Exception as exc:
|
|
361
|
+
logger.warning(
|
|
362
|
+
"[DIVERSE-SELECT] pytest pipeline failed, "
|
|
363
|
+
"falling back to all tests: %s",
|
|
364
|
+
exc,
|
|
365
|
+
)
|
|
366
|
+
return _fallback_execute_pytest(args)
|
|
367
|
+
|
|
368
|
+
|
|
283
369
|
def _fallback_execute(args: argparse.Namespace) -> int:
|
|
284
370
|
"""Execute all tests without selection (graceful degradation)."""
|
|
285
371
|
logger.info("[DIVERSE-SELECT] Running all tests (no selection)")
|
|
@@ -290,7 +376,6 @@ def _fallback_execute(args: argparse.Namespace) -> int:
|
|
|
290
376
|
"--outputdir",
|
|
291
377
|
str(args.output_dir),
|
|
292
378
|
]
|
|
293
|
-
# Pass through any robot options from after --
|
|
294
379
|
if args.robot_passthrough:
|
|
295
380
|
robot_args.extend(args.robot_passthrough)
|
|
296
381
|
robot_args.append(str(args.suite))
|
|
@@ -302,13 +387,30 @@ def _fallback_execute(args: argparse.Namespace) -> int:
|
|
|
302
387
|
return 2
|
|
303
388
|
|
|
304
389
|
|
|
390
|
+
def _fallback_execute_pytest(args: argparse.Namespace) -> int:
|
|
391
|
+
"""Execute all pytest tests without selection (graceful degradation)."""
|
|
392
|
+
logger.info("[DIVERSE-SELECT] Running all pytest tests (no selection)")
|
|
393
|
+
try:
|
|
394
|
+
import pytest
|
|
395
|
+
|
|
396
|
+
pytest_args = [str(args.suite)]
|
|
397
|
+
if args.robot_passthrough:
|
|
398
|
+
pytest_args.extend(args.robot_passthrough)
|
|
399
|
+
return pytest.main(pytest_args)
|
|
400
|
+
except Exception as exc:
|
|
401
|
+
logger.error(
|
|
402
|
+
"[DIVERSE-SELECT] Fallback execution failed: %s", exc,
|
|
403
|
+
)
|
|
404
|
+
return 2
|
|
405
|
+
|
|
406
|
+
|
|
305
407
|
def build_parser() -> argparse.ArgumentParser:
|
|
306
408
|
"""Build the argument parser."""
|
|
307
409
|
parser = argparse.ArgumentParser(
|
|
308
410
|
prog="testcase-select",
|
|
309
411
|
description=(
|
|
310
412
|
"Vector-based diverse test case selection "
|
|
311
|
-
"for Robot Framework"
|
|
413
|
+
"for Robot Framework and pytest"
|
|
312
414
|
),
|
|
313
415
|
)
|
|
314
416
|
parser.add_argument(
|
|
@@ -328,10 +430,10 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
328
430
|
def _split_robot_passthrough(
|
|
329
431
|
argv: list[str],
|
|
330
432
|
) -> tuple[list[str], list[str]]:
|
|
331
|
-
"""Split argv at -- into our args and
|
|
433
|
+
"""Split argv at -- into our args and passthrough args.
|
|
332
434
|
|
|
333
|
-
Returns (our_args,
|
|
334
|
-
is empty.
|
|
435
|
+
Returns (our_args, passthrough_args). If no -- is present,
|
|
436
|
+
passthrough_args is empty.
|
|
335
437
|
"""
|
|
336
438
|
try:
|
|
337
439
|
sep = argv.index("--")
|
|
@@ -343,7 +445,7 @@ def _split_robot_passthrough(
|
|
|
343
445
|
def main(argv: list[str] | None = None) -> int:
|
|
344
446
|
"""Main CLI entry point.
|
|
345
447
|
|
|
346
|
-
Arguments after -- are passed through to
|
|
448
|
+
Arguments after -- are passed through to the test runner:
|
|
347
449
|
testcase-select run --suite tests/ --k 20 \\
|
|
348
450
|
-- --variable ENV:staging --loglevel DEBUG --include smoke
|
|
349
451
|
"""
|
|
@@ -15,10 +15,16 @@ class CacheInvalidator:
|
|
|
15
15
|
def __init__(self, hash_store_path: Path) -> None:
|
|
16
16
|
self._hash_store_path = hash_store_path
|
|
17
17
|
|
|
18
|
-
def has_changes(
|
|
18
|
+
def has_changes(
|
|
19
|
+
self,
|
|
20
|
+
suite_path: Path,
|
|
21
|
+
glob_patterns: tuple[str, ...] | None = None,
|
|
22
|
+
) -> bool:
|
|
19
23
|
"""Compare current file hashes with stored hashes.
|
|
20
24
|
|
|
21
25
|
Returns True if there are changes or no stored hashes exist.
|
|
26
|
+
If glob_patterns is provided, uses those instead of the default
|
|
27
|
+
.robot/.csv patterns.
|
|
22
28
|
"""
|
|
23
29
|
if not self._hash_store_path.exists():
|
|
24
30
|
logger.info(
|
|
@@ -28,7 +34,7 @@ class CacheInvalidator:
|
|
|
28
34
|
return True
|
|
29
35
|
|
|
30
36
|
stored = json.loads(self._hash_store_path.read_text())
|
|
31
|
-
current = self._compute_hashes(suite_path)
|
|
37
|
+
current = self._compute_hashes(suite_path, glob_patterns)
|
|
32
38
|
|
|
33
39
|
if current != stored:
|
|
34
40
|
changed = set(current.keys()) ^ set(stored.keys())
|
|
@@ -49,19 +55,31 @@ class CacheInvalidator:
|
|
|
49
55
|
)
|
|
50
56
|
return False
|
|
51
57
|
|
|
52
|
-
def save_hashes(
|
|
58
|
+
def save_hashes(
|
|
59
|
+
self,
|
|
60
|
+
suite_path: Path,
|
|
61
|
+
glob_patterns: tuple[str, ...] | None = None,
|
|
62
|
+
) -> None:
|
|
53
63
|
"""Save current file hashes to the hash store."""
|
|
54
|
-
hashes = self._compute_hashes(suite_path)
|
|
64
|
+
hashes = self._compute_hashes(suite_path, glob_patterns)
|
|
55
65
|
self._hash_store_path.parent.mkdir(parents=True, exist_ok=True)
|
|
56
66
|
self._hash_store_path.write_text(json.dumps(hashes, indent=2))
|
|
57
67
|
|
|
58
|
-
def _compute_hashes(
|
|
59
|
-
|
|
68
|
+
def _compute_hashes(
|
|
69
|
+
self,
|
|
70
|
+
suite_path: Path,
|
|
71
|
+
glob_patterns: tuple[str, ...] | None = None,
|
|
72
|
+
) -> dict[str, str]:
|
|
73
|
+
"""Compute MD5 hashes for source files.
|
|
74
|
+
|
|
75
|
+
By default hashes .robot and .csv files. Pass glob_patterns
|
|
76
|
+
to override (e.g., ("**/*.py",) for pytest).
|
|
77
|
+
"""
|
|
60
78
|
hashes: dict[str, str] = {}
|
|
61
79
|
target = suite_path if suite_path.is_dir() else suite_path.parent
|
|
62
80
|
|
|
63
81
|
if suite_path.is_dir():
|
|
64
|
-
patterns = ["*.robot", "*.csv"]
|
|
82
|
+
patterns = list(glob_patterns) if glob_patterns else ["*.robot", "*.csv"]
|
|
65
83
|
for pattern in patterns:
|
|
66
84
|
for p in sorted(target.rglob(pattern)):
|
|
67
85
|
md5 = hashlib.md5(p.read_bytes()).hexdigest()
|