robotframework-testselection 0.1.0__py3-none-any.whl

TestSelection/selection/strategy.py

@@ -0,0 +1,142 @@
+"""Selection strategy protocol and domain value objects."""
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Protocol, runtime_checkable
+
+import numpy as np
+from numpy.typing import NDArray
+
+
+@runtime_checkable
+class SelectionStrategy(Protocol):
+    """Protocol for diversity selection algorithms.
+
+    All implementations take a (N, d) embedding matrix and return
+    k indices into that matrix.
+    """
+
+    @property
+    def name(self) -> str: ...
+
+    def select(
+        self, vectors: NDArray[np.float32], k: int, seed: int = 42
+    ) -> list[int]: ...
+
+
+@dataclass(frozen=True)
+class SelectedTest:
+    """A test case that was selected by the diversity algorithm."""
+
+    name: str
+    id: str
+    suite: str
+    is_datadriver: bool
+
+
+@dataclass(frozen=True)
+class DiversityMetrics:
+    """Statistics describing the diversity of a selected test subset."""
+
+    avg_pairwise_distance: float
+    min_pairwise_distance: float
+    suite_coverage: int
+    suite_total: int
+
+    @property
+    def suite_coverage_ratio(self) -> float:
+        if self.suite_total == 0:
+            return 0.0
+        return self.suite_coverage / self.suite_total
+
+
+@dataclass(frozen=True)
+class TagFilter:
+    """Pre-filter criteria applied before diversity selection."""
+
+    include_tags: frozenset[str] = frozenset()
+    exclude_tags: frozenset[str] = frozenset()
+    include_datadriver: bool = True
+
+    def matches(self, tags: frozenset[str], is_datadriver: bool) -> bool:
+        if not self.include_datadriver and is_datadriver:
+            return False
+        normalized_tags = frozenset(t.lower() for t in tags)
+        if self.include_tags and not (normalized_tags & self.include_tags):
+            return False
+        return not (self.exclude_tags and (normalized_tags & self.exclude_tags))
+
+
+@dataclass(frozen=True)
+class SelectionResult:
+    """Aggregate root: the output of the selection pipeline stage."""
+
+    strategy: str
+    k: int
+    seed: int
+    total_tests: int
+    filtered_tests: int
+    selected: tuple[SelectedTest, ...]
+    diversity_metrics: DiversityMetrics
+
+    def to_json(self, path: Path) -> None:
+        data = {
+            "strategy": self.strategy,
+            "k": self.k,
+            "seed": self.seed,
+            "total_tests": self.total_tests,
+            "filtered_tests": self.filtered_tests,
+            "selected": [
+                {
+                    "name": t.name,
+                    "id": t.id,
+                    "suite": t.suite,
+                    "is_datadriver": t.is_datadriver,
+                }
+                for t in self.selected
+            ],
+            "diversity_metrics": {
+                "avg_pairwise_distance": self.diversity_metrics.avg_pairwise_distance,
+                "min_pairwise_distance": self.diversity_metrics.min_pairwise_distance,
+                "suite_coverage": self.diversity_metrics.suite_coverage,
+                "suite_total": self.diversity_metrics.suite_total,
+            },
+        }
+        path.write_text(json.dumps(data, indent=2))
+
+    @classmethod
+    def from_json(cls, path: Path) -> SelectionResult:
+        data = json.loads(path.read_text())
+        selected = tuple(
+            SelectedTest(
+                name=t["name"],
+                id=t["id"],
+                suite=t.get("suite", ""),
+                is_datadriver=t.get("is_datadriver", False),
+            )
+            for t in data["selected"]
+        )
+        return cls(
+            strategy=data["strategy"],
+            k=data["k"],
+            seed=data.get("seed", 42),
+            total_tests=data["total_tests"],
+            filtered_tests=data.get("filtered_tests", data["total_tests"]),
+            selected=selected,
+            diversity_metrics=DiversityMetrics(
+                avg_pairwise_distance=data.get("diversity_metrics", {}).get(
+                    "avg_pairwise_distance", 0.0
+                ),
+                min_pairwise_distance=data.get("diversity_metrics", {}).get(
+                    "min_pairwise_distance", 0.0
+                ),
+                suite_coverage=data.get("diversity_metrics", {}).get(
+                    "suite_coverage", 0
+                ),
+                suite_total=data.get("diversity_metrics", {}).get(
+                    "suite_total", 0
+                ),
+            ),
+        )

TestSelection/shared/__init__.py

@@ -0,0 +1 @@
+"""Shared kernel: value objects, types, and configuration."""

TestSelection/shared/config.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from TestSelection.shared.types import NOISE_PREFIXES
+
+
+@dataclass(frozen=True)
+class PipelineConfig:
+    """Configuration for a complete pipeline run."""
+
+    model_name: str = "all-MiniLM-L6-v2"
+    resolve_depth: int = 0
+    k: int = 50
+    strategy: str = "fps"
+    seed: int = 42
+    output_dir: Path = Path("./results")
+    force_reindex: bool = False
+
+
+@dataclass(frozen=True)
+class TextBuilderConfig:
+    """Configuration for TextRepresentationBuilder."""
+
+    resolve_depth: int = 0
+    include_tags: bool = True
+    include_suite_name: bool = False
+    noise_prefixes: frozenset[str] = field(
+        default_factory=lambda: frozenset(NOISE_PREFIXES)
+    )

TestSelection/shared/types.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class TestCaseId:
+    """Stable identity for a test case, derived from source file + name."""
+
+    value: str
+
+    @classmethod
+    def from_source_and_name(cls, source: str, name: str) -> TestCaseId:
+        raw = f"{source}::{name}"
+        return cls(value=hashlib.md5(raw.encode()).hexdigest())
+
+
+@dataclass(frozen=True)
+class Tag:
+    """A Robot Framework test tag, normalized for comparison."""
+
+    value: str
+
+    @property
+    def normalized(self) -> str:
+        return self.value.lower().strip()
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, Tag):
+            return self.normalized == other.normalized
+        return NotImplemented
+
+    def __hash__(self) -> int:
+        return hash(self.normalized)
+
+
+@dataclass(frozen=True)
+class SuitePath:
+    """Path to a .robot file or test directory."""
+
+    value: Path
+
+    @property
+    def is_directory(self) -> bool:
+        return self.value.is_dir()
+
+
+@dataclass(frozen=True)
+class FileHash:
+    """Content hash of a .robot file for change detection."""
+
+    path: str
+    md5: str
+
+
+@dataclass(frozen=True)
+class TextRepresentation:
+    """Embeddable text representation of a test case.
+
+    Built from test name, tags, keyword names, and semantic arguments.
+    Excludes DOM locators, variable placeholders, and XPaths.
+    """
+
+    text: str
+    resolve_depth: int = 0
+    includes_tags: bool = True
+    includes_keyword_args: bool = True
+
+
+NOISE_PREFIXES = ("id:", "css:", "xpath:", "//", "${", "@{", "%{", "&{")
+
+
+@dataclass(frozen=True)
+class KeywordTree:
+    """Resolved keyword call tree for a single keyword invocation."""
+
+    keyword_name: str
+    args: tuple[str, ...]
+    children: tuple[KeywordTree, ...] = ()
+
+    def flatten(self) -> str:
+        """Convert to natural language, filtering noise arguments."""
+        kw = self.keyword_name.replace("_", " ")
+        semantic_args = [
+            a
+            for a in self.args
+            if not any(a.startswith(p) for p in NOISE_PREFIXES)
+        ]
+        text = kw
+        if semantic_args:
+            text += f" with {', '.join(semantic_args)}"
+        children_text = " ".join(c.flatten() for c in self.children)
+        return f"{text} {children_text}".strip()
+
+
+@dataclass(frozen=True)
+class UserKeywordRef:
+    """Domain representation of a user keyword from robot.api."""
+
+    name: str
+    normalized_name: str
+    body_items: tuple
+
+
+@dataclass(frozen=True)
+class TestCaseRecord:
+    """Domain entity representing a parsed test case ready for embedding."""
+
+    test_id: TestCaseId
+    name: str
+    tags: frozenset[Tag]
+    suite_source: SuitePath
+    suite_name: str
+    text_representation: TextRepresentation
+    is_datadriver: bool = False

robotframework_testselection-0.1.0.dist-info/METADATA

@@ -0,0 +1,408 @@
+Metadata-Version: 2.4
+Name: robotframework-testselection
+Version: 0.1.0
+Summary: Vector-based diverse test case selection for Robot Framework
+Project-URL: Homepage, https://github.com/manykarim/robotframework-testselection
+Project-URL: Documentation, https://github.com/manykarim/robotframework-testselection#readme
+Project-URL: Repository, https://github.com/manykarim/robotframework-testselection
+Project-URL: Issues, https://github.com/manykarim/robotframework-testselection/issues
+Project-URL: Changelog, https://github.com/manykarim/robotframework-testselection/blob/main/CHANGELOG.md
+Author: Many Kasiriha
+License: Apache-2.0
+License-File: LICENSE
+Keywords: diversity,embeddings,machine-learning,nlp,robotframework,test-selection,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: Robot Framework
+Classifier: Framework :: Robot Framework :: Library
+Classifier: Framework :: Robot Framework :: Tool
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Requires-Dist: numpy>=1.24
+Requires-Dist: robotframework>=7.0
+Requires-Dist: scikit-learn>=1.3
+Provides-Extra: all
+Requires-Dist: apricot-select>=0.6; extra == 'all'
+Requires-Dist: chromadb>=0.4; extra == 'all'
+Requires-Dist: dppy>=0.3; extra == 'all'
+Requires-Dist: mypy; extra == 'all'
+Requires-Dist: pytest-benchmark; extra == 'all'
+Requires-Dist: pytest-cov; extra == 'all'
+Requires-Dist: pytest>=8.0; extra == 'all'
+Requires-Dist: ruff; extra == 'all'
+Requires-Dist: scikit-learn-extra>=0.3; extra == 'all'
+Requires-Dist: sentence-transformers>=2.2; extra == 'all'
+Provides-Extra: chromadb
+Requires-Dist: chromadb>=0.4; extra == 'chromadb'
+Provides-Extra: dev
+Requires-Dist: mypy; extra == 'dev'
+Requires-Dist: pytest-benchmark; extra == 'dev'
+Requires-Dist: pytest-cov; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Provides-Extra: selection-extras
+Requires-Dist: apricot-select>=0.6; extra == 'selection-extras'
+Requires-Dist: dppy>=0.3; extra == 'selection-extras'
+Requires-Dist: scikit-learn-extra>=0.3; extra == 'selection-extras'
+Provides-Extra: vectorize
+Requires-Dist: sentence-transformers>=2.2; extra == 'vectorize'
+Description-Content-Type: text/markdown
+
+# robotframework-testselection
+
+Vector-based diverse test case selection for Robot Framework. Embeds test cases as semantic vectors and selects maximally diverse subsets to reduce test suite execution time while preserving coverage breadth.
+
+## How It Works
+
+The system operates as a **3-stage pipeline**:
+
+```
+Vectorize              Select                Execute
+(.robot files) ──►  (embeddings.npz   ──►  (selected_tests.json
+                     test_manifest.json)     + robot --prerunmodifier)
+```
+
+1. **Vectorize** — Parses `.robot` files via the Robot Framework API, converts each test case to a natural-language text representation (name + tags + resolved keyword tree), then encodes with `all-MiniLM-L6-v2` (384-dim sentence embeddings).
+2. **Select** — Loads embedding vectors and applies a diversity-maximizing selection algorithm (default: Farthest Point Sampling) to choose *k* tests that are as semantically different from each other as possible.
+3. **Execute** — Runs the selected tests via Robot Framework using a `PreRunModifier` (for standard tests) and a `Listener v3` (for DataDriver-generated tests).
+
+If any stage fails, the pipeline **gracefully degrades** by running all tests (exit code 2).
+
+## Installation
+
+```bash
+# From PyPI
+pip install robotframework-testselection
+
+# With sentence-transformers for vectorization
+pip install robotframework-testselection[vectorize]
+
+# With all optional selection algorithms
+pip install robotframework-testselection[selection-extras]
+
+# Everything
+pip install robotframework-testselection[all]
+```
+
+### Development Install (from source)
+
+Requires Python 3.10+ and [uv](https://docs.astral.sh/uv/).
+
+```bash
+uv sync --extra vectorize
+uv sync --extra all
+```
+
+### Dependency Groups
+
+| Group | Packages | Purpose |
+|-------|----------|---------|
+| *(base)* | `robotframework`, `numpy`, `scikit-learn` | Core pipeline |
+| `vectorize` | `sentence-transformers` | Embedding model (Stage 1) |
+| `selection-extras` | `scikit-learn-extra`, `dppy`, `apricot-select` | k-Medoids, DPP, Facility Location strategies |
+| `chromadb` | `chromadb` | Alternative vector storage |
+| `dev` | `pytest`, `pytest-cov`, `pytest-benchmark`, `ruff`, `mypy` | Development |
+
+## Quick Start
+
+### Full Pipeline (one command)
+
+```bash
+testcase-select run \
+  --suite tests/robot/ \
+  --k 20 \
+  --strategy fps \
+  --seed 42 \
+  --output-dir ./results/
+```
+
+### Stage-by-Stage
+
+```bash
+# Stage 1: Vectorize
+testcase-select vectorize \
+  --suite tests/robot/ \
+  --output ./artifacts/ \
+  --model all-MiniLM-L6-v2 \
+  --resolve-depth 2
+
+# Stage 2: Select
+testcase-select select \
+  --artifacts ./artifacts/ \
+  --k 20 \
+  --strategy fps \
+  --seed 42 \
+  --output selected_tests.json
+
+# Stage 3: Execute
+testcase-select execute \
+  --suite tests/robot/ \
+  --selection selected_tests.json \
+  --output-dir ./results/
+```
+
+### With DataDriver CSV Files
+
+```bash
+testcase-select vectorize \
+  --suite tests/robot/ \
+  --output ./artifacts/ \
+  --datadriver-csv tests/data/login.csv tests/data/search.csv
+```
+
+### Direct Robot Framework Integration
+
+You can also use the components directly with `robot`:
+
+```bash
+# PreRunModifier for standard tests
+robot --prerunmodifier TestSelection.execution.prerun_modifier.DiversePreRunModifier:selected_tests.json tests/
+
+# Listener v3 for DataDriver tests
+robot --listener TestSelection.execution.listener.DiverseDataDriverListener:selected_tests.json tests/
+
+# Both together
+robot \
+  --prerunmodifier TestSelection.execution.prerun_modifier.DiversePreRunModifier:selected_tests.json \
+  --listener TestSelection.execution.listener.DiverseDataDriverListener:selected_tests.json \
+  tests/
+```
+
+## Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `DIVERSE_K` | `50` | Number of tests to select |
+| `DIVERSE_STRATEGY` | `fps` | Selection algorithm |
+| `DIVERSE_SEED` | `42` | Random seed for reproducibility |
+| `DIVERSE_OUTPUT` | *(none)* | Output file path for selection JSON |
+
+## Selection Strategies
+
+| Strategy | Name | Dependencies | Description |
+|----------|------|-------------|-------------|
+| **FPS** | `fps` | *(base)* | Farthest Point Sampling. Greedy farthest-first traversal. O(N*k*d). 2-approximation guarantee for max-min dispersion. **Default.** |
+| **FPS Multi-Start** | `fps_multi` | *(base)* | Runs FPS from multiple random starting points, keeps the result with the highest minimum pairwise distance. Mitigates initial-point sensitivity. |
+| **k-Medoids** | `kmedoids` | `selection-extras` | PAM algorithm for medoid-based clustering. Better centroid representativeness. |
+| **DPP** | `dpp` | `selection-extras` | Determinantal Point Process. Probabilistic repulsion-based sampling. |
+| **Facility Location** | `facility` | `selection-extras` | Submodular facility location maximization. Optimizes for both diversity and representativeness. |
+
+## Tag Filtering
+
+Filter tests by tags before selection:
+
+```bash
+# Include only smoke and regression tests
+testcase-select select \
+  --artifacts ./artifacts/ \
+  --k 20 \
+  --include-tags smoke regression
+
+# Exclude slow tests
+testcase-select select \
+  --artifacts ./artifacts/ \
+  --k 20 \
+  --exclude-tags slow manual
+
+# Exclude DataDriver tests
+testcase-select select \
+  --artifacts ./artifacts/ \
+  --k 20 \
+  --no-datadriver
+```
+
+## Caching
+
+Stage 1 uses **content-hash caching**. It computes MD5 hashes of all `.robot` and `.csv` files and stores them alongside artifacts. On subsequent runs, vectorization is skipped when no source files have changed. Use `--force` to bypass the cache:
+
+```bash
+testcase-select vectorize --suite tests/ --output ./artifacts/ --force
+```
+
+## Artifacts
+
+The pipeline produces these artifacts:
+
+| File | Stage | Format | Contents |
+|------|-------|--------|----------|
+| `embeddings.npz` | Vectorize | NumPy compressed | N x 384 float32 matrix |
+| `test_manifest.json` | Vectorize | JSON | Test metadata (names, tags, suites, IDs) |
+| `file_hashes.json` | Vectorize | JSON | Source file MD5 hashes for cache invalidation |
+| `selected_tests.json` | Select | JSON | Selected test list + diversity metrics |
+
+### Selection Output Format
+
+```json
+{
+  "strategy": "fps",
+  "k": 20,
+  "seed": 42,
+  "total_tests": 150,
+  "filtered_tests": 120,
+  "selected": [
+    {
+      "name": "Login With Valid Credentials",
+      "id": "a1b2c3d4...",
+      "suite": "tests/login.robot",
+      "is_datadriver": false
+    }
+  ],
+  "diversity_metrics": {
+    "avg_pairwise_distance": 0.8234,
+    "min_pairwise_distance": 0.4512,
+    "suite_coverage": 8,
+    "suite_total": 10
+  }
+}
+```
+
+## CI/CD Integration
+
+Pre-built configurations are provided in `config/`:
+
+### GitHub Actions
+
+```yaml
+# .github/workflows/diverse-tests.yml
+# Copy from config/github-actions.yml
+```
+
+Features:
+- 3-job pipeline with artifact transfer between stages
+- Content-hash caching (`actions/cache@v4`) to skip vectorization
+- Automatic PR annotations with selection summary
+- `workflow_dispatch` for manual runs with custom k/strategy
+
+### GitLab CI
+
+```yaml
+# .gitlab-ci.yml
+# Copy from config/gitlab-ci.yml
+```
+
+### Jenkins
+
+```groovy
+// Jenkinsfile
+// Copy from config/Jenkinsfile
+```
+
+## Project Structure
+
+```
+src/TestSelection/
+  shared/              # Shared kernel (types, config)
+    types.py           # TestCaseId, Tag, TestCaseRecord, KeywordTree, ...
+    config.py          # PipelineConfig, TextBuilderConfig
+  parsing/             # Bounded context: Robot Framework parsing
+    suite_collector.py # RobotApiAdapter (TestSuite.from_file_system)
+    keyword_resolver.py# Recursive keyword tree resolution
+    text_builder.py    # Natural language text representation
+    datadriver_reader.py # DataDriver CSV reader
+  embedding/           # Bounded context: vector embedding
+    ports.py           # EmbeddingModel protocol
+    embedder.py        # SentenceTransformerAdapter (ACL)
+    models.py          # EmbeddingMatrix aggregate, ManifestEntry
+  selection/           # Bounded context: diversity selection
+    strategy.py        # SelectionStrategy protocol, SelectionResult
+    fps.py             # FarthestPointSampling, FPSMultiStart
+    kmedoids.py        # KMedoidsSelection (optional)
+    dpp.py             # DPPSelection (optional)
+    facility.py        # FacilityLocationSelection (optional)
+    registry.py        # StrategyRegistry with auto-discovery
+    filtering.py       # Tag-based pre-selection filtering
+  execution/           # Bounded context: Robot Framework execution
+    prerun_modifier.py # DiversePreRunModifier (SuiteVisitor)
+    listener.py        # DiverseDataDriverListener (Listener v3)
+    runner.py          # ExecutionRunner
+  pipeline/            # Orchestration layer
+    vectorize.py       # Stage 1 orchestrator
+    select.py          # Stage 2 orchestrator
+    execute.py         # Stage 3 orchestrator
+    cache.py           # Content-hash cache invalidator
+    artifacts.py       # Artifact storage and validation
+    errors.py          # Domain error hierarchy
+  cli.py               # CLI entry point (argparse)
+
+tests/
+  fixtures/            # Sample .robot and .csv files
+  unit/                # Unit tests
+  integration/         # Integration tests
+  benchmarks/          # Performance benchmarks
+```
+
+## Development
+
+```bash
+# Install with dev dependencies
+uv sync --extra dev
+
+# Run all tests
+uv run pytest tests/ -v --benchmark-disable
+
+# Run unit tests only
+uv run pytest tests/unit/ -v
+
+# Run integration tests only
+uv run pytest tests/integration/ -v
+
+# Run benchmarks
+uv run pytest tests/benchmarks/ -v
+
+# Lint
+uv run ruff check src/
+
+# Type check
+uv run mypy src/
+```
+
+### Test Markers
+
+```bash
+# Skip slow tests (ML model loading)
+uv run pytest -m "not slow"
+
+# Only integration tests
+uv run pytest -m integration
+
+# Only benchmarks
+uv run pytest -m benchmark
+```
+
+## Performance
+
+Benchmarked on 384-dimensional normalized vectors:
+
+| N (tests) | FPS Time | FPS Multi (5 starts) |
+|-----------|----------|---------------------|
+| 100 | 12 ms | — |
+| 500 | 99 ms | — |
+| 1,000 | 304 ms | 376 ms (3 starts) |
+| 5,000 | 8.0 s | — |
+
+Keyword resolution: ~18 us/depth-1 resolve, ~61 us/depth-3 resolve. Text building for 100 tests: ~458 us.
+
+## Architecture Decision Records
+
+| ADR | Title |
+|-----|-------|
+| [ADR-001](docs/adr/ADR-001-pipeline-architecture.md) | 3-Stage Pipeline Architecture |
+| [ADR-002](docs/adr/ADR-002-embedding-model-and-storage.md) | Embedding Model and Storage |
+| [ADR-003](docs/adr/ADR-003-selection-algorithm-strategy.md) | Selection Algorithm Strategy Pattern |
+| [ADR-004](docs/adr/ADR-004-robot-framework-integration.md) | Robot Framework Integration |
+| [ADR-005](docs/adr/ADR-005-project-structure-and-uv.md) | Project Structure and uv |
+| [ADR-006](docs/adr/ADR-006-text-representation-strategy.md) | Text Representation Strategy |
+| [ADR-007](docs/adr/ADR-007-cicd-integration-and-caching.md) | CI/CD Integration and Caching |
+| [ADR-008](docs/adr/ADR-008-reliability-and-observability.md) | Reliability and Observability |
+
+## License
+
+Apache 2.0