robotframework-testselection 0.1.0__py3-none-any.whl

TestSelection/__init__.py

@@ -0,0 +1,3 @@
+"""Vector-based diverse test case selection for Robot Framework."""
+
+__version__ = "0.1.0"

TestSelection/cli.py

@@ -0,0 +1,256 @@
+"""CLI entry points for the diverse test selection pipeline."""
+from __future__ import annotations
+
+import argparse
+import logging
+import os
+import sys
+from pathlib import Path
+
+logger = logging.getLogger("TestSelection")
+
+
+def _setup_logging(verbose: bool = False) -> None:
+    level = logging.DEBUG if verbose else logging.INFO
+    logging.basicConfig(
+        level=level,
+        format="%(message)s",
+    )
+
+
+def _add_vectorize_parser(subparsers: argparse._SubParsersAction) -> None:
+    p = subparsers.add_parser("vectorize", help="Stage 1: vectorize test suite")
+    p.add_argument("--suite", required=True, type=Path, help="Path to .robot suite")
+    p.add_argument("--output", required=True, type=Path, help="Artifact output dir")
+    p.add_argument("--model", default="all-MiniLM-L6-v2", help="Embedding model name")
+    p.add_argument("--resolve-depth", type=int, default=0, help="Keyword resolve depth")
+    p.add_argument("--force", action="store_true", help="Force re-indexing")
+    p.add_argument(
+        "--datadriver-csv", nargs="*", type=Path,
+        help="DataDriver CSV files",
+    )
+    p.set_defaults(func=_cmd_vectorize)
+
+
+def _add_select_parser(subparsers: argparse._SubParsersAction) -> None:
+    default_k = int(os.environ.get("DIVERSE_K", "50"))
+    default_strategy = os.environ.get("DIVERSE_STRATEGY", "fps")
+    default_seed = int(os.environ.get("DIVERSE_SEED", "42"))
+    default_output = os.environ.get("DIVERSE_OUTPUT", "")
+
+    p = subparsers.add_parser("select", help="Stage 2: select diverse subset")
+    p.add_argument("--artifacts", required=True, type=Path, help="Artifact directory")
+    p.add_argument("--k", type=int, default=default_k, help="Number of tests to select")
+    p.add_argument("--strategy", default=default_strategy, help="Selection strategy")
+    p.add_argument(
+        "--output",
+        type=Path,
+        default=Path(default_output) if default_output else None,
+        help="Output selection file",
+    )
+    p.add_argument(
+        "--include-tags", nargs="*",
+        help="Include only tests with these tags",
+    )
+    p.add_argument("--exclude-tags", nargs="*", help="Exclude tests with these tags")
+    p.add_argument("--seed", type=int, default=default_seed, help="Random seed")
+    p.add_argument(
+        "--no-datadriver",
+        action="store_true",
+        help="Exclude DataDriver tests",
+    )
+    p.set_defaults(func=_cmd_select)
+
+
+def _add_execute_parser(subparsers: argparse._SubParsersAction) -> None:
+    p = subparsers.add_parser("execute", help="Stage 3: execute selected tests")
+    p.add_argument("--suite", required=True, type=Path, help="Path to .robot suite")
+    p.add_argument(
+        "--selection", required=True, type=Path, help="Selection JSON file"
+    )
+    p.add_argument(
+        "--output-dir", type=Path, default=Path("./results"),
+        help="Output dir",
+    )
+    p.add_argument("--robot-args", nargs="*", help="Extra robot arguments")
+    p.set_defaults(func=_cmd_execute)
+
+
+def _add_run_parser(subparsers: argparse._SubParsersAction) -> None:
+    default_k = int(os.environ.get("DIVERSE_K", "50"))
+    default_strategy = os.environ.get("DIVERSE_STRATEGY", "fps")
+    default_seed = int(os.environ.get("DIVERSE_SEED", "42"))
+
+    p = subparsers.add_parser("run", help="Full pipeline: vectorize + select + execute")
+    p.add_argument("--suite", required=True, type=Path, help="Path to .robot suite")
+    p.add_argument("--k", type=int, default=default_k, help="Number of tests to select")
+    p.add_argument("--strategy", default=default_strategy, help="Selection strategy")
+    p.add_argument(
+        "--output-dir", type=Path, default=Path("./results"),
+        help="Output dir",
+    )
+    p.add_argument(
+        "--model", default="all-MiniLM-L6-v2",
+        help="Embedding model name",
+    )
+    p.add_argument("--resolve-depth", type=int, default=0, help="Keyword resolve depth")
+    p.add_argument("--force", action="store_true", help="Force re-indexing")
+    p.add_argument("--seed", type=int, default=default_seed, help="Random seed")
+    p.add_argument("--robot-args", nargs="*", help="Extra robot arguments")
+    p.set_defaults(func=_cmd_run)
+
+
+def _cmd_vectorize(args: argparse.Namespace) -> int:
+    from TestSelection.pipeline.vectorize import run_vectorize
+
+    try:
+        indexed = run_vectorize(
+            suite_path=args.suite,
+            artifact_dir=args.output,
+            model_name=args.model,
+            resolve_depth=args.resolve_depth,
+            force=args.force,
+            datadriver_csvs=args.datadriver_csv,
+        )
+        if indexed:
+            logger.info("[DIVERSE-SELECT] Vectorization complete")
+        else:
+            logger.info("[DIVERSE-SELECT] Vectorization skipped (no changes)")
+        return 0
+    except Exception as exc:
+        logger.error("[DIVERSE-SELECT] Vectorization failed: %s", exc)
+        return 2
+
+
+def _cmd_select(args: argparse.Namespace) -> int:
+    from TestSelection.pipeline.select import run_select
+
+    try:
+        result = run_select(
+            artifact_dir=args.artifacts,
+            k=args.k,
+            strategy=args.strategy,
+            output_file=args.output,
+            include_tags=args.include_tags,
+            exclude_tags=args.exclude_tags,
+            seed=args.seed,
+            include_datadriver=not args.no_datadriver,
+        )
+        logger.info(
+            "[DIVERSE-SELECT] Selected %d tests (strategy=%s)",
+            len(result.selected),
+            result.strategy,
+        )
+        return 0
+    except Exception as exc:
+        logger.error("[DIVERSE-SELECT] Selection failed: %s", exc)
+        return 2
+
+
+def _cmd_execute(args: argparse.Namespace) -> int:
+    from TestSelection.pipeline.execute import run_execute
+
+    return run_execute(
+        suite_path=args.suite,
+        selection_file=args.selection,
+        output_dir=str(args.output_dir),
+        extra_robot_args=args.robot_args,
+    )
+
+
+def _cmd_run(args: argparse.Namespace) -> int:
+    from TestSelection.pipeline.execute import run_execute
+    from TestSelection.pipeline.select import run_select
+    from TestSelection.pipeline.vectorize import run_vectorize
+
+    artifact_dir = args.output_dir / ".artifacts"
+
+    # Stage 1: Vectorize
+    try:
+        run_vectorize(
+            suite_path=args.suite,
+            artifact_dir=artifact_dir,
+            model_name=args.model,
+            resolve_depth=args.resolve_depth,
+            force=args.force,
+        )
+    except Exception as exc:
+        logger.warning(
+            "[DIVERSE-SELECT] Vectorization failed, falling back to all tests: %s",
+            exc,
+        )
+        return _fallback_execute(args)
+
+    # Stage 2: Select
+    selection_file = artifact_dir / "selected_tests.json"
+    try:
+        run_select(
+            artifact_dir=artifact_dir,
+            k=args.k,
+            strategy=args.strategy,
+            output_file=selection_file,
+            seed=args.seed,
+        )
+    except Exception as exc:
+        logger.warning(
+            "[DIVERSE-SELECT] Selection failed, falling back to all tests: %s",
+            exc,
+        )
+        return _fallback_execute(args)
+
+    # Stage 3: Execute
+    return run_execute(
+        suite_path=args.suite,
+        selection_file=selection_file,
+        output_dir=str(args.output_dir),
+        extra_robot_args=args.robot_args,
+    )
+
+
+def _fallback_execute(args: argparse.Namespace) -> int:
+    """Execute all tests without selection (graceful degradation)."""
+    logger.info("[DIVERSE-SELECT] Running all tests (no selection)")
+    try:
+        import robot
+
+        robot_args = [
+            "--outputdir",
+            str(args.output_dir),
+            str(args.suite),
+        ]
+        return robot.run_cli(robot_args, exit=False)  # type: ignore[attr-defined]
+    except Exception as exc:
+        logger.error("[DIVERSE-SELECT] Fallback execution failed: %s", exc)
+        return 2
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Build the argument parser."""
+    parser = argparse.ArgumentParser(
+        prog="testcase-select",
+        description="Vector-based diverse test case selection for Robot Framework",
+    )
+    parser.add_argument("--verbose", "-v", action="store_true", help="Verbose logging")
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+    _add_vectorize_parser(subparsers)
+    _add_select_parser(subparsers)
+    _add_execute_parser(subparsers)
+    _add_run_parser(subparsers)
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Main CLI entry point."""
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    _setup_logging(getattr(args, "verbose", False))
+
+    if not args.command:
+        parser.print_help()
+        return 0
+
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

TestSelection/embedding/__init__.py

@@ -0,0 +1 @@
+"""Embedding bounded context: vector embedding of test case text representations."""

TestSelection/embedding/embedder.py

@@ -0,0 +1,43 @@
+"""SentenceTransformer adapter -- ACL wrapping sentence-transformers."""
+from __future__ import annotations
+
+import numpy as np
+from numpy.typing import NDArray
+
+
+class SentenceTransformerAdapter:
+    """ACL: Wraps sentence-transformers behind the EmbeddingModel protocol.
+
+    Default model: all-MiniLM-L6-v2 (22M params, 384 dims, CPU-friendly).
+    Alternative: all-mpnet-base-v2 (768 dims) for higher quality.
+    """
+
+    def __init__(
+        self,
+        model_name: str = "all-MiniLM-L6-v2",
+        *,
+        show_progress_bar: bool = False,
+    ) -> None:
+        from sentence_transformers import SentenceTransformer
+
+        self._model = SentenceTransformer(model_name)
+        self._model_name = model_name
+        self._show_progress_bar = show_progress_bar
+
+    @property
+    def model_name(self) -> str:
+        return self._model_name
+
+    @property
+    def embedding_dim(self) -> int:
+        dim = self._model.get_sentence_embedding_dimension()
+        assert isinstance(dim, int)
+        return dim
+
+    def encode(self, texts: list[str]) -> NDArray[np.float32]:
+        result: NDArray[np.float32] = self._model.encode(
+            texts,
+            show_progress_bar=self._show_progress_bar,
+            normalize_embeddings=True,
+        )
+        return result

TestSelection/embedding/models.py

@@ -0,0 +1,198 @@
+"""Core domain objects for the Embedding bounded context."""
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+import numpy as np
+from numpy.typing import NDArray
+
+
+@dataclass(frozen=True)
+class Embedding:
+    """A single test case embedding vector (value object)."""
+
+    vector: NDArray[np.float32]
+
+    @property
+    def dimensionality(self) -> int:
+        return int(self.vector.shape[0])
+
+    @property
+    def is_normalized(self) -> bool:
+        norm = float(np.linalg.norm(self.vector))
+        return abs(norm - 1.0) < 1e-6
+
+
+@dataclass(frozen=True)
+class ManifestEntry:
+    """Metadata for a single test in the artifact manifest."""
+
+    id: str
+    name: str
+    tags: tuple[str, ...]
+    suite: str
+    suite_name: str
+    is_datadriver: bool
+
+
+@dataclass(frozen=True)
+class ArtifactManifest:
+    """Describes the contents of an EmbeddingArtifact."""
+
+    model: str
+    embedding_dim: int
+    test_count: int
+    resolve_depth: int
+    tests: tuple[ManifestEntry, ...]
+
+
+@dataclass(frozen=True)
+class EmbeddingArtifact:
+    """Portable artifact produced by the Embedding Context.
+
+    Consists of two files:
+    - embeddings.npz: numpy archive with 'vectors' (N x dim) and 'ids' (N,)
+    - test_manifest.json: human-readable metadata
+    """
+
+    embeddings_path: Path
+    manifest_path: Path
+    model_name: str
+    embedding_dim: int
+    test_count: int
+
+
+class EmbeddingMatrix:
+    """Aggregate root: holds all embeddings for a test suite.
+
+    Invariants:
+    - vectors.shape[0] == len(test_ids)  (one vector per test)
+    - vectors.shape[1] == embedding_dim  (consistent dimensionality)
+    - All vectors are L2-normalized for cosine distance computation
+    """
+
+    def __init__(
+        self,
+        model_name: str,
+        embedding_dim: int,
+        vectors: NDArray[np.float32],
+        test_ids: tuple[str, ...],
+    ) -> None:
+        self.model_name = model_name
+        self.embedding_dim = embedding_dim
+        self.vectors = vectors
+        self.test_ids = test_ids
+        self.validate_dimensions()
+
+    @property
+    def test_count(self) -> int:
+        return len(self.test_ids)
+
+    def validate_dimensions(self) -> None:
+        """Assert that vectors shape matches test_ids length and embedding_dim."""
+        if self.vectors.shape[0] != len(self.test_ids):
+            raise ValueError(
+                f"Vector count {self.vectors.shape[0]} != "
+                f"test ID count {len(self.test_ids)}"
+            )
+        if self.vectors.shape[1] != self.embedding_dim:
+            raise ValueError(
+                f"Vector dim {self.vectors.shape[1]} != "
+                f"expected dim {self.embedding_dim}"
+            )
+
+    def to_artifact(
+        self,
+        output_dir: Path,
+        manifest_entries: tuple[ManifestEntry, ...],
+        resolve_depth: int = 0,
+    ) -> EmbeddingArtifact:
+        """Serialize to portable artifact files."""
+        output_dir.mkdir(parents=True, exist_ok=True)
+        emb_path = output_dir / "embeddings.npz"
+        manifest_path = output_dir / "test_manifest.json"
+
+        np.savez_compressed(
+            emb_path,
+            vectors=self.vectors,
+            ids=np.array(list(self.test_ids)),
+        )
+
+        manifest = ArtifactManifest(
+            model=self.model_name,
+            embedding_dim=self.embedding_dim,
+            test_count=self.test_count,
+            resolve_depth=resolve_depth,
+            tests=manifest_entries,
+        )
+        manifest_path.write_text(
+            json.dumps(
+                {
+                    "model": manifest.model,
+                    "embedding_dim": manifest.embedding_dim,
+                    "test_count": manifest.test_count,
+                    "resolve_depth": manifest.resolve_depth,
+                    "tests": [
+                        {
+                            "id": e.id,
+                            "name": e.name,
+                            "tags": list(e.tags),
+                            "suite": e.suite,
+                            "suite_name": e.suite_name,
+                            "is_datadriver": e.is_datadriver,
+                        }
+                        for e in manifest.tests
+                    ],
+                },
+                indent=2,
+            )
+        )
+
+        return EmbeddingArtifact(
+            embeddings_path=emb_path,
+            manifest_path=manifest_path,
+            model_name=self.model_name,
+            embedding_dim=self.embedding_dim,
+            test_count=self.test_count,
+        )
+
+    @classmethod
+    def from_artifact(
+        cls, artifact_dir: Path
+    ) -> tuple[EmbeddingMatrix, ArtifactManifest]:
+        """Load an EmbeddingMatrix and ArtifactManifest from artifact files."""
+        emb_path = artifact_dir / "embeddings.npz"
+        manifest_path = artifact_dir / "test_manifest.json"
+
+        data = np.load(emb_path, allow_pickle=True)
+        vectors: NDArray[np.float32] = data["vectors"]
+        ids: tuple[str, ...] = tuple(str(i) for i in data["ids"])
+
+        raw = json.loads(manifest_path.read_text())
+        manifest = ArtifactManifest(
+            model=raw["model"],
+            embedding_dim=raw["embedding_dim"],
+            test_count=raw["test_count"],
+            resolve_depth=raw.get("resolve_depth", 0),
+            tests=tuple(
+                ManifestEntry(
+                    id=t["id"],
+                    name=t["name"],
+                    tags=tuple(t.get("tags", [])),
+                    suite=t.get("suite", ""),
+                    suite_name=t.get("suite_name", ""),
+                    is_datadriver=t.get("is_datadriver", False),
+                )
+                for t in raw["tests"]
+            ),
+        )
+
+        matrix = cls(
+            model_name=raw["model"],
+            embedding_dim=raw["embedding_dim"],
+            vectors=vectors,
+            test_ids=ids,
+        )
+        return matrix, manifest

TestSelection/embedding/ports.py

@@ -0,0 +1,24 @@
+"""Embedding model protocol -- the port for embedding adapters."""
+from __future__ import annotations
+
+from typing import Protocol, runtime_checkable
+
+import numpy as np
+from numpy.typing import NDArray
+
+
+@runtime_checkable
+class EmbeddingModel(Protocol):
+    """Protocol for embedding models.
+
+    Decouples the domain from sentence-transformers or any other
+    embedding backend. Implementations must be L2-normalizing.
+    """
+
+    @property
+    def model_name(self) -> str: ...
+
+    @property
+    def embedding_dim(self) -> int: ...
+
+    def encode(self, texts: list[str]) -> NDArray[np.float32]: ...

TestSelection/execution/__init__.py

@@ -0,0 +1 @@
+"""Execution context: Robot Framework PreRunModifier and Listener v3."""

TestSelection/execution/listener.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+
+class DiverseDataDriverListener:
+    """Listener v3 that filters DataDriver-generated tests.
+
+    Must run AFTER DataDriver. Lower priority = runs later.
+
+    Usage::
+
+        robot --listener module.DiverseDataDriverListener:file tests/
+    """
+
+    ROBOT_LISTENER_API_VERSION = 3
+    ROBOT_LISTENER_PRIORITY = 50  # lower than default -> runs after DataDriver
+
+    def __init__(self, selection_file: str) -> None:
+        data = json.loads(Path(selection_file).read_text())
+        self._selected_dd_names: set[str] = set(
+            t["name"]
+            for t in data["selected"]
+            if t.get("is_datadriver", False)
+        )
+        self._has_dd_tests = bool(self._selected_dd_names)
+        self._stats = {"suites_processed": 0, "tests_filtered": 0}
+
+    def start_suite(self, data, result) -> None:  # noqa: ARG002
+        if not self._has_dd_tests:
+            return
+        if len(data.tests) <= 1:
+            return  # not a DataDriver suite
+        original_count = len(data.tests)
+        matches = [t for t in data.tests if t.name in self._selected_dd_names]
+        if matches:
+            data.tests = matches
+            self._stats["suites_processed"] += 1
+            self._stats["tests_filtered"] += original_count - len(matches)
+
+    @property
+    def stats(self) -> dict[str, int]:
+        return dict(self._stats)

TestSelection/execution/prerun_modifier.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from robot.api import SuiteVisitor
+
+
+class DiversePreRunModifier(SuiteVisitor):
+    """PreRunModifier that filters standard (non-DataDriver) tests.
+
+    Usage CLI::
+
+        robot --prerunmodifier module.DiversePreRunModifier:file tests/
+
+    Usage programmatic:
+        suite.visit(DiversePreRunModifier('selected_tests.json'))
+    """
+
+    def __init__(self, selection_file: str) -> None:
+        data = json.loads(Path(selection_file).read_text())
+        self._selected_names: set[str] = set(
+            t["name"]
+            for t in data["selected"]
+            if not t.get("is_datadriver", False)
+        )
+        self._stats = {"kept": 0, "removed": 0}
+
+    def start_suite(self, suite) -> None:  # type: ignore[override]
+        original = len(suite.tests)
+        suite.tests = [t for t in suite.tests if t.name in self._selected_names]
+        self._stats["kept"] += len(suite.tests)
+        self._stats["removed"] += original - len(suite.tests)
+
+    def end_suite(self, suite) -> None:  # type: ignore[override]
+        suite.suites = [s for s in suite.suites if s.test_count > 0]
+
+    def visit_test(self, test) -> None:  # type: ignore[override]
+        pass  # skip internals for performance
+
+    @property
+    def stats(self) -> dict[str, int]:
+        return dict(self._stats)

TestSelection/execution/runner.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+
+class ExecutionRunner:
+    """Orchestrates Robot Framework execution with test selection.
+
+    Builds the appropriate robot CLI arguments based on whether
+    the selection includes DataDriver tests.
+    """
+
+    def __init__(
+        self,
+        suite_path: str | Path,
+        selection_file: str | Path,
+        output_dir: str | Path = "./results",
+    ) -> None:
+        self._suite_path = Path(suite_path)
+        self._selection_file = Path(selection_file)
+        self._output_dir = Path(output_dir)
+        self._selection_data = json.loads(self._selection_file.read_text())
+        self._has_datadriver = any(
+            t.get("is_datadriver", False)
+            for t in self._selection_data["selected"]
+        )
+
+    def build_robot_args(self) -> list[str]:
+        """Build robot CLI arguments with --prerunmodifier and --listener as needed."""
+        args: list[str] = [
+            "--outputdir",
+            str(self._output_dir),
+            "--prerunmodifier",
+            f"TestSelection.execution.prerun_modifier.DiversePreRunModifier:{self._selection_file}",
+        ]
+        if self._has_datadriver:
+            args.extend([
+                "--listener",
+                f"TestSelection.execution.listener.DiverseDataDriverListener:{self._selection_file}",
+            ])
+        return args
+
+    def execute(self, extra_args: list[str] | None = None) -> int:
+        """Run robot.run_cli with built arguments. Returns exit code."""
+        import robot
+
+        args = self.build_robot_args()
+        if extra_args:
+            args.extend(extra_args)
+        args.append(str(self._suite_path))
+        return robot.run_cli(args, exit=False)  # type: ignore[attr-defined]
+
+    def generate_report(self, return_code: int) -> dict:
+        """Generate selection_report.json with execution metadata."""
+        selected_count = len(self._selection_data["selected"])
+        dd_count = sum(
+            1
+            for t in self._selection_data["selected"]
+            if t.get("is_datadriver", False)
+        )
+        report = {
+            "return_code": return_code,
+            "suite_path": str(self._suite_path),
+            "selection_file": str(self._selection_file),
+            "selected_tests": selected_count,
+            "datadriver_tests": dd_count,
+            "standard_tests": selected_count - dd_count,
+            "strategy": self._selection_data.get("strategy", "unknown"),
+            "status": "pass" if return_code == 0 else "fail",
+        }
+        self._output_dir.mkdir(parents=True, exist_ok=True)
+        report_path = self._output_dir / "selection_report.json"
+        report_path.write_text(json.dumps(report, indent=2))
+        return report