testless 0.1.5__py3-none-any.whl

testless/__init__.py

@@ -0,0 +1,3 @@
+"""testless — analyze test value, coverage, duplication, and generate LLM planfiles."""
+
+__version__ = "0.1.5"

testless/analyze/dead_tests.py

@@ -0,0 +1,44 @@
+"""Detect tests that contribute no unique coverage."""
+
+from __future__ import annotations
+
+from testless.models.coverage_map import CoverageMap
+from testless.models.findings import DeadTestFinding, TestMeta
+
+
+def find_dead_tests(
+    tests: list[TestMeta],
+    cov_map: CoverageMap,
+    min_unique_lines: int = 1,
+) -> list[DeadTestFinding]:
+    """
+    Return tests whose unique coverage falls below *min_unique_lines*.
+
+    A test is considered "dead" if every line it touches is already covered
+    by at least one other test.
+    """
+    all_node_ids = [t.node_id for t in tests]
+    findings: list[DeadTestFinding] = []
+
+    for test in tests:
+        total_lines = 0
+        unique_lines = 0
+
+        for fc in cov_map.files.values():
+            covered = fc.lines_covered_by(test.node_id)
+            unique = fc.unique_lines_for(test.node_id, all_node_ids)
+            total_lines += len(covered)
+            unique_lines += len(unique)
+
+        if total_lines > 0 and unique_lines < min_unique_lines:
+            findings.append(
+                DeadTestFinding(
+                    node_id=test.node_id,
+                    file=test.file,
+                    unique_lines=unique_lines,
+                    total_lines=total_lines,
+                    reason="no unique coverage",
+                )
+            )
+
+    return findings

testless/analyze/duplicate_tests.py

@@ -0,0 +1,102 @@
+"""Detect duplicate tests using coverage overlap, AST similarity, fixtures and names."""
+
+from __future__ import annotations
+
+import ast
+import difflib
+from pathlib import Path
+
+from testless.models.coverage_map import CoverageMap
+from testless.models.findings import DuplicateFinding, TestMeta
+from testless.collect.fixture_index import FixtureIndex
+
+
+def _ast_similarity(file_a: str, name_a: str, file_b: str, name_b: str) -> float:
+    """Compute normalized edit-distance similarity between two test function bodies."""
+
+    def _get_source(file: str, name: str) -> str:
+        path = Path(file)
+        if not path.exists():
+            return ""
+        try:
+            source = path.read_text(encoding="utf-8")
+            tree = ast.parse(source)
+        except (SyntaxError, UnicodeDecodeError):
+            return ""
+        for node in ast.walk(tree):
+            if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) and node.name == name:
+                return ast.unparse(node)
+        return ""
+
+    src_a = _get_source(file_a, name_a)
+    src_b = _get_source(file_b, name_b)
+    if not src_a or not src_b:
+        return 0.0
+    return difflib.SequenceMatcher(None, src_a, src_b).ratio()
+
+
+def _name_similarity(name_a: str, name_b: str) -> float:
+    """Normalized edit-distance similarity between two test names."""
+    return difflib.SequenceMatcher(None, name_a, name_b).ratio()
+
+
+def _duplicate_score(
+    cov_overlap: float,
+    ast_sim: float,
+    fix_overlap: float,
+    name_sim: float,
+) -> float:
+    return (
+        0.45 * cov_overlap
+        + 0.25 * ast_sim
+        + 0.15 * fix_overlap
+        + 0.15 * name_sim
+    )
+
+
+def find_duplicates(
+    tests: list[TestMeta],
+    cov_map: CoverageMap,
+    fixture_index: FixtureIndex,
+    min_score: float = 0.85,
+) -> list[DuplicateFinding]:
+    """Return pairs of tests that are likely duplicates."""
+    findings: list[DuplicateFinding] = []
+    seen: set[frozenset[str]] = set()
+
+    for i, test_a in enumerate(tests):
+        for test_b in tests[i + 1 :]:
+            pair = frozenset({test_a.node_id, test_b.node_id})
+            if pair in seen:
+                continue
+            seen.add(pair)
+
+            cov_overlap = cov_map.overlap(test_a.node_id, test_b.node_id)
+            fix_overlap = fixture_index.fixture_overlap(test_a.node_id, test_b.node_id)
+
+            # Derive file and function name from node_id
+            parts_a = test_a.node_id.split("::")
+            parts_b = test_b.node_id.split("::")
+            file_a = parts_a[0] if parts_a else test_a.file
+            file_b = parts_b[0] if parts_b else test_b.file
+            name_a = parts_a[-1] if parts_a else test_a.name
+            name_b = parts_b[-1] if parts_b else test_b.name
+
+            ast_sim = _ast_similarity(file_a, name_a, file_b, name_b)
+            name_sim = _name_similarity(name_a, name_b)
+
+            score = _duplicate_score(cov_overlap, ast_sim, fix_overlap, name_sim)
+            if score >= min_score:
+                findings.append(
+                    DuplicateFinding(
+                        test_a=test_a.node_id,
+                        test_b=test_b.node_id,
+                        score=round(score, 3),
+                        coverage_overlap=round(cov_overlap, 3),
+                        ast_similarity=round(ast_sim, 3),
+                        fixture_overlap=round(fix_overlap, 3),
+                        name_similarity=round(name_sim, 3),
+                    )
+                )
+
+    return sorted(findings, key=lambda f: f.score, reverse=True)

testless/analyze/missing_tests.py

@@ -0,0 +1,98 @@
+"""Suggest missing tests based on endpoints, services, and modules."""
+
+from __future__ import annotations
+
+from testless.collect.endpoint_inventory import EndpointInfo, ServiceInfo
+from testless.models.findings import MissingTestFinding, TestMeta
+
+
+# Endpoint paths that should always have smoke test coverage
+_CRITICAL_PATHS = {"/health", "/login", "/logout", "/search", "/checkout", "/billing", "/register"}
+
+
+def _has_smoke_test(endpoint: EndpointInfo, tests: list[TestMeta]) -> bool:
+    """Heuristic: check if any test name references the endpoint path."""
+    slug = endpoint.path.strip("/").replace("/", "_").replace("-", "_")
+    for test in tests:
+        if slug.lower() in test.name.lower() or slug.lower() in test.file.lower():
+            return True
+    return False
+
+
+def _has_service_test(service: ServiceInfo, tests: list[TestMeta]) -> bool:
+    """Heuristic: check if any test imports or references the service module."""
+    module_slug = service.module.split(".")[-1]
+    for test in tests:
+        if module_slug.lower() in test.name.lower() or module_slug.lower() in test.file.lower():
+            return True
+    return False
+
+
+def find_missing_tests(
+    tests: list[TestMeta],
+    endpoints: list[EndpointInfo],
+    services: list[ServiceInfo],
+) -> list[MissingTestFinding]:
+    """Return suggested missing tests based on inventory."""
+    findings: list[MissingTestFinding] = []
+
+    for endpoint in endpoints:
+        if not _has_smoke_test(endpoint, tests):
+            priority = "high" if endpoint.path in _CRITICAL_PATHS else "medium"
+            findings.append(
+                MissingTestFinding(
+                    target=f"{endpoint.method} {endpoint.path}",
+                    test_type="smoke",
+                    description=(
+                        f"No smoke test found for {endpoint.method} {endpoint.path} "
+                        f"(handler: {endpoint.handler}). "
+                        "Add tests for 2xx, 4xx, and 5xx responses."
+                    ),
+                    priority=priority,
+                    suggested_file=f"tests/smoke/test_{endpoint.handler}.py",
+                )
+            )
+
+    for service in services:
+        if not _has_service_test(service, tests):
+            if service.has_sql:
+                findings.append(
+                    MissingTestFinding(
+                        target=service.module,
+                        test_type="testql",
+                        description=(
+                            f"Service {service.module} uses SQL but has no detected tests. "
+                            "Consider adding TestQL contract tests for critical queries."
+                        ),
+                        priority="high",
+                        suggested_file=f"tests/service/test_{service.module.split('.')[-1]}.py",
+                    )
+                )
+            if service.has_http_client:
+                findings.append(
+                    MissingTestFinding(
+                        target=service.module,
+                        test_type="contract",
+                        description=(
+                            f"Service {service.module} makes HTTP calls but has no contract tests. "
+                            "Add service contract tests for timeout and dependency failure scenarios."
+                        ),
+                        priority="medium",
+                        suggested_file=f"tests/service/test_{service.module.split('.')[-1]}_contract.py",
+                    )
+                )
+            if service.has_retry:
+                findings.append(
+                    MissingTestFinding(
+                        target=service.module,
+                        test_type="resilience",
+                        description=(
+                            f"Service {service.module} uses retry/backoff but has no resilience tests. "
+                            "Add tests for circuit-breaker and retry-exhaustion scenarios."
+                        ),
+                        priority="medium",
+                        suggested_file=f"tests/service/test_{service.module.split('.')[-1]}_resilience.py",
+                    )
+                )
+
+    return findings

testless/analyze/refactor_candidates.py

@@ -0,0 +1,87 @@
+"""Identify tests that are good candidates for refactoring."""
+
+from __future__ import annotations
+
+import ast
+from pathlib import Path
+
+from testless.models.findings import RefactorFinding, TestMeta
+
+
+_MAX_ASSERTIONS = 10
+_MAX_DURATION_S = 5.0
+_MAX_LINES = 80
+
+
+def _count_assertions(file: str, func_name: str) -> int:
+    path = Path(file)
+    if not path.exists():
+        return 0
+    try:
+        source = path.read_text(encoding="utf-8")
+        tree = ast.parse(source)
+    except (SyntaxError, UnicodeDecodeError):
+        return 0
+    for node in ast.walk(tree):
+        if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) and node.name == func_name:
+            return sum(
+                1
+                for child in ast.walk(node)
+                if isinstance(child, ast.Assert)
+                or (
+                    isinstance(child, ast.Call)
+                    and isinstance(child.func, ast.Attribute)
+                    and child.func.attr.startswith("assert")
+                )
+            )
+    return 0
+
+
+def _count_lines(file: str, func_name: str) -> int:
+    path = Path(file)
+    if not path.exists():
+        return 0
+    try:
+        source = path.read_text(encoding="utf-8")
+        tree = ast.parse(source)
+    except (SyntaxError, UnicodeDecodeError):
+        return 0
+    for node in ast.walk(tree):
+        if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) and node.name == func_name:
+            end = getattr(node, "end_lineno", node.lineno)
+            return end - node.lineno + 1
+    return 0
+
+
+def find_refactor_candidates(tests: list[TestMeta]) -> list[RefactorFinding]:
+    """Return tests that warrant refactoring."""
+    findings: list[RefactorFinding] = []
+
+    for test in tests:
+        parts = test.node_id.split("::")
+        file = parts[0] if parts else test.file
+        func_name = parts[-1] if len(parts) > 1 else test.name
+
+        reasons: list[str] = []
+
+        if test.duration > _MAX_DURATION_S:
+            reasons.append(f"slow test ({test.duration:.1f}s > {_MAX_DURATION_S}s threshold)")
+
+        assertions = _count_assertions(file, func_name)
+        if assertions > _MAX_ASSERTIONS:
+            reasons.append(f"too many assertions ({assertions} > {_MAX_ASSERTIONS})")
+
+        lines = _count_lines(file, func_name)
+        if lines > _MAX_LINES:
+            reasons.append(f"test function too long ({lines} lines > {_MAX_LINES})")
+
+        if reasons:
+            findings.append(
+                RefactorFinding(
+                    node_id=test.node_id,
+                    file=file,
+                    reason="; ".join(reasons),
+                )
+            )
+
+    return findings

testless/analyze/service_risk.py

@@ -0,0 +1,41 @@
+"""Assess service risk based on missing tests and critical characteristics."""
+
+from __future__ import annotations
+
+from testless.collect.endpoint_inventory import ServiceInfo
+from testless.models.findings import MissingTestFinding
+
+
+def score_service_risk(
+    service: ServiceInfo,
+    missing: list[MissingTestFinding],
+) -> dict[str, object]:
+    """
+    Return a risk assessment dict for a service.
+
+    Higher score = higher risk of undetected failures.
+    """
+    risk_score = 0.0
+    reasons: list[str] = []
+
+    # Missing tests for this service
+    service_missing = [m for m in missing if m.target == service.module]
+    if service_missing:
+        risk_score += 0.4 * len(service_missing)
+        reasons.append(f"{len(service_missing)} missing test type(s)")
+
+    if service.has_sql:
+        risk_score += 0.3
+        reasons.append("uses SQL without verified query tests")
+    if service.has_http_client:
+        risk_score += 0.2
+        reasons.append("makes external HTTP calls")
+    if service.has_retry:
+        risk_score += 0.1
+        reasons.append("uses retry/backoff logic")
+
+    return {
+        "module": service.module,
+        "risk_score": round(min(risk_score, 1.0), 2),
+        "reasons": reasons,
+    }

testless/cli.py

@@ -0,0 +1,306 @@
+"""CLI entry-point for testless."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import click
+
+from testless.config import load_config
+
+
+@click.group()
+@click.option("--config", "-c", default=None, help="Path to .testless.yml config file.")
+@click.pass_context
+def main(ctx: click.Context, config: str | None) -> None:
+    """testless — analyze test value, coverage, duplication, and generate LLM planfiles."""
+    ctx.ensure_object(dict)
+    ctx.obj["config"] = load_config(config)
+
+
+# ---------------------------------------------------------------------------
+# pretest scan
+# ---------------------------------------------------------------------------
+
+@main.command()
+@click.option("--out", default=None, help="Override coverage output directory.")
+@click.pass_context
+def scan(ctx: click.Context, out: str | None) -> None:
+    """Run pytest with coverage contexts and collect test metadata."""
+    from testless.collect.pytest_runner import run_pytest
+
+    cfg = ctx.obj["config"]
+    coverage_dir = out or cfg.coverage_dir
+
+    click.echo(f"Running pytest with coverage contexts → {coverage_dir}")
+    tests, cov_json = run_pytest(
+        packages=cfg.packages,
+        test_dirs=cfg.test_dirs,
+        coverage_dir=coverage_dir,
+        extra_args=cfg.pytest_args,
+    )
+    click.echo(f"Collected {len(tests)} tests. Coverage JSON: {cov_json}")
+
+
+# ---------------------------------------------------------------------------
+# pretest duplicates
+# ---------------------------------------------------------------------------
+
+@main.command()
+@click.option("--min-overlap", default=None, type=float, help="Minimum duplicate score (0–1).")
+@click.option("--coverage-json", default=None, help="Path to coverage.json (default: auto-detect).")
+@click.pass_context
+def duplicates(ctx: click.Context, min_overlap: float | None, coverage_json: str | None) -> None:
+    """Detect duplicate tests based on coverage overlap, AST similarity, and fixture use."""
+    from testless.collect.coverage_loader import load_coverage_json
+    from testless.collect.fixture_index import FixtureIndex
+    from testless.analyze.duplicate_tests import find_duplicates
+    from testless.reporters.console import print_report
+    from testless.models.findings import AnalysisReport, TestMeta
+
+    cfg = ctx.obj["config"]
+    min_score = min_overlap if min_overlap is not None else cfg.min_duplicate_score
+
+    # Load coverage
+    cov_path = coverage_json or str(Path(cfg.coverage_dir) / "coverage.json")
+    cov_map = load_coverage_json(cov_path)
+
+    # Build minimal test list from coverage map contexts
+    all_node_ids: set[str] = set()
+    for fc in cov_map.files.values():
+        for tests_list in fc.line_to_tests.values():
+            all_node_ids.update(tests_list)
+
+    tests = [
+        TestMeta(
+            node_id=nid,
+            file=nid.split("::")[0],
+            name=nid.split("::")[-1],
+        )
+        for nid in sorted(all_node_ids)
+    ]
+
+    fixture_index = FixtureIndex()
+    for td in cfg.test_dirs:
+        if Path(td).exists():
+            fixture_index.scan_directory(td)
+
+    dups = find_duplicates(tests, cov_map, fixture_index, min_score=min_score)
+
+    report = AnalysisReport(duplicates=dups, tests=tests)
+    print_report(report)
+
+    if dups:
+        sys.exit(1)
+
+
+# ---------------------------------------------------------------------------
+# pretest missing
+# ---------------------------------------------------------------------------
+
+@main.command()
+@click.option(
+    "--services",
+    "-s",
+    multiple=True,
+    help="Source directories to scan for endpoints/services.",
+)
+@click.pass_context
+def missing(ctx: click.Context, services: tuple[str, ...]) -> None:
+    """Suggest missing smoke, e2e, contract, and TestQL tests."""
+    from testless.collect.endpoint_inventory import EndpointInventory
+    from testless.analyze.missing_tests import find_missing_tests
+    from testless.reporters.console import print_report
+    from testless.models.findings import AnalysisReport
+
+    cfg = ctx.obj["config"]
+    dirs = list(services) if services else cfg.packages or ["."]
+
+    inventory = EndpointInventory()
+    for d in dirs:
+        if Path(d).exists():
+            inventory.scan_directory(d)
+        else:
+            click.echo(f"Warning: directory not found: {d}", err=True)
+
+    missing_findings = find_missing_tests([], inventory.endpoints, inventory.services)
+    report = AnalysisReport(missing_tests=missing_findings)
+    print_report(report)
+
+
+# ---------------------------------------------------------------------------
+# pretest planfiles
+# ---------------------------------------------------------------------------
+
+@main.command()
+@click.option("--out", default=None, help="Output directory for planfile YAML files.")
+@click.option("--coverage-json", default=None, help="Path to coverage.json.")
+@click.option(
+    "--services",
+    "-s",
+    multiple=True,
+    help="Source directories to scan for endpoints/services.",
+)
+@click.option("--with-prompts", is_flag=True, default=False, help="Embed LLM system prompts.")
+@click.pass_context
+def planfiles(
+    ctx: click.Context,
+    out: str | None,
+    coverage_json: str | None,
+    services: tuple[str, ...],
+    with_prompts: bool,
+) -> None:
+    """Generate planfile YAML tickets for LLM from all findings."""
+    from testless.collect.coverage_loader import load_coverage_json
+    from testless.collect.fixture_index import FixtureIndex
+    from testless.collect.endpoint_inventory import EndpointInventory
+    from testless.analyze.duplicate_tests import find_duplicates
+    from testless.analyze.dead_tests import find_dead_tests
+    from testless.analyze.missing_tests import find_missing_tests
+    from testless.analyze.refactor_candidates import find_refactor_candidates
+    from testless.tickets.builder import build_planfiles
+    from testless.tickets.serializer import write_planfiles, write_summary_json
+    from testless.tickets.prompts import attach_prompt
+    from testless.models.findings import AnalysisReport, TestMeta
+
+    cfg = ctx.obj["config"]
+    output_dir = out or cfg.planfiles_dir
+
+    # Load coverage
+    cov_path = coverage_json or str(Path(cfg.coverage_dir) / "coverage.json")
+    cov_map = load_coverage_json(cov_path)
+
+    all_node_ids: set[str] = set()
+    for fc in cov_map.files.values():
+        for tests_list in fc.line_to_tests.values():
+            all_node_ids.update(tests_list)
+
+    tests = [
+        TestMeta(
+            node_id=nid,
+            file=nid.split("::")[0],
+            name=nid.split("::")[-1],
+        )
+        for nid in sorted(all_node_ids)
+    ]
+
+    fixture_index = FixtureIndex()
+    for td in cfg.test_dirs:
+        if Path(td).exists():
+            fixture_index.scan_directory(td)
+
+    dirs = list(services) if services else cfg.packages or ["."]
+    inventory = EndpointInventory()
+    for d in dirs:
+        if Path(d).exists():
+            inventory.scan_directory(d)
+
+    report = AnalysisReport(
+        duplicates=find_duplicates(tests, cov_map, fixture_index, min_score=cfg.min_duplicate_score),
+        dead_tests=find_dead_tests(tests, cov_map),
+        missing_tests=find_missing_tests(tests, inventory.endpoints, inventory.services),
+        refactor_candidates=find_refactor_candidates(tests),
+        tests=tests,
+    )
+
+    tickets = build_planfiles(report)
+    if with_prompts:
+        tickets = [attach_prompt(t) for t in tickets]
+
+    written = write_planfiles(tickets, output_dir)
+    write_summary_json(tickets, output_dir)
+
+    click.echo(f"Generated {len(written)} planfile(s) in {output_dir}/")
+
+
+# ---------------------------------------------------------------------------
+# pretest doctor
+# ---------------------------------------------------------------------------
+
+@main.command()
+@click.argument("question", required=False)
+@click.option("--coverage-json", default=None, help="Path to coverage.json.")
+@click.option(
+    "--services",
+    "-s",
+    multiple=True,
+    help="Source directories to scan for endpoints/services.",
+)
+@click.pass_context
+def doctor(
+    ctx: click.Context,
+    question: str | None,
+    coverage_json: str | None,
+    services: tuple[str, ...],
+) -> None:
+    """Answer questions about test health (what to remove, add, or fix)."""
+    from testless.collect.coverage_loader import load_coverage_json
+    from testless.collect.fixture_index import FixtureIndex
+    from testless.collect.endpoint_inventory import EndpointInventory
+    from testless.analyze.duplicate_tests import find_duplicates
+    from testless.analyze.dead_tests import find_dead_tests
+    from testless.analyze.missing_tests import find_missing_tests
+    from testless.analyze.refactor_candidates import find_refactor_candidates
+    from testless.reporters.console import print_report
+    from testless.models.findings import AnalysisReport, TestMeta
+
+    cfg = ctx.obj["config"]
+
+    cov_path = coverage_json or str(Path(cfg.coverage_dir) / "coverage.json")
+    cov_map = load_coverage_json(cov_path)
+
+    all_node_ids: set[str] = set()
+    for fc in cov_map.files.values():
+        for tests_list in fc.line_to_tests.values():
+            all_node_ids.update(tests_list)
+
+    tests = [
+        TestMeta(
+            node_id=nid,
+            file=nid.split("::")[0],
+            name=nid.split("::")[-1],
+        )
+        for nid in sorted(all_node_ids)
+    ]
+
+    fixture_index = FixtureIndex()
+    for td in cfg.test_dirs:
+        if Path(td).exists():
+            fixture_index.scan_directory(td)
+
+    dirs = list(services) if services else cfg.packages or ["."]
+    inventory = EndpointInventory()
+    for d in dirs:
+        if Path(d).exists():
+            inventory.scan_directory(d)
+
+    report = AnalysisReport(
+        duplicates=find_duplicates(tests, cov_map, fixture_index, min_score=cfg.min_duplicate_score),
+        dead_tests=find_dead_tests(tests, cov_map),
+        missing_tests=find_missing_tests(tests, inventory.endpoints, inventory.services),
+        refactor_candidates=find_refactor_candidates(tests),
+        tests=tests,
+    )
+
+    if question:
+        q = question.lower()
+        click.echo(click.style(f"\nQuestion: {question}\n", bold=True))
+        if any(kw in q for kw in ("usuń", "remove", "delete", "usunąć")):
+            click.echo("Tests to consider removing:")
+            for d in report.dead_tests:
+                click.echo(f"  • {d.node_id} — {d.reason}")
+            for dup in report.duplicates:
+                click.echo(f"  • {dup.test_b} — duplicate of {dup.test_a} (score={dup.score:.2f})")
+            if not report.dead_tests and not report.duplicates:
+                click.echo("  Nothing to remove — suite looks clean.")
+        elif any(kw in q for kw in ("dodaj", "add", "missing", "brakuje")):
+            click.echo("Tests to consider adding:")
+            for m in report.missing_tests:
+                click.echo(f"  • [{m.test_type}] {m.target}: {m.description}")
+            if not report.missing_tests:
+                click.echo("  No obvious missing tests detected.")
+        else:
+            print_report(report)
+    else:
+        print_report(report)