testsmith-ai 0.2.0__py3-none-any.whl

testsmith/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.0"

testsmith/cli.py

@@ -0,0 +1,191 @@
+"""testsmith CLI entrypoint."""
+
+from __future__ import annotations
+
+import re
+import sys
+from pathlib import Path
+from typing import Optional
+
+import typer
+from rich.console import Console
+
+from .csv_writer import write_csv
+from .generator import generate_test_cases
+from .interview import run_interview
+from .loaders import build_context
+from .providers import get_provider
+
+app = typer.Typer(
+    add_completion=False, help="Generate QA test cases from text and documents."
+)
+console = Console()
+
+
+@app.command()
+def generate(
+    prompt: Optional[str] = typer.Option(
+        None, "--prompt", "-p", help="Plain text prompt / feature description."
+    ),
+    file: list[str] = typer.Option(
+        [],
+        "--file",
+        "-f",
+        help="Input source: local file (PDF/DOCX/MD/TXT) or URL. Repeatable.",
+    ),
+    out: Optional[Path] = typer.Option(
+        None,
+        "--out",
+        "-o",
+        help="Output CSV path. If omitted, a name is suggested by the LLM.",
+    ),
+    provider: Optional[str] = typer.Option(
+        None,
+        "--provider",
+        help="LLM provider: 'anthropic' or 'gemini'. Auto-detected from env if omitted.",
+    ),
+    model: Optional[str] = typer.Option(
+        None,
+        "--model",
+        "-m",
+        help="LLM model name (e.g. 'claude-sonnet-4-6', 'gemini-2.5-flash'). Defaults per provider.",
+    ),
+    temperature: Optional[float] = typer.Option(
+        None,
+        "--temperature",
+        "-t",
+        help="Sampling temperature (0.0–2.0). Lower = more deterministic.",
+    ),
+    top_p: Optional[float] = typer.Option(
+        None,
+        "--top-p",
+        help="Nucleus sampling top-p (0.0–1.0).",
+    ),
+    system_prompt: Optional[str] = typer.Option(
+        None,
+        "--system",
+        "-s",
+        help="Custom system prompt. Inline text or @path/to/file.txt. Replaces the default.",
+    ),
+    append_system: bool = typer.Option(
+        False,
+        "--append-system",
+        help="Append --system to the default system prompt instead of replacing it.",
+    ),
+    user_template: Optional[str] = typer.Option(
+        None,
+        "--user-template",
+        "-u",
+        help="Custom user prompt template. Inline text or @path/to/file.txt. Use {context} as a placeholder.",
+    ),
+    fmt: str = typer.Option(
+        "steps",
+        "--format",
+        help="Test step format: 'steps' (numbered steps) or 'bdd' (Given/When/Then, business-focused).",
+    ),
+    trace: bool = typer.Option(
+        False,
+        "--trace",
+        help="Add source traceability columns (document, section, quote, derivation) for debugging.",
+    ),
+    max_tokens: int = typer.Option(
+        16384,
+        "--max-tokens",
+        help="Maximum output tokens for LLM response. Increase for large prompts or thinking models.",
+    ),
+    interactive: bool = typer.Option(
+        False,
+        "--interactive",
+        "-i",
+        help="Let the LLM ask clarifying questions before generating test cases.",
+    ),
+):
+    """Generate test cases and write them to a CSV file."""
+    if fmt not in ("steps", "bdd"):
+        console.print("[red]Error:[/red] --format must be 'steps' or 'bdd'.")
+        raise typer.Exit(code=2)
+
+    system_prompt = _resolve_text_arg(system_prompt)
+    user_template = _resolve_text_arg(user_template)
+
+    if not prompt and not file and sys.stdin.isatty() is False:
+        prompt = sys.stdin.read().strip() or None
+
+    if not prompt and not file:
+        console.print(
+            "[red]Error:[/red] provide --prompt and/or --file (or pipe text via stdin)."
+        )
+        raise typer.Exit(code=2)
+
+    console.print(f"[cyan]Loading context[/cyan] ({len(file)} source(s))...")
+    context, load_errors = build_context(prompt, list(file))
+    for err in load_errors:
+        console.print(f"[yellow]Warning:[/yellow] failed to load source: {err}")
+    if not context.strip():
+        console.print("[red]Error:[/red] context is empty after loading.")
+        raise typer.Exit(code=2)
+
+    try:
+        llm = get_provider(provider, model=model, temperature=temperature, top_p=top_p)
+    except Exception as e:
+        console.print(f"[red]Provider error:[/red] {e}")
+        raise typer.Exit(code=2)
+
+    if interactive:
+        if not sys.stdin.isatty():
+            console.print(
+                "[yellow]Stdin is piped — skipping interactive mode.[/yellow]"
+            )
+        else:
+            context = run_interview(context, provider=llm, console=console)
+
+    console.print(f"[cyan]Generating test cases via {llm.name} ({llm.model})...[/cyan]")
+    try:
+        rows, suggested = generate_test_cases(
+            context,
+            provider=llm,
+            system_prompt=system_prompt,
+            user_template=user_template,
+            append_system=append_system,
+            fmt=fmt,
+            max_tokens=max_tokens,
+            trace=trace,
+        )
+    except Exception as e:
+        console.print(f"[red]Generation failed:[/red] {e}")
+        raise typer.Exit(code=1)
+
+    if out is None:
+        out = _resolve_output_path(suggested)
+
+    count = write_csv(rows, out, extra_columns=trace)
+    console.print(f"[green]Wrote {count} test case(s) to[/green] {out}")
+
+
+_SLUG_RE = re.compile(r"[^a-z0-9]+")
+
+
+def _slugify(value: str) -> str:
+    slug = _SLUG_RE.sub("-", value.lower()).strip("-")
+    return slug[:60] or "test-cases"
+
+
+def _resolve_output_path(suggested: str | None) -> Path:
+    base = _slugify(suggested) if suggested else "test-cases"
+    path = Path(f"{base}.csv")
+    n = 2
+    while path.exists():
+        path = Path(f"{base}_{n}.csv")
+        n += 1
+    return path
+
+
+def _resolve_text_arg(value: Optional[str]) -> Optional[str]:
+    """Allow '@path/to/file' to load text from a file."""
+    if value and value.startswith("@"):
+        return Path(value[1:]).expanduser().read_text(encoding="utf-8")
+    return value
+
+
+if __name__ == "__main__":
+    app()

testsmith/csv_writer.py

@@ -0,0 +1,58 @@
+"""Write test cases to CSV."""
+
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+
+from .generator import CSV_COLUMNS
+
+
+def _flatten(row: dict, parent_key: str = "") -> dict:
+    """Flatten nested dicts into dot-separated keys.
+
+    Example: {"source": {"document": "PRD"}} → {"source.document": "PRD"}
+    """
+    items: dict = {}
+    for key, value in row.items():
+        full_key = f"{parent_key}.{key}" if parent_key else key
+        if isinstance(value, dict):
+            items.update(_flatten(value, full_key))
+        else:
+            items[full_key] = value
+    return items
+
+
+def write_csv(rows: list[dict], out_path: Path, extra_columns: bool = False) -> int:
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+
+    # Flatten nested objects (e.g. source.document, source.section)
+    flat_rows = [_flatten(row) for row in rows]
+
+    if extra_columns:
+        # Discover extra columns beyond the standard set, preserving order of first appearance
+        extra: list[str] = []
+        seen = set(CSV_COLUMNS)
+        for row in flat_rows:
+            for key in row:
+                if key not in seen:
+                    seen.add(key)
+                    extra.append(key)
+        fieldnames = CSV_COLUMNS + extra
+    else:
+        fieldnames = CSV_COLUMNS
+
+    with out_path.open("w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore")
+        writer.writeheader()
+        for row in flat_rows:
+            writer.writerow({col: _stringify(row.get(col, "")) for col in fieldnames})
+    return len(rows)
+
+
+def _stringify(value) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, list):
+        return " | ".join(str(v) for v in value)
+    return str(value)

testsmith/generator.py

@@ -0,0 +1,168 @@
+"""Call Claude to generate test cases as structured JSON."""
+
+from __future__ import annotations
+
+import json
+import re
+
+from .providers import LLMProvider, get_provider
+
+CSV_COLUMNS = [
+    "ID",
+    "Title",
+    "Preconditions",
+    "Steps",
+    "Expected Result",
+    "Priority",
+    "Type",
+]
+
+_STEPS_GUIDANCE_DEFAULT = '- Steps: numbered steps, each on its own line (use "\\n" inside the JSON string). Example: "1. Open app\\n2. Click login\\n3. Enter credentials".'
+
+_STEPS_GUIDANCE_BDD = """\
+- Steps: write in BDD format using Given / When / Then keywords, each on its own line (use "\\n" inside the JSON string).
+  Each step MUST start with one of: "Given", "When", "Then", "And", "But".
+  Example: "Given user has an active subscription\\nWhen the subscription renewal date arrives\\nThen the subscription is renewed automatically\\nAnd the user receives a confirmation email"
+
+  CRITICAL — Business-focused language rules for BDD steps:
+  • Steps MUST describe business intent, outcomes, and domain actions — NOT UI interactions.
+  • NEVER use UI-action words: click, tap, press, scroll, hover, swipe, drag, select (dropdown),
+    type, enter (into field), navigate, open, close, toggle, check (checkbox), uncheck,
+    fill in, submit (button), expand, collapse.
+  • INSTEAD of "When user clicks the checkout button" → "When user initiates checkout"
+  • INSTEAD of "Given user navigates to profile page" → "Given user is viewing their profile"
+  • INSTEAD of "When user types email in the login field" → "When user provides login credentials"
+  • INSTEAD of "Then user scrolls to the bottom" → "Then user reviews the full content"
+  • "Given" sets up the business state or context (not the UI state).
+  • "When" describes the business action or event (not the UI gesture).
+  • "Then" asserts the business outcome or side-effect (not what appears on screen).
+  • If a verification is about data, say what the DATA should be — not what the SCREEN shows."""
+
+
+def _build_output_contract(fmt: str = "steps", trace: bool = False) -> str:
+    steps_guidance = _STEPS_GUIDANCE_BDD if fmt == "bdd" else _STEPS_GUIDANCE_DEFAULT
+    trace_guidance = _TRACE_GUIDANCE_TEXT if trace else ""
+    return f"""Return ONLY a JSON object (no prose, no markdown fences) with EXACTLY these keys:
+- "suggested_filename": a short, descriptive, kebab-case filename (no extension, no path,
+  max 60 chars) reflecting the feature under test. Examples: "login-social-auth",
+  "checkout-guest-flow", "password-reset-email".
+- "test_cases": a JSON array where each element is an object with AT LEAST these keys:
+{json.dumps(CSV_COLUMNS)}
+(Additional keys are allowed and will be preserved in the JSON but omitted from the CSV.)
+
+Field guidance for each test case:
+- ID: "TC-001", "TC-002", ... sequential.
+- Title: short imperative summary.
+- Preconditions: setup/state required; use "None" if not applicable.
+{steps_guidance}
+- Expected Result: the observable outcome.
+- Priority: one of P0, P1, P2, P3.
+- Type: one of Functional, Negative, Edge, UI, Integration, Performance, Security, Accessibility.
+{trace_guidance}"""
+
+
+_TRACE_GUIDANCE_TEXT = """
+IMPORTANT — Source traceability (required):
+Each test case MUST also include a "source" object with these keys:
+- "document": which source document or design file the test was derived from
+- "section": specific section, heading, rule ID, or component/screen name
+- "quote": verbatim excerpt (≤ 50 words) from the source that justifies this test.
+  For design sources (e.g. Figma) where no text is quotable, describe the visual element
+  or interaction pattern instead (e.g. "Toggle switch for Delivery option in Deal Method section").
+- "derivation": one sentence explaining how the test was derived (e.g. boundary test, negative case, happy path)"""
+
+
+# Default contract for backward compatibility
+OUTPUT_CONTRACT = _build_output_contract("steps")
+
+DEFAULT_SYSTEM_PROMPT = f"""You are a senior QA engineer. Given product context (requirements, design docs,
+user prompts), produce a comprehensive set of test cases covering happy paths,
+edge cases, negative tests, and non-functional concerns where relevant.
+
+{OUTPUT_CONTRACT}
+"""
+
+
+def _build_default_system_prompt(fmt: str = "steps") -> str:
+    contract = _build_output_contract(fmt)
+    return (
+        "You are a senior QA engineer. Given product context (requirements, design docs,\n"
+        "user prompts), produce a comprehensive set of test cases covering happy paths,\n"
+        "edge cases, negative tests, and non-functional concerns where relevant.\n\n"
+        f"{contract}\n"
+    )
+
+
+DEFAULT_USER_TEMPLATE = (
+    "Product context:\n\n{context}\n\nGenerate the test cases now as a JSON array."
+)
+
+
+def build_system_prompt(
+    custom: str | None,
+    append: bool = False,
+    fmt: str = "steps",
+    trace: bool = False,
+) -> str:
+    contract = _build_output_contract(fmt, trace=trace)
+    default = _build_default_system_prompt(fmt)
+    if not custom:
+        return default
+    if append:
+        return f"{default}\n\nAdditional instructions:\n{custom}"
+    # Custom replaces default, but we always enforce the output contract
+    # so the CSV stays parseable.
+    return f"{custom}\n\n{contract}"
+
+
+def build_user_prompt(context: str, template: str | None) -> str:
+    tmpl = template or DEFAULT_USER_TEMPLATE
+    if "{context}" in tmpl:
+        return tmpl.format(context=context)
+    return f"{tmpl}\n\nProduct context:\n\n{context}"
+
+
+def generate_test_cases(
+    context: str,
+    provider: LLMProvider | None = None,
+    system_prompt: str | None = None,
+    user_template: str | None = None,
+    append_system: bool = False,
+    fmt: str = "steps",
+    max_tokens: int = 16384,
+    trace: bool = False,
+) -> tuple[list[dict], str | None]:
+    provider = provider or get_provider()
+    system = build_system_prompt(
+        system_prompt, append=append_system, fmt=fmt, trace=trace
+    )
+    user = build_user_prompt(context, user_template)
+    text = provider.complete(system=system, user=user, max_tokens=max_tokens)
+    return _parse_response(text)
+
+
+def _parse_response(text: str) -> tuple[list[dict], str | None]:
+    text = text.strip()
+    # Strip accidental code fences.
+    fence = re.match(r"^```(?:json)?\s*(.*?)\s*```$", text, re.DOTALL)
+    if fence:
+        text = fence.group(1).strip()
+
+    # Try object form first ({"suggested_filename": ..., "test_cases": [...]})
+    if text.startswith("{"):
+        data = json.loads(text)
+        rows = data.get("test_cases")
+        if not isinstance(rows, list):
+            raise ValueError("Model response missing 'test_cases' array")
+        name = data.get("suggested_filename")
+        return rows, name if isinstance(name, str) and name.strip() else None
+
+    # Back-compat: bare array
+    if not text.startswith("["):
+        match = re.search(r"\[.*\]", text, re.DOTALL)
+        if match:
+            text = match.group(0)
+    data = json.loads(text)
+    if not isinstance(data, list):
+        raise ValueError("Model did not return a JSON array or object")
+    return data, None

testsmith/interview.py

@@ -0,0 +1,130 @@
+"""Adaptive interview: LLM asks clarifying questions one at a time, only when needed."""
+
+from __future__ import annotations
+
+import json
+import re
+
+from rich.console import Console
+from rich.prompt import Prompt
+
+from .providers import LLMProvider
+
+INTERVIEW_SYSTEM_PROMPT = """You are a senior QA engineer preparing to write test cases.
+You will review product context and decide whether you need a clarification from the
+user before you can write high-quality test cases.
+
+Rules:
+- Ask a clarifying question ONLY if the answer would MEANINGFULLY change the test cases.
+  Do NOT ask about things you can reasonably assume or that are already clear.
+- Ask about things like: user roles, platforms, acceptance criteria, edge cases,
+  out-of-scope items, non-functional concerns (a11y, perf, security), integrations.
+- Ask ONE focused question at a time. Do not batch multiple questions.
+- Stop asking as soon as you have enough to write solid test cases. It is perfectly
+  fine — and often correct — to ask zero questions.
+
+Return ONLY a JSON object (no prose, no markdown fences) with EXACTLY these keys:
+- "need_clarification": boolean
+- "question": string (the next question to ask, or "" if need_clarification is false)
+- "reason": string (short rationale; why you need this, or why you are ready to proceed)
+"""
+
+
+def run_interview(
+    context: str,
+    provider: LLMProvider,
+    console: Console,
+    max_turns: int = 5,
+) -> str:
+    """Adaptively ask clarifying questions one at a time until the LLM is confident."""
+    console.print(
+        "[cyan]Checking context for ambiguity...[/cyan] "
+        "[dim](type [cyan]done[/cyan] at any prompt to stop early)[/dim]"
+    )
+
+    answers: list[tuple[str, str]] = []
+    asked: set[str] = set()
+
+    for turn in range(1, max_turns + 1):
+        enriched = _build_context_with_answers(context, answers)
+        try:
+            raw = provider.complete(
+                system=INTERVIEW_SYSTEM_PROMPT,
+                user=(
+                    f"Product context:\n\n{enriched}\n\n"
+                    "Decide if you need one more clarifying question. "
+                    "Return the JSON object now."
+                ),
+                max_tokens=4096,
+            )
+            if not raw or not raw.strip():
+                raise ValueError("empty response from model")
+            decision = _parse_decision(raw)
+        except Exception as e:
+            console.print(
+                f"[yellow]Clarification check failed ({e}); proceeding.[/yellow]"
+            )
+            break
+
+        if not decision.get("need_clarification"):
+            if turn == 1:
+                console.print(
+                    "[green]Context looks clear — no questions needed.[/green]"
+                )
+            else:
+                console.print(
+                    "[green]Enough context gathered — generating now.[/green]"
+                )
+            break
+
+        question = (decision.get("question") or "").strip()
+        if not question or question in asked:
+            break
+        asked.add(question)
+
+        try:
+            ans = Prompt.ask(
+                f"[green]?[/green] {question}", default="", show_default=False
+            )
+        except (EOFError, KeyboardInterrupt):
+            console.print(
+                "\n[yellow]Interview aborted — generating with current answers.[/yellow]"
+            )
+            break
+
+        ans = ans.strip()
+        if ans.lower() == "done":
+            break
+        if not ans or ans.lower() == "skip":
+            # Record the skip so the model doesn't re-ask the same thing.
+            answers.append((question, "(user skipped)"))
+            continue
+        answers.append((question, ans))
+    else:
+        console.print(
+            f"[yellow]Reached max {max_turns} questions — proceeding.[/yellow]"
+        )
+
+    return _build_context_with_answers(context, answers)
+
+
+def _build_context_with_answers(context: str, answers: list[tuple[str, str]]) -> str:
+    if not answers:
+        return context
+    addendum = "\n\n".join(f"Q: {q}\nA: {a}" for q, a in answers)
+    return f"{context}\n\n---\nClarifications from the user:\n\n{addendum}"
+
+
+def _parse_decision(text: str) -> dict:
+    text = text.strip()
+    fence = re.match(r"^```(?:json)?\s*(.*?)\s*```$", text, re.DOTALL)
+    if fence:
+        text = fence.group(1).strip()
+    if not text.startswith("{"):
+        match = re.search(r"\{.*\}", text, re.DOTALL)
+        if match:
+            text = match.group(0)
+    data = json.loads(text)
+    if not isinstance(data, dict):
+        raise ValueError("Expected a JSON object")
+    return data

testsmith/loaders.py

@@ -0,0 +1,27 @@
+"""Build the combined context string from a prompt and a list of references.
+
+Loading logic lives in `testsmith.sources`. This module only composes
+`LoadedDoc`s into the final context passed to the LLM.
+"""
+
+from __future__ import annotations
+
+from .sources import SourceError, load
+
+_SEPARATOR = "\n\n---\n\n"
+
+
+def build_context(prompt: str | None, refs: list[str]) -> tuple[str, list[str]]:
+    """Build context and return (context_string, list_of_error_messages)."""
+    parts: list[str] = []
+    errors: list[str] = []
+    if prompt:
+        parts.append(f"## User Prompt\n{prompt}")
+    for ref in refs:
+        try:
+            doc = load(ref)
+            parts.append(f"## {doc.title}\n{doc.text}")
+        except SourceError as e:
+            parts.append(f"## {ref}\n[ERROR loading source: {e}]")
+            errors.append(f"{ref}: {e}")
+    return _SEPARATOR.join(parts), errors