lingtest-cli 0.2.0__py3-none-any.whl

lingtest/__init__.py

@@ -0,0 +1,19 @@
+"""LingTest public API."""
+
+from lingtest.ai.models import AmbiguityAnalysis, FailureDiagnosis, GeneratedTestCase, TestCasePlan
+from lingtest.models import RunReport, TestCase, TestResult
+from lingtest.openapi import generate_cases
+from lingtest.runner import run_cases
+
+__all__ = [
+    "AmbiguityAnalysis",
+    "FailureDiagnosis",
+    "GeneratedTestCase",
+    "RunReport",
+    "TestCase",
+    "TestCasePlan",
+    "TestResult",
+    "generate_cases",
+    "run_cases",
+]
+__version__ = "0.2.0"

lingtest/ai/__init__.py

@@ -0,0 +1,5 @@
+"""AI-assisted testing capabilities."""
+
+from lingtest.ai.service import analyze_prd, diagnose_failures, generate_test_plan
+
+__all__ = ["analyze_prd", "diagnose_failures", "generate_test_plan"]

lingtest/ai/models.py

@@ -0,0 +1,73 @@
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+Severity = Literal["low", "medium", "high", "critical"]
+CaseType = Literal["functional", "boundary", "exception", "api"]
+
+
+class Ambiguity(BaseModel):
+    title: str = Field(min_length=1)
+    description: str = Field(min_length=1)
+    severity: Severity
+    evidence: str = ""
+    question: str = Field(min_length=1)
+
+
+class Contradiction(BaseModel):
+    title: str = Field(min_length=1)
+    statements: list[str] = Field(min_length=2)
+    severity: Severity
+    recommendation: str = Field(min_length=1)
+
+
+class AmbiguityAnalysis(BaseModel):
+    summary: str = Field(min_length=1)
+    ambiguities: list[Ambiguity] = Field(default_factory=list)
+    contradictions: list[Contradiction] = Field(default_factory=list)
+    missing_requirements: list[str] = Field(default_factory=list)
+    testability_risks: list[str] = Field(default_factory=list)
+
+
+class GeneratedTestCase(BaseModel):
+    id: str = Field(min_length=1)
+    title: str = Field(min_length=1)
+    case_type: CaseType
+    priority: Literal["P0", "P1", "P2", "P3"] = "P1"
+    preconditions: list[str] = Field(default_factory=list)
+    steps: list[str] = Field(min_length=1)
+    expected_result: str = Field(min_length=1)
+    source_requirement: str = ""
+    method: str | None = None
+    path: str | None = None
+    expected_status: int | None = Field(default=None, ge=100, le=599)
+    headers: dict[str, str] = Field(default_factory=dict)
+    query: dict[str, Any] = Field(default_factory=dict)
+    body: Any | None = None
+
+
+class TestCasePlan(BaseModel):
+    summary: str = Field(min_length=1)
+    risks: list[str] = Field(default_factory=list)
+    test_cases: list[GeneratedTestCase] = Field(min_length=1)
+
+
+class FailureDiagnosis(BaseModel):
+    case_id: str = Field(min_length=1)
+    failure_type: Literal[
+        "product_defect",
+        "test_defect",
+        "environment",
+        "data",
+        "dependency",
+        "unknown",
+    ]
+    confidence: float = Field(ge=0, le=1)
+    diagnosis: str = Field(min_length=1)
+    evidence: list[str] = Field(default_factory=list)
+    suggestions: list[str] = Field(default_factory=list)
+
+
+class DiagnosisReport(BaseModel):
+    summary: str = Field(min_length=1)
+    diagnoses: list[FailureDiagnosis] = Field(default_factory=list)

lingtest/ai/prompts.py

@@ -0,0 +1,43 @@
+ANALYZE_SYSTEM = """You are a senior software quality architect.
+Analyze product requirements for ambiguity, contradictions, missing requirements, and
+testability risks. Do not invent requirements. Return one JSON object matching this shape:
+{
+  "summary": "string",
+  "ambiguities": [{"title":"string","description":"string","severity":"low|medium|high|critical",
+    "evidence":"string","question":"string"}],
+  "contradictions": [{"title":"string","statements":["string","string"],
+    "severity":"low|medium|high|critical","recommendation":"string"}],
+  "missing_requirements": ["string"],
+  "testability_risks": ["string"]
+}"""
+
+GENERATE_SYSTEM = """You are a senior test designer. Generate reviewable test cases grounded
+only in the supplied document. Cover normal, boundary, and exception behavior. For OpenAPI
+documents, include API cases with method, path, expected_status, headers, query, and body.
+Never place credentials in a case. Return one JSON object matching this shape:
+{
+  "summary": "string",
+  "risks": ["string"],
+  "test_cases": [{
+    "id":"string","title":"string","case_type":"functional|boundary|exception|api",
+    "priority":"P0|P1|P2|P3","preconditions":["string"],"steps":["string"],
+    "expected_result":"string","source_requirement":"string",
+    "method":null,"path":null,"expected_status":null,"headers":{},"query":{},"body":null
+  }]
+}"""
+
+DIAGNOSE_SYSTEM = """You are a senior test failure analyst. Diagnose only failed or errored
+test results. Separate product defects from test defects, environment, data, and dependency
+issues. Ground each diagnosis in supplied evidence and avoid false certainty. Return one JSON
+object matching this shape:
+{
+  "summary":"string",
+  "diagnoses":[{
+    "case_id":"string",
+    "failure_type":"product_defect|test_defect|environment|data|dependency|unknown",
+    "confidence":0.0,
+    "diagnosis":"string",
+    "evidence":["string"],
+    "suggestions":["string"]
+  }]
+}"""

lingtest/ai/provider.py

@@ -0,0 +1,87 @@
+import os
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+import httpx
+
+
+class ProviderError(RuntimeError):
+    """Raised when an LLM provider request fails."""
+
+
+class LLMProvider(Protocol):
+    def complete(self, messages: list[dict[str, str]], *, json_mode: bool = True) -> str: ...
+
+
+@dataclass(slots=True)
+class OpenAICompatibleProvider:
+    api_key: str
+    model: str
+    base_url: str = "https://api.openai.com/v1"
+    timeout: float = 60.0
+    temperature: float = 0.2
+
+    def complete(self, messages: list[dict[str, str]], *, json_mode: bool = True) -> str:
+        payload: dict[str, Any] = {
+            "model": self.model,
+            "messages": messages,
+            "temperature": self.temperature,
+        }
+        if json_mode:
+            payload["response_format"] = {"type": "json_object"}
+        try:
+            response = httpx.post(
+                f"{self.base_url.rstrip('/')}/chat/completions",
+                headers={
+                    "Authorization": f"Bearer {self.api_key}",
+                    "Content-Type": "application/json",
+                },
+                json=payload,
+                timeout=self.timeout,
+            )
+            response.raise_for_status()
+            data = response.json()
+            return data["choices"][0]["message"]["content"]
+        except (httpx.HTTPError, KeyError, TypeError, ValueError) as exc:
+            raise ProviderError(f"LLM request failed: {exc}") from exc
+
+
+PROVIDERS = {
+    "openai": ("https://api.openai.com/v1", "gpt-4.1-mini", "OPENAI_API_KEY"),
+    "deepseek": ("https://api.deepseek.com/v1", "deepseek-chat", "DEEPSEEK_API_KEY"),
+    "qwen": (
+        "https://dashscope.aliyuncs.com/compatible-mode/v1",
+        "qwen-plus",
+        "DASHSCOPE_API_KEY",
+    ),
+}
+
+
+def create_provider(
+    provider: str,
+    *,
+    model: str | None = None,
+    base_url: str | None = None,
+    api_key: str | None = None,
+    timeout: float = 60.0,
+) -> OpenAICompatibleProvider:
+    name = provider.lower()
+    if name == "custom":
+        if not base_url or not model:
+            raise ValueError("Custom provider requires --base-url and --model")
+        env_name = "LINGTEST_API_KEY"
+        default_url, default_model = base_url, model
+    elif name in PROVIDERS:
+        default_url, default_model, env_name = PROVIDERS[name]
+    else:
+        raise ValueError(f"Unknown provider {provider!r}; use openai, deepseek, qwen, or custom")
+
+    resolved_key = api_key or os.getenv(env_name) or os.getenv("LINGTEST_API_KEY")
+    if not resolved_key:
+        raise ValueError(f"Missing API key; set {env_name} or LINGTEST_API_KEY")
+    return OpenAICompatibleProvider(
+        api_key=resolved_key,
+        model=model or default_model,
+        base_url=base_url or default_url,
+        timeout=timeout,
+    )

lingtest/ai/service.py

@@ -0,0 +1,84 @@
+import json
+from pathlib import Path
+
+from lingtest.ai.models import AmbiguityAnalysis, DiagnosisReport, TestCasePlan
+from lingtest.ai.prompts import ANALYZE_SYSTEM, DIAGNOSE_SYSTEM, GENERATE_SYSTEM
+from lingtest.ai.provider import LLMProvider
+from lingtest.ai.structured import request_structured
+from lingtest.models import RunReport
+
+MAX_DOCUMENT_CHARS = 60_000
+
+
+def read_document(path: str | Path) -> str:
+    document = Path(path)
+    if document.suffix.lower() not in {".txt", ".md", ".json", ".yaml", ".yml"}:
+        raise ValueError("Supported document types: .txt, .md, .json, .yaml, .yml")
+    content = document.read_text(encoding="utf-8")
+    if not content.strip():
+        raise ValueError("Document is empty")
+    return content[:MAX_DOCUMENT_CHARS]
+
+
+def analyze_prd(
+    path: str | Path, provider: LLMProvider, *, language: str = "zh-CN"
+) -> AmbiguityAnalysis:
+    content = read_document(path)
+    return request_structured(
+        provider,
+        [
+            {"role": "system", "content": ANALYZE_SYSTEM},
+            {
+                "role": "user",
+                "content": f"Respond in {language}. Analyze this document:\n\n{content}",
+            },
+        ],
+        AmbiguityAnalysis,
+    )
+
+
+def generate_test_plan(
+    path: str | Path,
+    provider: LLMProvider,
+    *,
+    language: str = "zh-CN",
+) -> TestCasePlan:
+    content = read_document(path)
+    return request_structured(
+        provider,
+        [
+            {"role": "system", "content": GENERATE_SYSTEM},
+            {
+                "role": "user",
+                "content": f"Respond in {language}. Generate tests for this document:\n\n{content}",
+            },
+        ],
+        TestCasePlan,
+    )
+
+
+def diagnose_failures(
+    report: RunReport,
+    provider: LLMProvider,
+    *,
+    language: str = "zh-CN",
+) -> DiagnosisReport:
+    failures = [result for result in report.results if result.status != "passed"]
+    if not failures:
+        return DiagnosisReport(summary="No failed or errored test cases to diagnose.", diagnoses=[])
+    evidence = json.dumps(
+        [result.model_dump(mode="json") for result in failures],
+        ensure_ascii=False,
+        indent=2,
+    )
+    return request_structured(
+        provider,
+        [
+            {"role": "system", "content": DIAGNOSE_SYSTEM},
+            {
+                "role": "user",
+                "content": f"Respond in {language}. Diagnose these test results:\n\n{evidence}",
+            },
+        ],
+        DiagnosisReport,
+    )

lingtest/ai/structured.py

@@ -0,0 +1,86 @@
+import json
+from typing import TypeVar
+
+from pydantic import BaseModel, ValidationError
+
+from lingtest.ai.provider import LLMProvider
+
+T = TypeVar("T", bound=BaseModel)
+
+
+class StructuredOutputError(ValueError):
+    """Raised when an LLM response cannot satisfy the requested schema."""
+
+
+def parse_json_object(raw: str) -> dict:
+    text = raw.strip()
+    if text.startswith("```"):
+        lines = text.splitlines()
+        text = "\n".join(lines[1:-1] if lines[-1].strip() == "```" else lines[1:])
+    try:
+        value = json.loads(text)
+    except json.JSONDecodeError:
+        value = json.loads(_extract_object(text))
+    if not isinstance(value, dict):
+        raise StructuredOutputError("LLM response must be a JSON object")
+    return value
+
+
+def request_structured(
+    provider: LLMProvider,
+    messages: list[dict[str, str]],
+    schema: type[T],
+    *,
+    retries: int = 1,
+) -> T:
+    current = list(messages)
+    last_error: Exception | None = None
+    for attempt in range(retries + 1):
+        raw = provider.complete(current, json_mode=True)
+        try:
+            return schema.model_validate(parse_json_object(raw))
+        except (json.JSONDecodeError, StructuredOutputError, ValidationError) as exc:
+            last_error = exc
+            if attempt >= retries:
+                break
+            current.extend(
+                [
+                    {"role": "assistant", "content": raw},
+                    {
+                        "role": "user",
+                        "content": (
+                            "The response failed schema validation. Return only a corrected JSON "
+                            f"object. Validation error: {exc}"
+                        ),
+                    },
+                ]
+            )
+    raise StructuredOutputError(f"LLM output failed validation: {last_error}") from last_error
+
+
+def _extract_object(text: str) -> str:
+    start = text.find("{")
+    if start < 0:
+        raise StructuredOutputError("LLM response does not contain a JSON object")
+    depth = 0
+    in_string = False
+    escaped = False
+    for index in range(start, len(text)):
+        char = text[index]
+        if in_string:
+            if escaped:
+                escaped = False
+            elif char == "\\":
+                escaped = True
+            elif char == '"':
+                in_string = False
+            continue
+        if char == '"':
+            in_string = True
+        elif char == "{":
+            depth += 1
+        elif char == "}":
+            depth -= 1
+            if depth == 0:
+                return text[start : index + 1]
+    raise StructuredOutputError("LLM response contains an incomplete JSON object")

lingtest/cli.py

@@ -0,0 +1,171 @@
+import json
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from lingtest import __version__
+from lingtest.ai.provider import ProviderError, create_provider
+from lingtest.ai.service import analyze_prd, diagnose_failures, generate_test_plan
+from lingtest.ai.structured import StructuredOutputError
+from lingtest.io import read_cases, read_report, write_cases, write_junit, write_report
+from lingtest.openapi import generate_cases
+from lingtest.runner import run_cases
+
+app = typer.Typer(help="AI-assisted testing for PRDs and OpenAPI.", no_args_is_help=True)
+
+
+def version_callback(value: bool) -> None:
+    if value:
+        typer.echo(__version__)
+        raise typer.Exit()
+
+
+@app.callback()
+def main(
+    version: Annotated[
+        bool | None,
+        typer.Option("--version", callback=version_callback, is_eager=True, help="Show version."),
+    ] = None,
+) -> None:
+    pass
+
+
+@app.command()
+def generate(
+    spec: Annotated[Path, typer.Argument(help="OpenAPI JSON or YAML file.")],
+    output: Annotated[Path, typer.Option("-o", "--output")] = Path("lingtest-cases.json"),
+) -> None:
+    """Generate editable API test cases."""
+    try:
+        cases = generate_cases(spec)
+    except (OSError, ValueError, json.JSONDecodeError) as exc:
+        typer.echo(f"Error: {exc}", err=True)
+        raise typer.Exit(2) from exc
+    write_cases(cases, output)
+    typer.echo(f"Generated {len(cases)} cases -> {output}")
+
+
+@app.command()
+def analyze(
+    document: Annotated[Path, typer.Argument(help="PRD or requirements document.")],
+    output: Annotated[Path, typer.Option("-o", "--output")] = Path("lingtest-analysis.json"),
+    provider: Annotated[str, typer.Option(help="openai, deepseek, qwen, or custom.")] = "openai",
+    model: Annotated[str | None, typer.Option()] = None,
+    base_url: Annotated[str | None, typer.Option()] = None,
+    language: Annotated[str, typer.Option()] = "zh-CN",
+) -> None:
+    """Find ambiguities, contradictions, missing requirements, and testability risks."""
+    try:
+        result = analyze_prd(
+            document,
+            _create_provider(provider, model, base_url),
+            language=language,
+        )
+        _write_model(result, output)
+    except (OSError, ValueError, ProviderError, StructuredOutputError) as exc:
+        _fail(exc)
+    typer.echo(
+        f"Found {len(result.ambiguities)} ambiguities and "
+        f"{len(result.contradictions)} contradictions -> {output}"
+    )
+
+
+@app.command("ai-generate")
+def ai_generate(
+    document: Annotated[Path, typer.Argument(help="PRD, requirements, or OpenAPI document.")],
+    output: Annotated[Path, typer.Option("-o", "--output")] = Path("lingtest-ai-cases.json"),
+    provider: Annotated[str, typer.Option(help="openai, deepseek, qwen, or custom.")] = "openai",
+    model: Annotated[str | None, typer.Option()] = None,
+    base_url: Annotated[str | None, typer.Option()] = None,
+    language: Annotated[str, typer.Option()] = "zh-CN",
+) -> None:
+    """Use an LLM to generate functional, boundary, exception, and API cases."""
+    try:
+        result = generate_test_plan(
+            document,
+            _create_provider(provider, model, base_url),
+            language=language,
+        )
+        _write_model(result, output)
+    except (OSError, ValueError, ProviderError, StructuredOutputError) as exc:
+        _fail(exc)
+    typer.echo(f"Generated {len(result.test_cases)} AI test cases -> {output}")
+
+
+@app.command()
+def diagnose(
+    report_file: Annotated[Path, typer.Argument(help="LingTest JSON run report.")],
+    output: Annotated[Path, typer.Option("-o", "--output")] = Path("lingtest-diagnosis.json"),
+    provider: Annotated[str, typer.Option(help="openai, deepseek, qwen, or custom.")] = "openai",
+    model: Annotated[str | None, typer.Option()] = None,
+    base_url: Annotated[str | None, typer.Option()] = None,
+    language: Annotated[str, typer.Option()] = "zh-CN",
+) -> None:
+    """Use an LLM to diagnose failed and errored test results."""
+    try:
+        result = diagnose_failures(
+            read_report(report_file),
+            _create_provider(provider, model, base_url),
+            language=language,
+        )
+        _write_model(result, output)
+    except (OSError, ValueError, ProviderError, StructuredOutputError) as exc:
+        _fail(exc)
+    typer.echo(f"Diagnosed {len(result.diagnoses)} failures -> {output}")
+
+
+@app.command("run")
+def run_command(
+    cases_file: Annotated[Path, typer.Argument(help="Generated cases JSON file.")],
+    base_url: Annotated[str, typer.Option("--base-url", envvar="LINGTEST_BASE_URL")],
+    output: Annotated[Path, typer.Option("-o", "--output")] = Path("lingtest-report.json"),
+    junit: Annotated[Path | None, typer.Option("--junit")] = None,
+    timeout: Annotated[float, typer.Option(min=0.1)] = 10.0,
+    header: Annotated[
+        list[str] | None, typer.Option("--header", "-H", help="HTTP header as NAME:VALUE.")
+    ] = None,
+) -> None:
+    """Run generated test cases against an API."""
+    try:
+        headers = _parse_headers(header or [])
+        report = run_cases(
+            read_cases(cases_file), base_url, timeout=timeout, extra_headers=headers
+        )
+    except (OSError, ValueError, json.JSONDecodeError) as exc:
+        typer.echo(f"Error: {exc}", err=True)
+        raise typer.Exit(2) from exc
+    write_report(report, output)
+    if junit:
+        write_junit(report, junit)
+    typer.echo(f"{report.passed} passed, {report.failed} failed -> {output}")
+    if report.failed:
+        raise typer.Exit(1)
+
+
+def _parse_headers(values: list[str]) -> dict[str, str]:
+    headers: dict[str, str] = {}
+    for value in values:
+        if ":" not in value:
+            raise ValueError(f"Invalid header {value!r}; expected NAME:VALUE")
+        name, content = value.split(":", 1)
+        headers[name.strip()] = content.strip()
+    return headers
+
+
+def _create_provider(provider: str, model: str | None, base_url: str | None):
+    return create_provider(provider, model=model, base_url=base_url)
+
+
+def _write_model(model, path: Path) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(model.model_dump_json(indent=2), encoding="utf-8")
+
+
+def _fail(exc: Exception) -> None:
+    typer.echo(f"Error: {exc}", err=True)
+    raise typer.Exit(2) from exc
+
+
+if __name__ == "__main__":
+    app()

lingtest/io.py

@@ -0,0 +1,48 @@
+import json
+from pathlib import Path
+from xml.etree.ElementTree import Element, ElementTree, SubElement
+
+from lingtest.models import RunReport, TestCase
+
+
+def write_cases(cases: list[TestCase], path: Path) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(
+        json.dumps([case.model_dump(mode="json") for case in cases], indent=2),
+        encoding="utf-8",
+    )
+
+
+def read_cases(path: Path) -> list[TestCase]:
+    return [TestCase.model_validate(item) for item in json.loads(path.read_text(encoding="utf-8"))]
+
+
+def write_report(report: RunReport, path: Path) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(report.model_dump_json(indent=2), encoding="utf-8")
+
+
+def read_report(path: Path) -> RunReport:
+    return RunReport.model_validate_json(path.read_text(encoding="utf-8"))
+
+
+def write_junit(report: RunReport, path: Path) -> None:
+    suite = Element(
+        "testsuite",
+        name="lingtest",
+        tests=str(len(report.results)),
+        failures=str(report.failed),
+    )
+    for result in report.results:
+        case = SubElement(
+            suite,
+            "testcase",
+            name=result.title,
+            classname=result.case_id,
+            time=f"{result.duration_ms / 1000:.6f}",
+        )
+        if result.status != "passed":
+            failure = SubElement(case, "failure", message=result.error or result.status)
+            failure.text = result.error or ""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    ElementTree(suite).write(path, encoding="utf-8", xml_declaration=True)

lingtest/models.py

@@ -0,0 +1,40 @@
+from datetime import UTC, datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+
+class TestCase(BaseModel):
+    id: str
+    title: str
+    method: str
+    path: str
+    expected_status: int
+    headers: dict[str, str] = Field(default_factory=dict)
+    query: dict[str, Any] = Field(default_factory=dict)
+    body: Any | None = None
+
+
+class TestResult(BaseModel):
+    case_id: str
+    title: str
+    status: Literal["passed", "failed", "error"]
+    duration_ms: float
+    expected_status: int
+    actual_status: int | None = None
+    error: str | None = None
+
+
+class RunReport(BaseModel):
+    started_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+    base_url: str
+    results: list[TestResult]
+
+    @property
+    def passed(self) -> int:
+        return sum(result.status == "passed" for result in self.results)
+
+    @property
+    def failed(self) -> int:
+        return len(self.results) - self.passed
+

lingtest/openapi.py

@@ -0,0 +1,108 @@
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from lingtest.models import TestCase
+
+HTTP_METHODS = {"get", "post", "put", "patch", "delete", "head", "options"}
+
+
+def load_spec(path: Path) -> dict[str, Any]:
+    raw = path.read_text(encoding="utf-8")
+    data = json.loads(raw) if path.suffix.lower() == ".json" else yaml.safe_load(raw)
+    if not isinstance(data, dict) or "paths" not in data:
+        raise ValueError("Input is not an OpenAPI document: missing 'paths'")
+    return data
+
+
+def generate_cases(path: str | Path) -> list[TestCase]:
+    spec = load_spec(Path(path))
+    cases: list[TestCase] = []
+    for route, path_item in spec["paths"].items():
+        if not isinstance(path_item, dict):
+            continue
+        common_parameters = path_item.get("parameters", [])
+        for method, operation in path_item.items():
+            if method.lower() not in HTTP_METHODS or not isinstance(operation, dict):
+                continue
+            parameters = [*common_parameters, *operation.get("parameters", [])]
+            rendered_path, query, headers = _parameters(route, parameters)
+            expected = _expected_status(operation.get("responses", {}), method)
+            operation_id = operation.get("operationId") or _slug(f"{method}_{route}")
+            cases.append(
+                TestCase(
+                    id=operation_id,
+                    title=operation.get("summary") or operation_id.replace("_", " ").title(),
+                    method=method.upper(),
+                    path=rendered_path,
+                    expected_status=expected,
+                    query=query,
+                    headers=headers,
+                    body=_request_body(operation.get("requestBody", {})),
+                )
+            )
+    return cases
+
+
+def _parameters(
+    route: str, parameters: list[dict[str, Any]]
+) -> tuple[str, dict[str, Any], dict[str, str]]:
+    query: dict[str, Any] = {}
+    headers: dict[str, str] = {}
+    for parameter in parameters:
+        name = parameter.get("name")
+        if not name:
+            continue
+        value = _example(parameter.get("schema", {}), parameter.get("example"))
+        location = parameter.get("in")
+        if location == "path":
+            route = route.replace(f"{{{name}}}", str(value))
+        elif location == "query":
+            query[name] = value
+        elif location == "header":
+            headers[name] = str(value)
+    return route, query, headers
+
+
+def _request_body(request_body: dict[str, Any]) -> Any | None:
+    content = request_body.get("content", {})
+    media = content.get("application/json")
+    if not isinstance(media, dict):
+        return None
+    if "example" in media:
+        return media["example"]
+    return _example(media.get("schema", {}))
+
+
+def _example(schema: dict[str, Any], explicit: Any = None) -> Any:
+    if explicit is not None:
+        return explicit
+    if "example" in schema:
+        return schema["example"]
+    if "default" in schema:
+        return schema["default"]
+    if "enum" in schema and schema["enum"]:
+        return schema["enum"][0]
+    kind = schema.get("type")
+    if kind == "object":
+        return {name: _example(value) for name, value in schema.get("properties", {}).items()}
+    if kind == "array":
+        return [_example(schema.get("items", {}))]
+    return {"integer": 1, "number": 1.0, "boolean": True, "string": "test"}.get(kind, "test")
+
+
+def _expected_status(responses: dict[str, Any], method: str) -> int:
+    success = sorted(
+        int(code) for code in responses if str(code).isdigit() and 200 <= int(code) < 300
+    )
+    if success:
+        return success[0]
+    return 201 if method.lower() == "post" else 200
+
+
+def _slug(value: str) -> str:
+    return re.sub(r"[^a-z0-9]+", "_", value.lower()).strip("_")
+

lingtest/runner.py

@@ -0,0 +1,48 @@
+from time import perf_counter
+
+import httpx
+
+from lingtest.models import RunReport, TestCase, TestResult
+
+
+def run_cases(
+    cases: list[TestCase],
+    base_url: str,
+    *,
+    timeout: float = 10.0,
+    extra_headers: dict[str, str] | None = None,
+) -> RunReport:
+    results: list[TestResult] = []
+    with httpx.Client(base_url=base_url.rstrip("/"), timeout=timeout) as client:
+        for case in cases:
+            started = perf_counter()
+            actual_status = None
+            error = None
+            status = "error"
+            try:
+                response = client.request(
+                    case.method,
+                    case.path,
+                    params=case.query,
+                    headers={**case.headers, **(extra_headers or {})},
+                    json=case.body,
+                )
+                actual_status = response.status_code
+                status = "passed" if actual_status == case.expected_status else "failed"
+                if status == "failed":
+                    error = f"expected HTTP {case.expected_status}, got HTTP {actual_status}"
+            except httpx.HTTPError as exc:
+                error = str(exc)
+            results.append(
+                TestResult(
+                    case_id=case.id,
+                    title=case.title,
+                    status=status,
+                    duration_ms=(perf_counter() - started) * 1000,
+                    expected_status=case.expected_status,
+                    actual_status=actual_status,
+                    error=error,
+                )
+            )
+    return RunReport(base_url=base_url, results=results)
+

lingtest_cli-0.2.0.dist-info/METADATA

@@ -0,0 +1,410 @@
+Metadata-Version: 2.4
+Name: lingtest-cli
+Version: 0.2.0
+Summary: AI testing agent for PRDs, OpenAPI specifications, and Vibe Coding workflows
+Project-URL: Homepage, https://github.com/sunweitest/lingtest-cli
+Project-URL: Repository, https://github.com/sunweitest/lingtest-cli
+Project-URL: Issues, https://github.com/sunweitest/lingtest-cli/issues
+Author: LingTest Contributors
+License-Expression: MIT
+License-File: LICENSE
+Keywords: ai,api,cli,openapi,pytest,testing,vibe-coding
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.11
+Requires-Dist: httpx<1,>=0.27
+Requires-Dist: pydantic<3,>=2.8
+Requires-Dist: pyyaml<7,>=6.0
+Requires-Dist: typer<1,>=0.12
+Provides-Extra: ai
+Provides-Extra: dev
+Requires-Dist: pytest-cov<7,>=5; extra == 'dev'
+Requires-Dist: pytest<9,>=8.2; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Requires-Dist: twine<7,>=5; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# LingTest CLI
+
+[简体中文](README.zh-CN.md) | English
+
+LingTest CLI is a local-first AI testing agent for Vibe Coding workflows. It
+analyzes PRDs, generates validated test plans from requirements and OpenAPI,
+runs deterministic HTTP tests, and diagnoses failures with structured evidence.
+
+The deterministic core does not require an LLM or API key. Generated cases are
+plain JSON, so they can be reviewed and changed before any request is sent.
+
+> Status: `0.2.0` alpha. The file format and Python API may evolve before 1.0.
+
+## What's New in 0.2.0
+
+- Independent OpenAI-compatible provider layer with no FastAPI, Redis, database,
+  or worker dependency.
+- Built-in provider presets for OpenAI, DeepSeek, and Qwen, plus custom
+  `base_url` and model support.
+- `lingtest analyze` for PRD ambiguity, contradiction, missing requirement, and
+  testability-risk analysis.
+- `lingtest ai-generate` for functional, boundary, exception, and API test plans.
+- `lingtest diagnose` for evidence-grounded failure classification and advice.
+- Pydantic validation for every AI result, with one controlled correction retry.
+- Provider credentials read from environment variables and excluded from cases
+  and reports.
+- Expanded Chinese and English documentation, product design, and Vibe Coding
+  testing article.
+
+## Features
+
+- Read OpenAPI specifications in YAML or JSON.
+- Discover GET, POST, PUT, PATCH, DELETE, HEAD, and OPTIONS operations.
+- Generate values from `example`, `default`, `enum`, and schema types.
+- Resolve path parameters and generate query, header, and JSON body values.
+- Select the first documented 2xx response as the expected status.
+- Run cases with HTTPX using a configurable base URL and timeout.
+- Add runtime headers without storing credentials in generated cases.
+- Write detailed JSON reports and optional JUnit XML.
+- Return CI-friendly exit codes.
+- Use the same core through a small Python API.
+- Analyze PRDs for ambiguity, contradictions, missing requirements, and testability risks.
+- Generate validated functional, boundary, exception, and API test plans with an LLM.
+- Diagnose failed HTTP tests with evidence-grounded structured output.
+- Connect to OpenAI, DeepSeek, Qwen, or a custom OpenAI-compatible provider.
+
+## Requirements
+
+- Python 3.11 or newer
+- An OpenAPI 3.x YAML or JSON document
+- Network access to the API under test when running cases
+
+## Installation
+
+### Standard `pip`
+
+Install into the active Python environment or virtual environment:
+
+```bash
+python -m pip install lingtest-cli
+lingtest --version
+```
+
+Use this when LingTest should share an existing project environment or when
+`pip` is your standard package manager.
+
+### Isolated CLI with `pipx` (recommended)
+
+```bash
+pipx install lingtest-cli
+```
+
+`pipx` creates a dedicated environment while making the `lingtest` command
+available globally. This avoids dependency conflicts with application projects.
+
+### Isolated CLI with `uv`
+
+```bash
+uv tool install lingtest-cli
+```
+
+### Add to a UV project
+
+```bash
+uv add --dev lingtest-cli
+uv run lingtest --help
+```
+
+All installation methods provide the same `lingtest` command and Python API.
+
+## Quick Start
+
+Generate cases:
+
+```bash
+lingtest generate openapi.yaml -o cases.json
+```
+
+Review `cases.json`, then run it:
+
+```bash
+lingtest run cases.json \
+  --base-url http://localhost:8000 \
+  --junit junit.xml \
+  -o report.json
+```
+
+PowerShell authentication example:
+
+```powershell
+lingtest run cases.json `
+  --base-url https://api.example.com `
+  -H "Authorization:Bearer $env:API_TOKEN" `
+  --junit junit.xml
+```
+
+Use `-H` multiple times to add more headers:
+
+```bash
+lingtest run cases.json --base-url https://api.example.com \
+  -H "Authorization:Bearer ${API_TOKEN}" \
+  -H "X-Tenant-ID:demo"
+```
+
+Analyze requirements and generate an AI test plan:
+
+```bash
+export OPENAI_API_KEY="..."
+lingtest analyze requirements.md -o analysis.json
+lingtest ai-generate requirements.md -o ai-cases.json
+```
+
+Diagnose failed cases from a LingTest JSON report:
+
+```bash
+lingtest diagnose report.json -o diagnosis.json
+```
+
+Use `--provider deepseek`, `--provider qwen`, or a custom OpenAI-compatible
+endpoint with `--provider custom --base-url URL --model MODEL`.
+
+## Test Data Flow
+
+```mermaid
+flowchart LR
+    PRD["PRD / Requirements"] --> Analyze["lingtest analyze"]
+    Analyze --> Analysis["analysis.json<br/>ambiguities and risks"]
+
+    PRD --> AIGenerate["lingtest ai-generate"]
+    OpenAPI["OpenAPI YAML / JSON"] --> AIGenerate
+    AIGenerate --> AIPlan["ai-cases.json<br/>validated test plan"]
+    AIPlan --> Review["Human review"]
+
+    OpenAPI --> Generate["lingtest generate"]
+    Generate --> Cases["cases.json<br/>executable HTTP cases"]
+    Review -.->|planned 0.3 conversion| Cases
+
+    Cases --> Run["lingtest run"]
+    Runtime["Base URL + runtime headers"] --> Run
+    Run --> JSONReport["report.json<br/>original evidence"]
+    Run --> JUnit["junit.xml<br/>CI result"]
+
+    JSONReport --> Diagnose["lingtest diagnose"]
+    Diagnose --> Diagnosis["diagnosis.json<br/>classification, evidence, advice"]
+```
+
+The solid path is implemented in `0.2.0`. AI plans remain review artifacts;
+automatic conversion from an approved AI plan to executable HTTP cases is
+planned for `0.3`. Diagnosis produces a separate file and never overwrites the
+original run report.
+
+## Commands
+
+### `lingtest analyze`
+
+Finds ambiguities, contradictions, missing requirements, and testability risks
+in `.txt`, `.md`, `.json`, `.yaml`, and `.yml` documents.
+
+### `lingtest ai-generate`
+
+Generates a validated plan containing functional, boundary, exception, and API
+test cases. AI cases are review artifacts and are never executed implicitly.
+
+### `lingtest diagnose`
+
+Reads a LingTest JSON run report and classifies failed cases using the supplied
+execution evidence.
+
+### `lingtest generate`
+
+```text
+lingtest generate SPEC [-o OUTPUT]
+```
+
+`SPEC` is an OpenAPI YAML or JSON file. The default output is
+`lingtest-cases.json`.
+
+### `lingtest run`
+
+```text
+lingtest run CASES_FILE --base-url URL [OPTIONS]
+```
+
+Important options:
+
+| Option | Description | Default |
+| --- | --- | --- |
+| `--base-url` | Base URL of the API under test. It can also be provided through `LINGTEST_BASE_URL`. | Required |
+| `-o`, `--output` | JSON report path. | `lingtest-report.json` |
+| `--junit` | Optional JUnit XML report path. | Disabled |
+| `--timeout` | Per-request timeout in seconds. | `10.0` |
+| `-H`, `--header` | Runtime HTTP header in `NAME:VALUE` form. Repeatable. | None |
+
+Exit codes:
+
+| Code | Meaning |
+| --- | --- |
+| `0` | All test cases passed. |
+| `1` | One or more test cases failed or produced a request error. |
+| `2` | The command input or case file is invalid. |
+
+## Case File
+
+Generated files are JSON arrays. A case looks like this:
+
+```json
+{
+  "id": "get_user",
+  "title": "Get user",
+  "method": "GET",
+  "path": "/users/42",
+  "expected_status": 200,
+  "headers": {},
+  "query": {
+    "verbose": true
+  },
+  "body": null
+}
+```
+
+Edit `expected_status`, request values, or titles before execution. Do not put
+long-lived credentials in this file; inject them with `--header` and environment
+variables at runtime.
+
+## Reports
+
+The JSON report contains the start time, base URL, and a result for each case:
+
+- case ID and title
+- `passed`, `failed`, or `error` status
+- request duration in milliseconds
+- expected and actual HTTP status
+- a concise error message
+
+JUnit output can be uploaded to GitHub Actions, Jenkins, GitLab CI, Azure
+Pipelines, and other systems that support JUnit XML.
+
+## Python API
+
+```python
+from lingtest import generate_cases, run_cases
+
+cases = generate_cases("openapi.yaml")
+report = run_cases(cases, "http://localhost:8000")
+
+print(f"{report.passed} passed, {report.failed} failed")
+for result in report.results:
+    print(result.case_id, result.status, result.duration_ms)
+```
+
+The public data models are `TestCase`, `TestResult`, and `RunReport`.
+
+## CI Example
+
+```yaml
+name: API tests
+
+on: [push, pull_request]
+
+jobs:
+  lingtest:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+      - run: uv tool install lingtest-cli
+      - run: lingtest generate openapi.yaml -o cases.json
+      - run: lingtest run cases.json --base-url "${{ secrets.TEST_API_URL }}" --junit junit.xml
+```
+
+The target API must be reachable from the runner. Start the service in an
+earlier step or point the command at a deployed test environment.
+
+## AI Providers
+
+| Provider | Option | API key environment variable | Default model |
+| --- | --- | --- | --- |
+| OpenAI | `--provider openai` | `OPENAI_API_KEY` | `gpt-4.1-mini` |
+| DeepSeek | `--provider deepseek` | `DEEPSEEK_API_KEY` | `deepseek-chat` |
+| Qwen | `--provider qwen` | `DASHSCOPE_API_KEY` | `qwen-plus` |
+| Custom | `--provider custom` | `LINGTEST_API_KEY` | Set with `--model` |
+
+Provider responses are parsed as JSON and validated with Pydantic. LingTest
+requests one correction when a response fails validation. Documents are sent
+to the configured provider; review its privacy policy before processing
+confidential requirements.
+
+## Current Limitations
+
+- `$ref` resolution and advanced schema composition are not implemented yet.
+- Assertions currently compare the HTTP status only.
+- Cases run sequentially.
+- OpenAPI security schemes are not applied automatically.
+- PDF and DOCX document extraction are not implemented.
+- AI-generated cases are review artifacts and are not automatically converted
+  into executable HTTP cases.
+
+See [product-design.html](product-design.html) for implemented scope, design
+decisions, and the roadmap.
+
+The Chinese essay [AI Agent 时代如何打造高质量软件](docs/blog/ai-testing-for-vibe-coding.zh-CN.md)
+explains the Vibe Coding quality problem behind this project.
+
+## Development
+
+Clone the repository and install all development dependencies:
+
+```bash
+uv sync --extra dev
+uv run pytest
+uv run ruff check .
+```
+
+Build and validate distributions:
+
+```bash
+uv build
+uv run twine check dist/*
+```
+
+Run the development CLI:
+
+```bash
+uv run lingtest --version
+uv run lingtest generate examples/openapi.yaml -o cases.json
+```
+
+## Release
+
+1. Update the version in `pyproject.toml` and `src/lingtest/__init__.py`.
+2. Run tests, Ruff, `uv build`, and `twine check`.
+3. Commit the release and create a matching Git tag.
+4. Publish with `UV_PUBLISH_TOKEN`:
+
+```bash
+uv publish
+```
+
+## Security
+
+Only run generated cases against systems you are authorized to test. Runtime
+headers are not written to the report by version 0.2, but generated case files
+and reports should still be treated as project data.
+
+## Roadmap
+
+- Better OpenAPI schema and `$ref` support
+- Configuration files and named environments
+- Filtering by tag, operation, path, and method
+- JSON-path, schema, header, and latency assertions
+- Authentication strategies
+- Controlled concurrency and retries
+- Static HTML reports
+- Conversion of reviewed AI API cases into deterministic executable cases
+- pytest export and a stable plugin API
+
+## License
+
+[MIT](LICENSE)

lingtest_cli-0.2.0.dist-info/RECORD

@@ -0,0 +1,17 @@
+lingtest/__init__.py,sha256=Kj7egmnbwx5-G6wnVN9KGLGLHdDwXMRlgg45Qz0SVyI,491
+lingtest/cli.py,sha256=CMn3RIYlN3HLARGqBTaLJHiPKROHZFGG-wzMjhndyQk,6352
+lingtest/io.py,sha256=ogAFoQ1kVM7f62poEJAMWZAdE3Y6LoDNdeEzxpdqdIE,1591
+lingtest/models.py,sha256=WTU44cyK2FubVmvCHknoe8chOzWUoY78wERRz3UQfbg,948
+lingtest/openapi.py,sha256=rhut-vnzsDXtfj0fs26jeaUx6bhBLXPpSd2Dd10Bf6Q,3825
+lingtest/runner.py,sha256=NTvyxx8HyQMEMJ_fm7VWbEwMjMss1rs2aqlPKHv0PuM,1636
+lingtest/ai/__init__.py,sha256=oDLWAjkgT_VYLgkeqm1HVuzfafrHPVLpHTxm8ZtRo8o,194
+lingtest/ai/models.py,sha256=MXKkvr_-WPPVsc5JEBf4MaTzDjLKM4Isk85-aPM0MDg,2314
+lingtest/ai/prompts.py,sha256=Aitq_RcKGi9l54aL_Dx7Uffuq_CH573BfqLQcA15QaE,1947
+lingtest/ai/provider.py,sha256=f6yvz4V5FhjvvN0LNYWbBAC-EVuyfaOpLXRrYBPce-0,2819
+lingtest/ai/service.py,sha256=t9gWoT4rTBBc991FY6uXaicIImxx-emAhTpGViKXfw4,2513
+lingtest/ai/structured.py,sha256=OL8CA2TNWMRUpimN3Q7G8UfGMy5BGtxEDvD_bmqnUHY,2685
+lingtest_cli-0.2.0.dist-info/METADATA,sha256=AuonYvq_3xcnGksSxlfoknWYkH-DabQWxGk5TanhuSE,12271
+lingtest_cli-0.2.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+lingtest_cli-0.2.0.dist-info/entry_points.txt,sha256=mtYoLIDZU9DPvmZFLRqT4ZGXumUeHJF67XvFuiWBiII,46
+lingtest_cli-0.2.0.dist-info/licenses/LICENSE,sha256=EBImwImAFQQnHaM-j_RcQNC50PWI251smhca_ly_Sy8,1079
+lingtest_cli-0.2.0.dist-info/RECORD,,

lingtest_cli-0.2.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.31.0
+Root-Is-Purelib: true
+Tag: py3-none-any

lingtest_cli-0.2.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+lingtest = lingtest.cli:app

lingtest_cli-0.2.0.dist-info/licenses/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2026 LingTest Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+