validra 0.1.2__py3-none-any.whl

app/api/routes/generation.py

@@ -0,0 +1,142 @@
+import json
+import logging
+
+from fastapi import APIRouter, Request
+from fastapi.responses import StreamingResponse
+
+from app.api.schemas.requests import TestRequest
+from app.engine.orchestrator import Orchestrator
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=["Execution"])
+
+
+@router.post("/generateAndRun", summary="Generates Test Cases and Run")
+def generate_and_run(request: TestRequest, req: Request):
+    """
+    Generate and execute automated API tests, streaming results via SSE.
+
+    Events emitted:
+    - `{"phase": "generating"}` — LLM is generating test cases
+    - `{"phase": "running", "progress": N, "total": T, "result": {...}}` — one test completed
+    - `{"phase": "done", "summary": {...}}` — all tests finished
+    - `{"phase": "error", "message": "..."}` — unrecoverable failure
+
+    ## Test Types
+
+    ### FUZZ
+    Generates invalid, unexpected, or edge-case payloads to test input validation.
+
+    ### AUTH
+    Mutates headers to test authentication and authorization scenarios
+    (missing tokens, expired credentials, malformed values).
+
+    ### PEN
+    Simulates penetration test payloads: injection, privilege escalation,
+    parameter pollution, ID tampering, encoding tricks.
+
+    ## Provider Config
+    Pass `provider` and optionally `provider_config` to select and configure
+    the LLM backend per request. Unset fields fall back to environment defaults.
+    Unknown keys in `provider_config` are rejected with a 400 error.
+    """
+    plugin_registry = req.app.state.plugin_registry
+    provider_registry = req.app.state.provider_registry
+    executor = req.app.state.executor
+    validator = req.app.state.validator
+
+    try:
+        plugin = plugin_registry.get(request.test_type.upper())
+    except KeyError as e:
+        msg = str(e)
+        def error_stream():
+            yield f"data: {json.dumps({'phase': 'error', 'message': msg})}\n\n"
+        return StreamingResponse(error_stream(), media_type="text/event-stream")
+
+    try:
+        provider = provider_registry.get(request.provider.lower())
+    except KeyError as e:
+        msg = str(e)
+        def error_stream():
+            yield f"data: {json.dumps({'phase': 'error', 'message': msg})}\n\n"
+        return StreamingResponse(error_stream(), media_type="text/event-stream")
+
+    try:
+        merged = {**provider.default_config().model_dump(), **(request.provider_config or {})}
+        provider_config = provider.config_class(**merged)
+    except Exception as e:
+        msg = f"Invalid provider_config: {e}"
+        def error_stream():
+            yield f"data: {json.dumps({'phase': 'error', 'message': msg})}\n\n"
+        return StreamingResponse(error_stream(), media_type="text/event-stream")
+
+    orchestrator = Orchestrator(plugin, executor, validator, provider, provider_config)
+
+    safe_input = {
+        "payload": request.payload,
+        "headers": request.headers,
+        "meta": request.payload_meta or {},
+    }
+
+    safe_request = {
+        "endpoint": request.endpoint,
+        "method": request.method,
+        "headers": request.headers,
+        "validate": request.validate,
+        "meta": request.payload_meta or {},
+    }
+
+    def event_stream():
+        # Phase 1: Warming up (instant feedback to the client)
+        yield f"data: {json.dumps({'phase': 'warming_up'})}\n\n"
+
+        # Phase 2: Generate all test cases (single LLM call)
+        yield f"data: {json.dumps({'phase': 'generating'})}\n\n"
+
+        try:
+            tests = orchestrator.generate(safe_input, request.max_cases)
+            tests = tests[: request.max_cases]
+        except Exception as e:
+            logger.exception("Error generating test cases")
+            yield f"data: {json.dumps({'phase': 'error', 'message': f'Error generating test cases: {e}'})}\n\n"
+            return
+
+        if not tests:
+            yield f"data: {json.dumps({'phase': 'error', 'message': 'No test cases were generated. LLM may be unavailable or returned invalid output.'})}\n\n"
+            return
+
+        # Phase 3+4: Execute and validate each test, yielding granular step events
+        results = []
+        try:
+            for step in orchestrator.run_stream(safe_request, tests):
+                event = step["event"]
+                if event in ("executing", "validating"):
+                    yield f"data: {json.dumps({'phase': event, 'progress': step['progress'], 'total': step['total']})}\n\n"
+                elif event == "result":
+                    results.append(step["result"])
+                    yield f"data: {json.dumps({'phase': 'result', 'progress': step['progress'], 'total': step['total'], 'result': step['result']}, default=str)}\n\n"
+        except Exception as e:
+            logger.exception("Error executing test cases")
+            yield f"data: {json.dumps({'phase': 'error', 'message': f'Error executing test cases: {e}'})}\n\n"
+            return
+
+        # Phase 5: Final summary
+        success_count = sum(1 for r in results if r["success"])
+        total_duration = sum(r["duration_ms"] for r in results)
+        summary = {
+            "total": len(results),
+            "success": success_count,
+            "failed": len(results) - success_count,
+            "total_duration_ms": total_duration,
+        }
+        yield f"data: {json.dumps({'phase': 'done', 'summary': summary})}\n\n"
+
+    return StreamingResponse(
+        event_stream(),
+        media_type="text/event-stream",
+        headers={
+            "Cache-Control": "no-cache",
+            "X-Accel-Buffering": "no",
+        },
+    )

app/api/routes/validation.py

@@ -0,0 +1,34 @@
+from fastapi import APIRouter, HTTPException, Request
+
+from app.api.schemas.requests import ValidateRequest
+
+router = APIRouter(tags=["Validation"])
+
+
+@router.post("/validate", summary="Validate Response from /generateAndRun")
+def validate(request: ValidateRequest, req: Request):
+    provider_registry = req.app.state.provider_registry
+    validator = req.app.state.validator
+
+    try:
+        provider = provider_registry.get(request.provider.lower())
+    except KeyError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+    try:
+        merged = {**provider.default_config().model_dump(), **(request.provider_config or {})}
+        provider_config = provider.config_class(**merged)
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=f"Invalid provider_config: {e}")
+
+    try:
+        result = validator.validate(
+            test=request.test,
+            response=request.response,
+            meta=request.meta or {},
+            provider=provider,
+            provider_config=provider_config,
+        )
+        return {"validation": result}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Validation error: {str(e)}")

app/api/schemas/requests.py

@@ -0,0 +1,159 @@
+from pydantic import BaseModel, Field
+from typing import Optional, Dict, Any
+
+
+_PROVIDER_CONFIG_DESCRIPTION = """
+Per-request provider overrides. Fields depend on the selected provider:
+
+**ollama**
+- `model` (str) — default: `llama3:8b-instruct-q4_0`
+- `temperature` (float) — default: `0.7`
+- `max_tokens` (int) — default: `700`
+- `top_p` (float) — default: `0.9`
+- `url` (str) — default: `http://localhost:11434/api/generate`
+- `timeout` (int, seconds) — default: `300`
+
+**openai**
+- `model` (str) — default: `gpt-4o`
+- `temperature` (float) — default: `0.7`
+- `max_tokens` (int) — default: `700`
+- `timeout` (int, seconds) — default: `60`
+- `api_key` (str) — required
+- `base_url` (str) — default: `https://api.openai.com/v1/chat/completions`
+
+**anthropic**
+- `model` (str) — default: `claude-sonnet-4-6`
+- `temperature` (float) — default: `0.7`
+- `max_tokens` (int) — default: `700`
+- `timeout` (int, seconds) — default: `60`
+- `api_key` (str) — required
+- `base_url` (str) — default: `https://api.anthropic.com/v1/messages`
+- `anthropic_version` (str) — default: `2023-06-01`
+""".strip()
+
+
+class TestRequest(BaseModel):
+    endpoint: str
+    method: str
+    headers: dict = {}
+    payload: dict
+    payload_meta: Optional[Dict[str, Any]] = None
+    test_type: str
+    max_cases: int = Field(
+        default=10,
+        ge=3,
+        le=100,
+        description="Maximum number of test cases to generate (3-100)",
+    )
+    validate: bool = True
+    provider: str = Field(
+        default="ollama",
+        description="LLM provider: ollama | openai | anthropic",
+    )
+    provider_config: Optional[Dict[str, Any]] = Field(
+        default=None,
+        description=_PROVIDER_CONFIG_DESCRIPTION,
+    )
+
+    model_config = {
+        "json_schema_extra": {
+            "examples": [
+                {
+                    "endpoint": "https://jsonplaceholder.typicode.com/posts",
+                    "method": "POST",
+                    "headers": {"Content-Type": "application/json"},
+                    "payload": {"title": "Validra Test", "body": "Testing fuzzy payload generation", "userId": 30},
+                    "payload_meta": {
+                        "body": "required, alphanumeric [1-50]",
+                        "title": "optional, alphanumeric [1-50]",
+                        "userId": "numeric [1-999]",
+                    },
+                    "test_type": "FUZZ",
+                    "max_cases": 10,
+                    "validate": True,
+                    "provider": "ollama",
+                    "provider_config": {
+                        "model": "llama3:8b-instruct-q4_0",
+                        "temperature": 0.5,
+                        "top_p": 0.9,
+                        "max_tokens": 700,
+                        "timeout": 300,
+                        "url": "http://localhost:11434/api/generate",
+                    },
+                },
+                {
+                    "endpoint": "https://jsonplaceholder.typicode.com/posts",
+                    "method": "POST",
+                    "headers": {"Content-Type": "application/json"},
+                    "payload": {"title": "Validra Test", "body": "Testing fuzzy payload generation", "userId": 30},
+                    "test_type": "FUZZ",
+                    "max_cases": 10,
+                    "validate": True,
+                    "provider": "openai",
+                    "provider_config": {
+                        "api_key": "sk-...",
+                        "model": "gpt-4o",
+                        "temperature": 0.7,
+                        "max_tokens": 700,
+                        "timeout": 60,
+                    },
+                },
+                {
+                    "endpoint": "https://jsonplaceholder.typicode.com/posts",
+                    "method": "POST",
+                    "headers": {"Content-Type": "application/json"},
+                    "payload": {"title": "Validra Test", "body": "Testing fuzzy payload generation", "userId": 30},
+                    "test_type": "FUZZ",
+                    "max_cases": 10,
+                    "validate": True,
+                    "provider": "anthropic",
+                    "provider_config": {
+                        "api_key": "sk-ant-...",
+                        "model": "claude-sonnet-4-6",
+                        "temperature": 0.7,
+                        "max_tokens": 700,
+                        "timeout": 60,
+                    },
+                },
+            ]
+        }
+    }
+
+
+class ValidateRequest(BaseModel):
+    test: Dict[str, Any]
+    response: Dict[str, Any]
+    meta: Optional[Dict[str, Any]] = None
+    provider: str = Field(
+        default="ollama",
+        description="LLM provider: ollama | openai | anthropic",
+    )
+    provider_config: Optional[Dict[str, Any]] = Field(
+        default=None,
+        description=_PROVIDER_CONFIG_DESCRIPTION,
+    )
+
+    model_config = {
+        "json_schema_extra": {
+            "examples": [
+                {
+                    "test": {
+                        "id": "tc-001",
+                        "description": "Body too long",
+                        "payload": {
+                            "title": "Validra Test",
+                            "body": "Testing fuzzy payload generation",
+                            "userId": 30,
+                        },
+                    },
+                    "response": {"status_code": 201, "body": {"id": 101}},
+                    "meta": {
+                        "body": "required, alphanumeric [1-50]",
+                        "title": "optional, alphanumeric [1-50]",
+                        "userId": "numeric [1-999]",
+                    },
+                    "provider": "ollama",
+                }
+            ]
+        }
+    }

app/api/schemas/responses.py

@@ -0,0 +1,30 @@
+from pydantic import BaseModel
+from typing import Any, Dict, List, Optional
+
+
+class ValidationResult(BaseModel):
+    dstatus: str
+    reason: str
+    confidence: float
+
+
+class TestResult(BaseModel):
+    id: str
+    description: Optional[str]
+    request: Dict[str, Any]
+    response: Dict[str, Any]
+    success: bool
+    duration_ms: int
+    validation: Optional[ValidationResult]
+
+
+class Summary(BaseModel):
+    total: int
+    success: int
+    failed: int
+    total_duration_ms: int
+
+
+class GenerationResponse(BaseModel):
+    tests: List[TestResult]
+    summary: Summary

app/cli.py

@@ -0,0 +1,5 @@
+import uvicorn
+
+
+def serve():
+    uvicorn.run("app.main:app", host="0.0.0.0", port=8000, reload=False)

app/config/settings.py

@@ -0,0 +1,21 @@
+from pydantic_settings import BaseSettings
+from typing import Optional
+
+
+class Settings(BaseSettings):
+    default_provider: str = "ollama"
+    executor_timeout: int = 60
+
+    # Deployment-specific (changes between local/docker)
+    ollama_url: str = "http://localhost:11434/api/generate"
+
+    # Secrets — no defaults, must be set to use that provider
+    openai_api_key: Optional[str] = None
+    anthropic_api_key: Optional[str] = None
+
+    class Config:
+        env_file = ".env"
+        env_file_encoding = "utf-8"
+
+
+settings = Settings()

app/engine/executor.py

@@ -0,0 +1,33 @@
+import requests
+
+
+class Executor:
+
+    def execute(self, request: dict, payload: dict, headers: dict = None) -> dict:
+        from app.config.settings import settings
+
+        url = request["endpoint"]
+        method = request.get("method", "POST")
+        timeout = settings.executor_timeout
+
+        try:
+            if method == "POST":
+                response = requests.post(url, json=payload, headers=headers, timeout=timeout)
+            elif method == "GET":
+                response = requests.get(url, params=payload, headers=headers, timeout=timeout)
+            else:
+                raise Exception(f"Unsupported HTTP method: {method}")
+
+            try:
+                body = response.json()
+            except Exception:
+                body = response.text
+
+            return {"status_code": response.status_code, "body": body}
+
+        except requests.exceptions.Timeout:
+            return {"status_code": 408, "error": "timeout"}
+        except requests.exceptions.ConnectionError:
+            return {"status_code": 503, "error": "connection_error"}
+        except Exception as e:
+            return {"status_code": 500, "error": str(e)}

app/engine/orchestrator.py

@@ -0,0 +1,136 @@
+import time
+
+from app.engine.executor import Executor
+from app.validator.base import BaseValidator
+
+
+class Orchestrator:
+
+    def __init__(
+        self,
+        plugin,
+        executor: Executor,
+        validator: BaseValidator,
+        provider,
+        provider_config,
+    ):
+        self.plugin = plugin
+        self.executor = executor
+        self.validator = validator
+        self.provider = provider
+        self.provider_config = provider_config
+
+    def generate(self, payload: dict, max_cases: int) -> list:
+        return self.plugin.generate(
+            example=payload,
+            previous_cases=[],
+            max_cases=max_cases,
+            meta=payload.get("meta", {}),
+            provider=self.provider,
+            provider_config=self.provider_config,
+        )
+
+    def run_stream(self, request: dict, tests: list):
+        """Generator yielding typed step events for each test.
+
+        Event shapes:
+          {"event": "executing",  "progress": N, "total": T}
+          {"event": "validating", "progress": N, "total": T}
+          {"event": "result",     "progress": N, "total": T, "result": {...}}
+        """
+        validate_enabled = request.get("validate", True)
+        meta = request.get("meta", {})
+        total = len(tests)
+
+        for idx, test in enumerate(tests, start=1):
+            payload = test.get("payload", {})
+            test_headers = test.get("headers")
+
+            yield {"event": "executing", "progress": idx, "total": total}
+
+            start = time.time()
+            response = self.executor.execute(request, payload, headers=test_headers)
+            duration = int((time.time() - start) * 1000)
+            success = 200 <= response.get("status_code", 500) < 300
+
+            validation_result = None
+            if validate_enabled:
+                yield {"event": "validating", "progress": idx, "total": total}
+                validation_result = self.validator.validate(
+                    test=test,
+                    response=response,
+                    meta=meta,
+                    provider=self.provider,
+                    provider_config=self.provider_config,
+                )
+
+            yield {
+                "event": "result",
+                "progress": idx,
+                "total": total,
+                "result": {
+                    "id": f"tc-{idx:03}",
+                    "description": test.get("description"),
+                    "request": {
+                        "headers": test_headers if test_headers is not None else request.get("headers", {}),
+                        "body": payload,                
+                    },
+                    "response": response,
+                    "success": success,
+                    "duration_ms": duration,
+                    "validation": validation_result,
+                },
+            }
+
+    def run(self, request: dict, tests: list) -> dict:
+        validate_enabled = request.get("validate", True)
+        meta = request.get("meta", {})
+        enriched_tests = []
+        success_count = 0
+        total_duration = 0
+
+        for idx, test in enumerate(tests, start=1):
+            payload = test.get("payload", {})
+            test_headers = test.get("headers")
+
+            start = time.time()
+            response = self.executor.execute(request, payload, headers=test_headers)
+            duration = int((time.time() - start) * 1000)
+            total_duration += duration
+
+            success = 200 <= response.get("status_code", 500) < 300
+            if success:
+                success_count += 1
+
+            validation_result = None
+            if validate_enabled:
+                validation_result = self.validator.validate(
+                    test=test,
+                    response=response,
+                    meta=meta,
+                    provider=self.provider,
+                    provider_config=self.provider_config,
+                )
+
+            enriched_tests.append({
+                "id": f"tc-{idx:03}",
+                "description": test.get("description"),
+                "request": {
+                    "headers": test_headers if test_headers is not None else request.get("headers", {}),
+                    "body": payload,
+                },
+                "response": response,
+                "success": success,
+                "duration_ms": duration,
+                "validation": validation_result,
+            })
+
+        return {
+            "tests": enriched_tests,
+            "summary": {
+                "total": len(enriched_tests),
+                "success": success_count,
+                "failed": len(enriched_tests) - success_count,
+                "total_duration_ms": total_duration,
+            },
+        }

app/main.py

@@ -0,0 +1,95 @@
+import logging
+import traceback
+from contextlib import asynccontextmanager
+from pathlib import Path
+
+from fastapi import FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.openapi.docs import get_swagger_ui_html
+from fastapi.responses import FileResponse, JSONResponse
+from fastapi.staticfiles import StaticFiles
+
+logger = logging.getLogger("validra")
+
+from app.api.routes import generation, validation
+from app.engine.executor import Executor
+from app.plugins.fuzz.plugin import FuzzPlugin
+from app.plugins.pen.plugin import PenTestPlugin
+from app.plugins.registry import PluginRegistry
+from app.plugins.security.plugin import SecurityPlugin
+from app.providers.anthropic.provider import AnthropicProvider
+from app.providers.ollama.provider import OllamaProvider
+from app.providers.openai.provider import OpenAIProvider
+from app.providers.registry import ProviderRegistry
+from app.validator.llm_validator import LLMValidator
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # ── Plugins ──────────────────────────────────────────────────────────────
+    plugin_registry = PluginRegistry()
+    plugin_registry.register("FUZZ", FuzzPlugin())
+    plugin_registry.register("AUTH", SecurityPlugin())
+    plugin_registry.register("PEN", PenTestPlugin())
+    app.state.plugin_registry = plugin_registry
+
+    # ── Providers ────────────────────────────────────────────────────────────
+    provider_registry = ProviderRegistry()
+    provider_registry.register("ollama", OllamaProvider())
+    provider_registry.register("openai", OpenAIProvider())
+    provider_registry.register("anthropic", AnthropicProvider())
+    app.state.provider_registry = provider_registry
+
+    # ── Shared singletons ────────────────────────────────────────────────────
+    app.state.executor = Executor()
+    app.state.validator = LLMValidator()
+
+    yield
+
+
+def create_app() -> FastAPI:
+    app = FastAPI(
+        title="Validra",
+        swagger_ui_parameters={"defaultModelsExpandDepth": -1},
+        docs_url=None,
+        redoc_url=None,
+        lifespan=lifespan,
+    )
+
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=["http://localhost:3000"],
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    _static_dir = Path(__file__).parent / "static"
+    app.mount("/static", StaticFiles(directory=str(_static_dir)), name="static")
+    app.include_router(generation.router)
+    app.include_router(validation.router)
+
+    @app.get("/docs", include_in_schema=False)
+    async def custom_swagger_ui_html():
+        return get_swagger_ui_html(
+            openapi_url=app.openapi_url,
+            title="Validra",
+            swagger_favicon_url="/favicon.ico",
+            swagger_ui_parameters={
+                "defaultModelsExpandDepth": -1,
+                "defaultModelExpandDepth": -1,
+            },
+        )
+
+    @app.get("/favicon.ico", include_in_schema=False)
+    async def favicon():
+        return FileResponse(str(Path(__file__).parent / "static" / "favicon.ico"))
+
+    @app.exception_handler(Exception)
+    async def unhandled_exception_handler(_req: Request, exc: Exception):
+        logger.error("Unhandled exception:\n%s", traceback.format_exc())
+        return JSONResponse(status_code=500, content={"detail": str(exc)})
+
+    return app
+
+
+app = create_app()

app/plugins/base.py

@@ -0,0 +1,18 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+
+
+class BasePlugin(ABC):
+    name: str = "base"
+
+    @abstractmethod
+    def generate(
+        self,
+        example: dict,
+        previous_cases: list,
+        max_cases: int,
+        meta: Optional[dict],
+        provider,
+        provider_config,
+    ) -> list:
+        raise NotImplementedError