nat-engine 1__py3-none-any.whl

mannf/product/llm/prompts.py

@@ -0,0 +1,403 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Prompt templates for LLM-augmented test generation."""
+
+from __future__ import annotations
+
+EDGE_CASE_PROMPT = """\
+You are a software testing expert specialising in API testing.
+
+Given the following API endpoint specification, generate {n} edge-case test inputs \
+that might expose bugs, boundary violations, or unexpected behaviour.
+
+Endpoint:
+  Method: {method}
+  Path: {path}
+  Description: {description}
+  Parameters: {parameters}
+  Request body schema: {schema}
+
+Respond with a valid JSON array of test case objects.  Each object must have:
+  - "inputs": dict of input fields and their values
+  - "expected_behavior": short string describing the expected outcome
+  - "test_type": one of "edge_case", "boundary", "negative"
+
+Return ONLY the JSON array with no surrounding markdown or explanation."""
+
+ADVERSARIAL_PROMPT = """\
+You are a security researcher specialising in API penetration testing.
+
+Generate {n} adversarial or malicious test inputs for the following API endpoint \
+that might reveal security vulnerabilities (injection, auth bypass, IDOR, etc.).
+
+Endpoint:
+  Method: {method}
+  Path: {path}
+  Description: {description}
+  Parameters: {parameters}
+  Request body schema: {schema}
+
+Respond with a valid JSON array of test case objects.  Each object must have:
+  - "inputs": dict of input fields and their values
+  - "expected_behavior": short string describing the expected outcome
+  - "test_type": one of "injection", "auth_bypass", "idor", "adversarial"
+
+Return ONLY the JSON array with no surrounding markdown or explanation."""
+
+SEMANTIC_PROMPT = """\
+You are an API testing expert.
+
+Based on the description of the following API endpoint, generate {n} semantically \
+meaningful test scenarios that reflect realistic usage, business logic edge cases, \
+and common developer mistakes.
+
+Endpoint:
+  Method: {method}
+  Path: {path}
+  Description: {description}
+  Parameters: {parameters}
+  Request body schema: {schema}
+
+Respond with a valid JSON array of test case objects.  Each object must have:
+  - "inputs": dict of input fields and their values
+  - "expected_behavior": short string describing the expected outcome
+  - "test_type": one of "semantic", "business_logic", "realistic"
+
+Return ONLY the JSON array with no surrounding markdown or explanation."""
+
+MUTATION_PROMPT = """\
+You are a fuzzing expert.
+
+Given the following test case, suggest {n} smart mutations that are more likely to \
+expose bugs than purely random perturbations.  Consider boundary values, type coercions, \
+encoding variations, and semantic inversions.
+
+Original test case:
+  Target: {target}
+  Inputs: {inputs}
+  Expected behaviour: {expected_behavior}
+
+Respond with a valid JSON array of mutated test case objects.  Each object must have:
+  - "inputs": dict with the mutated input fields
+  - "expected_behavior": short string describing what this mutation tests
+  - "mutation_rationale": one-sentence explanation of why this mutation is interesting
+
+Return ONLY the JSON array with no surrounding markdown or explanation."""
+
+UI_FORM_FILL_PROMPT = """\
+You are a software testing expert specialising in web application testing.
+
+Given the following web form specification, generate {n} sets of realistic form fill \
+values that represent valid user inputs for happy-path testing.
+
+Context:
+  Locale: {locale}
+  Application domain: {domain_context}
+
+Form:
+  Page URL: {page_url}
+  Form action: {form_action}
+  Form method: {form_method}
+  Fields:
+{fields_description}
+
+Field-type guidance — generate realistic values per type:
+  - email      → use pattern firstname.lastname@domain.com (e.g. jane.smith@example.com)
+  - password   → strong password with upper-case, lower-case, digits, and symbols \
+(e.g. Tr0ub4dor&3)
+  - tel        → locale-appropriate phone number for the given locale \
+(e.g. +1-555-867-5309 for en-US)
+  - date       → realistic recent date in YYYY-MM-DD format (within the last 5 years)
+  - number     → a plausible integer or decimal within the expected range for the field \
+label/context
+  - url        → a well-formed HTTPS URL (e.g. https://example.com/path)
+  - text/other → a realistic value that matches the field label or placeholder context
+
+Constraint rules:
+  - Every field marked required=True MUST receive a non-empty value.
+  - When a field specifies a pattern attribute, the generated value MUST match that \
+regex pattern.
+
+Respond with a valid JSON array of test case objects.  Each object must have:
+  - "inputs": dict mapping field selectors to fill values
+  - "description": short string describing the user scenario
+  - "test_type": "happy_path"
+
+Return ONLY the JSON array with no surrounding markdown or explanation."""
+
+UI_NEGATIVE_PATH_PROMPT = """\
+You are a software testing expert specialising in form validation and edge-case testing.
+
+Given the following web form specification, generate {n} sets of invalid or edge-case \
+form fill values that might expose validation bugs, error handling issues, or security \
+vulnerabilities.
+
+Form:
+  Page URL: {page_url}
+  Form action: {form_action}
+  Form method: {form_method}
+  Fields:
+{fields_description}
+
+Violation strategy — for each test case choose a deliberate violation:
+  - empty_required  : leave a field with required=True blank or omit it entirely
+  - format_violation: supply a value that does NOT match the field's pattern attribute \
+(e.g. for an email field try "not-an-email", for a tel field try "abc")
+  - overflow        : supply an extremely long string (500+ chars) or an out-of-range number
+  - unicode         : supply Unicode characters, RTL text, or emoji
+  - special_chars   : supply shell/HTML special characters (<, >, ", ', ;, --)
+  - boundary        : supply the minimum or maximum allowed value (0, -1, empty string, \
+max-int)
+  - negative        : supply a semantically wrong value (negative age, future birth date)
+  - xss             : supply a script-injection string (e.g. <script>alert(1)</script>)
+  - injection       : supply an SQL or command-injection string (e.g. ' OR '1'='1)
+
+Per-field strategy hints:
+  - email fields    : try malformed addresses (missing @, double dots, bare username)
+  - password fields : try empty string, whitespace-only, extremely long passwords
+  - number fields   : try non-numeric strings, negative values, extreme values (1e308)
+  - date fields     : try invalid dates (2024-02-30), far-future/past dates, wrong format
+  - required fields : always generate at least one test case that omits or empties the field
+
+Expected behaviour guidance — set "expected_behavior" to one of:
+  "should show client-side validation error" | "should return 422" | \
+"should return 400" | "should sanitise and store safely" | "should reject with 403"
+
+Respond with a valid JSON array of test case objects.  Each object must have:
+  - "inputs": dict mapping field selectors to fill values
+  - "expected_behavior": short string describing the expected validation response
+  - "test_type": one of "empty_required", "format_violation", "overflow", "unicode", \
+"special_chars", "boundary", "negative", "xss", "injection"
+
+Return ONLY the JSON array with no surrounding markdown or explanation."""
+
+UI_FLOW_SUGGESTION_PROMPT = """\
+You are a software testing expert specialising in user journey and end-to-end testing.
+
+Given the following web application structure (discovered pages, forms, and navigation \
+links), suggest {n} multi-step user journey test scenarios that would provide good \
+coverage of the application's functionality.
+
+Application:
+  Base URL: {base_url}
+  Pages:
+{pages_description}
+  Navigation edges:
+{edges_description}
+
+Journey composition rules:
+  - Prefer journeys that are 3–7 steps long and cross at least one page boundary.
+  - Include a mix of critical paths (login, checkout, registration) and exploratory paths.
+  - When a page contains a form, the journey MUST include fill steps for each visible \
+field followed by a click on the submit button selector.  Do NOT use "submit_form" — \
+form submission must be represented as a "click" action on the submit button selector.
+  - Valid action values are: navigate, fill, click.  No other actions are permitted.
+
+Respond with a valid JSON array of test scenario objects.  Each object must have:
+  - "name": short scenario name
+  - "steps": array of step objects, each with "action" (navigate/fill/click), \
+    "selector" (CSS selector or URL for navigate), and optionally "value" (for fill)
+  - "description": one-sentence description of what this journey tests
+  - "priority": one of "critical", "high", "medium", "low"
+
+Return ONLY the JSON array with no surrounding markdown or explanation."""
+
+UI_SECURITY_PAYLOAD_PROMPT = """\
+You are a web application security expert specialising in browser-level penetration testing.
+
+Given the following web form specification, generate {n} targeted security test payloads \
+that attempt to expose injection vulnerabilities, XSS, path traversal, SSRF, or other \
+OWASP Top 10 weaknesses by injecting attack strings into form fields.
+
+Form:
+  Page URL: {page_url}
+  Form action: {form_action}
+  Form method: {form_method}
+  Fields:
+{fields_description}
+
+Attack categories to cover (choose the most relevant per field):
+  - xss             : inject script/event-handler payloads to test for reflected/stored XSS
+  - sql_injection   : inject SQL metacharacters to test for SQL injection
+  - nosql_injection : inject NoSQL operator objects (e.g. {{"$gt":""}}) for NoSQL injection
+  - path_traversal  : inject directory traversal sequences (e.g. ../../etc/passwd) for file fields
+  - crlf_injection  : inject \\r\\n sequences to test for header/log injection
+  - ssrf            : inject internal URLs (e.g. http://127.0.0.1) for URL-accepting fields
+  - proto_pollution : inject __proto__ keys for fields that feed JSON objects
+  - unicode_attack  : inject Unicode homoglyphs or RTL overrides to bypass filters
+
+Field-type guidance:
+  - text/textarea/search : xss, sql_injection, crlf_injection, unicode_attack
+  - url fields           : ssrf, path_traversal
+  - email fields         : xss, sql_injection (via the local part)
+  - password fields      : sql_injection, unicode_attack
+
+Constraint rules:
+  - Each test case MUST target exactly one field with an attack payload.
+  - All other required fields should receive safe, valid values.
+  - Choose payloads that have a realistic chance of bypassing basic sanitization.
+
+Respond with a valid JSON array of test case objects.  Each object must have:
+  - "inputs": dict mapping exactly the attacking field's selector to the attack payload \
+and all other fillable fields to safe values
+  - "description": one sentence describing what vulnerability this tests
+  - "payload_type": one of "xss", "sql_injection", "nosql_injection", "path_traversal", \
+"crlf_injection", "ssrf", "proto_pollution", "unicode_attack"
+  - "target_field": the selector of the field receiving the attack payload
+
+Return ONLY the JSON array with no surrounding markdown or explanation."""
+
+ROOT_CAUSE_ANALYSIS_PROMPT = """\
+You are an expert software reliability engineer specialising in web application \
+test automation and root cause analysis.
+
+A browser-based autonomous test has detected a failure.  Using the information \
+below, identify the most likely root cause and suggest where an engineer should \
+start debugging.
+
+Failure details:
+  URL: {url}
+  Page key: {page}
+  Error message: {error}
+  Task ID: {task_id}
+  Fault likelihood (belief state): {fault_likelihood}
+
+Belief history for this page (most recent first):
+{belief_history}
+
+Scenario steps that were executing when the failure occurred:
+{scenario_steps}
+
+DOM snapshot (truncated):
+{dom_snapshot}
+
+Page metadata:
+{page_metadata}
+
+Based on the above, provide a structured JSON response with the following fields:
+  - "likely_cause": one-sentence description of the most probable root cause
+  - "suggested_fix": concise actionable suggestion for where to start debugging \
+(e.g. "Check the /api/checkout timeout configuration — the belief trend shows \
+sustained degradation over 3 runs")
+  - "confidence": float 0.0–1.0 indicating your confidence in this root cause \
+(use lower values when limited context is available)
+  - "files_to_check": list of source file paths or component names most likely \
+to contain the defect (empty list when unknown)
+
+Return ONLY the JSON object with no surrounding markdown or explanation."""
+
+TEST_PLAN_PROMPT = """\
+You are a senior API security and quality engineer.
+
+Given the following structured summary of an API specification, produce a \
+comprehensive test plan as a JSON object.
+
+API summary:
+{api_summary}
+
+Your test plan MUST be a JSON object with exactly these top-level fields:
+  - "endpoint_groups": array of group objects (see below) — order by descending priority
+  - "estimated_coverage": float 0.0–1.0 indicating estimated coverage if the full plan is executed
+
+Each group object must have:
+  - "name": short human-readable group name (e.g. "Authentication", "User CRUD")
+  - "endpoints": list of strings in "METHOD /path" format belonging to this group
+  - "priority": one of "critical", "high", "medium", "low"
+  - "risk_assessment": one-to-two sentence description of the risk profile for this group
+  - "suggested_strategies": list of concise test strategy strings \
+(e.g. "boundary value analysis", "auth bypass", "IDOR", "SQL injection", "rate limiting")
+  - "edge_cases": list of concise edge-case or business-logic scenario strings
+
+Grouping rules:
+  - Group endpoints by functional domain (auth, users, payments, admin, etc.)
+  - Mark as "critical" any group that handles authentication, authorisation, payments, or PII
+  - Mark as "high" any group with complex business logic or data mutation
+  - Mark as "medium" read-only endpoints with filtering/pagination
+  - Mark as "low" health-check, metadata, or purely informational endpoints
+
+Return ONLY the JSON object with no surrounding markdown or explanation."""
+
+NL_TEST_AUTHORING_PROMPT = """\
+You are a senior software testing engineer specialising in both API and browser-based \
+end-to-end testing.
+
+A user has described what they want to test in plain English.  Your job is to translate \
+that description into a list of concrete, executable test scenarios that can be run by \
+the NAT testing framework.
+
+User description: {description}
+
+Context (may be empty):
+  Base URL: {base_url}
+  Auth type: {auth_type}
+  Known endpoints: {known_endpoints}
+
+Generate a JSON array of test scenario objects.  Each scenario MUST be one of two types:
+
+**API scenario** (for REST/HTTP tests):
+  - "type": "api"
+  - "name": short scenario name
+  - "description": one-sentence description of what this tests
+  - "method": HTTP method (GET, POST, PUT, PATCH, DELETE)
+  - "path": URL path (relative to base_url, e.g. "/users/123")
+  - "headers": dict of request headers (may be empty)
+  - "body": dict of request body fields (may be empty)
+  - "assertions": list of assertion strings describing expected behaviour \
+(e.g. "status 200", "body.id is not null", "status 403 when not authenticated")
+  - "priority": one of "critical", "high", "medium", "low"
+
+**Browser scenario** (for UI/end-to-end tests):
+  - "type": "browser"
+  - "name": short scenario name
+  - "description": one-sentence description of what this tests
+  - "steps": array of step objects, each with:
+      - "action": one of "navigate", "fill", "click"
+      - "selector": CSS selector or URL (for navigate)
+      - "value": string value (required for fill, omit for navigate/click)
+  - "assertions": list of assertion strings describing expected behaviour
+  - "priority": one of "critical", "high", "medium", "low"
+
+Rules:
+  - Choose API scenarios for backend/REST tests, browser scenarios for UI/form tests.
+  - Generate between 2 and 8 scenarios that together give good coverage of the description.
+  - Prefer security-relevant scenarios (auth, IDOR, injection) when the description \
+mentions access control, tenants, or sensitive data.
+  - For browser scenarios, each form submission MUST be a "click" on the submit button \
+(do NOT use "submit_form").
+
+Return ONLY the JSON array with no surrounding markdown or explanation."""
+
+UI_ACCESSIBILITY_PROMPT = """\
+You are an accessibility testing expert specialising in WCAG 2.1 compliance and \
+assistive technology compatibility.
+
+Given the following web page specification, generate {n} accessibility-focused test \
+cases that verify the page is usable by people relying on keyboard navigation, screen \
+readers, and other assistive technologies.
+
+Page:
+  URL: {page_url}
+  Fields / interactive elements:
+{fields_description}
+
+Test categories to cover:
+  - keyboard_nav   : verify that all interactive elements are reachable and operable \
+using Tab / Shift-Tab / Enter / Space / arrow keys without a mouse
+  - aria_labels    : verify that every input, button, and landmark has a non-empty, \
+meaningful aria-label or aria-labelledby attribute
+  - screen_reader  : verify that dynamic content changes (errors, confirmations) are \
+announced via aria-live regions or role="alert"
+  - focus_management: verify that focus moves to a logical next element after form \
+submission, dialog open/close, or page navigation
+
+For each test case provide:
+  - "inputs": dict mapping field/element selectors to the simulated interaction or \
+expected attribute value to verify
+  - "description": one sentence describing what accessibility property is being verified
+  - "test_type": one of "keyboard_nav", "aria_labels", "screen_reader", "focus_management"
+
+Respond with a valid JSON array of {n} test case objects.
+Return ONLY the JSON array with no surrounding markdown or explanation."""