altqa-cli 0.1.0__py3-none-any.whl

altqa/__init__.py

@@ -0,0 +1,3 @@
+"""AltQA — AI-powered QA automation CLI."""
+
+__version__ = "0.1.0"

altqa/agents/accessibility_auditor.py

@@ -0,0 +1,125 @@
+"""AccessibilityAuditorAgent — AI-powered WCAG accessibility auditing.
+
+Crawls pages and uses AI to identify accessibility violations, mapping
+them to WCAG 2.1 criteria with concrete fix suggestions.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+from agno.agent import Agent
+from pydantic import BaseModel, Field
+
+from altqa.config.models import resolve_model
+from altqa.config.schema import AltQAConfig
+
+log = logging.getLogger("altqa.agents")
+
+_PROMPT_PATH = Path(__file__).parent / "prompts" / "accessibility_audit.md"
+
+
+class A11yIssue(BaseModel):
+    severity: str = "warning"
+    wcag_criterion: str = ""
+    description: str = ""
+    element: str = ""
+    fix: str = ""
+
+
+class A11yAuditResult(BaseModel):
+    page_url: str = ""
+    score: str = "unknown"
+    issues: list[A11yIssue] = Field(default_factory=list)
+    summary: str = ""
+    passed_checks: list[str] = Field(default_factory=list)
+
+
+class AccessibilityAuditorAgent:
+    """Audit web pages for WCAG 2.1 accessibility compliance."""
+
+    def __init__(self, config: AltQAConfig) -> None:
+        self.config = config
+        self.model = resolve_model(config.llm)
+        self._system_prompt = _PROMPT_PATH.read_text()
+
+    def audit_html(self, html: str, page_url: str = "") -> A11yAuditResult:
+        """Audit raw HTML content for accessibility issues."""
+        html_snippet = html[:6000] if len(html) > 6000 else html
+
+        user_prompt = (
+            f"## Page to audit\n"
+            f"**URL:** {page_url}\n\n"
+            f"```html\n{html_snippet}\n```"
+        )
+
+        from agno.utils.log import logger as agno_logger
+        prev_handlers = agno_logger.handlers[:]
+        prev_propagate = agno_logger.propagate
+        agno_logger.handlers = [logging.NullHandler()]
+        agno_logger.propagate = False
+        try:
+            agent = Agent(
+                model=self.model,
+                instructions=[self._system_prompt],
+                markdown=False,
+            )
+            response = agent.run(user_prompt)
+            content = _clean(response.content or "{}")
+            data = json.loads(content)
+        except Exception as exc:
+            log.warning("Accessibility audit failed for %s: %s", page_url, exc)
+            return A11yAuditResult(page_url=page_url, summary=f"Audit failed: {exc}")
+        finally:
+            agno_logger.handlers = prev_handlers
+            agno_logger.propagate = prev_propagate
+
+        issues = [
+            A11yIssue(
+                severity=i.get("severity", "warning"),
+                wcag_criterion=i.get("wcag_criterion", ""),
+                description=i.get("description", ""),
+                element=i.get("element", ""),
+                fix=i.get("fix", ""),
+            )
+            for i in data.get("issues", [])
+        ]
+
+        return A11yAuditResult(
+            page_url=page_url,
+            score=data.get("score", "unknown"),
+            issues=issues,
+            summary=data.get("summary", ""),
+            passed_checks=data.get("passed_checks", []),
+        )
+
+    def audit_page(self, page, page_url: str = "") -> A11yAuditResult:
+        """Audit a live Playwright page."""
+        html = page.content()
+        url = page_url or page.url
+        return self.audit_html(html, url)
+
+    def audit_many(
+        self,
+        pages_html: list[tuple[str, str]],
+    ) -> list[A11yAuditResult]:
+        """Audit multiple pages. Each tuple is (html, page_url)."""
+        results: list[A11yAuditResult] = []
+        for html, url in pages_html:
+            result = self.audit_html(html, url)
+            results.append(result)
+            log.info("A11y audit %s: score=%s, %d issues", url, result.score, len(result.issues))
+        return results
+
+
+def _clean(content: str) -> str:
+    content = content.strip()
+    if content.startswith("```"):
+        first_newline = content.index("\n") if "\n" in content else 3
+        content = content[first_newline + 1:]
+    if content.endswith("```"):
+        content = content[:-3]
+    return content.strip()

altqa/agents/assertion_interpreter.py

@@ -0,0 +1,145 @@
+"""AssertionInterpreterAgent — convert natural language assertions to concrete checks.
+
+Users write:
+    assertions:
+      - "the success toast message should appear"
+      - "price should be greater than 0"
+
+This agent converts them into structured assertions the engine can execute.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+from agno.agent import Agent
+
+from altqa.config.models import resolve_model
+from altqa.config.schema import AltQAConfig
+
+log = logging.getLogger("altqa.agents")
+
+NL_ASSERTION_PROMPT = """\
+You are a Playwright test assertion expert. Convert natural language assertions into structured checks.
+
+Available assertion types:
+- url_contains: check if the current URL contains a string. Fields: value
+- url_equals: check if the URL exactly matches. Fields: value
+- element_visible: check if an element is visible. Fields: selector
+- element_absent: check if an element is NOT present. Fields: selector
+- text_equals: check if an element's text exactly matches. Fields: selector, value
+- text_contains: check if an element's text contains a substring. Fields: selector, value
+
+Rules:
+1. Choose the most appropriate assertion type.
+2. Use robust CSS selectors (prefer ID, name, role, text content over class names).
+3. If the assertion mentions "should appear" or "should be visible", use element_visible.
+4. If it mentions "should not appear" or "should be gone", use element_absent.
+5. If it mentions URL or redirect, use url_contains.
+6. If it mentions specific text content, use text_contains or text_equals.
+7. For complex assertions that don't map directly, use element_visible with the best selector.
+
+Return ONLY valid JSON — an array of assertion objects:
+```json
+[
+  {"type": "element_visible", "selector": "#success-toast"},
+  {"type": "text_contains", "selector": ".price", "value": "$"}
+]
+```
+"""
+
+
+class AssertionInterpreterAgent:
+    """Convert natural language assertions to structured assertion defs."""
+
+    def __init__(self, config: AltQAConfig) -> None:
+        self.config = config
+        self.model = resolve_model(config.llm)
+
+    def interpret(self, nl_assertion: str) -> list[dict[str, str]]:
+        """Convert a single NL assertion into one or more structured assertions."""
+        user_prompt = f"Convert this assertion to a structured check:\n\n\"{nl_assertion}\""
+
+        from agno.utils.log import logger as agno_logger
+        prev_handlers = agno_logger.handlers[:]
+        prev_propagate = agno_logger.propagate
+        agno_logger.handlers = [logging.NullHandler()]
+        agno_logger.propagate = False
+        try:
+            agent = Agent(
+                model=self.model,
+                instructions=[NL_ASSERTION_PROMPT],
+                markdown=False,
+            )
+            response = agent.run(user_prompt)
+            content = _clean(response.content or "[]")
+            data = json.loads(content)
+            if isinstance(data, dict):
+                data = [data]
+            return data
+        except Exception as exc:
+            log.debug("NL assertion interpretation failed: %s", exc)
+            return self._fallback(nl_assertion)
+        finally:
+            agno_logger.handlers = prev_handlers
+            agno_logger.propagate = prev_propagate
+
+    def interpret_many(self, assertions: list[str]) -> list[dict[str, str]]:
+        """Convert multiple NL assertions."""
+        results: list[dict[str, str]] = []
+        for nl in assertions:
+            results.extend(self.interpret(nl))
+        return results
+
+    def _fallback(self, text: str) -> list[dict[str, str]]:
+        """Heuristic fallback for common assertion patterns."""
+        lower = text.lower()
+
+        if "visible" in lower or "appear" in lower or "show" in lower:
+            selector = self._guess_selector(lower)
+            return [{"type": "element_visible", "selector": selector}]
+
+        if "not" in lower and ("visible" in lower or "appear" in lower or "present" in lower):
+            selector = self._guess_selector(lower)
+            return [{"type": "element_absent", "selector": selector}]
+
+        if "url" in lower or "redirect" in lower or "navigate" in lower:
+            value = self._extract_url_hint(lower)
+            return [{"type": "url_contains", "value": value}]
+
+        if "text" in lower or "contain" in lower or "say" in lower:
+            return [{"type": "element_visible", "selector": "body"}]
+
+        return [{"type": "element_visible", "selector": "body"}]
+
+    def _guess_selector(self, text: str) -> str:
+        if "success" in text or "toast" in text:
+            return ".success, .toast, [role='alert'], #success"
+        if "error" in text or "alert" in text:
+            return ".error, .alert, [role='alert']"
+        if "modal" in text or "dialog" in text:
+            return "[role='dialog'], .modal"
+        if "button" in text:
+            return "button"
+        return "body"
+
+    def _extract_url_hint(self, text: str) -> str:
+        if "dashboard" in text:
+            return "/dashboard"
+        if "login" in text:
+            return "/login"
+        if "home" in text:
+            return "/"
+        return "/"
+
+
+def _clean(content: str) -> str:
+    content = content.strip()
+    if content.startswith("```"):
+        first_newline = content.index("\n") if "\n" in content else 3
+        content = content[first_newline + 1:]
+    if content.endswith("```"):
+        content = content[:-3]
+    return content.strip()

altqa/agents/data_generator.py

@@ -0,0 +1,184 @@
+"""TestDataGeneratorAgent — AI-powered realistic test data for forms.
+
+Instead of hardcoded "test123" values, generates contextual test data:
+- Valid data for happy paths
+- Invalid data for negative tests (wrong formats, boundary values)
+- Edge case data (long strings, special chars, XSS, SQLi)
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+from agno.agent import Agent
+
+from altqa.config.models import resolve_model
+from altqa.config.schema import AltQAConfig
+
+log = logging.getLogger("altqa.agents")
+
+DATA_GEN_PROMPT = """\
+You are a QA test data expert. Generate realistic test data for form fields.
+
+Given a list of form fields (with their names/types), generate multiple data sets:
+1. **valid**: Realistic data that should pass all validation
+2. **invalid**: Data that should trigger validation errors (wrong formats, missing required)
+3. **boundary**: Edge cases (empty strings, max length, min/max numbers, unicode, special chars)
+4. **security**: Potentially dangerous inputs (SQL injection, XSS, path traversal) for security testing
+
+Rules:
+- Use realistic, contextual values (not "test123")
+- For email fields: use real-looking emails for valid, malformed for invalid
+- For password fields: use strong passwords for valid, too-short for invalid
+- For numeric fields: use 0, -1, MAX_INT, decimals for boundary
+- For text fields: use very long strings (200+ chars), unicode, emojis for boundary
+- Security data must be actual payloads (e.g. `' OR 1=1--`, `<script>alert(1)</script>`)
+
+Return ONLY valid JSON:
+```json
+{
+  "valid": [
+    {"field_name": "value", ...}
+  ],
+  "invalid": [
+    {"field_name": "bad_value", "_expect_error": "description of expected validation error", ...}
+  ],
+  "boundary": [
+    {"field_name": "edge_value", "_scenario": "empty field", ...}
+  ],
+  "security": [
+    {"field_name": "payload", "_test_type": "sql_injection", ...}
+  ]
+}
+```
+"""
+
+
+class TestDataGeneratorAgent:
+    """Generate realistic test data for form fields using AI."""
+
+    def __init__(self, config: AltQAConfig) -> None:
+        self.config = config
+        self.model = resolve_model(config.llm)
+
+    def generate(
+        self,
+        fields: list[dict[str, str]],
+        form_context: str = "",
+    ) -> dict[str, list[dict[str, Any]]]:
+        """Generate test data sets for a list of form fields.
+
+        Args:
+            fields: List of dicts like [{"name": "email", "type": "email"}, ...]
+            form_context: Optional description of the form purpose
+
+        Returns:
+            Dict with keys: valid, invalid, boundary, security — each a list of data dicts.
+        """
+        user_prompt = self._build_prompt(fields, form_context)
+
+        from agno.utils.log import logger as agno_logger
+        prev_handlers = agno_logger.handlers[:]
+        prev_propagate = agno_logger.propagate
+        agno_logger.handlers = [logging.NullHandler()]
+        agno_logger.propagate = False
+        try:
+            agent = Agent(
+                model=self.model,
+                instructions=[DATA_GEN_PROMPT],
+                markdown=False,
+            )
+            response = agent.run(user_prompt)
+            content = _clean(response.content or "{}")
+            return json.loads(content)
+        except Exception as exc:
+            log.warning("Test data generation failed: %s", exc)
+            return self._fallback(fields)
+        finally:
+            agno_logger.handlers = prev_handlers
+            agno_logger.propagate = prev_propagate
+
+    def generate_for_page(
+        self,
+        page_html: str,
+        page_url: str = "",
+    ) -> dict[str, list[dict[str, Any]]]:
+        """Extract form fields from HTML and generate test data."""
+        fields = self._extract_fields_from_html(page_html)
+        if not fields:
+            return {"valid": [], "invalid": [], "boundary": [], "security": []}
+        return self.generate(fields, form_context=f"Form on page: {page_url}")
+
+    def _build_prompt(self, fields: list[dict[str, str]], context: str) -> str:
+        parts = ["## Form Fields\n"]
+        if context:
+            parts.append(f"**Context:** {context}\n")
+        for f in fields:
+            parts.append(f"- **{f.get('name', 'unknown')}**: type={f.get('type', 'text')}")
+        return "\n".join(parts)
+
+    def _extract_fields_from_html(self, html: str) -> list[dict[str, str]]:
+        """Simple extraction of input fields from HTML."""
+        import re
+        fields: list[dict[str, str]] = []
+        for match in re.finditer(r'<input[^>]*>', html, re.IGNORECASE):
+            tag = match.group()
+            name_m = re.search(r'name=["\']([^"\']+)', tag)
+            type_m = re.search(r'type=["\']([^"\']+)', tag)
+            if name_m:
+                fields.append({
+                    "name": name_m.group(1),
+                    "type": type_m.group(1) if type_m else "text",
+                })
+        return fields
+
+    def _fallback(self, fields: list[dict[str, str]]) -> dict[str, list[dict[str, Any]]]:
+        """Deterministic fallback when AI is unavailable."""
+        valid: dict[str, Any] = {}
+        invalid: dict[str, Any] = {}
+        boundary: dict[str, Any] = {}
+        security: dict[str, Any] = {}
+
+        for f in fields:
+            name = f.get("name", "field")
+            ftype = f.get("type", "text")
+
+            if ftype == "email":
+                valid[name] = "testuser@example.com"
+                invalid[name] = "not-an-email"
+                boundary[name] = ""
+                security[name] = "test@example.com' OR 1=1--"
+            elif ftype == "password":
+                valid[name] = "SecureP@ss123!"
+                invalid[name] = "123"
+                boundary[name] = ""
+                security[name] = "' OR 1=1--"
+            elif "number" in ftype or "price" in name or "stock" in name:
+                valid[name] = "100"
+                invalid[name] = "abc"
+                boundary[name] = "-1"
+                security[name] = "999999999999"
+            else:
+                valid[name] = f"Test {name.title()}"
+                invalid[name] = ""
+                boundary[name] = "A" * 256
+                security[name] = "<script>alert(1)</script>"
+
+        return {
+            "valid": [valid],
+            "invalid": [{**invalid, "_expect_error": "Validation should fail"}],
+            "boundary": [{**boundary, "_scenario": "Edge case values"}],
+            "security": [{**security, "_test_type": "injection_tests"}],
+        }
+
+
+def _clean(content: str) -> str:
+    content = content.strip()
+    if content.startswith("```"):
+        first_newline = content.index("\n") if "\n" in content else 3
+        content = content[first_newline + 1:]
+    if content.endswith("```"):
+        content = content[:-3]
+    return content.strip()

altqa/agents/failure_analyzer.py

@@ -0,0 +1,180 @@
+"""FailureAnalyzerAgent — AI-powered bug report generation from test failures.
+
+Analyses test execution results and produces structured bug reports
+ready to be filed as Jira sub-tasks via the BugReporter.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+from agno.agent import Agent
+from pydantic import BaseModel, Field
+
+from altqa.config.models import resolve_model
+from altqa.config.schema import AltQAConfig
+from altqa.flows.runner import FlowResult, StepResult
+
+log = logging.getLogger("altqa.agents")
+
+_PROMPT_PATH = Path(__file__).parent / "prompts" / "analyze_failure.md"
+
+
+class BugReport(BaseModel):
+    """Structured bug report generated by AI analysis."""
+    summary: str = ""
+    description: str = ""
+    severity: str = "major"
+    steps_to_reproduce: list[str] = Field(default_factory=list)
+    expected_result: str = ""
+    actual_result: str = ""
+    affected_area: str = ""
+    probable_fix: str = ""
+    user_impact: str = ""
+    regression_test: str = ""
+    environment: str = ""
+
+
+class FailureAnalyzerAgent:
+    """Analyse test failures and generate structured bug reports."""
+
+    def __init__(self, config: AltQAConfig) -> None:
+        self.config = config
+        self.model = resolve_model(config.llm)
+        self._system_prompt = _PROMPT_PATH.read_text()
+
+    def analyze_step_failure(
+        self,
+        flow_name: str,
+        failed_step: StepResult,
+        all_steps: list[StepResult],
+        page_url: str = "",
+    ) -> BugReport:
+        """Generate a BugReport for a single failed step."""
+        user_prompt = self._build_step_prompt(flow_name, failed_step, all_steps, page_url)
+        return self._run(user_prompt, flow_name)
+
+    def analyze_flow_result(self, flow_result: FlowResult) -> list[BugReport]:
+        """Generate BugReports for all failed steps in a FlowResult."""
+        reports: list[BugReport] = []
+
+        for failed_step in flow_result.failed_steps:
+            try:
+                report = self.analyze_step_failure(
+                    flow_name=flow_result.flow_name,
+                    failed_step=failed_step,
+                    all_steps=flow_result.steps,
+                )
+                reports.append(report)
+                log.info("Bug report generated for step %d of %s", failed_step.step_number, flow_result.flow_name)
+            except Exception as exc:
+                log.error("Failed to analyse step %d: %s", failed_step.step_number, exc)
+                reports.append(self._fallback_report(flow_result.flow_name, failed_step))
+
+        return reports
+
+    def severity_to_priority(self, severity: str) -> str:
+        """Map AI severity to Jira priority."""
+        mapping = {
+            "critical": "Highest",
+            "major": "High",
+            "minor": "Medium",
+        }
+        return mapping.get(severity.lower(), "Medium")
+
+    # ── Internal ───────────────────────────────────────
+
+    def _run(self, user_prompt: str, context_name: str) -> BugReport:
+        agent = Agent(
+            model=self.model,
+            instructions=[self._system_prompt],
+            markdown=False,
+        )
+
+        response = agent.run(user_prompt)
+        content = self._clean(response.content or "{}")
+
+        try:
+            data = json.loads(content)
+            return BugReport(
+                summary=data.get("summary", ""),
+                description=data.get("description", ""),
+                severity=data.get("severity", "major"),
+                steps_to_reproduce=data.get("steps_to_reproduce", []),
+                expected_result=data.get("expected_result", ""),
+                actual_result=data.get("actual_result", ""),
+                affected_area=data.get("affected_area", ""),
+                probable_fix=data.get("probable_fix", ""),
+                user_impact=data.get("user_impact", ""),
+                regression_test=data.get("regression_test", ""),
+                environment=data.get("environment", ""),
+            )
+        except (json.JSONDecodeError, Exception) as exc:
+            log.warning("AI analysis failed for %s: %s", context_name, exc)
+            return BugReport(summary=f"Test failure in {context_name}", severity="major")
+
+    def _build_step_prompt(
+        self,
+        flow_name: str,
+        failed_step: StepResult,
+        all_steps: list[StepResult],
+        page_url: str,
+    ) -> str:
+        ar = failed_step.action_result
+
+        preceding_steps = [
+            f"  {s.step_number}. [{s.action}] {s.description} — {'PASS' if s.passed else 'FAIL'}"
+            for s in all_steps
+            if s.step_number <= failed_step.step_number
+        ]
+
+        parts = [
+            f"## Test Failure Details\n",
+            f"**Flow:** {flow_name}",
+            f"**Failed Step:** #{failed_step.step_number} — {failed_step.description}",
+            f"**Action:** {failed_step.action}",
+            f"**Error:** {ar.error or 'Unknown error'}",
+            f"**Duration:** {ar.duration_ms:.0f}ms",
+        ]
+
+        if page_url:
+            parts.append(f"**Page URL:** {page_url}")
+
+        if ar.details:
+            parts.append(f"**Action details:** {json.dumps(ar.details)}")
+
+        if ar.screenshot_path:
+            parts.append(f"**Screenshot:** {ar.screenshot_path}")
+
+        parts.append(f"\n**Execution history:**")
+        parts.extend(preceding_steps)
+
+        parts.append(f"\n**App URL:** {self.config.app_url}")
+
+        return "\n".join(parts)
+
+    @staticmethod
+    def _fallback_report(flow_name: str, failed_step: StepResult) -> BugReport:
+        ar = failed_step.action_result
+        return BugReport(
+            summary=f"[{flow_name}] Step {failed_step.step_number} failed: {failed_step.action}",
+            description=f"Action '{failed_step.action}' failed with error: {ar.error}",
+            severity="major",
+            steps_to_reproduce=[f"Run flow '{flow_name}' and observe step {failed_step.step_number}"],
+            expected_result=f"Step {failed_step.step_number} ({failed_step.description}) should succeed",
+            actual_result=ar.error or "Step failed",
+            affected_area=flow_name,
+        )
+
+    @staticmethod
+    def _clean(content: str) -> str:
+        content = content.strip()
+        if content.startswith("```"):
+            first_newline = content.index("\n") if "\n" in content else 3
+            content = content[first_newline + 1:]
+        if content.endswith("```"):
+            content = content[:-3]
+        return content.strip()