qa360 2.0.11 → 2.0.13

package/dist/core/crawler/page-analyzer.js

@@ -0,0 +1,709 @@
+/**
+ * QA360 Page Analyzer
+ *
+ * Analyzes web pages to discover elements, forms, and patterns
+ */
+import { chromium } from '@playwright/test';
+import { generateSelectorFromElement } from './selector-generator.js';
+/**
+ * Page Analyzer class
+ */
+export class PageAnalyzer {
+    browser;
+    context;
+    page;
+    options;
+    constructor(options) {
+        this.options = {
+            timeout: 30000,
+            headless: true,
+            waitForNetworkIdle: true,
+            ...options,
+        };
+    }
+    /**
+     * Initialize browser
+     */
+    async initBrowser() {
+        this.browser = await chromium.launch({
+            headless: this.options.headless ?? true,
+            args: ['--no-sandbox', '--disable-dev-shm-usage'],
+        });
+        this.context = await this.browser.newContext({
+            viewport: { width: 1920, height: 1080 },
+            userAgent: 'QA360-Crawler/1.0',
+        });
+        this.page = await this.context.newPage();
+        // Set default timeout
+        this.page.setDefaultTimeout(this.options.timeout);
+    }
+    /**
+     * Perform authentication if configured
+     */
+    async performAuth() {
+        if (!this.options.auth)
+            return;
+        const auth = this.options.auth;
+        if (auth.type === 'basic') {
+            // Set basic auth headers
+            const credentials = Buffer.from(`${auth.username}:${auth.password}`).toString('base64');
+            await this.page.setExtraHTTPHeaders({
+                Authorization: `Basic ${credentials}`,
+            });
+        }
+        else if (auth.type === 'bearer' && auth.token) {
+            await this.page.setExtraHTTPHeaders({
+                Authorization: `Bearer ${auth.token}`,
+            });
+        }
+        else if (auth.type === 'cookie' && auth.cookies) {
+            await this.context.addCookies(auth.cookies.map(c => ({
+                name: c.name,
+                value: c.value,
+                domain: c.domain || new URL(this.options.baseUrl).hostname,
+                path: '/',
+            })));
+        }
+        else if (auth.type === 'form' && auth.loginUrl) {
+            // Perform form login
+            await this.page.goto(auth.loginUrl);
+            const usernameSelector = auth.usernameSelector || 'input[name="username"], input[name="email"], input[type="email"]';
+            const passwordSelector = auth.passwordSelector || 'input[name="password"], input[type="password"]';
+            const submitSelector = auth.submitSelector || 'button[type="submit"]';
+            await this.page.fill(usernameSelector, auth.username || '');
+            await this.page.fill(passwordSelector, auth.password || '');
+            await this.page.click(submitSelector);
+            // Wait for navigation
+            if (this.options.waitForNetworkIdle) {
+                await this.page.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => { });
+            }
+        }
+    }
+    /**
+     * Analyze a single page
+     */
+    async analyze(url, depth) {
+        if (!this.browser) {
+            await this.initBrowser();
+            await this.performAuth();
+        }
+        const startTime = Date.now();
+        try {
+            // Navigate to page
+            const response = await this.page.goto(url, {
+                waitUntil: this.options.waitForNetworkIdle ? 'networkidle' : 'domcontentloaded',
+                timeout: this.options.timeout,
+            });
+            const loadTime = Date.now() - startTime;
+            const status = response?.status() || 0;
+            // Get page info
+            const title = await this.page.title();
+            const path = url.replace(new URL(this.options.baseUrl).origin, '') || '/';
+            // Get meta description
+            const description = (await this.page
+                .locator('meta[name="description"]')
+                .getAttribute('content')
+                .catch(() => undefined)) || undefined;
+            // Discover all elements
+            const elements = await this.discoverElements();
+            // Analyze navigation
+            const navigation = await this.analyzeNavigation();
+            // Accessibility scan
+            const accessibility = await this.runAccessibilityScan();
+            // Screenshot if requested
+            let screenshot;
+            if (this.options.screenshots) {
+                const buffer = await this.page.screenshot({ fullPage: false });
+                screenshot = `data:image/png;base64,${buffer.toString('base64')}`;
+            }
+            // Detect page type
+            const { pageType, pageTypeConfidence } = this.detectPageType(elements, title, path);
+            return {
+                url,
+                path,
+                title,
+                depth,
+                status,
+                loadTime,
+                description,
+                elements,
+                navigation,
+                screenshot,
+                accessibility,
+                pageType,
+                pageTypeConfidence,
+            };
+        }
+        catch (error) {
+            // Return minimal info on error
+            return {
+                url,
+                path: url.replace(new URL(this.options.baseUrl).origin, ''),
+                title: 'Error',
+                depth,
+                status: 0,
+                loadTime: Date.now() - startTime,
+                elements: {
+                    buttons: [],
+                    links: [],
+                    forms: [],
+                    inputs: [],
+                    selects: [],
+                    checkboxes: [],
+                    radios: [],
+                },
+                navigation: {
+                    main: undefined,
+                    footer: undefined,
+                    breadcrumb: undefined,
+                },
+                pageType: 'other',
+                pageTypeConfidence: 0,
+            };
+        }
+    }
+    /**
+     * Discover all interactive elements on the page
+     */
+    async discoverElements() {
+        const buttons = await this.discoverButtons();
+        const links = await this.discoverLinks();
+        const forms = await this.discoverForms();
+        const inputs = await this.discoverInputs();
+        const selects = await this.discoverSelects();
+        const checkboxes = await this.discoverCheckboxes();
+        const radios = await this.discoverRadios();
+        return {
+            buttons,
+            links,
+            forms,
+            inputs,
+            selects,
+            checkboxes,
+            radios,
+        };
+    }
+    /**
+     * Discover buttons
+     */
+    async discoverButtons() {
+        const elements = await this.page.$$('button, input[type="button"], input[type="submit"], [role="button"], a[role="button"]');
+        const buttons = [];
+        for (const el of elements) {
+            try {
+                const info = await generateSelectorFromElement(el, this.page);
+                if (info.selector !== 'unknown') {
+                    buttons.push(info);
+                }
+            }
+            catch {
+                // Skip failed elements
+            }
+        }
+        return buttons;
+    }
+    /**
+     * Discover links
+     */
+    async discoverLinks() {
+        const elements = await this.page.$$('[href]');
+        const links = [];
+        const baseUrl = new URL(this.options.baseUrl);
+        for (const el of elements) {
+            try {
+                const info = await generateSelectorFromElement(el, this.page);
+                const href = await el.getAttribute('href');
+                if (!href || href.startsWith('#') || href.startsWith('javascript:')) {
+                    continue;
+                }
+                // Resolve relative URLs
+                let url;
+                try {
+                    url = new URL(href, baseUrl.origin).href;
+                }
+                catch {
+                    continue;
+                }
+                const internal = url.startsWith(baseUrl.origin);
+                links.push({
+                    ...info,
+                    url,
+                    internal,
+                    visited: false,
+                });
+            }
+            catch {
+                // Skip failed elements
+            }
+        }
+        return links;
+    }
+    /**
+     * Discover forms
+     */
+    async discoverForms() {
+        const forms = await this.page.$$('[role="form"], form, .form, [data-form]');
+        const discovered = [];
+        for (const formEl of forms) {
+            try {
+                const selector = await generateSelectorFromElement(formEl, this.page).then(i => i.selector);
+                // Get form fields
+                const fields = await this.discoverFormFields(formEl);
+                if (fields.length === 0)
+                    continue;
+                // Get submit button
+                let submitButton;
+                const submitBtn = await formEl.$('button[type="submit"], input[type="submit"], [type="submit"]');
+                if (submitBtn) {
+                    submitButton = await generateSelectorFromElement(submitBtn, this.page);
+                }
+                // Detect form purpose
+                const purpose = this.detectFormPurpose(fields, selector);
+                const confidence = this.calculateFormPurposeConfidence(fields, purpose);
+                discovered.push({
+                    selector,
+                    purpose,
+                    fields,
+                    submitButton,
+                    confidence,
+                });
+            }
+            catch {
+                // Skip failed forms
+            }
+        }
+        return discovered;
+    }
+    /**
+     * Discover fields within a form
+     */
+    async discoverFormFields(formEl) {
+        const fields = [];
+        // Get all input, select, textarea elements
+        const inputs = await formEl.$$('input:not([type="hidden"]):not([type="submit"]):not([type="button"]), select, textarea');
+        for (const input of inputs) {
+            try {
+                const info = await generateSelectorFromElement(input, this.page);
+                // Get field-specific attributes
+                const inputType = await input.getAttribute('type') || 'text';
+                const name = await input.getAttribute('name');
+                const required = (await input.getAttribute('required')) !== null;
+                const placeholder = await input.getAttribute('placeholder');
+                // Get validation attributes
+                const validation = {};
+                const min = await input.getAttribute('min');
+                const max = await input.getAttribute('max');
+                const pattern = await input.getAttribute('pattern');
+                const minLength = await input.getAttribute('minlength');
+                const maxLength = await input.getAttribute('maxlength');
+                if (min)
+                    validation.min = min;
+                if (max)
+                    validation.max = max;
+                if (pattern)
+                    validation.pattern = pattern;
+                if (minLength)
+                    validation.minLength = parseInt(minLength, 10);
+                if (maxLength)
+                    validation.maxLength = parseInt(maxLength, 10);
+                // Get options for selects
+                let options;
+                if (inputType === 'select-one' || inputType === 'select-multiple') {
+                    options = await input.$$eval('option', (opts) => opts
+                        .map(o => o.value || o.text)
+                        .filter(Boolean));
+                }
+                fields.push({
+                    ...info,
+                    inputType,
+                    name: name || undefined,
+                    required,
+                    placeholder: placeholder || undefined,
+                    options,
+                    validation: Object.keys(validation).length > 0 ? validation : undefined,
+                });
+            }
+            catch {
+                // Skip failed fields
+            }
+        }
+        return fields;
+    }
+    /**
+     * Detect form purpose based on fields
+     */
+    detectFormPurpose(fields, selector) {
+        const fieldNames = fields.map(f => f.name?.toLowerCase() || '');
+        const selectorLower = selector.toLowerCase();
+        // Check for login form
+        if ((fieldNames.includes('password') || fieldNames.includes('pass')) &&
+            (fieldNames.includes('email') ||
+                fieldNames.includes('username') ||
+                fieldNames.includes('user'))) {
+            return 'login';
+        }
+        // Check for signup form
+        if ((fieldNames.includes('password') || fieldNames.includes('confirm')) &&
+            (fieldNames.includes('email') || fieldNames.includes('username'))) {
+            return 'signup';
+        }
+        // Check for search form
+        if (selectorLower.includes('search') || fieldNames.includes('q') || fieldNames.includes('search')) {
+            return 'search';
+        }
+        // Check for checkout form
+        if (fieldNames.includes('card') ||
+            fieldNames.includes('cvv') ||
+            fieldNames.includes('expiry')) {
+            return 'checkout';
+        }
+        // Check for contact form
+        if (fieldNames.includes('message') || fieldNames.includes('subject')) {
+            return 'contact';
+        }
+        return 'other';
+    }
+    /**
+     * Calculate confidence for form purpose detection
+     */
+    calculateFormPurposeConfidence(fields, purpose) {
+        // High confidence for login/signup with password field
+        if (purpose === 'login' || purpose === 'signup') {
+            if (fields.some(f => f.inputType === 'password')) {
+                return 0.9;
+            }
+        }
+        // Medium confidence for forms with clear naming
+        if (purpose !== 'other') {
+            return 0.7;
+        }
+        return 0.5;
+    }
+    /**
+     * Discover standalone input fields (not in forms)
+     */
+    async discoverInputs() {
+        const inputs = await this.page.$$('input:not([type="hidden"]):not([type="submit"]):not(form input), textarea:not(form textarea)');
+        const fields = [];
+        for (const input of inputs) {
+            try {
+                const info = await generateSelectorFromElement(input, this.page);
+                const inputType = await input.getAttribute('type') || 'text';
+                const name = await input.getAttribute('name');
+                const required = (await input.getAttribute('required')) !== null;
+                const placeholder = await input.getAttribute('placeholder');
+                fields.push({
+                    ...info,
+                    inputType,
+                    name: name || undefined,
+                    required,
+                    placeholder: placeholder || undefined,
+                });
+            }
+            catch {
+                // Skip failed
+            }
+        }
+        return fields;
+    }
+    /**
+     * Discover select elements
+     */
+    async discoverSelects() {
+        const selects = await this.page.$$('select:not(form select)');
+        const fields = [];
+        for (const select of selects) {
+            try {
+                const info = await generateSelectorFromElement(select, this.page);
+                const name = await select.getAttribute('name');
+                const required = (await select.getAttribute('required')) !== null;
+                // Get options
+                const options = await select.$$eval('option', opts => opts.map(o => o.value).filter(Boolean));
+                fields.push({
+                    ...info,
+                    inputType: 'select-one',
+                    name: name || undefined,
+                    required,
+                    options: options.length > 0 ? options : undefined,
+                });
+            }
+            catch {
+                // Skip failed
+            }
+        }
+        return fields;
+    }
+    /**
+     * Discover checkboxes
+     */
+    async discoverCheckboxes() {
+        const checkboxSelector = "input[type=\"checkbox\"]";
+        const checkboxes = await this.page.$$(checkboxSelector);
+        const elements = [];
+        for (const el of checkboxes) {
+            try {
+                const info = await generateSelectorFromElement(el, this.page);
+                if (info.selector !== 'unknown') {
+                    elements.push(info);
+                }
+            }
+            catch {
+                // Skip failed
+            }
+        }
+        return elements;
+    }
+    /**
+     * Discover radio buttons
+     */
+    async discoverRadios() {
+        const radioSelector = "input[type=\"radio\"]";
+        const radios = await this.page.$$(radioSelector);
+        const elements = [];
+        for (const el of radios) {
+            try {
+                const info = await generateSelectorFromElement(el, this.page);
+                if (info.selector !== 'unknown') {
+                    elements.push(info);
+                }
+            }
+            catch {
+                // Skip failed
+            }
+        }
+        return elements;
+    }
+    /**
+     * Analyze page navigation structure
+     */
+    async analyzeNavigation() {
+        const navigation = {};
+        // Main navigation (nav, menu, navbar)
+        const mainNav = await this.page.$('nav, [role="navigation"], .nav, .navigation, .navbar, .menu');
+        if (mainNav) {
+            const navSelector = await generateSelectorFromElement(mainNav, this.page).then(i => i.selector);
+            const links = await mainNav.$$('[href]');
+            const items = [];
+            for (const link of links) {
+                try {
+                    const info = await generateSelectorFromElement(link, this.page);
+                    const href = await link.getAttribute('href');
+                    if (href) {
+                        const url = new URL(href, new URL(this.options.baseUrl).origin).href;
+                        items.push({
+                            ...info,
+                            url,
+                            internal: url.startsWith(new URL(this.options.baseUrl).origin),
+                            visited: false,
+                        });
+                    }
+                }
+                catch {
+                    // Skip
+                }
+            }
+            navigation.main = { selector: navSelector, items };
+        }
+        // Breadcrumb
+        const breadcrumb = await this.page.$('[aria-label="breadcrumb"], .breadcrumb, .breadcrumbs, ol.breadcrumb');
+        if (breadcrumb) {
+            const crumbSelector = await generateSelectorFromElement(breadcrumb, this.page).then(i => i.selector);
+            const items = await breadcrumb.$$('[href], span, li');
+            const breadcrumbs = [];
+            for (const item of items) {
+                try {
+                    const text = await item.textContent();
+                    const href = await item.getAttribute('href');
+                    if (text && text.trim()) {
+                        breadcrumbs.push({
+                            text: text.trim(),
+                            url: href || '',
+                        });
+                    }
+                }
+                catch {
+                    // Skip
+                }
+            }
+            if (breadcrumbs.length > 0) {
+                navigation.breadcrumb = { selector: crumbSelector, items: breadcrumbs };
+            }
+        }
+        // Pagination
+        const pagination = await this.page.$('[aria-label="pagination"], .pagination, [role="navigation"] nav');
+        if (pagination) {
+            const pageSelector = await generateSelectorFromElement(pagination, this.page).then(i => i.selector);
+            const nextLink = await pagination.$('a[rel="next"], .next, [aria-label="next"]');
+            const prevLink = await pagination.$('a[rel="prev"], .prev, [aria-label="previous"]');
+            const paginationInfo = {
+                selector: pageSelector,
+            };
+            if (nextLink) {
+                const info = await generateSelectorFromElement(nextLink, this.page);
+                const href = await nextLink.getAttribute('href');
+                if (href) {
+                    paginationInfo.nextPage = {
+                        ...info,
+                        url: new URL(href, new URL(this.options.baseUrl).origin).href,
+                        internal: true,
+                        visited: false,
+                    };
+                }
+            }
+            if (prevLink) {
+                const info = await generateSelectorFromElement(prevLink, this.page);
+                const href = await prevLink.getAttribute('href');
+                if (href) {
+                    paginationInfo.prevPage = {
+                        ...info,
+                        url: new URL(href, new URL(this.options.baseUrl).origin).href,
+                        internal: true,
+                        visited: false,
+                    };
+                }
+            }
+            navigation.pagination = paginationInfo;
+        }
+        // Footer
+        const footer = await this.page.$('footer, [role="contentinfo"], .footer');
+        if (footer) {
+            const footerSelector = await generateSelectorFromElement(footer, this.page).then(i => i.selector);
+            const links = await footer.$$('[href]');
+            const items = [];
+            for (const link of links) {
+                try {
+                    const info = await generateSelectorFromElement(link, this.page);
+                    const href = await link.getAttribute('href');
+                    if (href) {
+                        const url = new URL(href, new URL(this.options.baseUrl).origin).href;
+                        items.push({
+                            ...info,
+                            url,
+                            internal: url.startsWith(new URL(this.options.baseUrl).origin),
+                            visited: false,
+                        });
+                    }
+                }
+                catch {
+                    // Skip
+                }
+            }
+            navigation.footer = { selector: footerSelector, items };
+        }
+        return navigation;
+    }
+    /**
+     * Run accessibility scan using axe-core
+     */
+    async runAccessibilityScan() {
+        try {
+            // Inject axe-core
+            await this.page.addScriptTag({
+                url: 'https://unpkg.com/axe-core@4.8.2/axe.min.js',
+            });
+            // Run axe
+            const results = await this.page.evaluate(() => {
+                return new Promise((resolve) => {
+                    // @ts-ignore
+                    if (typeof axe !== 'undefined') {
+                        // @ts-ignore
+                        axe.run((err, results) => {
+                            if (err) {
+                                resolve({ violations: [], passes: [], incomplete: [] });
+                            }
+                            else {
+                                resolve(results);
+                            }
+                        });
+                    }
+                    else {
+                        resolve({ violations: [], passes: [], incomplete: [] });
+                    }
+                });
+            });
+            const axeResults = results;
+            // Process violations
+            const violations = (axeResults.violations || []).map((v) => ({
+                id: v.id,
+                impact: v.impact || 'moderate',
+                description: v.description || v.help || 'Accessibility issue',
+                nodes: v.nodes?.length || 0,
+                selectors: (v.nodes || []).slice(0, 5).map((n) => n.target?.[0] || '').filter(Boolean),
+            }));
+            // Calculate score
+            const criticalCount = violations.filter((v) => v.impact === 'critical').length;
+            const seriousCount = violations.filter((v) => v.impact === 'serious').length;
+            const moderateCount = violations.filter((v) => v.impact === 'moderate').length;
+            const minorCount = violations.filter((v) => v.impact === 'minor').length;
+            const score = Math.max(0, 100 - (criticalCount * 25 + seriousCount * 10 + moderateCount * 5 + minorCount * 1));
+            return {
+                score: Math.round(score),
+                violations,
+                passes: axeResults.passes?.length || 0,
+                incomplete: axeResults.incomplete?.length || 0,
+            };
+        }
+        catch {
+            return undefined;
+        }
+    }
+    /**
+     * Detect page type
+     */
+    detectPageType(elements, title, path) {
+        const pathLower = path.toLowerCase();
+        const titleLower = title.toLowerCase();
+        // Homepage
+        if (path === '/' || path === '' || pathLower === '/home' || pathLower === '/index') {
+            return { pageType: 'homepage', pageTypeConfidence: 0.95 };
+        }
+        // Login page
+        if (pathLower.includes('/login') ||
+            pathLower.includes('/signin') ||
+            pathLower.includes('/auth') ||
+            titleLower.includes('login') ||
+            titleLower.includes('sign in')) {
+            return { pageType: 'login', pageTypeConfidence: 0.9 };
+        }
+        // Signup page
+        if (pathLower.includes('/signup') ||
+            pathLower.includes('/register') ||
+            pathLower.includes('/join') ||
+            titleLower.includes('sign up') ||
+            titleLower.includes('register')) {
+            return { pageType: 'signup', pageTypeConfidence: 0.9 };
+        }
+        // Dashboard
+        if (pathLower.includes('/dashboard') ||
+            pathLower.includes('/my-account') ||
+            pathLower.includes('/profile') ||
+            titleLower.includes('dashboard')) {
+            return { pageType: 'dashboard', pageTypeConfidence: 0.85 };
+        }
+        // Listing page (multiple items, pagination)
+        if (elements.links.length > 20 || elements.forms.some(f => f.purpose === 'filter')) {
+            return { pageType: 'listing', pageTypeConfidence: 0.7 };
+        }
+        // Form page
+        if (elements.forms.length > 0) {
+            return { pageType: 'form', pageTypeConfidence: 0.75 };
+        }
+        return { pageType: 'other', pageTypeConfidence: 0.5 };
+    }
+    /**
+     * Clean up resources
+     */
+    async cleanup() {
+        try {
+            if (this.page)
+                await this.page.close();
+            if (this.context)
+                await this.context.close();
+            if (this.browser)
+                await this.browser.close();
+        }
+        catch {
+            // Ignore cleanup errors
+        }
+    }
+}