qa360 2.1.7 → 2.2.0

package/dist/core/crawler/page-analyzer.js

@@ -1,709 +0,0 @@
-/**
- * QA360 Page Analyzer
- *
- * Analyzes web pages to discover elements, forms, and patterns
- */
-import { chromium } from '@playwright/test';
-import { generateSelectorFromElement } from './selector-generator.js';
-/**
- * Page Analyzer class
- */
-export class PageAnalyzer {
-    browser;
-    context;
-    page;
-    options;
-    constructor(options) {
-        this.options = {
-            timeout: 30000,
-            headless: true,
-            waitForNetworkIdle: true,
-            ...options,
-        };
-    }
-    /**
-     * Initialize browser
-     */
-    async initBrowser() {
-        this.browser = await chromium.launch({
-            headless: this.options.headless ?? true,
-            args: ['--no-sandbox', '--disable-dev-shm-usage'],
-        });
-        this.context = await this.browser.newContext({
-            viewport: { width: 1920, height: 1080 },
-            userAgent: 'QA360-Crawler/1.0',
-        });
-        this.page = await this.context.newPage();
-        // Set default timeout
-        this.page.setDefaultTimeout(this.options.timeout);
-    }
-    /**
-     * Perform authentication if configured
-     */
-    async performAuth() {
-        if (!this.options.auth)
-            return;
-        const auth = this.options.auth;
-        if (auth.type === 'basic') {
-            // Set basic auth headers
-            const credentials = Buffer.from(`${auth.username}:${auth.password}`).toString('base64');
-            await this.page.setExtraHTTPHeaders({
-                Authorization: `Basic ${credentials}`,
-            });
-        }
-        else if (auth.type === 'bearer' && auth.token) {
-            await this.page.setExtraHTTPHeaders({
-                Authorization: `Bearer ${auth.token}`,
-            });
-        }
-        else if (auth.type === 'cookie' && auth.cookies) {
-            await this.context.addCookies(auth.cookies.map(c => ({
-                name: c.name,
-                value: c.value,
-                domain: c.domain || new URL(this.options.baseUrl).hostname,
-                path: '/',
-            })));
-        }
-        else if (auth.type === 'form' && auth.loginUrl) {
-            // Perform form login
-            await this.page.goto(auth.loginUrl);
-            const usernameSelector = auth.usernameSelector || 'input[name="username"], input[name="email"], input[type="email"]';
-            const passwordSelector = auth.passwordSelector || 'input[name="password"], input[type="password"]';
-            const submitSelector = auth.submitSelector || 'button[type="submit"]';
-            await this.page.fill(usernameSelector, auth.username || '');
-            await this.page.fill(passwordSelector, auth.password || '');
-            await this.page.click(submitSelector);
-            // Wait for navigation
-            if (this.options.waitForNetworkIdle) {
-                await this.page.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => { });
-            }
-        }
-    }
-    /**
-     * Analyze a single page
-     */
-    async analyze(url, depth) {
-        if (!this.browser) {
-            await this.initBrowser();
-            await this.performAuth();
-        }
-        const startTime = Date.now();
-        try {
-            // Navigate to page
-            const response = await this.page.goto(url, {
-                waitUntil: this.options.waitForNetworkIdle ? 'networkidle' : 'domcontentloaded',
-                timeout: this.options.timeout,
-            });
-            const loadTime = Date.now() - startTime;
-            const status = response?.status() || 0;
-            // Get page info
-            const title = await this.page.title();
-            const path = url.replace(new URL(this.options.baseUrl).origin, '') || '/';
-            // Get meta description
-            const description = (await this.page
-                .locator('meta[name="description"]')
-                .getAttribute('content')
-                .catch(() => undefined)) || undefined;
-            // Discover all elements
-            const elements = await this.discoverElements();
-            // Analyze navigation
-            const navigation = await this.analyzeNavigation();
-            // Accessibility scan
-            const accessibility = await this.runAccessibilityScan();
-            // Screenshot if requested
-            let screenshot;
-            if (this.options.screenshots) {
-                const buffer = await this.page.screenshot({ fullPage: false });
-                screenshot = `data:image/png;base64,${buffer.toString('base64')}`;
-            }
-            // Detect page type
-            const { pageType, pageTypeConfidence } = this.detectPageType(elements, title, path);
-            return {
-                url,
-                path,
-                title,
-                depth,
-                status,
-                loadTime,
-                description,
-                elements,
-                navigation,
-                screenshot,
-                accessibility,
-                pageType,
-                pageTypeConfidence,
-            };
-        }
-        catch (error) {
-            // Return minimal info on error
-            return {
-                url,
-                path: url.replace(new URL(this.options.baseUrl).origin, ''),
-                title: 'Error',
-                depth,
-                status: 0,
-                loadTime: Date.now() - startTime,
-                elements: {
-                    buttons: [],
-                    links: [],
-                    forms: [],
-                    inputs: [],
-                    selects: [],
-                    checkboxes: [],
-                    radios: [],
-                },
-                navigation: {
-                    main: undefined,
-                    footer: undefined,
-                    breadcrumb: undefined,
-                },
-                pageType: 'other',
-                pageTypeConfidence: 0,
-            };
-        }
-    }
-    /**
-     * Discover all interactive elements on the page
-     */
-    async discoverElements() {
-        const buttons = await this.discoverButtons();
-        const links = await this.discoverLinks();
-        const forms = await this.discoverForms();
-        const inputs = await this.discoverInputs();
-        const selects = await this.discoverSelects();
-        const checkboxes = await this.discoverCheckboxes();
-        const radios = await this.discoverRadios();
-        return {
-            buttons,
-            links,
-            forms,
-            inputs,
-            selects,
-            checkboxes,
-            radios,
-        };
-    }
-    /**
-     * Discover buttons
-     */
-    async discoverButtons() {
-        const elements = await this.page.$$('button, input[type="button"], input[type="submit"], [role="button"], a[role="button"]');
-        const buttons = [];
-        for (const el of elements) {
-            try {
-                const info = await generateSelectorFromElement(el, this.page);
-                if (info.selector !== 'unknown') {
-                    buttons.push(info);
-                }
-            }
-            catch {
-                // Skip failed elements
-            }
-        }
-        return buttons;
-    }
-    /**
-     * Discover links
-     */
-    async discoverLinks() {
-        const elements = await this.page.$$('[href]');
-        const links = [];
-        const baseUrl = new URL(this.options.baseUrl);
-        for (const el of elements) {
-            try {
-                const info = await generateSelectorFromElement(el, this.page);
-                const href = await el.getAttribute('href');
-                if (!href || href.startsWith('#') || href.startsWith('javascript:')) {
-                    continue;
-                }
-                // Resolve relative URLs
-                let url;
-                try {
-                    url = new URL(href, baseUrl.origin).href;
-                }
-                catch {
-                    continue;
-                }
-                const internal = url.startsWith(baseUrl.origin);
-                links.push({
-                    ...info,
-                    url,
-                    internal,
-                    visited: false,
-                });
-            }
-            catch {
-                // Skip failed elements
-            }
-        }
-        return links;
-    }
-    /**
-     * Discover forms
-     */
-    async discoverForms() {
-        const forms = await this.page.$$('[role="form"], form, .form, [data-form]');
-        const discovered = [];
-        for (const formEl of forms) {
-            try {
-                const selector = await generateSelectorFromElement(formEl, this.page).then(i => i.selector);
-                // Get form fields
-                const fields = await this.discoverFormFields(formEl);
-                if (fields.length === 0)
-                    continue;
-                // Get submit button
-                let submitButton;
-                const submitBtn = await formEl.$('button[type="submit"], input[type="submit"], [type="submit"]');
-                if (submitBtn) {
-                    submitButton = await generateSelectorFromElement(submitBtn, this.page);
-                }
-                // Detect form purpose
-                const purpose = this.detectFormPurpose(fields, selector);
-                const confidence = this.calculateFormPurposeConfidence(fields, purpose);
-                discovered.push({
-                    selector,
-                    purpose,
-                    fields,
-                    submitButton,
-                    confidence,
-                });
-            }
-            catch {
-                // Skip failed forms
-            }
-        }
-        return discovered;
-    }
-    /**
-     * Discover fields within a form
-     */
-    async discoverFormFields(formEl) {
-        const fields = [];
-        // Get all input, select, textarea elements
-        const inputs = await formEl.$$('input:not([type="hidden"]):not([type="submit"]):not([type="button"]), select, textarea');
-        for (const input of inputs) {
-            try {
-                const info = await generateSelectorFromElement(input, this.page);
-                // Get field-specific attributes
-                const inputType = await input.getAttribute('type') || 'text';
-                const name = await input.getAttribute('name');
-                const required = (await input.getAttribute('required')) !== null;
-                const placeholder = await input.getAttribute('placeholder');
-                // Get validation attributes
-                const validation = {};
-                const min = await input.getAttribute('min');
-                const max = await input.getAttribute('max');
-                const pattern = await input.getAttribute('pattern');
-                const minLength = await input.getAttribute('minlength');
-                const maxLength = await input.getAttribute('maxlength');
-                if (min)
-                    validation.min = min;
-                if (max)
-                    validation.max = max;
-                if (pattern)
-                    validation.pattern = pattern;
-                if (minLength)
-                    validation.minLength = parseInt(minLength, 10);
-                if (maxLength)
-                    validation.maxLength = parseInt(maxLength, 10);
-                // Get options for selects
-                let options;
-                if (inputType === 'select-one' || inputType === 'select-multiple') {
-                    options = await input.$$eval('option', (opts) => opts
-                        .map(o => o.value || o.text)
-                        .filter(Boolean));
-                }
-                fields.push({
-                    ...info,
-                    inputType,
-                    name: name || undefined,
-                    required,
-                    placeholder: placeholder || undefined,
-                    options,
-                    validation: Object.keys(validation).length > 0 ? validation : undefined,
-                });
-            }
-            catch {
-                // Skip failed fields
-            }
-        }
-        return fields;
-    }
-    /**
-     * Detect form purpose based on fields
-     */
-    detectFormPurpose(fields, selector) {
-        const fieldNames = fields.map(f => f.name?.toLowerCase() || '');
-        const selectorLower = selector.toLowerCase();
-        // Check for login form
-        if ((fieldNames.includes('password') || fieldNames.includes('pass')) &&
-            (fieldNames.includes('email') ||
-                fieldNames.includes('username') ||
-                fieldNames.includes('user'))) {
-            return 'login';
-        }
-        // Check for signup form
-        if ((fieldNames.includes('password') || fieldNames.includes('confirm')) &&
-            (fieldNames.includes('email') || fieldNames.includes('username'))) {
-            return 'signup';
-        }
-        // Check for search form
-        if (selectorLower.includes('search') || fieldNames.includes('q') || fieldNames.includes('search')) {
-            return 'search';
-        }
-        // Check for checkout form
-        if (fieldNames.includes('card') ||
-            fieldNames.includes('cvv') ||
-            fieldNames.includes('expiry')) {
-            return 'checkout';
-        }
-        // Check for contact form
-        if (fieldNames.includes('message') || fieldNames.includes('subject')) {
-            return 'contact';
-        }
-        return 'other';
-    }
-    /**
-     * Calculate confidence for form purpose detection
-     */
-    calculateFormPurposeConfidence(fields, purpose) {
-        // High confidence for login/signup with password field
-        if (purpose === 'login' || purpose === 'signup') {
-            if (fields.some(f => f.inputType === 'password')) {
-                return 0.9;
-            }
-        }
-        // Medium confidence for forms with clear naming
-        if (purpose !== 'other') {
-            return 0.7;
-        }
-        return 0.5;
-    }
-    /**
-     * Discover standalone input fields (not in forms)
-     */
-    async discoverInputs() {
-        const inputs = await this.page.$$('input:not([type="hidden"]):not([type="submit"]):not(form input), textarea:not(form textarea)');
-        const fields = [];
-        for (const input of inputs) {
-            try {
-                const info = await generateSelectorFromElement(input, this.page);
-                const inputType = await input.getAttribute('type') || 'text';
-                const name = await input.getAttribute('name');
-                const required = (await input.getAttribute('required')) !== null;
-                const placeholder = await input.getAttribute('placeholder');
-                fields.push({
-                    ...info,
-                    inputType,
-                    name: name || undefined,
-                    required,
-                    placeholder: placeholder || undefined,
-                });
-            }
-            catch {
-                // Skip failed
-            }
-        }
-        return fields;
-    }
-    /**
-     * Discover select elements
-     */
-    async discoverSelects() {
-        const selects = await this.page.$$('select:not(form select)');
-        const fields = [];
-        for (const select of selects) {
-            try {
-                const info = await generateSelectorFromElement(select, this.page);
-                const name = await select.getAttribute('name');
-                const required = (await select.getAttribute('required')) !== null;
-                // Get options
-                const options = await select.$$eval('option', opts => opts.map(o => o.value).filter(Boolean));
-                fields.push({
-                    ...info,
-                    inputType: 'select-one',
-                    name: name || undefined,
-                    required,
-                    options: options.length > 0 ? options : undefined,
-                });
-            }
-            catch {
-                // Skip failed
-            }
-        }
-        return fields;
-    }
-    /**
-     * Discover checkboxes
-     */
-    async discoverCheckboxes() {
-        const checkboxSelector = "input[type=\"checkbox\"]";
-        const checkboxes = await this.page.$$(checkboxSelector);
-        const elements = [];
-        for (const el of checkboxes) {
-            try {
-                const info = await generateSelectorFromElement(el, this.page);
-                if (info.selector !== 'unknown') {
-                    elements.push(info);
-                }
-            }
-            catch {
-                // Skip failed
-            }
-        }
-        return elements;
-    }
-    /**
-     * Discover radio buttons
-     */
-    async discoverRadios() {
-        const radioSelector = "input[type=\"radio\"]";
-        const radios = await this.page.$$(radioSelector);
-        const elements = [];
-        for (const el of radios) {
-            try {
-                const info = await generateSelectorFromElement(el, this.page);
-                if (info.selector !== 'unknown') {
-                    elements.push(info);
-                }
-            }
-            catch {
-                // Skip failed
-            }
-        }
-        return elements;
-    }
-    /**
-     * Analyze page navigation structure
-     */
-    async analyzeNavigation() {
-        const navigation = {};
-        // Main navigation (nav, menu, navbar)
-        const mainNav = await this.page.$('nav, [role="navigation"], .nav, .navigation, .navbar, .menu');
-        if (mainNav) {
-            const navSelector = await generateSelectorFromElement(mainNav, this.page).then(i => i.selector);
-            const links = await mainNav.$$('[href]');
-            const items = [];
-            for (const link of links) {
-                try {
-                    const info = await generateSelectorFromElement(link, this.page);
-                    const href = await link.getAttribute('href');
-                    if (href) {
-                        const url = new URL(href, new URL(this.options.baseUrl).origin).href;
-                        items.push({
-                            ...info,
-                            url,
-                            internal: url.startsWith(new URL(this.options.baseUrl).origin),
-                            visited: false,
-                        });
-                    }
-                }
-                catch {
-                    // Skip
-                }
-            }
-            navigation.main = { selector: navSelector, items };
-        }
-        // Breadcrumb
-        const breadcrumb = await this.page.$('[aria-label="breadcrumb"], .breadcrumb, .breadcrumbs, ol.breadcrumb');
-        if (breadcrumb) {
-            const crumbSelector = await generateSelectorFromElement(breadcrumb, this.page).then(i => i.selector);
-            const items = await breadcrumb.$$('[href], span, li');
-            const breadcrumbs = [];
-            for (const item of items) {
-                try {
-                    const text = await item.textContent();
-                    const href = await item.getAttribute('href');
-                    if (text && text.trim()) {
-                        breadcrumbs.push({
-                            text: text.trim(),
-                            url: href || '',
-                        });
-                    }
-                }
-                catch {
-                    // Skip
-                }
-            }
-            if (breadcrumbs.length > 0) {
-                navigation.breadcrumb = { selector: crumbSelector, items: breadcrumbs };
-            }
-        }
-        // Pagination
-        const pagination = await this.page.$('[aria-label="pagination"], .pagination, [role="navigation"] nav');
-        if (pagination) {
-            const pageSelector = await generateSelectorFromElement(pagination, this.page).then(i => i.selector);
-            const nextLink = await pagination.$('a[rel="next"], .next, [aria-label="next"]');
-            const prevLink = await pagination.$('a[rel="prev"], .prev, [aria-label="previous"]');
-            const paginationInfo = {
-                selector: pageSelector,
-            };
-            if (nextLink) {
-                const info = await generateSelectorFromElement(nextLink, this.page);
-                const href = await nextLink.getAttribute('href');
-                if (href) {
-                    paginationInfo.nextPage = {
-                        ...info,
-                        url: new URL(href, new URL(this.options.baseUrl).origin).href,
-                        internal: true,
-                        visited: false,
-                    };
-                }
-            }
-            if (prevLink) {
-                const info = await generateSelectorFromElement(prevLink, this.page);
-                const href = await prevLink.getAttribute('href');
-                if (href) {
-                    paginationInfo.prevPage = {
-                        ...info,
-                        url: new URL(href, new URL(this.options.baseUrl).origin).href,
-                        internal: true,
-                        visited: false,
-                    };
-                }
-            }
-            navigation.pagination = paginationInfo;
-        }
-        // Footer
-        const footer = await this.page.$('footer, [role="contentinfo"], .footer');
-        if (footer) {
-            const footerSelector = await generateSelectorFromElement(footer, this.page).then(i => i.selector);
-            const links = await footer.$$('[href]');
-            const items = [];
-            for (const link of links) {
-                try {
-                    const info = await generateSelectorFromElement(link, this.page);
-                    const href = await link.getAttribute('href');
-                    if (href) {
-                        const url = new URL(href, new URL(this.options.baseUrl).origin).href;
-                        items.push({
-                            ...info,
-                            url,
-                            internal: url.startsWith(new URL(this.options.baseUrl).origin),
-                            visited: false,
-                        });
-                    }
-                }
-                catch {
-                    // Skip
-                }
-            }
-            navigation.footer = { selector: footerSelector, items };
-        }
-        return navigation;
-    }
-    /**
-     * Run accessibility scan using axe-core
-     */
-    async runAccessibilityScan() {
-        try {
-            // Inject axe-core
-            await this.page.addScriptTag({
-                url: 'https://unpkg.com/axe-core@4.8.2/axe.min.js',
-            });
-            // Run axe
-            const results = await this.page.evaluate(() => {
-                return new Promise((resolve) => {
-                    // @ts-ignore
-                    if (typeof axe !== 'undefined') {
-                        // @ts-ignore
-                        axe.run((err, results) => {
-                            if (err) {
-                                resolve({ violations: [], passes: [], incomplete: [] });
-                            }
-                            else {
-                                resolve(results);
-                            }
-                        });
-                    }
-                    else {
-                        resolve({ violations: [], passes: [], incomplete: [] });
-                    }
-                });
-            });
-            const axeResults = results;
-            // Process violations
-            const violations = (axeResults.violations || []).map((v) => ({
-                id: v.id,
-                impact: v.impact || 'moderate',
-                description: v.description || v.help || 'Accessibility issue',
-                nodes: v.nodes?.length || 0,
-                selectors: (v.nodes || []).slice(0, 5).map((n) => n.target?.[0] || '').filter(Boolean),
-            }));
-            // Calculate score
-            const criticalCount = violations.filter((v) => v.impact === 'critical').length;
-            const seriousCount = violations.filter((v) => v.impact === 'serious').length;
-            const moderateCount = violations.filter((v) => v.impact === 'moderate').length;
-            const minorCount = violations.filter((v) => v.impact === 'minor').length;
-            const score = Math.max(0, 100 - (criticalCount * 25 + seriousCount * 10 + moderateCount * 5 + minorCount * 1));
-            return {
-                score: Math.round(score),
-                violations,
-                passes: axeResults.passes?.length || 0,
-                incomplete: axeResults.incomplete?.length || 0,
-            };
-        }
-        catch {
-            return undefined;
-        }
-    }
-    /**
-     * Detect page type
-     */
-    detectPageType(elements, title, path) {
-        const pathLower = path.toLowerCase();
-        const titleLower = title.toLowerCase();
-        // Homepage
-        if (path === '/' || path === '' || pathLower === '/home' || pathLower === '/index') {
-            return { pageType: 'homepage', pageTypeConfidence: 0.95 };
-        }
-        // Login page
-        if (pathLower.includes('/login') ||
-            pathLower.includes('/signin') ||
-            pathLower.includes('/auth') ||
-            titleLower.includes('login') ||
-            titleLower.includes('sign in')) {
-            return { pageType: 'login', pageTypeConfidence: 0.9 };
-        }
-        // Signup page
-        if (pathLower.includes('/signup') ||
-            pathLower.includes('/register') ||
-            pathLower.includes('/join') ||
-            titleLower.includes('sign up') ||
-            titleLower.includes('register')) {
-            return { pageType: 'signup', pageTypeConfidence: 0.9 };
-        }
-        // Dashboard
-        if (pathLower.includes('/dashboard') ||
-            pathLower.includes('/my-account') ||
-            pathLower.includes('/profile') ||
-            titleLower.includes('dashboard')) {
-            return { pageType: 'dashboard', pageTypeConfidence: 0.85 };
-        }
-        // Listing page (multiple items, pagination)
-        if (elements.links.length > 20 || elements.forms.some(f => f.purpose === 'filter')) {
-            return { pageType: 'listing', pageTypeConfidence: 0.7 };
-        }
-        // Form page
-        if (elements.forms.length > 0) {
-            return { pageType: 'form', pageTypeConfidence: 0.75 };
-        }
-        return { pageType: 'other', pageTypeConfidence: 0.5 };
-    }
-    /**
-     * Clean up resources
-     */
-    async cleanup() {
-        try {
-            if (this.page)
-                await this.page.close();
-            if (this.context)
-                await this.context.close();
-            if (this.browser)
-                await this.browser.close();
-        }
-        catch {
-            // Ignore cleanup errors
-        }
-    }
-}