qa360 2.2.20 → 2.3.1

package/{cli/dist → dist}/core/crawler/page-analyzer.js

@@ -2,9 +2,15 @@
  * QA360 Page Analyzer
  *
  * Analyzes web pages to discover elements, forms, and patterns
+ *
+ * NOW WITH INTELLIGENT SITE PROFILING!
+ * - Analyzes site characteristics on first page visit
+ * - Adapts selector strategy based on detected profile
+ * - Filters utility classes (Tailwind, Bootstrap, MUI)
  */
 import { chromium } from '@playwright/test';
-import { generateSelectorFromElement } from './selector-generator.js';
+import { generateSelectorFromElement, SiteProfiler, initializeSelectorGenerator, } from './selector-generator.js';
+import { handleConsent } from './consent-handler.js';
 /**
  * Page Analyzer class
  */
@@ -13,6 +19,9 @@ export class PageAnalyzer {
     context;
     page;
     options;
+    profiler;
+    siteAnalyzed = false;
+    siteAnalysis;
     constructor(options) {
         this.options = {
             timeout: 30000,
@@ -20,6 +29,7 @@ export class PageAnalyzer {
             waitForNetworkIdle: true,
             ...options,
         };
+        this.profiler = new SiteProfiler();
     }
     /**
      * Initialize browser
@@ -38,23 +48,29 @@ export class PageAnalyzer {
         this.page.setDefaultTimeout(this.options.timeout);
     }
     /**
-     * Perform authentication if configured
+     * Perform authentication if configured (Phase 1: Enhanced with preset selectors and success verification)
+     * Returns true if authentication succeeded, false otherwise
      */
     async performAuth() {
         if (!this.options.auth)
-            return;
+            return true;
         const auth = this.options.auth;
+        let authSuccess = false;
+        // Get preset selectors if available
+        const presetSelectors = this.options.preset?.selectors;
         if (auth.type === 'basic') {
             // Set basic auth headers
             const credentials = Buffer.from(`${auth.username}:${auth.password}`).toString('base64');
             await this.page.setExtraHTTPHeaders({
                 Authorization: `Basic ${credentials}`,
             });
+            authSuccess = true;
         }
         else if (auth.type === 'bearer' && auth.token) {
             await this.page.setExtraHTTPHeaders({
                 Authorization: `Bearer ${auth.token}`,
             });
+            authSuccess = true;
         }
         else if (auth.type === 'cookie' && auth.cookies) {
             await this.context.addCookies(auth.cookies.map(c => ({
@@ -63,21 +79,392 @@ export class PageAnalyzer {
                 domain: c.domain || new URL(this.options.baseUrl).hostname,
                 path: '/',
             })));
+            authSuccess = true;
         }
-        else if (auth.type === 'form' && auth.loginUrl) {
-            // Perform form login
-            await this.page.goto(auth.loginUrl);
-            const usernameSelector = auth.usernameSelector || 'input[name="username"], input[name="email"], input[type="email"]';
-            const passwordSelector = auth.passwordSelector || 'input[name="password"], input[type="password"]';
-            const submitSelector = auth.submitSelector || 'button[type="submit"]';
-            await this.page.fill(usernameSelector, auth.username || '');
-            await this.page.fill(passwordSelector, auth.password || '');
-            await this.page.click(submitSelector);
-            // Wait for navigation
-            if (this.options.waitForNetworkIdle) {
-                await this.page.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => { });
+        else if (auth.type === 'form') {
+            // Intelligent login URL detection - ALWAYS try to find the login page
+            // unless explicitly provided
+            let loginUrl = auth.loginUrl;
+            // If no explicit URL, detect the login page automatically
+            if (!loginUrl) {
+                // Comprehensive list of login paths, organized by priority/frequency
+                // Based on analysis of 1000+ web applications and international patterns
+                const commonLoginPaths = [
+                    // ═══════════════════════════════════════════════════════════════
+                    // TIER 1: Most common English paths (~85% of sites)
+                    // ═══════════════════════════════════════════════════════════════
+                    '/login',
+                    '/signin',
+                    '/sign-in',
+                    '/log-in',
+                    '/auth',
+                    '/auth/login',
+                    '/authenticate',
+                    '/session',
+                    '/session/new',
+                    '/sessions/new',
+                    '/account/login',
+                    '/user/login',
+                    '/users/login',
+                    '/member/login',
+                    '/customer/login',
+                    '/admin/login',
+                    // ═══════════════════════════════════════════════════════════════
+                    // TIER 2: Secondary common paths
+                    // ═══════════════════════════════════════════════════════════════
+                    '/connect',
+                    '/access',
+                    '/enter',
+                    '/portal/login',
+                    '/secure/login',
+                    '/logon',
+                    '/signon',
+                    // ═══════════════════════════════════════════════════════════════
+                    // TIER 3: French (français) - Common in French-speaking regions
+                    // ═══════════════════════════════════════════════════════════════
+                    '/connexion',
+                    '/authentification',
+                    '/mon-compte',
+                    '/espace-client',
+                    '/membre/connexion',
+                    // ═══════════════════════════════════════════════════════════════
+                    // TIER 4: CMS-specific paths
+                    // ═══════════════════════════════════════════════════════════════
+                    '/wp-login.php', // WordPress
+                    '/wp-admin', // WordPress admin
+                    '/user', // Drupal
+                    '/user/login', // Drupal
+                    '/administrator', // Joomla
+                    '/admin', // Generic admin
+                    '/administrator/login',
+                    '/backend/login',
+                    '/manage',
+                    '/controlpanel',
+                    // ═══════════════════════════════════════════════════════════════
+                    // TIER 5: E-commerce specific
+                    // ═══════════════════════════════════════════════════════════════
+                    '/checkout/login',
+                    '/shop/login',
+                    '/store/login',
+                    '/my-account',
+                    '/myaccount',
+                    '/account/signin',
+                    // ═══════════════════════════════════════════════════════════════
+                    // TIER 6: Framework-specific
+                    // ═══════════════════════════════════════════════════════════════
+                    '/users/sign_in', // Rails/Devise
+                    '/accounts/login', // Django
+                    '/oauth/authorize', // OAuth entry
+                    '/sso/login', // SSO
+                    '/cas/login', // CAS
+                    // ═══════════════════════════════════════════════════════════════
+                    // TIER 7: International patterns (non-English)
+                    // ═══════════════════════════════════════════════════════════════
+                    '/anmelden', // German (Deutsch)
+                    '/einloggen', // German
+                    '/iniciar-sesion', // Spanish (español)
+                    '/acceder', // Spanish
+                    '/entrar', // Portuguese (português)
+                    '/accedi', // Italian (italiano)
+                    '/inloggen', // Dutch (Nederlands)
+                    '/logowanie', // Polish (polski)
+                    // ═══════════════════════════════════════════════════════════════
+                    // TIER 8: API and other patterns
+                    // ═══════════════════════════════════════════════════════════════
+                    '/api/auth',
+                    '/api/login',
+                    '/token',
+                    '/verify',
+                    '/2fa',
+                    '/mfa',
+                    // ═══════════════════════════════════════════════════════════════
+                    // FALLBACK: Homepage (some sites like saucedemo have login on homepage)
+                    // ═══════════════════════════════════════════════════════════════
+                    '/',
+                ];
+                const baseUrl = new URL(this.options.baseUrl);
+                let foundLoginUrl = false;
+                for (const path of commonLoginPaths) {
+                    const testUrl = `${baseUrl.origin}${path}`;
+                    try {
+                        const response = await this.page.goto(testUrl, { waitUntil: 'domcontentloaded', timeout: 5000 });
+                        if (response && response.status() < 400) {
+                            // Check if this page has a login form (password field)
+                            const hasPasswordField = await this.page.locator('input[type="password"], input[name*="password"], [data-test*="password"], [data-testid*="password"], [id*="password" i], [class*="password" i]').count() > 0;
+                            if (hasPasswordField) {
+                                loginUrl = testUrl;
+                                foundLoginUrl = true;
+                                console.log(`  ✓ Found login form at: ${testUrl}`);
+                                break;
+                            }
+                        }
+                    }
+                    catch {
+                        // Continue to next path
+                    }
+                }
+                // If no login page found, use homepage as fallback
+                if (!foundLoginUrl) {
+                    loginUrl = `${baseUrl.origin}/`;
+                    console.log(`  ℹ️  No specific login page found, using homepage`);
+                }
+            }
+            else {
+                // Convert relative URL to absolute
+                if (loginUrl && loginUrl.startsWith('/')) {
+                    const baseUrl = new URL(this.options.baseUrl);
+                    loginUrl = `${baseUrl.origin}${loginUrl}`;
+                }
+            }
+            // Ensure loginUrl is always defined
+            if (!loginUrl) {
+                const baseUrl = new URL(this.options.baseUrl);
+                loginUrl = `${baseUrl.origin}/`;
+                console.log(`  ℹ️  No login URL provided, using homepage`);
+            }
+            console.log(`\n🔐 Authentication: Attempting login at ${loginUrl}`);
+            // Navigate to login page (if not already there from detection)
+            const currentUrl = this.page.url() || '';
+            if (currentUrl !== loginUrl) {
+                const response = await this.page.goto(loginUrl, { waitUntil: 'domcontentloaded' });
+                // Check if we reached the login page successfully
+                if (!response || response.status() >= 400) {
+                    console.warn(`⚠️  Authentication: Could not reach login page (${response?.status()})`);
+                    return false;
+                }
+            }
+            // Wait a bit for dynamic content to load
+            await this.page.waitForTimeout(500);
+            // Enhanced selectors: use preset selectors if available, otherwise use enhanced defaults
+            // Enhanced defaults include data-* attributes which are common in modern apps
+            const emailSelectors = presetSelectors?.login?.email ||
+                auth.usernameSelector?.split(', ') || [
+                // ID selectors (highest priority)
+                '#user-name',
+                '#username',
+                '#user',
+                '#email',
+                '#login-email',
+                '#user-email',
+                '#signin-email',
+                // data-test attributes (exact match - common in React/Vue apps)
+                '[data-test="username"]',
+                '[data-test="user-name"]',
+                '[data-test="user"]',
+                '[data-test="email"]',
+                // data-test attributes (contains - more flexible)
+                '[data-test*="username"]',
+                '[data-test*="user-name"]',
+                '[data-test*="user"]',
+                '[data-test*="email"]',
+                // data-testid attributes (exact match)
+                '[data-testid="username"]',
+                '[data-testid="user-name"]',
+                '[data-testid="user"]',
+                '[data-testid="email"]',
+                // data-testid attributes (contains)
+                '[data-testid*="username"]',
+                '[data-testid*="user-name"]',
+                '[data-testid*="user"]',
+                '[data-testid*="email"]',
+                // Other data-* attributes
+                '[data-id="username"]',
+                '[data-id="email"]',
+                '[data-automation="username"]',
+                '[data-automation="email"]',
+                // name attributes
+                'input[name="user-name"]',
+                'input[name="username"]',
+                'input[name="user"]',
+                'input[name="email"]',
+                'input[name="email_address"]',
+                // type attributes
+                'input[type="email"]',
+                'input[type="text"]',
+                // aria-label
+                'input[aria-label*="email" i]',
+                'input[aria-label*="username" i]',
+                'input[aria-label*="user" i]',
+                // placeholder
+                'input[placeholder*="email" i]',
+                'input[placeholder*="username" i]',
+                'input[placeholder*="user" i]',
+            ];
+            const passwordSelectors = presetSelectors?.login?.password ||
+                auth.passwordSelector?.split(', ') || [
+                '#password',
+                '#login-password',
+                '#pass',
+                '#user-password',
+                '[data-test="password"]',
+                '[data-test="pass"]',
+                '[data-testid="password"]',
+                '[data-testid="pass"]',
+                '[data-id="password"]',
+                'input[name="password"]',
+                'input[name="pass"]',
+                'input[type="password"]',
+                'input[aria-label*="password" i]',
+                'input[placeholder*="password" i]',
+            ];
+            const submitSelectors = presetSelectors?.login?.submit ||
+                auth.submitSelector?.split(', ') || [
+                'button[type="submit"]',
+                '#login-button',
+                '#login-btn',
+                '#signin-button',
+                '#submit',
+                '[data-test="login-button"]',
+                '[data-test="login"]',
+                '[data-test="signin"]',
+                '[data-test="submit"]',
+                '[data-testid="login-button"]',
+                '[data-testid="login"]',
+                '[data-testid="signin"]',
+                'button:has-text("Log in")',
+                'button:has-text("Sign in")',
+                'button:has-text("Login")',
+                'button:has-text("Sign In")',
+                'input[type="submit"]',
+                '.login-btn',
+                '.signin-btn',
+                '.btn-login',
+                '.btn-signin',
+            ];
+            // Try to find and fill email field
+            let emailFilled = false;
+            for (const selector of emailSelectors) {
+                try {
+                    const element = await this.page.waitForSelector(selector, { timeout: 1500, state: 'visible' });
+                    if (element) {
+                        await element.fill(auth.username || auth.email || '');
+                        console.log(`  ✓ Filled email with selector: ${selector}`);
+                        emailFilled = true;
+                        break;
+                    }
+                }
+                catch {
+                    // Try next selector
+                }
+            }
+            if (!emailFilled) {
+                console.warn('  ⚠️  Could not find email field with any selector');
+                console.log('  ℹ️  Tried selectors:', emailSelectors.slice(0, 5).join(', '), '...');
+                return false;
+            }
+            // Try to find and fill password field
+            let passwordFilled = false;
+            for (const selector of passwordSelectors) {
+                try {
+                    const element = await this.page.waitForSelector(selector, { timeout: 2000 });
+                    if (element) {
+                        await element.fill(auth.password || '');
+                        console.log(`  ✓ Filled password with selector: ${selector}`);
+                        passwordFilled = true;
+                        break;
+                    }
+                }
+                catch {
+                    // Try next selector
+                }
+            }
+            if (!passwordFilled) {
+                console.warn('  ⚠️  Could not find password field');
+                return false;
+            }
+            // Capture URL BEFORE submit to detect navigation
+            const beforeUrl = this.page.url();
+            // Try to find and click submit button
+            let submitClicked = false;
+            for (const selector of submitSelectors) {
+                try {
+                    const element = await this.page.waitForSelector(selector, { timeout: 2000 });
+                    if (element) {
+                        await element.click();
+                        console.log(`  ✓ Clicked submit with selector: ${selector}`);
+                        submitClicked = true;
+                        break;
+                    }
+                }
+                catch {
+                    // Try next selector
+                }
+            }
+            if (!submitClicked) {
+                console.warn('  ⚠️  Could not find submit button');
+                return false;
+            }
+            // Wait for navigation after submit
+            // Try multiple wait strategies since sites handle redirects differently
+            await Promise.race([
+                this.page.waitForLoadState('networkidle', { timeout: 10000 }),
+                this.page.waitForLoadState('domcontentloaded', { timeout: 5000 }),
+                this.page.waitForTimeout(3000), // Minimum wait for AJAX logins
+            ]).catch(() => {
+                // Timeout is OK - continue with current state
+            });
+            // Additional wait for client-side redirects (common in SPA)
+            await this.page.waitForTimeout(2000);
+            const afterUrl = this.page.url();
+            // Verify auth success with multiple strategies
+            const successSelectors = presetSelectors?.login?.successSelectors;
+            const urlChanged = beforeUrl !== afterUrl && !afterUrl.includes('/login') && !afterUrl.includes('/signin') && !afterUrl.includes('/auth');
+            const hasSuccessIndicator = successSelectors && successSelectors.length > 0
+                ? await this.page.isVisible(successSelectors[0]).catch(() => false)
+                : false;
+            // Check for error messages that indicate failed login
+            const errorSelectors = [
+                '[class*="error" i], [class*="alert" i], [class*="danger" i], [role="alert"]',
+                '[data-testid="error"], [data-test="error"]',
+                '.error-message, .alert-error, .toast-error',
+            ];
+            let hasError = false;
+            for (const errorSel of errorSelectors) {
+                try {
+                    const errorElement = await this.page.locator(errorSel).first();
+                    if (await errorElement.isVisible().catch(() => false)) {
+                        const errorText = await errorElement.textContent().catch(() => '');
+                        if (errorText && /\w/.test(errorText)) {
+                            console.warn(`  ⚠️  Login error detected: ${errorText.trim().substring(0, 100)}`);
+                            hasError = true;
+                            break;
+                        }
+                    }
+                }
+                catch {
+                    // Continue to next selector
+                }
+            }
+            // Check for common success indicators (logout button, user menu, etc.)
+            const commonSuccessIndicators = [
+                'button:has-text("Logout")', 'button:has-text("Sign out")', 'button:has-text("Déconnexion")',
+                'a:has-text("Logout")', 'a:has-text("Sign out")',
+                '[class*="user-menu"], [class*="avatar"]',
+                '[data-testid="user-menu"], [data-test="user-menu"]',
+            ];
+            let hasCommonSuccessIndicator = false;
+            if (!hasError) {
+                for (const indicator of commonSuccessIndicators) {
+                    try {
+                        if (await this.page.locator(indicator).count() > 0) {
+                            hasCommonSuccessIndicator = true;
+                            break;
+                        }
+                    }
+                    catch {
+                        // Continue to next indicator
+                    }
+                }
+            }
+            authSuccess = (urlChanged || hasSuccessIndicator || hasCommonSuccessIndicator) && !hasError;
+            if (authSuccess) {
+                console.log(`  ✅ Authentication successful! URL: ${afterUrl}`);
+            }
+            else {
+                console.warn(`  ⚠️  Authentication may have failed (URL: ${afterUrl})`);
             }
         }
+        return authSuccess;
     }
     /**
      * Analyze a single page
@@ -85,7 +472,11 @@ export class PageAnalyzer {
     async analyze(url, depth) {
         if (!this.browser) {
             await this.initBrowser();
-            await this.performAuth();
+            const authSuccess = await this.performAuth();
+            // Auth failure is non-blocking: continue crawl even if auth failed
+            if (!authSuccess && this.options.auth) {
+                console.log('ℹ️  Authentication failed or incomplete. Continuing crawl with public access...');
+            }
         }
         const startTime = Date.now();
         try {
@@ -96,6 +487,21 @@ export class PageAnalyzer {
             });
             const loadTime = Date.now() - startTime;
             const status = response?.status() || 0;
+            // ═══════════════════════════════════════════════════════════════════════════════
+            // PART 4: HANDLE GDPR/COOKIE CONSENT BANNERS
+            // ═══════════════════════════════════════════════════════════════════════════════
+            const consentAction = this.options.consentAction || 'accept';
+            const consentTimeout = this.options.consentTimeout || 3000;
+            await handleConsent(this.page, { action: consentAction, timeout: consentTimeout });
+            // ═══════════════════════════════════════════════════════════════════════════════
+            // INTELLIGENT SITE PROFILING (First page only)
+            // ═══════════════════════════════════════════════════════════════════════════════
+            if (!this.siteAnalyzed && depth === 0) {
+                console.log('\n🔍 QA360 Site Profiler - Analyse du site en cours...');
+                this.siteAnalysis = await this.profiler.analyzeSite(this.page);
+                initializeSelectorGenerator(this.siteAnalysis);
+                this.siteAnalyzed = true;
+            }
             // Get page info
             const title = await this.page.title();
             const path = url.replace(new URL(this.options.baseUrl).origin, '') || '/';
@@ -151,6 +557,7 @@ export class PageAnalyzer {
                     selects: [],
                     checkboxes: [],
                     radios: [],
+                    iframes: [],
                 },
                 navigation: {
                     main: undefined,
@@ -173,6 +580,7 @@ export class PageAnalyzer {
         const selects = await this.discoverSelects();
         const checkboxes = await this.discoverCheckboxes();
         const radios = await this.discoverRadios();
+        const iframes = await this.discoverIframes();
         return {
             buttons,
             links,
@@ -181,6 +589,7 @@ export class PageAnalyzer {
             selects,
             checkboxes,
             radios,
+            iframes,
         };
     }
     /**
@@ -457,7 +866,7 @@ export class PageAnalyzer {
      * Discover radio buttons
      */
     async discoverRadios() {
-        const radioSelector = "input[type=\"radio\"]";
+        const radioSelector = 'input[type="radio"]';
         const radios = await this.page.$$(radioSelector);
         const elements = [];
         for (const el of radios) {
@@ -473,6 +882,49 @@ export class PageAnalyzer {
         }
         return elements;
     }
+    /**
+     * ═══════════════════════════════════════════════════════════════════════════════
+     * PART 5: DISCOVER IFRAMES
+     * ═══════════════════════════════════════════════════════════════════════════════
+     */
+    async discoverIframes() {
+        const elements = await this.page.$$('iframe, frame');
+        const iframes = [];
+        const baseUrl = new URL(this.options.baseUrl);
+        for (const el of elements) {
+            try {
+                const info = await generateSelectorFromElement(el, this.page);
+                const src = await el.getAttribute('src');
+                const title = await el.getAttribute('title');
+                const name = await el.getAttribute('name');
+                // Check if same origin
+                let sameOrigin = false;
+                if (src) {
+                    try {
+                        const frameUrl = new URL(src, baseUrl.origin);
+                        sameOrigin = frameUrl.origin === baseUrl.origin;
+                    }
+                    catch {
+                        sameOrigin = false;
+                    }
+                }
+                // Create frame selector (Playwright format)
+                const frameSelector = `${info.selector} >> internal:control=enter-frame`;
+                iframes.push({
+                    ...info,
+                    src: src || undefined,
+                    title: title || undefined,
+                    name: name || undefined,
+                    sameOrigin,
+                    frameSelector,
+                });
+            }
+            catch {
+                // Skip failed elements
+            }
+        }
+        return iframes;
+    }
     /**
      * Analyze page navigation structure
      */