qa360 2.1.7 → 2.2.0

package/core/src/crawler/page-analyzer.ts

@@ -0,0 +1,857 @@
+/**
+ * QA360 Page Analyzer
+ *
+ * Analyzes web pages to discover elements, forms, and patterns
+ */
+
+import { chromium, Browser, Page, BrowserContext } from '@playwright/test';
+import type {
+  CrawlOptions,
+  PageDefinition,
+  FormInfo,
+  FieldInfo,
+  ElementInfo,
+  LinkInfo,
+  A11ySnapshot,
+} from './types.js';
+import { generateSelectorFromElement, generateSelector } from './selector-generator.js';
+
+/**
+ * Page Analyzer class
+ */
+export class PageAnalyzer {
+  private browser?: Browser;
+  private context?: BrowserContext;
+  private page?: Page;
+  private options: CrawlOptions;
+
+  constructor(options: CrawlOptions) {
+    this.options = {
+      timeout: 30000,
+      headless: true,
+      waitForNetworkIdle: true,
+      ...options,
+    };
+  }
+
+  /**
+   * Initialize browser
+   */
+  private async initBrowser(): Promise<void> {
+    this.browser = await chromium.launch({
+      headless: this.options.headless ?? true,
+      args: ['--no-sandbox', '--disable-dev-shm-usage'],
+    });
+
+    this.context = await this.browser.newContext({
+      viewport: { width: 1920, height: 1080 },
+      userAgent: 'QA360-Crawler/1.0',
+    });
+
+    this.page = await this.context.newPage();
+
+    // Set default timeout
+    this.page.setDefaultTimeout(this.options.timeout!);
+  }
+
+  /**
+   * Perform authentication if configured
+   */
+  private async performAuth(): Promise<void> {
+    if (!this.options.auth) return;
+
+    const auth = this.options.auth;
+
+    if (auth.type === 'basic') {
+      // Set basic auth headers
+      const credentials = Buffer.from(`${auth.username}:${auth.password}`).toString('base64');
+      await this.page!.setExtraHTTPHeaders({
+        Authorization: `Basic ${credentials}`,
+      });
+    } else if (auth.type === 'bearer' && auth.token) {
+      await this.page!.setExtraHTTPHeaders({
+        Authorization: `Bearer ${auth.token}`,
+      });
+    } else if (auth.type === 'cookie' && auth.cookies) {
+      await this.context!.addCookies(
+        auth.cookies.map(c => ({
+          name: c.name,
+          value: c.value,
+          domain: c.domain || new URL(this.options.baseUrl).hostname,
+          path: '/',
+        }))
+      );
+    } else if (auth.type === 'form' && auth.loginUrl) {
+      // Perform form login
+      await this.page!.goto(auth.loginUrl);
+
+      const usernameSelector =
+        auth.usernameSelector || 'input[name="username"], input[name="email"], input[type="email"]';
+      const passwordSelector =
+        auth.passwordSelector || 'input[name="password"], input[type="password"]';
+      const submitSelector = auth.submitSelector || 'button[type="submit"]';
+
+      await this.page!.fill(usernameSelector, auth.username || '');
+      await this.page!.fill(passwordSelector, auth.password || '');
+      await this.page!.click(submitSelector);
+
+      // Wait for navigation
+      if (this.options.waitForNetworkIdle) {
+        await this.page!.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => {});
+      }
+    }
+  }
+
+  /**
+   * Analyze a single page
+   */
+  async analyze(url: string, depth: number): Promise<PageDefinition> {
+    if (!this.browser) {
+      await this.initBrowser();
+      await this.performAuth();
+    }
+
+    const startTime = Date.now();
+
+    try {
+      // Navigate to page
+      const response = await this.page!.goto(url, {
+        waitUntil: this.options.waitForNetworkIdle ? 'networkidle' : 'domcontentloaded',
+        timeout: this.options.timeout,
+      });
+
+      const loadTime = Date.now() - startTime;
+      const status = response?.status() || 0;
+
+      // Get page info
+      const title = await this.page!.title();
+      const path = url.replace(new URL(this.options.baseUrl).origin, '') || '/';
+
+      // Get meta description
+      const description = (await this.page!
+        .locator('meta[name="description"]')
+        .getAttribute('content')
+        .catch(() => undefined)) || undefined;
+
+      // Discover all elements
+      const elements = await this.discoverElements();
+
+      // Analyze navigation
+      const navigation = await this.analyzeNavigation();
+
+      // Accessibility scan
+      const accessibility = await this.runAccessibilityScan();
+
+      // Screenshot if requested
+      let screenshot: string | undefined;
+      if (this.options.screenshots) {
+        const buffer = await this.page!.screenshot({ fullPage: false });
+        screenshot = `data:image/png;base64,${buffer.toString('base64')}`;
+      }
+
+      // Detect page type
+      const { pageType, pageTypeConfidence } = this.detectPageType(elements, title, path);
+
+      return {
+        url,
+        path,
+        title,
+        depth,
+        status,
+        loadTime,
+        description,
+        elements,
+        navigation,
+        screenshot,
+        accessibility,
+        pageType,
+        pageTypeConfidence,
+      };
+    } catch (error) {
+      // Return minimal info on error
+      return {
+        url,
+        path: url.replace(new URL(this.options.baseUrl).origin, ''),
+        title: 'Error',
+        depth,
+        status: 0,
+        loadTime: Date.now() - startTime,
+        elements: {
+          buttons: [],
+          links: [],
+          forms: [],
+          inputs: [],
+          selects: [],
+          checkboxes: [],
+          radios: [],
+        },
+        navigation: {
+          main: undefined,
+          footer: undefined,
+          breadcrumb: undefined,
+        },
+        pageType: 'other',
+        pageTypeConfidence: 0,
+      };
+    }
+  }
+
+  /**
+   * Discover all interactive elements on the page
+   */
+  private async discoverElements(): Promise<PageDefinition['elements']> {
+    const buttons = await this.discoverButtons();
+    const links = await this.discoverLinks();
+    const forms = await this.discoverForms();
+    const inputs = await this.discoverInputs();
+    const selects = await this.discoverSelects();
+    const checkboxes = await this.discoverCheckboxes();
+    const radios = await this.discoverRadios();
+
+    return {
+      buttons,
+      links,
+      forms,
+      inputs,
+      selects,
+      checkboxes,
+      radios,
+    };
+  }
+
+  /**
+   * Discover buttons
+   */
+  private async discoverButtons(): Promise<ElementInfo[]> {
+    const elements = await this.page!.$$(
+      'button, input[type="button"], input[type="submit"], [role="button"], a[role="button"]'
+    );
+
+    const buttons: ElementInfo[] = [];
+
+    for (const el of elements) {
+      try {
+        const info = await generateSelectorFromElement(el, this.page);
+        if (info.selector !== 'unknown') {
+          buttons.push(info);
+        }
+      } catch {
+        // Skip failed elements
+      }
+    }
+
+    return buttons;
+  }
+
+  /**
+   * Discover links
+   */
+  private async discoverLinks(): Promise<LinkInfo[]> {
+    const elements = await this.page!.$$('[href]');
+
+    const links: LinkInfo[] = [];
+    const baseUrl = new URL(this.options.baseUrl);
+
+    for (const el of elements) {
+      try {
+        const info = await generateSelectorFromElement(el, this.page);
+        const href = await el.getAttribute('href');
+
+        if (!href || href.startsWith('#') || href.startsWith('javascript:')) {
+          continue;
+        }
+
+        // Resolve relative URLs
+        let url: string;
+        try {
+          url = new URL(href, baseUrl.origin).href;
+        } catch {
+          continue;
+        }
+
+        const internal = url.startsWith(baseUrl.origin);
+
+        links.push({
+          ...info,
+          url,
+          internal,
+          visited: false,
+        });
+      } catch {
+        // Skip failed elements
+      }
+    }
+
+    return links;
+  }
+
+  /**
+   * Discover forms
+   */
+  private async discoverForms(): Promise<FormInfo[]> {
+    const forms = await this.page!.$$('[role="form"], form, .form, [data-form]');
+
+    const discovered: FormInfo[] = [];
+
+    for (const formEl of forms) {
+      try {
+        const selector = await generateSelectorFromElement(formEl, this.page).then(
+          i => i.selector
+        );
+
+        // Get form fields
+        const fields = await this.discoverFormFields(formEl);
+
+        if (fields.length === 0) continue;
+
+        // Get submit button
+        let submitButton: ElementInfo | undefined;
+        const submitBtn = await formEl.$(
+          'button[type="submit"], input[type="submit"], [type="submit"]'
+        );
+        if (submitBtn) {
+          submitButton = await generateSelectorFromElement(submitBtn, this.page);
+        }
+
+        // Detect form purpose
+        const purpose = this.detectFormPurpose(fields, selector);
+        const confidence = this.calculateFormPurposeConfidence(fields, purpose);
+
+        discovered.push({
+          selector,
+          purpose,
+          fields,
+          submitButton,
+          confidence,
+        });
+      } catch {
+        // Skip failed forms
+      }
+    }
+
+    return discovered;
+  }
+
+  /**
+   * Discover fields within a form
+   */
+  private async discoverFormFields(formEl: any): Promise<FieldInfo[]> {
+    const fields: FieldInfo[] = [];
+
+    // Get all input, select, textarea elements
+    const inputs = await formEl.$$(
+      'input:not([type="hidden"]):not([type="submit"]):not([type="button"]), select, textarea'
+    );
+
+    for (const input of inputs) {
+      try {
+        const info = await generateSelectorFromElement(input, this.page);
+
+        // Get field-specific attributes
+        const inputType = await input.getAttribute('type') || 'text';
+        const name = await input.getAttribute('name');
+        const required = (await input.getAttribute('required')) !== null;
+        const placeholder = await input.getAttribute('placeholder');
+
+        // Get validation attributes
+        const validation: FieldInfo['validation'] = {};
+        const min = await input.getAttribute('min');
+        const max = await input.getAttribute('max');
+        const pattern = await input.getAttribute('pattern');
+        const minLength = await input.getAttribute('minlength');
+        const maxLength = await input.getAttribute('maxlength');
+
+        if (min) validation.min = min;
+        if (max) validation.max = max;
+        if (pattern) validation.pattern = pattern;
+        if (minLength) validation.minLength = parseInt(minLength, 10);
+        if (maxLength) validation.maxLength = parseInt(maxLength, 10);
+
+        // Get options for selects
+        let options: string[] | undefined;
+        if (inputType === 'select-one' || inputType === 'select-multiple') {
+          options = await input.$$eval('option', (opts: HTMLOptionElement[]) =>
+            opts
+              .map(o => o.value || o.text)
+              .filter(Boolean)
+          );
+        }
+
+        fields.push({
+          ...info,
+          inputType,
+          name: name || undefined,
+          required,
+          placeholder: placeholder || undefined,
+          options,
+          validation: Object.keys(validation).length > 0 ? validation : undefined,
+        });
+      } catch {
+        // Skip failed fields
+      }
+    }
+
+    return fields;
+  }
+
+  /**
+   * Detect form purpose based on fields
+   */
+  private detectFormPurpose(fields: FieldInfo[], selector: string): FormInfo['purpose'] {
+    const fieldNames = fields.map(f => f.name?.toLowerCase() || '');
+    const selectorLower = selector.toLowerCase();
+
+    // Check for login form
+    if (
+      (fieldNames.includes('password') || fieldNames.includes('pass')) &&
+      (fieldNames.includes('email') ||
+        fieldNames.includes('username') ||
+        fieldNames.includes('user'))
+    ) {
+      return 'login';
+    }
+
+    // Check for signup form
+    if (
+      (fieldNames.includes('password') || fieldNames.includes('confirm')) &&
+      (fieldNames.includes('email') || fieldNames.includes('username'))
+    ) {
+      return 'signup';
+    }
+
+    // Check for search form
+    if (selectorLower.includes('search') || fieldNames.includes('q') || fieldNames.includes('search')) {
+      return 'search';
+    }
+
+    // Check for checkout form
+    if (
+      fieldNames.includes('card') ||
+      fieldNames.includes('cvv') ||
+      fieldNames.includes('expiry')
+    ) {
+      return 'checkout';
+    }
+
+    // Check for contact form
+    if (fieldNames.includes('message') || fieldNames.includes('subject')) {
+      return 'contact';
+    }
+
+    return 'other';
+  }
+
+  /**
+   * Calculate confidence for form purpose detection
+   */
+  private calculateFormPurposeConfidence(fields: FieldInfo[], purpose: FormInfo['purpose']): number {
+    // High confidence for login/signup with password field
+    if (purpose === 'login' || purpose === 'signup') {
+      if (fields.some(f => f.inputType === 'password')) {
+        return 0.9;
+      }
+    }
+
+    // Medium confidence for forms with clear naming
+    if (purpose !== 'other') {
+      return 0.7;
+    }
+
+    return 0.5;
+  }
+
+  /**
+   * Discover standalone input fields (not in forms)
+   */
+  private async discoverInputs(): Promise<FieldInfo[]> {
+    const inputs = await this.page!.$$(
+      'input:not([type="hidden"]):not([type="submit"]):not(form input), textarea:not(form textarea)'
+    );
+
+    const fields: FieldInfo[] = [];
+
+    for (const input of inputs) {
+      try {
+        const info = await generateSelectorFromElement(input, this.page);
+        const inputType = await input.getAttribute('type') || 'text';
+        const name = await input.getAttribute('name');
+        const required = (await input.getAttribute('required')) !== null;
+        const placeholder = await input.getAttribute('placeholder');
+
+        fields.push({
+          ...info,
+          inputType,
+          name: name || undefined,
+          required,
+          placeholder: placeholder || undefined,
+        });
+      } catch {
+        // Skip failed
+      }
+    }
+
+    return fields;
+  }
+
+  /**
+   * Discover select elements
+   */
+  private async discoverSelects(): Promise<FieldInfo[]> {
+    const selects = await this.page!.$$('select:not(form select)');
+
+    const fields: FieldInfo[] = [];
+
+    for (const select of selects) {
+      try {
+        const info = await generateSelectorFromElement(select, this.page);
+        const name = await select.getAttribute('name');
+        const required = (await select.getAttribute('required')) !== null;
+
+        // Get options
+        const options = await select.$$eval('option', opts =>
+          opts.map(o => (o as HTMLOptionElement).value).filter(Boolean)
+        );
+
+        fields.push({
+          ...info,
+          inputType: 'select-one',
+          name: name || undefined,
+          required,
+          options: options.length > 0 ? options : undefined,
+        });
+      } catch {
+        // Skip failed
+      }
+    }
+
+    return fields;
+  }
+
+  /**
+   * Discover checkboxes
+   */
+  private async discoverCheckboxes(): Promise<ElementInfo[]> {
+    const checkboxSelector = "input[type=\"checkbox\"]"; const checkboxes = await this.page!.$$(checkboxSelector);
+
+    const elements: ElementInfo[] = [];
+
+    for (const el of checkboxes) {
+      try {
+        const info = await generateSelectorFromElement(el, this.page);
+        if (info.selector !== 'unknown') {
+          elements.push(info);
+        }
+      } catch {
+        // Skip failed
+      }
+    }
+
+    return elements;
+  }
+
+  /**
+   * Discover radio buttons
+   */
+  private async discoverRadios(): Promise<ElementInfo[]> {
+    const radioSelector = "input[type=\"radio\"]"; const radios = await this.page!.$$(radioSelector);
+
+    const elements: ElementInfo[] = [];
+
+    for (const el of radios) {
+      try {
+        const info = await generateSelectorFromElement(el, this.page);
+        if (info.selector !== 'unknown') {
+          elements.push(info);
+        }
+      } catch {
+        // Skip failed
+      }
+    }
+
+    return elements;
+  }
+
+  /**
+   * Analyze page navigation structure
+   */
+  private async analyzeNavigation(): Promise<PageDefinition['navigation']> {
+    const navigation: PageDefinition['navigation'] = {};
+
+    // Main navigation (nav, menu, navbar)
+    const mainNav = await this.page!.$('nav, [role="navigation"], .nav, .navigation, .navbar, .menu');
+    if (mainNav) {
+      const navSelector = await generateSelectorFromElement(mainNav, this.page).then(
+        i => i.selector
+      );
+
+      const links = await mainNav.$$('[href]');
+      const items: LinkInfo[] = [];
+
+      for (const link of links) {
+        try {
+          const info = await generateSelectorFromElement(link, this.page);
+          const href = await link.getAttribute('href');
+          if (href) {
+            const url = new URL(href, new URL(this.options.baseUrl).origin).href;
+            items.push({
+              ...info,
+              url,
+              internal: url.startsWith(new URL(this.options.baseUrl).origin),
+              visited: false,
+            });
+          }
+        } catch {
+          // Skip
+        }
+      }
+
+      navigation.main = { selector: navSelector, items };
+    }
+
+    // Breadcrumb
+    const breadcrumb = await this.page!.$(
+      '[aria-label="breadcrumb"], .breadcrumb, .breadcrumbs, ol.breadcrumb'
+    );
+    if (breadcrumb) {
+      const crumbSelector = await generateSelectorFromElement(breadcrumb, this.page).then(
+        i => i.selector
+      );
+
+      const items = await breadcrumb.$$('[href], span, li');
+      const breadcrumbs: Array<{ text: string; url: string }> = [];
+
+      for (const item of items) {
+        try {
+          const text = await item.textContent();
+          const href = await item.getAttribute('href');
+          if (text && text.trim()) {
+            breadcrumbs.push({
+              text: text.trim(),
+              url: href || '',
+            });
+          }
+        } catch {
+          // Skip
+        }
+      }
+
+      if (breadcrumbs.length > 0) {
+        navigation.breadcrumb = { selector: crumbSelector, items: breadcrumbs };
+      }
+    }
+
+    // Pagination
+    const pagination = await this.page!.$(
+      '[aria-label="pagination"], .pagination, [role="navigation"] nav'
+    );
+    if (pagination) {
+      const pageSelector = await generateSelectorFromElement(pagination, this.page).then(
+        i => i.selector
+      );
+
+      const nextLink = await pagination.$('a[rel="next"], .next, [aria-label="next"]');
+      const prevLink = await pagination.$('a[rel="prev"], .prev, [aria-label="previous"]');
+
+      const paginationInfo: PageDefinition['navigation']['pagination'] = {
+        selector: pageSelector,
+      };
+
+      if (nextLink) {
+        const info = await generateSelectorFromElement(nextLink, this.page);
+        const href = await nextLink.getAttribute('href');
+        if (href) {
+          paginationInfo.nextPage = {
+            ...info,
+            url: new URL(href, new URL(this.options.baseUrl).origin).href,
+            internal: true,
+            visited: false,
+          };
+        }
+      }
+
+      if (prevLink) {
+        const info = await generateSelectorFromElement(prevLink, this.page);
+        const href = await prevLink.getAttribute('href');
+        if (href) {
+          paginationInfo.prevPage = {
+            ...info,
+            url: new URL(href, new URL(this.options.baseUrl).origin).href,
+            internal: true,
+            visited: false,
+          };
+        }
+      }
+
+      navigation.pagination = paginationInfo;
+    }
+
+    // Footer
+    const footer = await this.page!.$('footer, [role="contentinfo"], .footer');
+    if (footer) {
+      const footerSelector = await generateSelectorFromElement(footer, this.page).then(
+        i => i.selector
+      );
+
+      const links = await footer.$$('[href]');
+      const items: LinkInfo[] = [];
+
+      for (const link of links) {
+        try {
+          const info = await generateSelectorFromElement(link, this.page);
+          const href = await link.getAttribute('href');
+          if (href) {
+            const url = new URL(href, new URL(this.options.baseUrl).origin).href;
+            items.push({
+              ...info,
+              url,
+              internal: url.startsWith(new URL(this.options.baseUrl).origin),
+              visited: false,
+            });
+          }
+        } catch {
+          // Skip
+        }
+      }
+
+      navigation.footer = { selector: footerSelector, items };
+    }
+
+    return navigation;
+  }
+
+  /**
+   * Run accessibility scan using axe-core
+   */
+  private async runAccessibilityScan(): Promise<A11ySnapshot | undefined> {
+    try {
+      // Inject axe-core
+      await this.page!.addScriptTag({
+        url: 'https://unpkg.com/axe-core@4.8.2/axe.min.js',
+      });
+
+      // Run axe
+      const results = await this.page!.evaluate(() => {
+        return new Promise((resolve) => {
+          // @ts-ignore
+          if (typeof axe !== 'undefined') {
+            // @ts-ignore
+            axe.run((err: any, results: any) => {
+              if (err) {
+                resolve({ violations: [], passes: [], incomplete: [] });
+              } else {
+                resolve(results);
+              }
+            });
+          } else {
+            resolve({ violations: [], passes: [], incomplete: [] });
+          }
+        });
+      });
+
+      const axeResults = results as any;
+
+      // Process violations
+      const violations = (axeResults.violations || []).map((v: any) => ({
+        id: v.id,
+        impact: v.impact || 'moderate',
+        description: v.description || v.help || 'Accessibility issue',
+        nodes: v.nodes?.length || 0,
+        selectors: (v.nodes || []).slice(0, 5).map((n: any) => n.target?.[0] || '').filter(Boolean),
+      }));
+
+      // Calculate score
+      const criticalCount = violations.filter((v: any) => v.impact === 'critical').length;
+      const seriousCount = violations.filter((v: any) => v.impact === 'serious').length;
+      const moderateCount = violations.filter((v: any) => v.impact === 'moderate').length;
+      const minorCount = violations.filter((v: any) => v.impact === 'minor').length;
+
+      const score = Math.max(
+        0,
+        100 - (criticalCount * 25 + seriousCount * 10 + moderateCount * 5 + minorCount * 1)
+      );
+
+      return {
+        score: Math.round(score),
+        violations,
+        passes: axeResults.passes?.length || 0,
+        incomplete: axeResults.incomplete?.length || 0,
+      };
+    } catch {
+      return undefined;
+    }
+  }
+
+  /**
+   * Detect page type
+   */
+  private detectPageType(
+    elements: PageDefinition['elements'],
+    title: string,
+    path: string
+  ): { pageType: PageDefinition['pageType']; pageTypeConfidence: number } {
+    const pathLower = path.toLowerCase();
+    const titleLower = title.toLowerCase();
+
+    // Homepage
+    if (path === '/' || path === '' || pathLower === '/home' || pathLower === '/index') {
+      return { pageType: 'homepage', pageTypeConfidence: 0.95 };
+    }
+
+    // Login page
+    if (
+      pathLower.includes('/login') ||
+      pathLower.includes('/signin') ||
+      pathLower.includes('/auth') ||
+      titleLower.includes('login') ||
+      titleLower.includes('sign in')
+    ) {
+      return { pageType: 'login', pageTypeConfidence: 0.9 };
+    }
+
+    // Signup page
+    if (
+      pathLower.includes('/signup') ||
+      pathLower.includes('/register') ||
+      pathLower.includes('/join') ||
+      titleLower.includes('sign up') ||
+      titleLower.includes('register')
+    ) {
+      return { pageType: 'signup', pageTypeConfidence: 0.9 };
+    }
+
+    // Dashboard
+    if (
+      pathLower.includes('/dashboard') ||
+      pathLower.includes('/my-account') ||
+      pathLower.includes('/profile') ||
+      titleLower.includes('dashboard')
+    ) {
+      return { pageType: 'dashboard', pageTypeConfidence: 0.85 };
+    }
+
+    // Listing page (multiple items, pagination)
+    if (elements.links.length > 20 || elements.forms.some(f => f.purpose === 'filter')) {
+      return { pageType: 'listing', pageTypeConfidence: 0.7 };
+    }
+
+    // Form page
+    if (elements.forms.length > 0) {
+      return { pageType: 'form', pageTypeConfidence: 0.75 };
+    }
+
+    return { pageType: 'other', pageTypeConfidence: 0.5 };
+  }
+
+  /**
+   * Clean up resources
+   */
+  async cleanup(): Promise<void> {
+    try {
+      if (this.page) await this.page.close();
+      if (this.context) await this.context.close();
+      if (this.browser) await this.browser.close();
+    } catch {
+      // Ignore cleanup errors
+    }
+  }
+}