qa360 2.2.1 → 2.2.13

package/core/src/crawler/page-analyzer.ts

@@ -1,857 +0,0 @@
-/**
- * QA360 Page Analyzer
- *
- * Analyzes web pages to discover elements, forms, and patterns
- */
-
-import { chromium, Browser, Page, BrowserContext } from '@playwright/test';
-import type {
-  CrawlOptions,
-  PageDefinition,
-  FormInfo,
-  FieldInfo,
-  ElementInfo,
-  LinkInfo,
-  A11ySnapshot,
-} from './types.js';
-import { generateSelectorFromElement, generateSelector } from './selector-generator.js';
-
-/**
- * Page Analyzer class
- */
-export class PageAnalyzer {
-  private browser?: Browser;
-  private context?: BrowserContext;
-  private page?: Page;
-  private options: CrawlOptions;
-
-  constructor(options: CrawlOptions) {
-    this.options = {
-      timeout: 30000,
-      headless: true,
-      waitForNetworkIdle: true,
-      ...options,
-    };
-  }
-
-  /**
-   * Initialize browser
-   */
-  private async initBrowser(): Promise<void> {
-    this.browser = await chromium.launch({
-      headless: this.options.headless ?? true,
-      args: ['--no-sandbox', '--disable-dev-shm-usage'],
-    });
-
-    this.context = await this.browser.newContext({
-      viewport: { width: 1920, height: 1080 },
-      userAgent: 'QA360-Crawler/1.0',
-    });
-
-    this.page = await this.context.newPage();
-
-    // Set default timeout
-    this.page.setDefaultTimeout(this.options.timeout!);
-  }
-
-  /**
-   * Perform authentication if configured
-   */
-  private async performAuth(): Promise<void> {
-    if (!this.options.auth) return;
-
-    const auth = this.options.auth;
-
-    if (auth.type === 'basic') {
-      // Set basic auth headers
-      const credentials = Buffer.from(`${auth.username}:${auth.password}`).toString('base64');
-      await this.page!.setExtraHTTPHeaders({
-        Authorization: `Basic ${credentials}`,
-      });
-    } else if (auth.type === 'bearer' && auth.token) {
-      await this.page!.setExtraHTTPHeaders({
-        Authorization: `Bearer ${auth.token}`,
-      });
-    } else if (auth.type === 'cookie' && auth.cookies) {
-      await this.context!.addCookies(
-        auth.cookies.map(c => ({
-          name: c.name,
-          value: c.value,
-          domain: c.domain || new URL(this.options.baseUrl).hostname,
-          path: '/',
-        }))
-      );
-    } else if (auth.type === 'form' && auth.loginUrl) {
-      // Perform form login
-      await this.page!.goto(auth.loginUrl);
-
-      const usernameSelector =
-        auth.usernameSelector || 'input[name="username"], input[name="email"], input[type="email"]';
-      const passwordSelector =
-        auth.passwordSelector || 'input[name="password"], input[type="password"]';
-      const submitSelector = auth.submitSelector || 'button[type="submit"]';
-
-      await this.page!.fill(usernameSelector, auth.username || '');
-      await this.page!.fill(passwordSelector, auth.password || '');
-      await this.page!.click(submitSelector);
-
-      // Wait for navigation
-      if (this.options.waitForNetworkIdle) {
-        await this.page!.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => {});
-      }
-    }
-  }
-
-  /**
-   * Analyze a single page
-   */
-  async analyze(url: string, depth: number): Promise<PageDefinition> {
-    if (!this.browser) {
-      await this.initBrowser();
-      await this.performAuth();
-    }
-
-    const startTime = Date.now();
-
-    try {
-      // Navigate to page
-      const response = await this.page!.goto(url, {
-        waitUntil: this.options.waitForNetworkIdle ? 'networkidle' : 'domcontentloaded',
-        timeout: this.options.timeout,
-      });
-
-      const loadTime = Date.now() - startTime;
-      const status = response?.status() || 0;
-
-      // Get page info
-      const title = await this.page!.title();
-      const path = url.replace(new URL(this.options.baseUrl).origin, '') || '/';
-
-      // Get meta description
-      const description = (await this.page!
-        .locator('meta[name="description"]')
-        .getAttribute('content')
-        .catch(() => undefined)) || undefined;
-
-      // Discover all elements
-      const elements = await this.discoverElements();
-
-      // Analyze navigation
-      const navigation = await this.analyzeNavigation();
-
-      // Accessibility scan
-      const accessibility = await this.runAccessibilityScan();
-
-      // Screenshot if requested
-      let screenshot: string | undefined;
-      if (this.options.screenshots) {
-        const buffer = await this.page!.screenshot({ fullPage: false });
-        screenshot = `data:image/png;base64,${buffer.toString('base64')}`;
-      }
-
-      // Detect page type
-      const { pageType, pageTypeConfidence } = this.detectPageType(elements, title, path);
-
-      return {
-        url,
-        path,
-        title,
-        depth,
-        status,
-        loadTime,
-        description,
-        elements,
-        navigation,
-        screenshot,
-        accessibility,
-        pageType,
-        pageTypeConfidence,
-      };
-    } catch (error) {
-      // Return minimal info on error
-      return {
-        url,
-        path: url.replace(new URL(this.options.baseUrl).origin, ''),
-        title: 'Error',
-        depth,
-        status: 0,
-        loadTime: Date.now() - startTime,
-        elements: {
-          buttons: [],
-          links: [],
-          forms: [],
-          inputs: [],
-          selects: [],
-          checkboxes: [],
-          radios: [],
-        },
-        navigation: {
-          main: undefined,
-          footer: undefined,
-          breadcrumb: undefined,
-        },
-        pageType: 'other',
-        pageTypeConfidence: 0,
-      };
-    }
-  }
-
-  /**
-   * Discover all interactive elements on the page
-   */
-  private async discoverElements(): Promise<PageDefinition['elements']> {
-    const buttons = await this.discoverButtons();
-    const links = await this.discoverLinks();
-    const forms = await this.discoverForms();
-    const inputs = await this.discoverInputs();
-    const selects = await this.discoverSelects();
-    const checkboxes = await this.discoverCheckboxes();
-    const radios = await this.discoverRadios();
-
-    return {
-      buttons,
-      links,
-      forms,
-      inputs,
-      selects,
-      checkboxes,
-      radios,
-    };
-  }
-
-  /**
-   * Discover buttons
-   */
-  private async discoverButtons(): Promise<ElementInfo[]> {
-    const elements = await this.page!.$$(
-      'button, input[type="button"], input[type="submit"], [role="button"], a[role="button"]'
-    );
-
-    const buttons: ElementInfo[] = [];
-
-    for (const el of elements) {
-      try {
-        const info = await generateSelectorFromElement(el, this.page);
-        if (info.selector !== 'unknown') {
-          buttons.push(info);
-        }
-      } catch {
-        // Skip failed elements
-      }
-    }
-
-    return buttons;
-  }
-
-  /**
-   * Discover links
-   */
-  private async discoverLinks(): Promise<LinkInfo[]> {
-    const elements = await this.page!.$$('[href]');
-
-    const links: LinkInfo[] = [];
-    const baseUrl = new URL(this.options.baseUrl);
-
-    for (const el of elements) {
-      try {
-        const info = await generateSelectorFromElement(el, this.page);
-        const href = await el.getAttribute('href');
-
-        if (!href || href.startsWith('#') || href.startsWith('javascript:')) {
-          continue;
-        }
-
-        // Resolve relative URLs
-        let url: string;
-        try {
-          url = new URL(href, baseUrl.origin).href;
-        } catch {
-          continue;
-        }
-
-        const internal = url.startsWith(baseUrl.origin);
-
-        links.push({
-          ...info,
-          url,
-          internal,
-          visited: false,
-        });
-      } catch {
-        // Skip failed elements
-      }
-    }
-
-    return links;
-  }
-
-  /**
-   * Discover forms
-   */
-  private async discoverForms(): Promise<FormInfo[]> {
-    const forms = await this.page!.$$('[role="form"], form, .form, [data-form]');
-
-    const discovered: FormInfo[] = [];
-
-    for (const formEl of forms) {
-      try {
-        const selector = await generateSelectorFromElement(formEl, this.page).then(
-          i => i.selector
-        );
-
-        // Get form fields
-        const fields = await this.discoverFormFields(formEl);
-
-        if (fields.length === 0) continue;
-
-        // Get submit button
-        let submitButton: ElementInfo | undefined;
-        const submitBtn = await formEl.$(
-          'button[type="submit"], input[type="submit"], [type="submit"]'
-        );
-        if (submitBtn) {
-          submitButton = await generateSelectorFromElement(submitBtn, this.page);
-        }
-
-        // Detect form purpose
-        const purpose = this.detectFormPurpose(fields, selector);
-        const confidence = this.calculateFormPurposeConfidence(fields, purpose);
-
-        discovered.push({
-          selector,
-          purpose,
-          fields,
-          submitButton,
-          confidence,
-        });
-      } catch {
-        // Skip failed forms
-      }
-    }
-
-    return discovered;
-  }
-
-  /**
-   * Discover fields within a form
-   */
-  private async discoverFormFields(formEl: any): Promise<FieldInfo[]> {
-    const fields: FieldInfo[] = [];
-
-    // Get all input, select, textarea elements
-    const inputs = await formEl.$$(
-      'input:not([type="hidden"]):not([type="submit"]):not([type="button"]), select, textarea'
-    );
-
-    for (const input of inputs) {
-      try {
-        const info = await generateSelectorFromElement(input, this.page);
-
-        // Get field-specific attributes
-        const inputType = await input.getAttribute('type') || 'text';
-        const name = await input.getAttribute('name');
-        const required = (await input.getAttribute('required')) !== null;
-        const placeholder = await input.getAttribute('placeholder');
-
-        // Get validation attributes
-        const validation: FieldInfo['validation'] = {};
-        const min = await input.getAttribute('min');
-        const max = await input.getAttribute('max');
-        const pattern = await input.getAttribute('pattern');
-        const minLength = await input.getAttribute('minlength');
-        const maxLength = await input.getAttribute('maxlength');
-
-        if (min) validation.min = min;
-        if (max) validation.max = max;
-        if (pattern) validation.pattern = pattern;
-        if (minLength) validation.minLength = parseInt(minLength, 10);
-        if (maxLength) validation.maxLength = parseInt(maxLength, 10);
-
-        // Get options for selects
-        let options: string[] | undefined;
-        if (inputType === 'select-one' || inputType === 'select-multiple') {
-          options = await input.$$eval('option', (opts: HTMLOptionElement[]) =>
-            opts
-              .map(o => o.value || o.text)
-              .filter(Boolean)
-          );
-        }
-
-        fields.push({
-          ...info,
-          inputType,
-          name: name || undefined,
-          required,
-          placeholder: placeholder || undefined,
-          options,
-          validation: Object.keys(validation).length > 0 ? validation : undefined,
-        });
-      } catch {
-        // Skip failed fields
-      }
-    }
-
-    return fields;
-  }
-
-  /**
-   * Detect form purpose based on fields
-   */
-  private detectFormPurpose(fields: FieldInfo[], selector: string): FormInfo['purpose'] {
-    const fieldNames = fields.map(f => f.name?.toLowerCase() || '');
-    const selectorLower = selector.toLowerCase();
-
-    // Check for login form
-    if (
-      (fieldNames.includes('password') || fieldNames.includes('pass')) &&
-      (fieldNames.includes('email') ||
-        fieldNames.includes('username') ||
-        fieldNames.includes('user'))
-    ) {
-      return 'login';
-    }
-
-    // Check for signup form
-    if (
-      (fieldNames.includes('password') || fieldNames.includes('confirm')) &&
-      (fieldNames.includes('email') || fieldNames.includes('username'))
-    ) {
-      return 'signup';
-    }
-
-    // Check for search form
-    if (selectorLower.includes('search') || fieldNames.includes('q') || fieldNames.includes('search')) {
-      return 'search';
-    }
-
-    // Check for checkout form
-    if (
-      fieldNames.includes('card') ||
-      fieldNames.includes('cvv') ||
-      fieldNames.includes('expiry')
-    ) {
-      return 'checkout';
-    }
-
-    // Check for contact form
-    if (fieldNames.includes('message') || fieldNames.includes('subject')) {
-      return 'contact';
-    }
-
-    return 'other';
-  }
-
-  /**
-   * Calculate confidence for form purpose detection
-   */
-  private calculateFormPurposeConfidence(fields: FieldInfo[], purpose: FormInfo['purpose']): number {
-    // High confidence for login/signup with password field
-    if (purpose === 'login' || purpose === 'signup') {
-      if (fields.some(f => f.inputType === 'password')) {
-        return 0.9;
-      }
-    }
-
-    // Medium confidence for forms with clear naming
-    if (purpose !== 'other') {
-      return 0.7;
-    }
-
-    return 0.5;
-  }
-
-  /**
-   * Discover standalone input fields (not in forms)
-   */
-  private async discoverInputs(): Promise<FieldInfo[]> {
-    const inputs = await this.page!.$$(
-      'input:not([type="hidden"]):not([type="submit"]):not(form input), textarea:not(form textarea)'
-    );
-
-    const fields: FieldInfo[] = [];
-
-    for (const input of inputs) {
-      try {
-        const info = await generateSelectorFromElement(input, this.page);
-        const inputType = await input.getAttribute('type') || 'text';
-        const name = await input.getAttribute('name');
-        const required = (await input.getAttribute('required')) !== null;
-        const placeholder = await input.getAttribute('placeholder');
-
-        fields.push({
-          ...info,
-          inputType,
-          name: name || undefined,
-          required,
-          placeholder: placeholder || undefined,
-        });
-      } catch {
-        // Skip failed
-      }
-    }
-
-    return fields;
-  }
-
-  /**
-   * Discover select elements
-   */
-  private async discoverSelects(): Promise<FieldInfo[]> {
-    const selects = await this.page!.$$('select:not(form select)');
-
-    const fields: FieldInfo[] = [];
-
-    for (const select of selects) {
-      try {
-        const info = await generateSelectorFromElement(select, this.page);
-        const name = await select.getAttribute('name');
-        const required = (await select.getAttribute('required')) !== null;
-
-        // Get options
-        const options = await select.$$eval('option', opts =>
-          opts.map(o => (o as HTMLOptionElement).value).filter(Boolean)
-        );
-
-        fields.push({
-          ...info,
-          inputType: 'select-one',
-          name: name || undefined,
-          required,
-          options: options.length > 0 ? options : undefined,
-        });
-      } catch {
-        // Skip failed
-      }
-    }
-
-    return fields;
-  }
-
-  /**
-   * Discover checkboxes
-   */
-  private async discoverCheckboxes(): Promise<ElementInfo[]> {
-    const checkboxSelector = "input[type=\"checkbox\"]"; const checkboxes = await this.page!.$$(checkboxSelector);
-
-    const elements: ElementInfo[] = [];
-
-    for (const el of checkboxes) {
-      try {
-        const info = await generateSelectorFromElement(el, this.page);
-        if (info.selector !== 'unknown') {
-          elements.push(info);
-        }
-      } catch {
-        // Skip failed
-      }
-    }
-
-    return elements;
-  }
-
-  /**
-   * Discover radio buttons
-   */
-  private async discoverRadios(): Promise<ElementInfo[]> {
-    const radioSelector = "input[type=\"radio\"]"; const radios = await this.page!.$$(radioSelector);
-
-    const elements: ElementInfo[] = [];
-
-    for (const el of radios) {
-      try {
-        const info = await generateSelectorFromElement(el, this.page);
-        if (info.selector !== 'unknown') {
-          elements.push(info);
-        }
-      } catch {
-        // Skip failed
-      }
-    }
-
-    return elements;
-  }
-
-  /**
-   * Analyze page navigation structure
-   */
-  private async analyzeNavigation(): Promise<PageDefinition['navigation']> {
-    const navigation: PageDefinition['navigation'] = {};
-
-    // Main navigation (nav, menu, navbar)
-    const mainNav = await this.page!.$('nav, [role="navigation"], .nav, .navigation, .navbar, .menu');
-    if (mainNav) {
-      const navSelector = await generateSelectorFromElement(mainNav, this.page).then(
-        i => i.selector
-      );
-
-      const links = await mainNav.$$('[href]');
-      const items: LinkInfo[] = [];
-
-      for (const link of links) {
-        try {
-          const info = await generateSelectorFromElement(link, this.page);
-          const href = await link.getAttribute('href');
-          if (href) {
-            const url = new URL(href, new URL(this.options.baseUrl).origin).href;
-            items.push({
-              ...info,
-              url,
-              internal: url.startsWith(new URL(this.options.baseUrl).origin),
-              visited: false,
-            });
-          }
-        } catch {
-          // Skip
-        }
-      }
-
-      navigation.main = { selector: navSelector, items };
-    }
-
-    // Breadcrumb
-    const breadcrumb = await this.page!.$(
-      '[aria-label="breadcrumb"], .breadcrumb, .breadcrumbs, ol.breadcrumb'
-    );
-    if (breadcrumb) {
-      const crumbSelector = await generateSelectorFromElement(breadcrumb, this.page).then(
-        i => i.selector
-      );
-
-      const items = await breadcrumb.$$('[href], span, li');
-      const breadcrumbs: Array<{ text: string; url: string }> = [];
-
-      for (const item of items) {
-        try {
-          const text = await item.textContent();
-          const href = await item.getAttribute('href');
-          if (text && text.trim()) {
-            breadcrumbs.push({
-              text: text.trim(),
-              url: href || '',
-            });
-          }
-        } catch {
-          // Skip
-        }
-      }
-
-      if (breadcrumbs.length > 0) {
-        navigation.breadcrumb = { selector: crumbSelector, items: breadcrumbs };
-      }
-    }
-
-    // Pagination
-    const pagination = await this.page!.$(
-      '[aria-label="pagination"], .pagination, [role="navigation"] nav'
-    );
-    if (pagination) {
-      const pageSelector = await generateSelectorFromElement(pagination, this.page).then(
-        i => i.selector
-      );
-
-      const nextLink = await pagination.$('a[rel="next"], .next, [aria-label="next"]');
-      const prevLink = await pagination.$('a[rel="prev"], .prev, [aria-label="previous"]');
-
-      const paginationInfo: PageDefinition['navigation']['pagination'] = {
-        selector: pageSelector,
-      };
-
-      if (nextLink) {
-        const info = await generateSelectorFromElement(nextLink, this.page);
-        const href = await nextLink.getAttribute('href');
-        if (href) {
-          paginationInfo.nextPage = {
-            ...info,
-            url: new URL(href, new URL(this.options.baseUrl).origin).href,
-            internal: true,
-            visited: false,
-          };
-        }
-      }
-
-      if (prevLink) {
-        const info = await generateSelectorFromElement(prevLink, this.page);
-        const href = await prevLink.getAttribute('href');
-        if (href) {
-          paginationInfo.prevPage = {
-            ...info,
-            url: new URL(href, new URL(this.options.baseUrl).origin).href,
-            internal: true,
-            visited: false,
-          };
-        }
-      }
-
-      navigation.pagination = paginationInfo;
-    }
-
-    // Footer
-    const footer = await this.page!.$('footer, [role="contentinfo"], .footer');
-    if (footer) {
-      const footerSelector = await generateSelectorFromElement(footer, this.page).then(
-        i => i.selector
-      );
-
-      const links = await footer.$$('[href]');
-      const items: LinkInfo[] = [];
-
-      for (const link of links) {
-        try {
-          const info = await generateSelectorFromElement(link, this.page);
-          const href = await link.getAttribute('href');
-          if (href) {
-            const url = new URL(href, new URL(this.options.baseUrl).origin).href;
-            items.push({
-              ...info,
-              url,
-              internal: url.startsWith(new URL(this.options.baseUrl).origin),
-              visited: false,
-            });
-          }
-        } catch {
-          // Skip
-        }
-      }
-
-      navigation.footer = { selector: footerSelector, items };
-    }
-
-    return navigation;
-  }
-
-  /**
-   * Run accessibility scan using axe-core
-   */
-  private async runAccessibilityScan(): Promise<A11ySnapshot | undefined> {
-    try {
-      // Inject axe-core
-      await this.page!.addScriptTag({
-        url: 'https://unpkg.com/axe-core@4.8.2/axe.min.js',
-      });
-
-      // Run axe
-      const results = await this.page!.evaluate(() => {
-        return new Promise((resolve) => {
-          // @ts-ignore
-          if (typeof axe !== 'undefined') {
-            // @ts-ignore
-            axe.run((err: any, results: any) => {
-              if (err) {
-                resolve({ violations: [], passes: [], incomplete: [] });
-              } else {
-                resolve(results);
-              }
-            });
-          } else {
-            resolve({ violations: [], passes: [], incomplete: [] });
-          }
-        });
-      });
-
-      const axeResults = results as any;
-
-      // Process violations
-      const violations = (axeResults.violations || []).map((v: any) => ({
-        id: v.id,
-        impact: v.impact || 'moderate',
-        description: v.description || v.help || 'Accessibility issue',
-        nodes: v.nodes?.length || 0,
-        selectors: (v.nodes || []).slice(0, 5).map((n: any) => n.target?.[0] || '').filter(Boolean),
-      }));
-
-      // Calculate score
-      const criticalCount = violations.filter((v: any) => v.impact === 'critical').length;
-      const seriousCount = violations.filter((v: any) => v.impact === 'serious').length;
-      const moderateCount = violations.filter((v: any) => v.impact === 'moderate').length;
-      const minorCount = violations.filter((v: any) => v.impact === 'minor').length;
-
-      const score = Math.max(
-        0,
-        100 - (criticalCount * 25 + seriousCount * 10 + moderateCount * 5 + minorCount * 1)
-      );
-
-      return {
-        score: Math.round(score),
-        violations,
-        passes: axeResults.passes?.length || 0,
-        incomplete: axeResults.incomplete?.length || 0,
-      };
-    } catch {
-      return undefined;
-    }
-  }
-
-  /**
-   * Detect page type
-   */
-  private detectPageType(
-    elements: PageDefinition['elements'],
-    title: string,
-    path: string
-  ): { pageType: PageDefinition['pageType']; pageTypeConfidence: number } {
-    const pathLower = path.toLowerCase();
-    const titleLower = title.toLowerCase();
-
-    // Homepage
-    if (path === '/' || path === '' || pathLower === '/home' || pathLower === '/index') {
-      return { pageType: 'homepage', pageTypeConfidence: 0.95 };
-    }
-
-    // Login page
-    if (
-      pathLower.includes('/login') ||
-      pathLower.includes('/signin') ||
-      pathLower.includes('/auth') ||
-      titleLower.includes('login') ||
-      titleLower.includes('sign in')
-    ) {
-      return { pageType: 'login', pageTypeConfidence: 0.9 };
-    }
-
-    // Signup page
-    if (
-      pathLower.includes('/signup') ||
-      pathLower.includes('/register') ||
-      pathLower.includes('/join') ||
-      titleLower.includes('sign up') ||
-      titleLower.includes('register')
-    ) {
-      return { pageType: 'signup', pageTypeConfidence: 0.9 };
-    }
-
-    // Dashboard
-    if (
-      pathLower.includes('/dashboard') ||
-      pathLower.includes('/my-account') ||
-      pathLower.includes('/profile') ||
-      titleLower.includes('dashboard')
-    ) {
-      return { pageType: 'dashboard', pageTypeConfidence: 0.85 };
-    }
-
-    // Listing page (multiple items, pagination)
-    if (elements.links.length > 20 || elements.forms.some(f => f.purpose === 'filter')) {
-      return { pageType: 'listing', pageTypeConfidence: 0.7 };
-    }
-
-    // Form page
-    if (elements.forms.length > 0) {
-      return { pageType: 'form', pageTypeConfidence: 0.75 };
-    }
-
-    return { pageType: 'other', pageTypeConfidence: 0.5 };
-  }
-
-  /**
-   * Clean up resources
-   */
-  async cleanup(): Promise<void> {
-    try {
-      if (this.page) await this.page.close();
-      if (this.context) await this.context.close();
-      if (this.browser) await this.browser.close();
-    } catch {
-      // Ignore cleanup errors
-    }
-  }
-}