qa360 2.3.0 → 2.3.1

package/dist/core/crawler/csp-handler.js

@@ -0,0 +1,415 @@
+/**
+ * CSP Handler
+ *
+ * P1 - Content Security Policy management
+ *
+ * Supports:
+ * - CSP header parsing and validation
+ * - CSP directive analysis
+ * - CSP violation detection
+ * - CSP report-only mode
+ * - CSP nonce and hash validation
+ *
+ * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP
+ */
+/**
+ * CSP Handler class
+ */
+export class CSPHandler {
+    page;
+    violations = [];
+    constructor(page) {
+        this.page = page;
+    }
+    /**
+     * Get CSP header from response
+     */
+    async getCSPHeader() {
+        const response = await this.page.evaluate(() => {
+            const metaTags = document.querySelectorAll('meta[http-equiv="Content-Security-Policy"]');
+            if (metaTags.length > 0) {
+                return metaTags[0].getAttribute('content');
+            }
+            return null;
+        });
+        if (response)
+            return response;
+        // Check via cdp (for actual headers)
+        try {
+            const cdp = this.page.context()?.cdp;
+            if (cdp) {
+                const headers = await this.page.evaluate(() => {
+                    // Try to get from fetch
+                    return fetch(window.location.href)
+                        .then(r => {
+                        const csp = r.headers.get('Content-Security-Policy');
+                        const cspReportOnly = r.headers.get('Content-Security-Policy-Report-Only');
+                        return csp || cspReportOnly || null;
+                    })
+                        .catch(() => null);
+                });
+                return headers;
+            }
+        }
+        catch {
+            // Ignore
+        }
+        return null;
+    }
+    /**
+     * Parse CSP header value
+     */
+    parseCSP(cspValue, reportOnly = false) {
+        const directives = new Map();
+        const parts = cspValue.split(';').map(s => s.trim()).filter(Boolean);
+        let reportEndpoint;
+        let reportUri;
+        for (const part of parts) {
+            const spaceIndex = part.indexOf(' ');
+            if (spaceIndex === -1)
+                continue;
+            const name = part.slice(0, spaceIndex).trim();
+            const valueStr = part.slice(spaceIndex + 1).trim();
+            const values = valueStr.split(/\s+/);
+            if (name === 'report-to' || name === 'report-uri') {
+                if (name === 'report-to')
+                    reportEndpoint = valueStr;
+                if (name === 'report-uri')
+                    reportUri = valueStr;
+                continue;
+            }
+            const directive = {
+                name,
+                values,
+                allowsUnsafeInline: values.includes("'unsafe-inline'"),
+                allowsUnsafeEval: values.includes("'unsafe-eval'"),
+                allowsSelf: values.includes("'self'"),
+                allowsWildcard: values.includes('*'),
+                hashes: values.filter(v => v.startsWith("'sha256-") || v.startsWith("'sha384-") || v.startsWith("'sha512-")),
+            };
+            // Extract nonce
+            const nonceValue = values.find(v => v.startsWith("'nonce-"));
+            if (nonceValue) {
+                directive.nonce = nonceValue.slice(7, -1);
+            }
+            directives.set(name, directive);
+        }
+        return {
+            raw: cspValue,
+            directives,
+            reportOnly,
+            reportEndpoint,
+            reportUri,
+        };
+    }
+    /**
+     * Get CSP policy from page
+     */
+    async getCSPPolicy() {
+        const cspHeader = await this.getCSPHeader();
+        if (!cspHeader)
+            return null;
+        // Check if it's report-only
+        const isReportOnly = cspHeader.includes('report-only') ||
+            await this.page.evaluate(() => {
+                const meta = document.querySelector('meta[http-equiv="Content-Security-Policy-Report-Only"]');
+                return meta !== null;
+            });
+        return this.parseCSP(cspHeader, isReportOnly);
+    }
+    /**
+     * Validate CSP policy
+     */
+    async validateCSP() {
+        const policy = await this.getCSPPolicy();
+        const issues = [];
+        const warnings = [];
+        const recommendations = [];
+        let score = 100;
+        if (!policy) {
+            return {
+                hasCSP: false,
+                securityScore: 0,
+                issues: ['No Content-Security-Policy header found'],
+                warnings: [],
+                recommendations: [
+                    'Implement a Content-Security-Policy header',
+                    'Start with report-only mode to test',
+                ],
+            };
+        }
+        // Check if report-only mode (good for testing, but not for production)
+        if (policy.reportOnly) {
+            warnings.push('CSP is in report-only mode (not enforcing)');
+            score -= 10;
+        }
+        // Check for unsafe-inline
+        for (const [name, directive] of policy.directives) {
+            if (directive.allowsUnsafeInline) {
+                if (name === 'script-src') {
+                    issues.push(`script-src allows 'unsafe-inline' (XSS risk)`);
+                    score -= 30;
+                }
+                else if (name === 'style-src') {
+                    warnings.push(`style-src allows 'unsafe-inline' (consider using nonces)`);
+                    score -= 10;
+                }
+            }
+            if (directive.allowsUnsafeEval) {
+                issues.push(`${name} allows 'unsafe-eval' (security risk)`);
+                score -= 20;
+            }
+            if (directive.allowsWildcard && name !== 'img-src' && name !== 'media-src') {
+                warnings.push(`${name} allows wildcard (*) (consider tightening)`);
+                score -= 10;
+            }
+            if (!directive.allowsSelf && !directive.allowsWildcard && directive.values.length === 0) {
+                warnings.push(`${name} has no sources defined`);
+            }
+        }
+        // Check for default-src
+        if (!policy.directives.has('default-src')) {
+            recommendations.push('Add default-src directive for fallback protection');
+            score -= 15;
+        }
+        // Check for script-src
+        if (!policy.directives.has('script-src')) {
+            recommendations.push('Add explicit script-src directive');
+            score -= 10;
+        }
+        // Check for object-src (prevents plugins)
+        if (!policy.directives.has('object-src') && !policy.directives.has('default-src')) {
+            recommendations.push('Add object-src \'none\' to prevent plugin execution');
+            score -= 5;
+        }
+        // Check for base-uri
+        if (!policy.directives.has('base-uri')) {
+            recommendations.push('Add base-uri directive to prevent base tag injection');
+        }
+        // Check for form-action
+        if (!policy.directives.has('form-action')) {
+            recommendations.push('Add form-action directive to restrict form submissions');
+        }
+        // Check for frame-ancestors (clickjacking protection)
+        if (!policy.directives.has('frame-ancestors')) {
+            recommendations.push('Add frame-ancestors directive to prevent clickjacking');
+            score -= 5;
+        }
+        // Check for upgrade-insecure-requests
+        if (!policy.directives.has('upgrade-insecure-requests')) {
+            recommendations.push('Consider upgrade-insecure-requests for HTTPS migration');
+        }
+        return {
+            hasCSP: true,
+            policy,
+            securityScore: Math.max(0, score),
+            issues,
+            warnings,
+            recommendations,
+        };
+    }
+    /**
+     * Get directive by name
+     */
+    async getDirective(directiveName) {
+        const policy = await this.getCSPPolicy();
+        if (!policy)
+            return null;
+        return policy.directives.get(directiveName) || null;
+    }
+    /**
+     * Check if a resource would be allowed by CSP
+     */
+    async checkResourceAllowed(resourceType, url) {
+        const policy = await this.getCSPPolicy();
+        if (!policy)
+            return true; // No CSP = allowed
+        // Find matching directive
+        let directive = policy.directives.get(`${resourceType}-src`);
+        if (!directive) {
+            directive = policy.directives.get('default-src');
+        }
+        if (!directive)
+            return true;
+        // Check if URL matches any source
+        const parsedUrl = new URL(url);
+        // Check 'self'
+        const pageUrl = new URL(this.page.url());
+        if (directive.allowsSelf && parsedUrl.origin === pageUrl.origin) {
+            return true;
+        }
+        // Check wildcard
+        if (directive.allowsWildcard) {
+            return true;
+        }
+        // Check specific sources
+        for (const value of directive.values) {
+            if (value.startsWith('http:') || value.startsWith('https:') || value.startsWith('/')) {
+                try {
+                    const sourcePattern = new URL(value);
+                    if (sourcePattern.origin === parsedUrl.origin) {
+                        return true;
+                    }
+                }
+                catch {
+                    // Might be a path pattern
+                    if (value.startsWith('/') && parsedUrl.pathname.startsWith(value)) {
+                        return true;
+                    }
+                }
+            }
+        }
+        return false;
+    }
+    /**
+     * Setup CSP violation monitoring
+     */
+    async setupViolationMonitoring() {
+        await this.page.evaluate(() => {
+            document.addEventListener('securitypolicyviolation', (event) => {
+                const e = event;
+                // Store violation for retrieval
+                const violations = window.__cspViolations || [];
+                violations.push({
+                    directive: e.violatedDirective,
+                    blockedURI: e.blockedURI,
+                    policy: e.originalPolicy,
+                    sourceFile: e.sourceFile,
+                    lineNumber: e.lineNumber,
+                    columnNumber: e.columnNumber,
+                    timestamp: Date.now(),
+                });
+                window.__cspViolations = violations;
+            });
+        });
+        // Listen for console messages about CSP
+        this.page.on('console', (msg) => {
+            const text = msg.text();
+            if (text.includes('Refused to') || text.includes('CSP')) {
+                this.violations.push({
+                    directive: 'unknown',
+                    timestamp: Date.now(),
+                });
+            }
+        });
+    }
+    /**
+     * Get collected violations
+     */
+    getViolations() {
+        return [...this.violations];
+    }
+    /**
+     * Clear violations
+     */
+    clearViolations() {
+        this.violations = [];
+    }
+    /**
+     * Generate a nonce value for inline scripts
+     */
+    generateNonce() {
+        const array = new Uint8Array(16);
+        crypto.getRandomValues(array);
+        return Array.from(array, b => b.toString(16).padStart(2, '0')).join('');
+    }
+    /**
+     * Inject nonce into inline scripts
+     */
+    async injectNonces() {
+        const nonces = new Map();
+        await this.page.evaluate((nonceMap) => {
+            const scripts = document.querySelectorAll('script:not([src])');
+            scripts.forEach((script, i) => {
+                const nonce = 'nonce-' + Math.random().toString(36).substring(7);
+                script.setAttribute('nonce', nonce);
+                nonceMap[`script-${i}`] = nonce;
+            });
+        }, {});
+        return nonces;
+    }
+    /**
+     * Get all CSP-related meta tags
+     */
+    async getMetaTags() {
+        return await this.page.evaluate(() => {
+            const metaTags = document.querySelectorAll('meta[http-equiv*="Content-Security-Policy" i]');
+            return Array.from(metaTags).map(meta => ({
+                httpEquiv: meta.getAttribute('http-equiv') || '',
+                content: meta.getAttribute('content') || '',
+            }));
+        });
+    }
+    /**
+     * Check for common CSP bypass patterns
+     */
+    async checkBypassPatterns() {
+        const bypasses = [];
+        const policy = await this.getCSPPolicy();
+        if (!policy)
+            return bypasses;
+        // Check for unsafe-inline with script-src (XSS risk)
+        const scriptSrc = policy.directives.get('script-src');
+        if (scriptSrc?.allowsUnsafeInline && !scriptSrc.nonce && scriptSrc.hashes.length === 0) {
+            bypasses.push('script-src with unsafe-inline allows arbitrary inline scripts');
+        }
+        // Check for data: in script-src
+        if (scriptSrc?.values.some((v) => v === 'data:')) {
+            bypasses.push('script-src allows data: URLs (can execute arbitrary scripts)');
+        }
+        // Check for unsafe-eval
+        if (scriptSrc?.allowsUnsafeEval) {
+            bypasses.push('script-src allows eval() (dynamic code execution risk)');
+        }
+        // Check for missing object-src (fallback to default-src which might be loose)
+        if (!policy.directives.has('object-src') && !policy.directives.has('default-src')) {
+            bypasses.push('Missing object-src allows plugin execution');
+        }
+        // Check for frame-src allowing potentially dangerous origins
+        const frameSrc = policy.directives.get('frame-src') || policy.directives.get('child-src');
+        if (frameSrc?.allowsWildcard) {
+            bypasses.push('frame-src allows any origin (iframe injection risk)');
+        }
+        return bypasses;
+    }
+    /**
+     * Compare two CSP policies
+     */
+    comparePolicies(policy1, policy2) {
+        const stricter = [];
+        const looser = [];
+        const onlyIn1 = [];
+        const onlyIn2 = [];
+        const allDirectives = new Set([
+            ...policy1.directives.keys(),
+            ...policy2.directives.keys(),
+        ]);
+        for (const directive of allDirectives) {
+            const d1 = policy1.directives.get(directive);
+            const d2 = policy2.directives.get(directive);
+            if (!d1) {
+                onlyIn2.push(directive);
+            }
+            else if (!d2) {
+                onlyIn1.push(directive);
+            }
+            else {
+                // Compare values
+                if (d1.values.length < d2.values.length) {
+                    stricter.push(directive);
+                }
+                else if (d1.values.length > d2.values.length) {
+                    looser.push(directive);
+                }
+            }
+        }
+        return { stricter, looser, onlyIn1, onlyIn2 };
+    }
+}
+/**
+ * Factory function to create CSP Handler
+ */
+export function createCSPHandler(page) {
+    return new CSPHandler(page);
+}
+export default CSPHandler;

package/dist/core/crawler/download-handler.d.ts

@@ -0,0 +1,155 @@
+/**
+ * Download Handler
+ *
+ * P1 - File download management
+ *
+ * Supports:
+ * - Click and wait for download
+ * - Download verification
+ * - Multiple downloads handling
+ * - Download timeout
+ * - File size validation
+ *
+ * @see https://playwright.dev/docs/downloads
+ */
+export interface DownloadConfig {
+    /** Download directory */
+    downloadDir?: string;
+    /** Timeout for download to start (ms) */
+    timeout?: number;
+    /** Save downloaded files */
+    saveFiles?: boolean;
+}
+export interface DownloadInfo {
+    /** Suggested filename */
+    filename: string;
+    /** Save path (if saved) */
+    path?: string;
+    /** Download URL */
+    url?: string;
+    /** File MIME type */
+    mimeType?: string;
+    /** File size in bytes */
+    size?: number;
+    /** Download timestamp */
+    timestamp: number;
+}
+export interface DownloadActionOptions {
+    /** Action to perform before download (click, hover, etc) */
+    action?: 'click' | 'dblclick' | 'hover';
+    /** Wait for download to complete */
+    waitForCompletion?: boolean;
+    /** Timeout (ms) */
+    timeout?: number;
+}
+export interface DownloadVerification {
+    /** Expected filename pattern */
+    filenamePattern?: string | RegExp;
+    /** Minimum file size (bytes) */
+    minSize?: number;
+    /** Maximum file size (bytes) */
+    maxSize?: number;
+    /** Expected MIME type */
+    mimeType?: string;
+}
+export interface DownloadResult {
+    success: boolean;
+    download?: DownloadInfo;
+    rawDownload?: any;
+    error?: string;
+}
+/**
+ * Download Handler class
+ */
+export declare class DownloadHandler {
+    private downloads;
+    private downloadDir;
+    private defaultTimeout;
+    private saveFiles;
+    private stats;
+    constructor(config?: DownloadConfig);
+    private ensureDownloadDir;
+    /**
+     * Setup download handling for a page
+     */
+    setupDownloadHandler(page: any): void;
+    /**
+     * Click element and wait for download
+     */
+    clickAndWaitForDownload(page: any, selector: string, options?: DownloadActionOptions): Promise<DownloadResult & {
+        download?: any;
+    }>;
+    /**
+     * Click multiple elements and wait for downloads
+     */
+    clickAndWaitForDownloads(page: any, selectors: string[], options?: DownloadActionOptions): Promise<Array<DownloadResult & {
+        download?: any;
+    }>>;
+    /**
+     * Wait for download by filename pattern
+     */
+    waitForDownload(page: any, filenamePattern: string | RegExp, timeout?: number): Promise<DownloadResult>;
+    /**
+     * Verify a download
+     */
+    verifyDownload(download: any, verification: any): Promise<any>;
+    /**
+     * Get all downloads
+     */
+    getAllDownloads(): DownloadInfo[];
+    /**
+     * Get download by filename
+     */
+    getDownload(filename: string): DownloadInfo | undefined;
+    /**
+     * Get last download
+     */
+    getLastDownload(): DownloadInfo | undefined;
+    /**
+     * Clear all downloads
+     */
+    clearDownloads(): void;
+    /**
+     * Delete downloaded files
+     */
+    clearDownloadFiles(): Promise<void>;
+    /**
+     * Get download directory
+     */
+    getDownloadDir(): string;
+    /**
+     * Set download directory
+     */
+    setDownloadDir(dir: string): void;
+    /**
+     * Get download count
+     */
+    get downloadCount(): number;
+    /**
+     * Assert download exists
+     */
+    assertDownloadExists(filenamePattern: string | RegExp): DownloadResult;
+    /**
+     * Get download statistics
+     */
+    getStats(): {
+        totalDownloads: number;
+        successfulDownloads: number;
+        failedDownloads: number;
+        totalSize: number;
+        averageSize: number;
+    };
+    /**
+     * Get download history
+     */
+    getHistory(): DownloadInfo[];
+    /**
+     * Reset statistics
+     */
+    resetStats(): void;
+}
+/**
+ * Factory function to create Download Handler
+ */
+export declare function createDownloadHandler(config?: DownloadConfig): DownloadHandler;
+export default DownloadHandler;