securl 1.4.1

package/dist/html-page-analysis.js

@@ -0,0 +1,459 @@
+import { URL } from "node:url";
+import { parse } from "node-html-parser";
+import { analyzeAiSurface, detectHtmlTechnologies } from "./htmlInsights.js";
+import { collectClientExposureSignals, collectPassiveLeakSignals, collectSameSiteHosts, extractHtmlTitle, getHtmlTitle, isAccessDeniedHtml, normalizeHtmlSignature, } from "./html-extraction.js";
+import { collectLibraryFingerprints } from "./libraryRisk.js";
+import { normalizeDiscoveredPath, rankDiscoveredPaths } from "./path-discovery.js";
+import { requestText } from "./network.js";
+import { headerValue, unique } from "./utils.js";
+const emptySriCoverage = () => ({
+    externalScripts: 0,
+    externalStylesheets: 0,
+    scriptsWithSri: 0,
+    stylesheetsWithSri: 0,
+    coveragePercent: 100,
+    issues: [],
+    strengths: [],
+});
+const hasValidSriPrefix = (integrity) => Boolean(integrity && /\bsha(?:256|384|512)-[A-Za-z0-9+/=]+/.test(integrity));
+const isAbsoluteExternalResource = (value, finalUrl) => {
+    if (!value || !/^(?:https?:)?\/\//i.test(value)) {
+        return false;
+    }
+    try {
+        return new URL(value, finalUrl).hostname !== finalUrl.hostname;
+    }
+    catch {
+        return false;
+    }
+};
+const calculateSriCoverage = (scriptElements, stylesheetElements, finalUrl) => {
+    const externalScripts = scriptElements.filter((script) => isAbsoluteExternalResource(script.getAttribute("src"), finalUrl));
+    const externalStylesheets = stylesheetElements.filter((link) => isAbsoluteExternalResource(link.getAttribute("href"), finalUrl));
+    const scriptsWithSri = externalScripts.filter((script) => hasValidSriPrefix(script.getAttribute("integrity"))).length;
+    const stylesheetsWithSri = externalStylesheets.filter((link) => hasValidSriPrefix(link.getAttribute("integrity"))).length;
+    const total = externalScripts.length + externalStylesheets.length;
+    const protectedTotal = scriptsWithSri + stylesheetsWithSri;
+    const coveragePercent = total ? Math.round((protectedTotal / total) * 100) : 100;
+    const issues = [];
+    const strengths = [];
+    if (total > 0 && coveragePercent === 0) {
+        issues.push("External scripts or stylesheets are loaded without Subresource Integrity coverage.");
+    }
+    else if (total > 0 && coveragePercent < 100) {
+        issues.push(`Subresource Integrity coverage is partial (${coveragePercent}%).`);
+    }
+    else if (total > 0) {
+        strengths.push("All externally hosted scripts and stylesheets include Subresource Integrity hashes.");
+    }
+    return {
+        externalScripts: externalScripts.length,
+        externalStylesheets: externalStylesheets.length,
+        scriptsWithSri,
+        stylesheetsWithSri,
+        coveragePercent,
+        issues,
+        strengths,
+    };
+};
+const extractVersion = (value, pattern) => value.match(pattern)?.slice(1).find(Boolean) || null;
+const extractVersionNear = (value, marker) => {
+    const index = value.toLowerCase().indexOf(marker.toLowerCase());
+    if (index === -1) {
+        return null;
+    }
+    return value.slice(index, index + marker.length + 80).match(/\d+\.\d+\.\d+|\d+\.\d+/)?.[0] || null;
+};
+const detectFrameworkVersionLeaks = (html, metaGenerator, externalScriptUrls, externalStylesheetUrls) => {
+    const signals = `${html}\n${metaGenerator || ""}\n${externalScriptUrls.join("\n")}\n${externalStylesheetUrls.join("\n")}`;
+    const lower = signals.toLowerCase();
+    const generator = (metaGenerator || "").toLowerCase();
+    const leaks = [];
+    const seen = new Set();
+    const addLeak = (framework, versionHint, evidence, risk = "medium") => {
+        const key = `${framework}:${versionHint || evidence}`;
+        if (seen.has(key))
+            return;
+        seen.add(key);
+        leaks.push({ framework, versionHint, evidence, risk });
+    };
+    if (/__REACT_VERSION__|react@|react-dom@/i.test(signals)) {
+        addLeak("React", extractVersion(signals, /react(?:-dom)?@(\d+\.\d+\.\d+)/i), "React version marker is visible in page source or asset references.");
+    }
+    if (/ng\.version\.full|@angular\/core/i.test(signals)) {
+        addLeak("Angular", extractVersionNear(signals, "@angular/core"), "Angular version marker is visible in page source.");
+    }
+    if (/Vue\.version|__VUE_VERSION__|\/vue@\d/i.test(signals)) {
+        addLeak("Vue", extractVersionNear(signals, "Vue.version") || extractVersionNear(signals, "__VUE_VERSION__") || extractVersionNear(signals, "/vue@"), "Vue version marker is visible in page source or asset references.");
+    }
+    if (lower.includes("__next_data__") || lower.includes("/_next/static")) {
+        addLeak("Next.js", null, "Next.js runtime markers are visible in the fetched HTML.");
+    }
+    if (/jQuery v/i.test(signals) || /jquery[-.@/](\d+\.\d+\.\d+)/i.test(signals)) {
+        addLeak("jQuery", extractVersion(signals, /jQuery v(\d+\.\d+\.\d+)|jquery[-.@/](\d+\.\d+\.\d+)/i), "jQuery version marker or asset path is visible.");
+    }
+    if (/bootstrap@|Bootstrap v/i.test(signals)) {
+        addLeak("Bootstrap", extractVersion(signals, /(?:bootstrap@|Bootstrap v)(\d+\.\d+\.\d+)/i), "Bootstrap version marker is visible.");
+    }
+    if (lower.includes("wp-content/") || lower.includes("wp-includes/") || generator.includes("wordpress")) {
+        addLeak("WordPress", extractVersion(signals, /WordPress\s+(\d+\.\d+(?:\.\d+)?)/i), "WordPress public page markers are visible.", "low");
+    }
+    if (lower.includes("sites/default/files/") || lower.includes("drupal.js") || generator.includes("drupal")) {
+        addLeak("Drupal", extractVersion(signals, /Drupal\s+(\d+\.\d+(?:\.\d+)?)/i), "Drupal public page markers are visible.", "low");
+    }
+    if (lower.includes("/media/joomla_version.xml") || generator.includes("joomla")) {
+        addLeak("Joomla", extractVersion(signals, /Joomla!?\s+(\d+\.\d+(?:\.\d+)?)/i), "Joomla public page markers are visible.", "low");
+    }
+    return leaks;
+};
+const summarizeEvidence = (values, limit = 6) => unique(values.filter((value) => Boolean(value && value.trim())).map((value) => value.trim())).slice(0, limit);
+const detectSuspiciousScriptSignals = (scriptElements, externalScriptUrls, finalUrl) => {
+    const signals = [];
+    const inlineBodies = scriptElements
+        .filter((script) => !script.getAttribute("src"))
+        .map((script) => script.text.slice(0, 20_000));
+    const inlineCorpus = inlineBodies.join("\n");
+    const obfuscationMarkers = summarizeEvidence([
+        /\beval\s*\(/.test(inlineCorpus) ? "eval(...)" : null,
+        /\batob\s*\(/.test(inlineCorpus) ? "atob(...)" : null,
+        /String\.fromCharCode\s*\(/.test(inlineCorpus) ? "String.fromCharCode(...)" : null,
+        /\bunescape\s*\(/.test(inlineCorpus) ? "unescape(...)" : null,
+        /[A-Za-z0-9+/]{180,}={0,2}/.test(inlineCorpus) ? "long base64-like string" : null,
+    ]);
+    if (obfuscationMarkers.length) {
+        signals.push({
+            category: "obfuscation",
+            severity: "warning",
+            title: "Obfuscated inline script markers visible",
+            detail: "The fetched page contains inline JavaScript patterns often used by packers, tag loaders, or injected scripts. Review the source before treating this as compromise proof.",
+            evidence: obfuscationMarkers,
+        });
+    }
+    const dynamicLoaderMarkers = summarizeEvidence([
+        /document\.write\s*\([^)]*<script/i.test(inlineCorpus) ? "document.write(<script...)" : null,
+        /createElement\s*\(\s*['"]script['"]\s*\)/i.test(inlineCorpus) ? "createElement('script')" : null,
+        /\.appendChild\s*\([^)]*script/i.test(inlineCorpus) ? "appendChild(script)" : null,
+    ]);
+    if (dynamicLoaderMarkers.length) {
+        signals.push({
+            category: "dynamic_loader",
+            severity: "info",
+            title: "Dynamic script loader markers visible",
+            detail: "The page dynamically creates or writes script elements. That is common for tag managers, but it increases review value when combined with weak CSP or unfamiliar third-party hosts.",
+            evidence: dynamicLoaderMarkers,
+        });
+    }
+    const suspiciousHosts = summarizeEvidence(externalScriptUrls
+        .map((url) => {
+        const parsed = new URL(url);
+        if (parsed.hostname === finalUrl.hostname)
+            return null;
+        return /(^|\.)xn--|\.duckdns\.org$|\.ddns\.net$|\.no-ip\./i.test(parsed.hostname)
+            ? parsed.hostname
+            : null;
+    }));
+    if (suspiciousHosts.length) {
+        signals.push({
+            category: "suspicious_host",
+            severity: "warning",
+            title: "Suspicious script host pattern visible",
+            detail: "One or more script hosts use punycode or dynamic-DNS style naming. Review ownership and expected use before trusting loaded code.",
+            evidence: suspiciousHosts,
+        });
+    }
+    return signals;
+};
+export async function fetchHtmlDocument(finalUrl) {
+    const response = await requestText(finalUrl);
+    const contentType = headerValue(response.headers, "content-type") || "";
+    if (!contentType.toLowerCase().includes("text/html")) {
+        return null;
+    }
+    const html = response.body;
+    return {
+        html,
+        pageTitle: getHtmlTitle(html),
+        signature: normalizeHtmlSignature(html),
+    };
+}
+export function analyzeHtmlSecurity(finalUrl, document) {
+    try {
+        if (!document) {
+            return {
+                fetched: false,
+                pageUrl: finalUrl.toString(),
+                pageTitle: null,
+                metaGenerator: null,
+                forms: [],
+                sameSiteHosts: [],
+                externalScriptDomains: [],
+                externalStylesheetDomains: [],
+                insecureResourceUrls: [],
+                inlineScriptCount: 0,
+                inlineStyleCount: 0,
+                missingSriScriptUrls: [],
+                sriCoverage: emptySriCoverage(),
+                firstPartyPaths: [],
+                passiveLeakSignals: [],
+                clientExposureSignals: [],
+                libraryFingerprints: [],
+                libraryRiskSignals: [],
+                frameworkVersionLeaks: [],
+                suspiciousScriptSignals: [],
+                detectedTechnologies: [],
+                aiSurface: {
+                    detected: false,
+                    assistantVisible: false,
+                    aiPageSignals: [],
+                    vendors: [],
+                    discoveredPaths: [],
+                    disclosures: [],
+                    privacySignals: [],
+                    governanceSignals: [],
+                    issues: ["Primary response was not HTML, so AI surface inspection was skipped."],
+                    strengths: [],
+                },
+                issues: ["Primary response was not HTML, so page content inspection was skipped."],
+                strengths: [],
+            };
+        }
+        const html = document.html;
+        const issues = [];
+        const strengths = [];
+        const root = parse(html);
+        const pageTitle = document.pageTitle || root.querySelector("title")?.text?.trim() || null;
+        const metaGenerator = root.querySelector('meta[name="generator"]')?.getAttribute("content") || null;
+        const forms = root.querySelectorAll("form").map((form) => {
+            const action = form.getAttribute("action") || null;
+            const method = (form.getAttribute("method") || "GET").toUpperCase();
+            const resolvedAction = action ? new URL(action, finalUrl).toString() : finalUrl.toString();
+            const actionHost = new URL(resolvedAction).hostname;
+            return {
+                action,
+                resolvedAction,
+                actionHost,
+                method,
+                insecureSubmission: resolvedAction.startsWith("http://"),
+                hasPasswordField: form.querySelectorAll('input[type="password"]').length > 0,
+                offOriginSubmission: actionHost !== finalUrl.hostname,
+            };
+        });
+        const scriptElements = root.querySelectorAll("script");
+        const externalScriptUrls = scriptElements
+            .map((script) => script.getAttribute("src"))
+            .filter(Boolean)
+            .map((src) => new URL(src, finalUrl).toString());
+        const stylesheetElements = root.querySelectorAll('link[rel~="stylesheet"]');
+        const externalStylesheetUrls = stylesheetElements
+            .map((link) => link.getAttribute("href"))
+            .filter(Boolean)
+            .map((href) => new URL(href, finalUrl).toString());
+        const anchorElements = root.querySelectorAll("a[href]");
+        const sameSiteHosts = collectSameSiteHosts(finalUrl, [
+            ...anchorElements.map((anchor) => anchor.getAttribute("href")),
+            ...scriptElements.map((script) => script.getAttribute("src")),
+            ...stylesheetElements.map((link) => link.getAttribute("href")),
+            ...forms.map((form) => form.action),
+        ]);
+        const firstPartyPaths = rankDiscoveredPaths([
+            ...anchorElements.map((anchor) => normalizeDiscoveredPath(anchor.getAttribute("href"), finalUrl)),
+            ...forms.map((form) => normalizeDiscoveredPath(form.action, finalUrl)),
+        ]);
+        const trainingLabMarkers = unique([
+            /(xss game|firing range|vulnweb|testfire|altoro mutual)/i.test(pageTitle || "")
+                ? `Title: ${pageTitle}`
+                : null,
+            ...firstPartyPaths
+                .filter((path) => /(xss|clickjacking|csrf|sql|sqli|mixedcontent|leakedcookie|dom)(?:\/|$|-|_)/i.test(path))
+                .slice(0, 6),
+        ]);
+        const insecureResourceUrls = unique([...externalScriptUrls, ...externalStylesheetUrls].filter((url) => url.startsWith("http://")));
+        const externalScriptDomains = unique(externalScriptUrls.map((url) => new URL(url).hostname).filter((hostname) => hostname !== finalUrl.hostname));
+        const externalStylesheetDomains = unique(externalStylesheetUrls.map((url) => new URL(url).hostname).filter((hostname) => hostname !== finalUrl.hostname));
+        const inlineScriptCount = scriptElements.filter((script) => !script.getAttribute("src")).length;
+        const inlineStyleCount = root.querySelectorAll("style").length;
+        const missingSriScriptUrls = scriptElements
+            .map((script) => {
+            const src = script.getAttribute("src");
+            if (!src) {
+                return null;
+            }
+            const resolved = new URL(src, finalUrl);
+            if (resolved.hostname === finalUrl.hostname || script.getAttribute("integrity")) {
+                return null;
+            }
+            return resolved.toString();
+        })
+            .filter(Boolean);
+        const sriCoverage = calculateSriCoverage(scriptElements, stylesheetElements, finalUrl);
+        const suspiciousScriptSignals = detectSuspiciousScriptSignals(scriptElements, externalScriptUrls, finalUrl);
+        const passiveLeakSignals = collectPassiveLeakSignals(html, finalUrl, metaGenerator || null, externalScriptUrls, externalStylesheetUrls);
+        const clientExposureSignals = collectClientExposureSignals(html, finalUrl);
+        if (forms.some((form) => form.hasPasswordField)) {
+            strengths.push("Login-like form elements are present for passive inspection.");
+        }
+        if (sameSiteHosts.length) {
+            strengths.push(`Page content referenced ${sameSiteHosts.length} same-site host${sameSiteHosts.length === 1 ? "" : "s"} for passive discovery.`);
+        }
+        if (forms.some((form) => form.insecureSubmission)) {
+            issues.push("At least one form appears to submit over HTTP.");
+        }
+        if (forms.some((form) => form.hasPasswordField && form.offOriginSubmission)) {
+            issues.push("A password form appears to submit to a different origin.");
+        }
+        if (insecureResourceUrls.length) {
+            issues.push("The page references insecure HTTP resources.");
+        }
+        if (inlineScriptCount > 0) {
+            issues.push(`Inline scripts detected (${inlineScriptCount}).`);
+        }
+        if (inlineStyleCount > 0) {
+            issues.push(`Inline style blocks detected (${inlineStyleCount}).`);
+        }
+        if (missingSriScriptUrls.length) {
+            issues.push("Some third-party scripts are missing Subresource Integrity attributes.");
+        }
+        issues.push(...sriCoverage.issues);
+        strengths.push(...sriCoverage.strengths);
+        for (const signal of passiveLeakSignals) {
+            if (signal.severity === "warning") {
+                issues.push(signal.title);
+            }
+        }
+        for (const signal of clientExposureSignals) {
+            if (signal.severity === "warning") {
+                issues.push(signal.title);
+            }
+        }
+        for (const signal of suspiciousScriptSignals) {
+            if (signal.severity === "warning") {
+                issues.push(signal.title);
+            }
+        }
+        if (trainingLabMarkers.length) {
+            issues.push("Page content suggests an intentionally vulnerable training or challenge surface.");
+        }
+        if (firstPartyPaths.length) {
+            strengths.push(`Discovered ${firstPartyPaths.length} same-origin navigation paths for low-noise follow-up scans.`);
+        }
+        if (passiveLeakSignals.length) {
+            strengths.push(`Passive pre-check identified ${passiveLeakSignals.length} leak or fingerprinting signal${passiveLeakSignals.length === 1 ? "" : "s"} worth review.`);
+        }
+        if (clientExposureSignals.length) {
+            strengths.push(`Client-side markup exposed ${clientExposureSignals.length} API or configuration signal${clientExposureSignals.length === 1 ? "" : "s"} for review.`);
+        }
+        if (!issues.length) {
+            strengths.push("No obvious passive HTML transport/content risks detected on the fetched page.");
+        }
+        return {
+            fetched: true,
+            pageUrl: finalUrl.toString(),
+            pageTitle,
+            metaGenerator: metaGenerator || null,
+            forms,
+            sameSiteHosts,
+            externalScriptDomains,
+            externalStylesheetDomains,
+            insecureResourceUrls,
+            inlineScriptCount,
+            inlineStyleCount,
+            missingSriScriptUrls,
+            sriCoverage,
+            firstPartyPaths,
+            passiveLeakSignals,
+            clientExposureSignals,
+            libraryFingerprints: collectLibraryFingerprints(externalScriptUrls),
+            libraryRiskSignals: [],
+            frameworkVersionLeaks: detectFrameworkVersionLeaks(html, metaGenerator || null, externalScriptUrls, externalStylesheetUrls),
+            suspiciousScriptSignals,
+            detectedTechnologies: detectHtmlTechnologies(html, finalUrl, metaGenerator || null, externalScriptUrls, externalStylesheetUrls),
+            aiSurface: analyzeAiSurface(html, externalScriptUrls, firstPartyPaths),
+            issues,
+            strengths,
+        };
+    }
+    catch (error) {
+        return {
+            fetched: false,
+            pageUrl: finalUrl.toString(),
+            pageTitle: null,
+            metaGenerator: null,
+            forms: [],
+            sameSiteHosts: [],
+            externalScriptDomains: [],
+            externalStylesheetDomains: [],
+            insecureResourceUrls: [],
+            inlineScriptCount: 0,
+            inlineStyleCount: 0,
+            missingSriScriptUrls: [],
+            sriCoverage: emptySriCoverage(),
+            firstPartyPaths: [],
+            passiveLeakSignals: [],
+            clientExposureSignals: [],
+            libraryFingerprints: [],
+            libraryRiskSignals: [],
+            frameworkVersionLeaks: [],
+            suspiciousScriptSignals: [],
+            detectedTechnologies: [],
+            aiSurface: {
+                detected: false,
+                assistantVisible: false,
+                aiPageSignals: [],
+                vendors: [],
+                discoveredPaths: [],
+                disclosures: [],
+                privacySignals: [],
+                governanceSignals: [],
+                issues: [error instanceof Error ? error.message : "AI surface inspection failed."],
+                strengths: [],
+            },
+            issues: [error instanceof Error ? error.message : "HTML inspection failed."],
+            strengths: [],
+        };
+    }
+}
+export function analyzeHtmlDocument(input, html) {
+    const finalUrl = typeof input === "string" ? new URL(input) : input;
+    const pageTitle = extractHtmlTitle(html);
+    return analyzeHtmlSecurity(finalUrl, { html, pageTitle });
+}
+export function detectAssessmentLimitation(statusCode, headers, html) {
+    if (statusCode === 401) {
+        return {
+            limited: true,
+            kind: "auth_required",
+            title: "Assessment limited by an authenticated response",
+            detail: "The target required authentication before serving the normal page, so this result reflects a restricted response rather than the full application surface.",
+        };
+    }
+    if (statusCode === 429) {
+        return {
+            limited: true,
+            kind: "rate_limited",
+            title: "Assessment limited by rate limiting",
+            detail: "The target rate-limited the scanner, so this result reflects a throttled response rather than a normal page render.",
+        };
+    }
+    if (statusCode >= 500) {
+        return {
+            limited: true,
+            kind: "service_unavailable",
+            title: "Assessment limited by service availability",
+            detail: `The target returned HTTP ${statusCode}, so this result reflects an unavailable or error response rather than the normal site posture.`,
+        };
+    }
+    if (statusCode === 403 && html && isAccessDeniedHtml(headers, html)) {
+        return {
+            limited: true,
+            kind: "blocked_edge_response",
+            title: "Assessment limited by a blocked edge response",
+            detail: "The target returned a generic blocked or protection-layer response, so missing headers on this page may not reflect the normal site posture.",
+        };
+    }
+    return {
+        limited: false,
+        kind: null,
+        title: null,
+        detail: null,
+    };
+}

package/dist/htmlInsights.d.ts

@@ -0,0 +1,23 @@
+import type { AiSurfaceInfo, AnalysisResult, ExecutiveSummaryInfo, IssueConfidence, TechnologyResult, ThirdPartyProvider, ThirdPartyTrustInfo } from "./types.js";
+interface AiVendorMatcher {
+    name: string;
+    pattern: RegExp;
+    evidence: string;
+    category: AiSurfaceInfo["vendors"][number]["category"];
+    confidence: IssueConfidence;
+}
+export declare const AI_VENDOR_MATCHERS: AiVendorMatcher[];
+interface ThirdPartyProviderMatcher {
+    pattern: RegExp;
+    name: string;
+    category: ThirdPartyProvider["category"];
+    risk: ThirdPartyProvider["risk"];
+    evidence: string;
+}
+export declare const THIRD_PARTY_PROVIDER_MATCHERS: ThirdPartyProviderMatcher[];
+export declare const detectHtmlTechnologies: (html: string, finalUrl: URL, metaGenerator: string | null, externalScriptUrls: string[], externalStylesheetUrls: string[]) => TechnologyResult[];
+export declare const analyzeAiSurface: (html: string, externalScriptUrls: string[], firstPartyPaths: string[]) => AiSurfaceInfo;
+export declare const analyzeThirdPartyTrust: (finalUrl: URL, htmlSecurity: Pick<AnalysisResult["htmlSecurity"], "externalScriptDomains" | "externalStylesheetDomains" | "missingSriScriptUrls">, aiSurface: AiSurfaceInfo) => ThirdPartyTrustInfo;
+export declare const buildExecutiveSummary: (result: Pick<AnalysisResult, "score" | "headers" | "thirdPartyTrust" | "aiSurface" | "domainSecurity" | "publicSignals" | "assessmentLimitation" | "htmlSecurity">) => ExecutiveSummaryInfo;
+export declare const mergeTechnologies: (...groups: Array<TechnologyResult[] | null | undefined>) => TechnologyResult[];
+export {};