penthera 1.1.0

package/lib/params.js

@@ -0,0 +1,235 @@
+/**
+ * HTTP Parameter Discovery (Arjun-style)
+ *
+ * Discovers active HTTP parameters via batched differential analysis.
+ * Instead of testing 1 parameter per request (slow), we send batches of
+ * 50+ parameters and use response differences to detect active ones.
+ *
+ * Algorithm:
+ *   1. Establish baseline response (no params)
+ *   2. Send batches of 50 candidate params
+ *   3. If response differs from baseline → active params in this batch
+ *   4. Binary-search the batch to isolate individual active params
+ *
+ * Inspired by: Arjun (6.2k stars), ParamSpider
+ */
+
+const PARAM_TIMEOUT = 8_000;
+
+async function safeFetch(url, opts = {}) {
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), opts.timeout || PARAM_TIMEOUT);
+  try {
+    const res = await fetch(url, { ...opts, signal: controller.signal, redirect: "manual" });
+    const body = await res.text().catch(() => "");
+    return { status: res.status, body, size: body.length, ok: true };
+  } catch {
+    return { status: 0, body: "", size: 0, ok: false };
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+function simpleHash(str) {
+  let hash = 0;
+  for (let i = 0; i < str.length; i++) {
+    hash = ((hash << 5) - hash + str.charCodeAt(i)) | 0;
+  }
+  return hash;
+}
+
+function isDifferent(baseline, response, tolerance = 0.05) {
+  if (baseline.status !== response.status) return true;
+  const sizeDiff = Math.abs(baseline.size - response.size);
+  if (sizeDiff > Math.max(50, baseline.size * tolerance)) return true;
+  if (simpleHash(baseline.body) !== simpleHash(response.body)) return true;
+  return false;
+}
+
+// ── Common parameter wordlist ────────────────────────────────────────────
+// Curated from SecLists, Arjun defaults, and CommonCrawl analysis.
+
+const COMMON_PARAMS = [
+  // Auth / identity
+  "id", "user", "username", "email", "password", "token", "key", "api_key",
+  "apikey", "auth", "session", "sid", "uid", "user_id", "userId",
+  // Pagination / filtering
+  "page", "limit", "offset", "sort", "order", "filter", "search", "query",
+  "q", "keyword", "per_page", "perPage", "size", "start", "end",
+  // Content / data
+  "name", "title", "body", "content", "text", "message", "description",
+  "comment", "note", "data", "value", "type", "status", "state",
+  // URLs / redirects
+  "url", "link", "href", "src", "redirect", "redirect_uri", "return",
+  "return_to", "next", "callback", "redir", "destination", "go", "target",
+  // Files / paths
+  "file", "filename", "path", "dir", "folder", "upload", "image", "img",
+  "photo", "avatar", "attachment", "document", "doc",
+  // Actions
+  "action", "cmd", "command", "method", "do", "func", "function",
+  "handler", "operation", "task", "step", "mode",
+  // Formatting
+  "format", "output", "view", "template", "layout", "theme", "lang",
+  "language", "locale", "timezone", "tz",
+  // Database / technical
+  "table", "column", "field", "select", "where", "from", "join",
+  "group", "having", "debug", "verbose", "trace", "log", "level",
+  // Common API
+  "version", "v", "format", "fields", "include", "exclude", "expand",
+  "embed", "populate", "with", "scope", "context",
+  // Security-relevant
+  "admin", "role", "permission", "access", "grant", "deny", "allow",
+  "block", "ban", "delete", "remove", "drop", "reset", "confirm",
+  // Payment / e-commerce
+  "amount", "price", "quantity", "qty", "total", "discount", "coupon",
+  "code", "promo", "plan", "subscription", "invoice", "order_id",
+  // Misc
+  "category", "tag", "label", "ref", "source", "medium", "campaign",
+  "from", "to", "date", "timestamp", "created", "updated",
+];
+
+// ── Batched parameter discovery ──────────────────────────────────────────
+
+/**
+ * Discover active HTTP parameters on an endpoint.
+ *
+ * @param {string} baseUrl
+ * @param {string} path - Endpoint path
+ * @param {object} opts - { method, batchSize, extraParams, timeout }
+ * @returns {{ params: string[], method: string, path: string }}
+ */
+export async function discoverParams(baseUrl, path, opts = {}) {
+  const method = opts.method || "GET";
+  const batchSize = opts.batchSize || 50;
+  const wordlist = [...COMMON_PARAMS, ...(opts.extraParams || [])];
+  const canary = "pnth3r4_" + Math.random().toString(36).slice(2, 8);
+
+  // 1. Baseline (no params)
+  const baselineUrl = `${baseUrl}${path}`;
+  const baseline = method === "GET"
+    ? await safeFetch(baselineUrl)
+    : await safeFetch(baselineUrl, {
+        method,
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({}),
+      });
+
+  if (!baseline.ok) return { params: [], method, path };
+
+  // Also get a "noise" baseline (with a definitely-fake param)
+  const noiseUrl = `${baseUrl}${path}?__pnth_noise__=${canary}`;
+  const noise = await safeFetch(noiseUrl);
+
+  // If adding a random param changes the response, the endpoint is too dynamic
+  const tooNoisy = isDifferent(baseline, noise, 0.01);
+
+  const activeParams = [];
+
+  // 2. Batch testing
+  for (let i = 0; i < wordlist.length; i += batchSize) {
+    const batch = wordlist.slice(i, i + batchSize);
+
+    let batchResponse;
+    if (method === "GET") {
+      const qs = batch.map((p) => `${p}=${canary}`).join("&");
+      batchResponse = await safeFetch(`${baseUrl}${path}?${qs}`);
+    } else {
+      const body = {};
+      for (const p of batch) body[p] = canary;
+      batchResponse = await safeFetch(`${baseUrl}${path}`, {
+        method,
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify(body),
+      });
+    }
+
+    if (!batchResponse.ok) continue;
+
+    // If batch response equals baseline, no active params here
+    if (!isDifferent(baseline, batchResponse)) continue;
+
+    // If endpoint is noisy, skip individual testing (too many false positives)
+    if (tooNoisy) continue;
+
+    // 3. Narrow down: test each param individually
+    for (const param of batch) {
+      let singleResponse;
+      if (method === "GET") {
+        singleResponse = await safeFetch(`${baseUrl}${path}?${param}=${canary}`);
+      } else {
+        singleResponse = await safeFetch(`${baseUrl}${path}`, {
+          method,
+          headers: { "Content-Type": "application/json" },
+          body: JSON.stringify({ [param]: canary }),
+        });
+      }
+
+      if (singleResponse.ok && isDifferent(baseline, singleResponse)) {
+        activeParams.push(param);
+      }
+    }
+  }
+
+  return { params: activeParams, method, path };
+}
+
+/**
+ * Discover parameters across multiple endpoints.
+ *
+ * @param {string} baseUrl
+ * @param {Array<{ path, status }>} endpoints
+ * @param {object} opts - { onPhase, maxEndpoints, extraParams }
+ * @returns {{ results: object[], findings: object[] }}
+ */
+export async function discoverAllParams(baseUrl, endpoints, opts = {}) {
+  const progress = opts.onPhase || (() => {});
+  const max = opts.maxEndpoints || 5;
+  const findings = [];
+  const results = [];
+
+  const candidates = endpoints
+    .filter((ep) => ep.status !== 404 && ep.status < 500)
+    .filter((ep) => ep.path.includes("/api/") || ep.path.includes("?"))
+    .slice(0, max);
+
+  if (candidates.length === 0) return { results, findings };
+
+  for (const ep of candidates) {
+    progress(`Discovering parameters on ${ep.path}...`);
+    const result = await discoverParams(baseUrl, ep.path, {
+      extraParams: opts.extraParams,
+    });
+    results.push(result);
+
+    if (result.params.length > 0) {
+      // Flag security-sensitive parameters
+      const sensitiveParams = result.params.filter((p) =>
+        /admin|role|permission|debug|token|secret|password|key|cmd|exec|file|path|redirect|url/i.test(p),
+      );
+
+      if (sensitiveParams.length > 0) {
+        findings.push({
+          severity: "medium",
+          title: `Sensitive parameters found on ${ep.path}`,
+          description: `Active params: ${sensitiveParams.join(", ")}`,
+          url: `${baseUrl}${ep.path}`,
+          category: "parameter-discovery",
+          source: "param-discovery",
+        });
+      }
+
+      if (result.params.length > 0) {
+        findings.push({
+          severity: "info",
+          title: `${result.params.length} active params on ${ep.path}`,
+          description: result.params.join(", "),
+          url: `${baseUrl}${ep.path}`,
+          category: "parameter-discovery",
+          source: "param-discovery",
+        });
+      }
+    }
+  }
+
+  return { results, findings };
+}

package/lib/plugins.js

@@ -0,0 +1,70 @@
+/**
+ * Plugin / templates API, load and run custom YAML security templates.
+ *
+ * Programmatic usage:
+ *   import { loadTemplatesFromPaths, runTemplateScan } from "penthera/plugins";
+ *   const templates = await loadTemplatesFromPaths(["./my-templates"]);
+ *   const findings = await runTemplateScan("https://app.example", templates);
+ */
+import { existsSync, statSync } from "node:fs";
+import { extname, resolve } from "node:path";
+import {
+  loadTemplatesFromDir,
+  executeTemplates,
+} from "./nuclei-loader.js";
+import {
+  runBuiltInTemplates,
+  executeTemplate,
+  BUILT_IN_TEMPLATES,
+} from "./templates.js";
+
+export { loadTemplatesFromDir, executeTemplates, runBuiltInTemplates, executeTemplate, BUILT_IN_TEMPLATES };
+
+/**
+ * Load templates from one or more directories (Nuclei-compatible YAML).
+ *
+ * @param {string[]} paths
+ * @param {object} opts - passed to loadTemplatesFromDir
+ */
+export async function loadTemplatesFromPaths(paths, opts = {}) {
+  const seen = new Set();
+  const templates = [];
+
+  for (const raw of paths) {
+    const dir = resolve(raw);
+    if (!existsSync(dir)) {
+      throw new Error(`Template path not found: ${dir}`);
+    }
+    if (!statSync(dir).isDirectory()) {
+      throw new Error(`Template path must be a directory: ${dir}`);
+    }
+
+    const loaded = await loadTemplatesFromDir(dir, opts);
+    for (const t of loaded) {
+      if (seen.has(t.id)) continue;
+      seen.add(t.id);
+      templates.push(t);
+    }
+  }
+
+  return templates;
+}
+
+/**
+ * Execute templates against a target URL.
+ *
+ * @param {string} baseUrl
+ * @param {object[]} templates
+ * @param {object} opts
+ */
+export async function runTemplateScan(baseUrl, templates, opts = {}) {
+  return executeTemplates(baseUrl, templates, opts);
+}
+
+/**
+ * Parse --templates CLI value (comma-separated paths).
+ */
+export function parseTemplatePaths(value) {
+  if (!value) return [];
+  return value.split(",").map((p) => p.trim()).filter(Boolean);
+}

package/lib/provenance.js

@@ -0,0 +1,82 @@
+/**
+ * Provenance and honest-confidence labelling for findings (principles #1 and #3).
+ *
+ * Every finding carries { source, method, collected_at, confidence }:
+ *   - source / method: which probe produced it,
+ *   - collected_at: when the scan ran,
+ *   - confidence: how sure we are it is real and reachable, not just its severity.
+ *
+ * Confidence is deliberately honest about what a scanner can and cannot judge:
+ *   confirmed           deterministic evidence (a header is absent, a cert expired)
+ *   likely              a strong signal with a small false-positive chance
+ *   potential           a heuristic or timing signal that needs corroboration
+ *   needs-human-review  exploitability can't be judged from a scan (e.g. IDOR)
+ *
+ * A scanner must not present a guess as a fact, so heuristic probes are labelled
+ * accordingly rather than being dressed up as certainties.
+ */
+
+// Probe source -> default confidence.
+const SOURCE_CONFIDENCE = {
+  "tls-check": "confirmed",
+  "header-audit": "confirmed",
+  "cookie-audit": "confirmed",
+  "secret-scan": "confirmed",
+  "sensitive-file-check": "confirmed",
+  "openapi-scan": "confirmed",
+  "email-dns-probe": "confirmed",
+  "framework-check": "likely",
+  "static-analysis": "likely",
+  "trust-boundary": "likely",
+  "auth-probe": "likely",
+  "oauth-probe": "likely",
+  "jwt-probe": "likely",
+  "injection-probe": "likely",
+  "template": "likely",
+  "nuclei-community": "likely",
+  "client-auth-probe": "needs-human-review",
+  "idor-probe": "needs-human-review",
+  "fuzzer": "potential",
+  "adaptive-probe": "potential",
+};
+
+// Category fallback when the source is unknown.
+const CATEGORY_CONFIDENCE = {
+  headers: "confirmed",
+  transport: "confirmed",
+  tls: "confirmed",
+  cookie: "confirmed",
+  secrets: "confirmed",
+  cve: "confirmed",
+  "js-vulnerability": "confirmed",
+  "email-auth": "confirmed",
+  auth: "likely",
+  "auth-bypass": "needs-human-review",
+  ssti: "likely",
+  sqli: "likely",
+  xss: "likely",
+  cmdi: "likely",
+  ssrf: "likely",
+  "open-redirect": "likely",
+  fuzzing: "potential",
+};
+
+/** Honest-confidence label for a finding. An explicit finding.confidence wins. */
+export function confidenceFor(f) {
+  if (f.confidence) return f.confidence;
+  return SOURCE_CONFIDENCE[f.source] || CATEGORY_CONFIDENCE[f.category] || "likely";
+}
+
+/** Stamp { source, method, collected_at, confidence } onto every finding. */
+export function stampProvenance(findings, collectedAt) {
+  return (findings || []).map((f) => {
+    const source = f.source || "penthera";
+    return {
+      ...f,
+      source,
+      method: f.method || source,
+      collected_at: f.collected_at || collectedAt || null,
+      confidence: confidenceFor(f),
+    };
+  });
+}

package/lib/recon.js

@@ -0,0 +1,193 @@
+/**
+ * OSINT Reconnaissance Module
+ *
+ * Passive information gathering from public sources, no interaction with the target.
+ *
+ * Sources:
+ *   - crt.sh (Certificate Transparency logs), subdomain discovery
+ *   - Wayback Machine CDX API, historical URL enumeration
+ *   - AlienVault OTX, threat intelligence URL list
+ *
+ * All sources are free, require no API key, and return JSON.
+ * Inspired by: subfinder, gau, waybackurls, ParamSpider
+ */
+
+const RECON_TIMEOUT = 15_000;
+
+async function safeFetch(url, opts = {}) {
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), opts.timeout || RECON_TIMEOUT);
+  try {
+    return await fetch(url, { signal: controller.signal, ...opts });
+  } catch {
+    return null;
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+// ── Subdomain Discovery via Certificate Transparency ─────────────────────
+
+/**
+ * Query crt.sh for all certificates issued to *.domain.
+ * Returns unique subdomains sorted alphabetically.
+ *
+ * @param {string} domain - e.g. "example.com"
+ * @returns {string[]} Discovered subdomains
+ */
+export async function discoverSubdomains(domain) {
+  const res = await safeFetch(
+    `https://crt.sh/?q=%25.${encodeURIComponent(domain)}&output=json`,
+    { timeout: 20_000 },
+  );
+  if (!res || !res.ok) return [];
+
+  let data;
+  try {
+    data = await res.json();
+  } catch {
+    return [];
+  }
+
+  const subdomains = new Set();
+  for (const entry of data) {
+    if (!entry.name_value) continue;
+    for (const name of entry.name_value.split("\n")) {
+      const clean = name.trim().toLowerCase().replace(/^\*\./, "");
+      if (clean.endsWith(domain) && clean !== domain) {
+        subdomains.add(clean);
+      }
+    }
+  }
+
+  return [...subdomains].sort();
+}
+
+// ── Historical URL Enumeration ───────────────────────────────────────────
+
+/**
+ * Fetch known URLs from the Wayback Machine CDX API.
+ * Collapses by URL key to avoid duplicates.
+ *
+ * @param {string} domain - e.g. "example.com"
+ * @param {object} opts - { limit }
+ * @returns {string[]} Historical URLs
+ */
+export async function fetchWaybackUrls(domain, opts = {}) {
+  const limit = opts.limit || 1000;
+  const res = await safeFetch(
+    `https://web.archive.org/cdx/search/cdx?url=*.${encodeURIComponent(domain)}/*&output=json&fl=original&collapse=urlkey&limit=${limit}`,
+    { timeout: 20_000 },
+  );
+  if (!res || !res.ok) return [];
+
+  let rows;
+  try {
+    rows = await res.json();
+  } catch {
+    return [];
+  }
+
+  // First row is the header ["original"]
+  return rows.slice(1).map((r) => r[0]).filter(Boolean);
+}
+
+/**
+ * Fetch known URLs from AlienVault OTX.
+ *
+ * @param {string} domain
+ * @param {object} opts - { limit }
+ * @returns {string[]}
+ */
+export async function fetchOtxUrls(domain, opts = {}) {
+  const limit = opts.limit || 200;
+  const res = await safeFetch(
+    `https://otx.alienvault.com/api/v1/indicators/domain/${encodeURIComponent(domain)}/url_list?limit=${limit}`,
+  );
+  if (!res || !res.ok) return [];
+
+  let data;
+  try {
+    data = await res.json();
+  } catch {
+    return [];
+  }
+
+  return (data.url_list || []).map((e) => e.url).filter(Boolean);
+}
+
+// ── Aggregate Recon ──────────────────────────────────────────────────────
+
+/**
+ * Run full OSINT recon for a domain.
+ * Returns subdomains, historical URLs, and extracted parameters.
+ *
+ * @param {string} domain - Target domain (e.g. "example.com")
+ * @param {object} opts - { onPhase }
+ * @returns {{ subdomains, urls, parameters, endpoints }}
+ */
+export async function runRecon(domain, opts = {}) {
+  const progress = opts.onPhase || (() => {});
+
+  progress("Querying Certificate Transparency logs (crt.sh)...");
+  const subdomains = await discoverSubdomains(domain);
+
+  progress("Fetching historical URLs (Wayback Machine)...");
+  const [waybackUrls, otxUrls] = await Promise.all([
+    fetchWaybackUrls(domain),
+    fetchOtxUrls(domain),
+  ]);
+
+  // Merge and deduplicate all discovered URLs
+  const allUrls = [...new Set([...waybackUrls, ...otxUrls])];
+
+  // Extract unique URL paths (for active scanning)
+  const endpoints = new Set();
+  const parameters = new Set();
+  for (const url of allUrls) {
+    try {
+      const parsed = new URL(url);
+      // Normalize path
+      const path = parsed.pathname.replace(/\/+$/, "") || "/";
+      if (path !== "/" && !path.match(/\.(css|js|png|jpg|gif|svg|ico|woff|ttf|eot)$/i)) {
+        endpoints.add(path);
+      }
+      // Extract parameter names
+      for (const key of parsed.searchParams.keys()) {
+        parameters.add(key);
+      }
+    } catch {
+      // Invalid URL, skip
+    }
+  }
+
+  return {
+    subdomains,
+    urls: allUrls,
+    endpoints: [...endpoints].sort(),
+    parameters: [...parameters].sort(),
+    sources: {
+      crtsh: subdomains.length,
+      wayback: waybackUrls.length,
+      otx: otxUrls.length,
+    },
+  };
+}
+
+/**
+ * Extract the root domain from a URL.
+ * e.g. "https://app.staging.example.com/path" → "example.com"
+ */
+export function extractDomain(urlString) {
+  try {
+    const hostname = new URL(urlString).hostname;
+    const parts = hostname.split(".");
+    // Handle two-part TLDs like .co.uk, .com.au
+    if (parts.length >= 3 && parts[parts.length - 2].length <= 3) {
+      return parts.slice(-3).join(".");
+    }
+    return parts.slice(-2).join(".");
+  } catch {
+    return urlString;
+  }
+}