sentinel-scanner 1.1.0-alpha.1 → 1.1.0

package/build/bin.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/bin.ts", "../src/commands/spider.ts", "../src/modules/spider/index.ts", "../src/utils/index.ts"],
+  "sourcesContent": ["#!/usr/bin/env node --no-warnings\n\nimport yargs from \"yargs\";\nimport { hideBin } from \"yargs/helpers\";\nimport { spiderCommand } from \"./commands/spider\";\n\nconst commandHandler = yargs(hideBin(process.argv));\n\ncommandHandler.demandCommand();\ncommandHandler.scriptName(\"sentinel-scanner\");\ncommandHandler.usage(\"Usage: $0 <command> [options]\");\ncommandHandler.help().alias(\"help\", \"h\");\ncommandHandler.version().alias(\"version\", \"v\");\ncommandHandler.strict();\ncommandHandler.showHelpOnFail(true);\n\n// Handle Commands\ncommandHandler.command(spiderCommand);\n\ncommandHandler.parse();\n", "import fs from \"node:fs\";\nimport path from \"node:path\";\nimport type { ArgumentsCamelCase, CommandModule } from \"yargs\";\nimport SpiderScanner from \"../modules/spider\";\nimport { createLogger } from \"../utils\";\n\nexport type SpiderScannerCLIOptions = {\n\turl: string;\n\tdepth?: number;\n\toutput?: string;\n\tconcurrency?: number;\n\ttimeout?: number;\n\tretries?: number;\n};\n\nconst cliLogger = createLogger(\"CLI\");\n\nexport const spiderCommand: CommandModule = {\n\tcommand: \"spider\",\n\tdescribe:\n\t\t\"Crawl a website and get an array of URLs which are internal to the website\",\n\tbuilder: (yargs) => {\n\t\treturn yargs\n\t\t\t.option(\"url\", {\n\t\t\t\talias: \"u\",\n\t\t\t\ttype: \"string\",\n\t\t\t\tdescription: \"The URL of the website to scan\",\n\t\t\t\tdemandOption: true,\n\t\t\t\tcoerce: (url) => {\n\t\t\t\t\ttry {\n\t\t\t\t\t\tnew URL(url);\n\n\t\t\t\t\t\treturn url;\n\t\t\t\t\t} catch (error) {\n\t\t\t\t\t\tthrow new Error(`Invalid URL: ${url}`);\n\t\t\t\t\t}\n\t\t\t\t},\n\t\t\t})\n\t\t\t.option(\"depth\", {\n\t\t\t\talias: \"d\",\n\t\t\t\ttype: \"number\",\n\t\t\t\tdescription: \"The maximum depth to crawl\",\n\t\t\t\tdefault: 250,\n\t\t\t\tcoerce: (depth) => {\n\t\t\t\t\tif (depth < 0) {\n\t\t\t\t\t\tthrow new Error(\"Depth must be a positive number\");\n\t\t\t\t\t}\n\n\t\t\t\t\tif (depth > 250) {\n\t\t\t\t\t\tthrow new Error(\"Depth must be less than 250\");\n\t\t\t\t\t}\n\n\t\t\t\t\treturn depth;\n\t\t\t\t},\n\t\t\t})\n\t\t\t.option(\"output\", {\n\t\t\t\talias: \"o\",\n\t\t\t\ttype: \"string\",\n\t\t\t\tdescription:\n\t\t\t\t\t\"The output file to write the results to. Must be a JSON file\",\n\t\t\t\tcoerce: (output) => {\n\t\t\t\t\ttry {\n\t\t\t\t\t\t// Should throw an error if the path is invalid\n\t\t\t\t\t\t// Should Be A JSON File\n\t\t\t\t\t\tconst resolvedPath = path.resolve(output);\n\t\t\t\t\t\tconst parsedPath = path.parse(resolvedPath);\n\n\t\t\t\t\t\tif (parsedPath.ext !== \".json\") {\n\t\t\t\t\t\t\tthrow new Error(\"Output file must be a JSON file\");\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tif (fs.existsSync(resolvedPath)) {\n\t\t\t\t\t\t\tthrow new Error(\"Output file already exists\");\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\treturn resolvedPath;\n\t\t\t\t\t} catch (error) {\n\t\t\t\t\t\tthrow new Error(`Invalid output file: ${output}`);\n\t\t\t\t\t}\n\t\t\t\t},\n\t\t\t\tdefault: getDefaultFilePath(),\n\t\t\t})\n\t\t\t.option(\"concurrency\", {\n\t\t\t\talias: \"c\",\n\t\t\t\ttype: \"number\",\n\t\t\t\tdescription: \"The number of concurrent requests to make\",\n\t\t\t\tdefault: 10,\n\t\t\t\tcoerce: (concurrency) => {\n\t\t\t\t\tif (concurrency < 1) {\n\t\t\t\t\t\tthrow new Error(\"Concurrency must be a positive number\");\n\t\t\t\t\t}\n\n\t\t\t\t\tif (concurrency > 20) {\n\t\t\t\t\t\tthrow new Error(\"Concurrency must be less than 20\");\n\t\t\t\t\t}\n\n\t\t\t\t\treturn concurrency;\n\t\t\t\t},\n\t\t\t})\n\t\t\t.option(\"timeout\", {\n\t\t\t\talias: \"t\",\n\t\t\t\ttype: \"number\",\n\t\t\t\tdescription: \"The timeout for each request in milliseconds\",\n\t\t\t\tdefault: 5000,\n\t\t\t\tcoerce: (timeout) => {\n\t\t\t\t\tif (timeout < 0) {\n\t\t\t\t\t\tthrow new Error(\"Timeout must be a positive number\");\n\t\t\t\t\t}\n\n\t\t\t\t\tif (timeout > 25_000) {\n\t\t\t\t\t\tthrow new Error(\"Timeout must be less than 25,000\");\n\t\t\t\t\t}\n\n\t\t\t\t\treturn timeout;\n\t\t\t\t},\n\t\t\t})\n\t\t\t.option(\"retries\", {\n\t\t\t\talias: \"r\",\n\t\t\t\ttype: \"number\",\n\t\t\t\tdescription: \"The number of retries for each request\",\n\t\t\t\tdefault: 3,\n\t\t\t\tcoerce: (retries) => {\n\t\t\t\t\tif (retries < 0) {\n\t\t\t\t\t\tthrow new Error(\"Retries must be a positive number\");\n\t\t\t\t\t}\n\n\t\t\t\t\tif (retries > 10) {\n\t\t\t\t\t\tthrow new Error(\"Retries must be less than 10\");\n\t\t\t\t\t}\n\n\t\t\t\t\treturn retries;\n\t\t\t\t},\n\t\t\t});\n\t},\n\thandler: async (args) => {\n\t\ttry {\n\t\t\tconst argData = args as ArgumentsCamelCase<SpiderScannerCLIOptions>;\n\n\t\t\tconst scanner = new SpiderScanner(argData.url, {\n\t\t\t\tdepth: argData.depth ?? 250,\n\t\t\t\tconcurrency: argData.concurrency ?? 10,\n\t\t\t\ttimeout: argData.timeout ?? 5000,\n\t\t\t\tretries: argData.retries ?? 3,\n\t\t\t});\n\n\t\t\tcliLogger.info(\"Starting to crawl website\");\n\n\t\t\tconst results = await scanner.crawl();\n\n\t\t\tif (argData.output) {\n\t\t\t\tfs.writeFileSync(argData.output, JSON.stringify(results, null, 2));\n\t\t\t\tcliLogger.info(`Results written to ${argData.output}`);\n\t\t\t} else {\n\t\t\t\tconst resolvedPath = getDefaultFilePath();\n\t\t\t\tfs.writeFileSync(resolvedPath, JSON.stringify(results, null, 2));\n\t\t\t\tcliLogger.info(`Results written to ${resolvedPath}`);\n\t\t\t}\n\t\t} catch (error) {\n\t\t\tif (error instanceof Error) {\n\t\t\t\tcliLogger.error(error.message);\n\t\t\t}\n\t\t\tcliLogger.error(\"Failed to run spider command\");\n\t\t\tprocess.exit(1);\n\t\t}\n\t},\n};\n\nconst getDefaultFilePath = () => {\n\ttry {\n\t\tconst resolvedDir = path.resolve(\"sentinel_output\");\n\t\t// Check If Directory Exists\n\t\tif (!fs.existsSync(resolvedDir)) {\n\t\t\tfs.mkdirSync(resolvedDir);\n\t\t}\n\n\t\tconst resolvedPath = path.resolve(\n\t\t\t`sentinel_output/spider_${Date.now()}.json`,\n\t\t);\n\t\t// Check If File Exists\n\t\tif (fs.existsSync(resolvedPath)) {\n\t\t\tthrow new Error(\"Output file already exists\");\n\t\t}\n\t\tconst parsedPath = path.parse(resolvedPath);\n\n\t\tif (parsedPath.ext !== \".json\") {\n\t\t\tthrow new Error(\"Output file must be a JSON file\");\n\t\t}\n\n\t\treturn resolvedPath;\n\t} catch (error) {\n\t\tthrow new Error(\"Invalid output file\");\n\t}\n};\n", "import fetch from \"isomorphic-fetch\";\nimport jsdom from \"jsdom\";\nimport UserAgent from \"user-agents\";\nimport { createLogger } from \"../../utils\";\n\nexport interface SpiderScannerOptions {\n\tdepth?: number;\n\tconcurrency?: number;\n\tretries?: number;\n\ttimeout?: number;\n}\n\nexport default class SpiderScanner {\n\tprivate header: Record<string, string> = {\n\t\t\"User-Agent\": new UserAgent().toString(),\n\t};\n\tprivate url: URL;\n\tprivate logger = createLogger(\"SpiderScanner\");\n\n\tprivate depth: number;\n\tprivate concurrency: number;\n\tprivate retries: number;\n\tprivate timeout: number;\n\n\tconstructor(url: string, options: SpiderScannerOptions = {}) {\n\t\tconst {\n\t\t\tdepth = 250,\n\t\t\tconcurrency = 5,\n\t\t\tretries = 3,\n\t\t\ttimeout = 5000,\n\t\t} = options;\n\t\tthis.depth = depth;\n\t\tthis.concurrency = concurrency;\n\t\tthis.retries = retries;\n\t\tthis.timeout = timeout;\n\n\t\ttry {\n\t\t\tthis.url = new URL(url);\n\t\t\tthis.logger.info(\n\t\t\t\t`Initialized with URL: ${url}, User-Agent: ${this.header[\"User-Agent\"]}`,\n\t\t\t);\n\t\t} catch (error) {\n\t\t\tif (error instanceof TypeError) {\n\t\t\t\tthis.logger.error(\"Invalid URL\");\n\t\t\t\tthrow new Error(\"Invalid URL\");\n\t\t\t}\n\t\t\tthis.logger.error(`Unexpected error in constructor: ${error}`);\n\t\t\tthrow error;\n\t\t}\n\t}\n\n\tprivate normalizeDomain(domain: string): string {\n\t\treturn domain.startsWith(\"www.\") ? domain.slice(4) : domain;\n\t}\n\n\tprivate convertRelativeUrlToAbsolute(url: string): string {\n\t\treturn new URL(url, this.url.toString()).toString();\n\t}\n\n\tprivate isInternalLink(url: string): boolean {\n\t\ttry {\n\t\t\tconst parsedUrl = new URL(url, this.url.href);\n\t\t\tif (![\"http:\", \"https:\"].includes(parsedUrl.protocol)) {\n\t\t\t\treturn false;\n\t\t\t}\n\t\t\tconst baseDomain = this.normalizeDomain(this.url.hostname);\n\t\t\tconst parsedDomain = this.normalizeDomain(parsedUrl.hostname);\n\t\t\treturn parsedDomain === baseDomain;\n\t\t} catch (error) {\n\t\t\tthis.logger.warn(`Error parsing URL: ${url} - ${error}`);\n\t\t\treturn false;\n\t\t}\n\t}\n\n\tprivate async fetchWithRetries(\n\t\turl: string,\n\t\tretries: number,\n\t): Promise<string | null> {\n\t\tfor (let attempt = 1; attempt <= retries; attempt++) {\n\t\t\tconst controller = new AbortController();\n\t\t\tconst timeoutId = setTimeout(() => controller.abort(), this.timeout);\n\n\t\t\ttry {\n\t\t\t\tthis.logger.debug(`Fetching URL (Attempt ${attempt}): ${url}`);\n\t\t\t\tconst randomUserAgent = new UserAgent().toString();\n\t\t\t\tthis.logger.info(`Changing User-Agent to: ${randomUserAgent}`);\n\t\t\t\tthis.header[\"User-Agent\"] = randomUserAgent;\n\t\t\t\tconst response = await fetch(url, {\n\t\t\t\t\theaders: this.header,\n\t\t\t\t\tsignal: controller.signal,\n\t\t\t\t\tredirect: \"follow\",\n\t\t\t\t});\n\n\t\t\t\tclearTimeout(timeoutId);\n\n\t\t\t\tif (response.ok) {\n\t\t\t\t\tthis.logger.info(`Successfully fetched URL: ${url}`);\n\t\t\t\t\treturn await response.text();\n\t\t\t\t}\n\n\t\t\t\tthis.logger.warn(`Failed to fetch URL (${response.status}): ${url}`);\n\t\t\t} catch (error) {\n\t\t\t\tif ((error as Error).name === \"AbortError\") {\n\t\t\t\t\tthis.logger.warn(`Fetch timed out: ${url}`);\n\t\t\t\t} else {\n\t\t\t\t\tthis.logger.error(`Error fetching URL: ${url} - ${error}`);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn null;\n\t}\n\n\tprivate extractLinks(html: string): string[] {\n\t\tconst { JSDOM } = jsdom;\n\t\tconst dom = new JSDOM(html);\n\t\tconst links = Array.from(dom.window.document.querySelectorAll(\"a\"));\n\t\tconst hrefs = links.map((link) => link.href);\n\t\tconst internalLinks = hrefs.filter((href) => this.isInternalLink(href));\n\t\tthis.logger.debug(\n\t\t\t`Extracted ${internalLinks.length} internal links from HTML content`,\n\t\t);\n\t\treturn internalLinks.map((link) => this.convertRelativeUrlToAbsolute(link));\n\t}\n\n\tpublic async crawl(): Promise<Array<string>> {\n\t\tconst visited = new Set<string>();\n\t\tconst queue = new Set<string>([this.url.href]);\n\t\tconst resultLinks = new Set<string>();\n\n\t\t// Assets to ignore\n\t\tconst assetExtensions = [\n\t\t\t\".css\",\n\t\t\t\".js\",\n\t\t\t\".png\",\n\t\t\t\".jpg\",\n\t\t\t\".jpeg\",\n\t\t\t\".gif\",\n\t\t\t\".svg\",\n\t\t\t\".ico\",\n\t\t\t\".webp\",\n\t\t\t\".mp4\",\n\t\t\t\".mp3\",\n\t\t\t\".wav\",\n\t\t\t\".avi\",\n\t\t\t\".mov\",\n\t\t\t\".webm\",\n\t\t\t\".pdf\",\n\t\t\t\".doc\",\n\t\t\t\".docx\",\n\t\t\t\".xls\",\n\t\t\t\".xlsx\",\n\t\t\t\".ppt\",\n\t\t\t\".pptx\",\n\t\t\t\".zip\",\n\t\t\t\".rar\",\n\t\t\t\".tar\",\n\t\t\t\".gz\",\n\t\t];\n\n\t\tconst fetchAndExtract = async (currentUrl: string) => {\n\t\t\tif (visited.has(currentUrl)) {\n\t\t\t\tthis.logger.debug(`Skipping already visited URL: ${currentUrl}`);\n\t\t\t\treturn;\n\t\t\t}\n\t\t\tvisited.add(currentUrl);\n\t\t\tthis.logger.info(`Visiting URL: ${currentUrl}`);\n\n\t\t\tconst html = await this.fetchWithRetries(currentUrl, this.retries);\n\t\t\tif (!html) return;\n\n\t\t\tconst links = this.extractLinks(html);\n\n\t\t\t// Filter out asset links\n\t\t\tfor (const link of links) {\n\t\t\t\tif (assetExtensions.some((ext) => link.endsWith(ext))) {\n\t\t\t\t\tthis.logger.debug(`Ignoring asset link: ${link}`);\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tthis.logger.debug(`Found link: ${link}`);\n\t\t\t}\n\n\t\t\tfor (const link of links) {\n\t\t\t\tif (!visited.has(link) && queue.size < this.depth) {\n\t\t\t\t\tqueue.add(link);\n\t\t\t\t\tthis.logger.debug(`Added to queue: ${link}`);\n\t\t\t\t}\n\t\t\t}\n\t\t\tresultLinks.add(currentUrl);\n\t\t};\n\n\t\tconst processBatch = async () => {\n\t\t\tconst batch = Array.from(queue).slice(0, this.concurrency);\n\t\t\tfor (const url of batch) {\n\t\t\t\tqueue.delete(url);\n\t\t\t}\n\t\t\tawait Promise.allSettled(batch.map((url) => fetchAndExtract(url)));\n\t\t};\n\n\t\tthis.logger.info(\n\t\t\t`Starting crawl with depth: ${this.depth}, concurrency: ${this.concurrency}`,\n\t\t);\n\t\twhile (queue.size > 0 && visited.size < this.depth) {\n\t\t\tawait processBatch();\n\t\t}\n\n\t\tthis.logger.info(\n\t\t\t`Crawling completed. Total pages visited: ${resultLinks.size}`,\n\t\t);\n\n\t\treturn Array.from(resultLinks);\n\t}\n}\n", "import winston from \"winston\";\n\nexport const createLogger = (label: string) =>\n\twinston.createLogger({\n\t\tlevels: {\n\t\t\terror: 0,\n\t\t\twarn: 1,\n\t\t\tinfo: 2,\n\t\t\thttp: 3,\n\t\t\tverbose: 4,\n\t\t\tdebug: 5,\n\t\t\tsilly: 6,\n\t\t},\n\t\tformat: winston.format.combine(\n\t\t\twinston.format.label({ label }),\n\t\t\twinston.format.colorize(),\n\t\t\twinston.format.timestamp({\n\t\t\t\tformat: () => {\n\t\t\t\t\treturn new Date().toLocaleString(\"en-US\");\n\t\t\t\t},\n\t\t\t}),\n\t\t\twinston.format.align(),\n\t\t\twinston.format.printf(\n\t\t\t\t(info) =>\n\t\t\t\t\t`\\x1b[34m(${info.label})\\x1b[0m \\x1b[33m${info.timestamp}\\x1b[0m [${info.level}]: ${info.message}`,\n\t\t\t),\n\t\t),\n\t\ttransports: [new winston.transports.Console()],\n\t});\n"],
+  "mappings": ";;;AAEA,OAAO,WAAW;AAClB,SAAS,eAAe;;;ACHxB,OAAO,QAAQ;AACf,OAAO,UAAU;;;ACDjB,OAAO,WAAW;AAClB,OAAO,WAAW;AAClB,OAAO,eAAe;;;ACFtB,OAAO,aAAa;AAEb,IAAM,eAAe,CAAC,UAC5B,QAAQ,aAAa;AAAA,EACpB,QAAQ;AAAA,IACP,OAAO;AAAA,IACP,MAAM;AAAA,IACN,MAAM;AAAA,IACN,MAAM;AAAA,IACN,SAAS;AAAA,IACT,OAAO;AAAA,IACP,OAAO;AAAA,EACR;AAAA,EACA,QAAQ,QAAQ,OAAO;AAAA,IACtB,QAAQ,OAAO,MAAM,EAAE,MAAM,CAAC;AAAA,IAC9B,QAAQ,OAAO,SAAS;AAAA,IACxB,QAAQ,OAAO,UAAU;AAAA,MACxB,QAAQ,MAAM;AACb,gBAAO,oBAAI,KAAK,GAAE,eAAe,OAAO;AAAA,MACzC;AAAA,IACD,CAAC;AAAA,IACD,QAAQ,OAAO,MAAM;AAAA,IACrB,QAAQ,OAAO;AAAA,MACd,CAAC,SACA,YAAY,KAAK,KAAK,oBAAoB,KAAK,SAAS,YAAY,KAAK,KAAK,MAAM,KAAK,OAAO;AAAA,IAClG;AAAA,EACD;AAAA,EACA,YAAY,CAAC,IAAI,QAAQ,WAAW,QAAQ,CAAC;AAC9C,CAAC;;;ADhBF,IAAqB,gBAArB,MAAmC;AAAA,EAC1B,SAAiC;AAAA,IACxC,cAAc,IAAI,UAAU,EAAE,SAAS;AAAA,EACxC;AAAA,EACQ;AAAA,EACA,SAAS,aAAa,eAAe;AAAA,EAErC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAER,YAAY,KAAa,UAAgC,CAAC,GAAG;AAC5D,UAAM;AAAA,MACL,QAAQ;AAAA,MACR,cAAc;AAAA,MACd,UAAU;AAAA,MACV,UAAU;AAAA,IACX,IAAI;AACJ,SAAK,QAAQ;AACb,SAAK,cAAc;AACnB,SAAK,UAAU;AACf,SAAK,UAAU;AAEf,QAAI;AACH,WAAK,MAAM,IAAI,IAAI,GAAG;AACtB,WAAK,OAAO;AAAA,QACX,yBAAyB,GAAG,iBAAiB,KAAK,OAAO,YAAY,CAAC;AAAA,MACvE;AAAA,IACD,SAAS,OAAO;AACf,UAAI,iBAAiB,WAAW;AAC/B,aAAK,OAAO,MAAM,aAAa;AAC/B,cAAM,IAAI,MAAM,aAAa;AAAA,MAC9B;AACA,WAAK,OAAO,MAAM,oCAAoC,KAAK,EAAE;AAC7D,YAAM;AAAA,IACP;AAAA,EACD;AAAA,EAEQ,gBAAgB,QAAwB;AAC/C,WAAO,OAAO,WAAW,MAAM,IAAI,OAAO,MAAM,CAAC,IAAI;AAAA,EACtD;AAAA,EAEQ,6BAA6B,KAAqB;AACzD,WAAO,IAAI,IAAI,KAAK,KAAK,IAAI,SAAS,CAAC,EAAE,SAAS;AAAA,EACnD;AAAA,EAEQ,eAAe,KAAsB;AAC5C,QAAI;AACH,YAAM,YAAY,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI;AAC5C,UAAI,CAAC,CAAC,SAAS,QAAQ,EAAE,SAAS,UAAU,QAAQ,GAAG;AACtD,eAAO;AAAA,MACR;AACA,YAAM,aAAa,KAAK,gBAAgB,KAAK,IAAI,QAAQ;AACzD,YAAM,eAAe,KAAK,gBAAgB,UAAU,QAAQ;AAC5D,aAAO,iBAAiB;AAAA,IACzB,SAAS,OAAO;AACf,WAAK,OAAO,KAAK,sBAAsB,GAAG,MAAM,KAAK,EAAE;AACvD,aAAO;AAAA,IACR;AAAA,EACD;AAAA,EAEA,MAAc,iBACb,KACA,SACyB;AACzB,aAAS,UAAU,GAAG,WAAW,SAAS,WAAW;AACpD,YAAM,aAAa,IAAI,gBAAgB;AACvC,YAAM,YAAY,WAAW,MAAM,WAAW,MAAM,GAAG,KAAK,OAAO;AAEnE,UAAI;AACH,aAAK,OAAO,MAAM,yBAAyB,OAAO,MAAM,GAAG,EAAE;AAC7D,cAAM,kBAAkB,IAAI,UAAU,EAAE,SAAS;AACjD,aAAK,OAAO,KAAK,2BAA2B,eAAe,EAAE;AAC7D,aAAK,OAAO,YAAY,IAAI;AAC5B,cAAM,WAAW,MAAM,MAAM,KAAK;AAAA,UACjC,SAAS,KAAK;AAAA,UACd,QAAQ,WAAW;AAAA,UACnB,UAAU;AAAA,QACX,CAAC;AAED,qBAAa,SAAS;AAEtB,YAAI,SAAS,IAAI;AAChB,eAAK,OAAO,KAAK,6BAA6B,GAAG,EAAE;AACnD,iBAAO,MAAM,SAAS,KAAK;AAAA,QAC5B;AAEA,aAAK,OAAO,KAAK,wBAAwB,SAAS,MAAM,MAAM,GAAG,EAAE;AAAA,MACpE,SAAS,OAAO;AACf,YAAK,MAAgB,SAAS,cAAc;AAC3C,eAAK,OAAO,KAAK,oBAAoB,GAAG,EAAE;AAAA,QAC3C,OAAO;AACN,eAAK,OAAO,MAAM,uBAAuB,GAAG,MAAM,KAAK,EAAE;AAAA,QAC1D;AAAA,MACD;AAAA,IACD;AACA,WAAO;AAAA,EACR;AAAA,EAEQ,aAAa,MAAwB;AAC5C,UAAM,EAAE,MAAM,IAAI;AAClB,UAAM,MAAM,IAAI,MAAM,IAAI;AAC1B,UAAM,QAAQ,MAAM,KAAK,IAAI,OAAO,SAAS,iBAAiB,GAAG,CAAC;AAClE,UAAM,QAAQ,MAAM,IAAI,CAAC,SAAS,KAAK,IAAI;AAC3C,UAAM,gBAAgB,MAAM,OAAO,CAAC,SAAS,KAAK,eAAe,IAAI,CAAC;AACtE,SAAK,OAAO;AAAA,MACX,aAAa,cAAc,MAAM;AAAA,IAClC;AACA,WAAO,cAAc,IAAI,CAAC,SAAS,KAAK,6BAA6B,IAAI,CAAC;AAAA,EAC3E;AAAA,EAEA,MAAa,QAAgC;AAC5C,UAAM,UAAU,oBAAI,IAAY;AAChC,UAAM,QAAQ,oBAAI,IAAY,CAAC,KAAK,IAAI,IAAI,CAAC;AAC7C,UAAM,cAAc,oBAAI,IAAY;AAGpC,UAAM,kBAAkB;AAAA,MACvB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAEA,UAAM,kBAAkB,OAAO,eAAuB;AACrD,UAAI,QAAQ,IAAI,UAAU,GAAG;AAC5B,aAAK,OAAO,MAAM,iCAAiC,UAAU,EAAE;AAC/D;AAAA,MACD;AACA,cAAQ,IAAI,UAAU;AACtB,WAAK,OAAO,KAAK,iBAAiB,UAAU,EAAE;AAE9C,YAAM,OAAO,MAAM,KAAK,iBAAiB,YAAY,KAAK,OAAO;AACjE,UAAI,CAAC,KAAM;AAEX,YAAM,QAAQ,KAAK,aAAa,IAAI;AAGpC,iBAAW,QAAQ,OAAO;AACzB,YAAI,gBAAgB,KAAK,CAAC,QAAQ,KAAK,SAAS,GAAG,CAAC,GAAG;AACtD,eAAK,OAAO,MAAM,wBAAwB,IAAI,EAAE;AAChD;AAAA,QACD;AACA,aAAK,OAAO,MAAM,eAAe,IAAI,EAAE;AAAA,MACxC;AAEA,iBAAW,QAAQ,OAAO;AACzB,YAAI,CAAC,QAAQ,IAAI,IAAI,KAAK,MAAM,OAAO,KAAK,OAAO;AAClD,gBAAM,IAAI,IAAI;AACd,eAAK,OAAO,MAAM,mBAAmB,IAAI,EAAE;AAAA,QAC5C;AAAA,MACD;AACA,kBAAY,IAAI,UAAU;AAAA,IAC3B;AAEA,UAAM,eAAe,YAAY;AAChC,YAAM,QAAQ,MAAM,KAAK,KAAK,EAAE,MAAM,GAAG,KAAK,WAAW;AACzD,iBAAW,OAAO,OAAO;AACxB,cAAM,OAAO,GAAG;AAAA,MACjB;AACA,YAAM,QAAQ,WAAW,MAAM,IAAI,CAAC,QAAQ,gBAAgB,GAAG,CAAC,CAAC;AAAA,IAClE;AAEA,SAAK,OAAO;AAAA,MACX,8BAA8B,KAAK,KAAK,kBAAkB,KAAK,WAAW;AAAA,IAC3E;AACA,WAAO,MAAM,OAAO,KAAK,QAAQ,OAAO,KAAK,OAAO;AACnD,YAAM,aAAa;AAAA,IACpB;AAEA,SAAK,OAAO;AAAA,MACX,4CAA4C,YAAY,IAAI;AAAA,IAC7D;AAEA,WAAO,MAAM,KAAK,WAAW;AAAA,EAC9B;AACD;;;ADpMA,IAAM,YAAY,aAAa,KAAK;AAE7B,IAAM,gBAA+B;AAAA,EAC3C,SAAS;AAAA,EACT,UACC;AAAA,EACD,SAAS,CAACA,WAAU;AACnB,WAAOA,OACL,OAAO,OAAO;AAAA,MACd,OAAO;AAAA,MACP,MAAM;AAAA,MACN,aAAa;AAAA,MACb,cAAc;AAAA,MACd,QAAQ,CAAC,QAAQ;AAChB,YAAI;AACH,cAAI,IAAI,GAAG;AAEX,iBAAO;AAAA,QACR,SAAS,OAAO;AACf,gBAAM,IAAI,MAAM,gBAAgB,GAAG,EAAE;AAAA,QACtC;AAAA,MACD;AAAA,IACD,CAAC,EACA,OAAO,SAAS;AAAA,MAChB,OAAO;AAAA,MACP,MAAM;AAAA,MACN,aAAa;AAAA,MACb,SAAS;AAAA,MACT,QAAQ,CAAC,UAAU;AAClB,YAAI,QAAQ,GAAG;AACd,gBAAM,IAAI,MAAM,iCAAiC;AAAA,QAClD;AAEA,YAAI,QAAQ,KAAK;AAChB,gBAAM,IAAI,MAAM,6BAA6B;AAAA,QAC9C;AAEA,eAAO;AAAA,MACR;AAAA,IACD,CAAC,EACA,OAAO,UAAU;AAAA,MACjB,OAAO;AAAA,MACP,MAAM;AAAA,MACN,aACC;AAAA,MACD,QAAQ,CAAC,WAAW;AACnB,YAAI;AAGH,gBAAM,eAAe,KAAK,QAAQ,MAAM;AACxC,gBAAM,aAAa,KAAK,MAAM,YAAY;AAE1C,cAAI,WAAW,QAAQ,SAAS;AAC/B,kBAAM,IAAI,MAAM,iCAAiC;AAAA,UAClD;AAEA,cAAI,GAAG,WAAW,YAAY,GAAG;AAChC,kBAAM,IAAI,MAAM,4BAA4B;AAAA,UAC7C;AAEA,iBAAO;AAAA,QACR,SAAS,OAAO;AACf,gBAAM,IAAI,MAAM,wBAAwB,MAAM,EAAE;AAAA,QACjD;AAAA,MACD;AAAA,MACA,SAAS,mBAAmB;AAAA,IAC7B,CAAC,EACA,OAAO,eAAe;AAAA,MACtB,OAAO;AAAA,MACP,MAAM;AAAA,MACN,aAAa;AAAA,MACb,SAAS;AAAA,MACT,QAAQ,CAAC,gBAAgB;AACxB,YAAI,cAAc,GAAG;AACpB,gBAAM,IAAI,MAAM,uCAAuC;AAAA,QACxD;AAEA,YAAI,cAAc,IAAI;AACrB,gBAAM,IAAI,MAAM,kCAAkC;AAAA,QACnD;AAEA,eAAO;AAAA,MACR;AAAA,IACD,CAAC,EACA,OAAO,WAAW;AAAA,MAClB,OAAO;AAAA,MACP,MAAM;AAAA,MACN,aAAa;AAAA,MACb,SAAS;AAAA,MACT,QAAQ,CAAC,YAAY;AACpB,YAAI,UAAU,GAAG;AAChB,gBAAM,IAAI,MAAM,mCAAmC;AAAA,QACpD;AAEA,YAAI,UAAU,MAAQ;AACrB,gBAAM,IAAI,MAAM,kCAAkC;AAAA,QACnD;AAEA,eAAO;AAAA,MACR;AAAA,IACD,CAAC,EACA,OAAO,WAAW;AAAA,MAClB,OAAO;AAAA,MACP,MAAM;AAAA,MACN,aAAa;AAAA,MACb,SAAS;AAAA,MACT,QAAQ,CAAC,YAAY;AACpB,YAAI,UAAU,GAAG;AAChB,gBAAM,IAAI,MAAM,mCAAmC;AAAA,QACpD;AAEA,YAAI,UAAU,IAAI;AACjB,gBAAM,IAAI,MAAM,8BAA8B;AAAA,QAC/C;AAEA,eAAO;AAAA,MACR;AAAA,IACD,CAAC;AAAA,EACH;AAAA,EACA,SAAS,OAAO,SAAS;AACxB,QAAI;AACH,YAAM,UAAU;AAEhB,YAAM,UAAU,IAAI,cAAc,QAAQ,KAAK;AAAA,QAC9C,OAAO,QAAQ,SAAS;AAAA,QACxB,aAAa,QAAQ,eAAe;AAAA,QACpC,SAAS,QAAQ,WAAW;AAAA,QAC5B,SAAS,QAAQ,WAAW;AAAA,MAC7B,CAAC;AAED,gBAAU,KAAK,2BAA2B;AAE1C,YAAM,UAAU,MAAM,QAAQ,MAAM;AAEpC,UAAI,QAAQ,QAAQ;AACnB,WAAG,cAAc,QAAQ,QAAQ,KAAK,UAAU,SAAS,MAAM,CAAC,CAAC;AACjE,kBAAU,KAAK,sBAAsB,QAAQ,MAAM,EAAE;AAAA,MACtD,OAAO;AACN,cAAM,eAAe,mBAAmB;AACxC,WAAG,cAAc,cAAc,KAAK,UAAU,SAAS,MAAM,CAAC,CAAC;AAC/D,kBAAU,KAAK,sBAAsB,YAAY,EAAE;AAAA,MACpD;AAAA,IACD,SAAS,OAAO;AACf,UAAI,iBAAiB,OAAO;AAC3B,kBAAU,MAAM,MAAM,OAAO;AAAA,MAC9B;AACA,gBAAU,MAAM,8BAA8B;AAC9C,cAAQ,KAAK,CAAC;AAAA,IACf;AAAA,EACD;AACD;AAEA,IAAM,qBAAqB,MAAM;AAChC,MAAI;AACH,UAAM,cAAc,KAAK,QAAQ,iBAAiB;AAElD,QAAI,CAAC,GAAG,WAAW,WAAW,GAAG;AAChC,SAAG,UAAU,WAAW;AAAA,IACzB;AAEA,UAAM,eAAe,KAAK;AAAA,MACzB,0BAA0B,KAAK,IAAI,CAAC;AAAA,IACrC;AAEA,QAAI,GAAG,WAAW,YAAY,GAAG;AAChC,YAAM,IAAI,MAAM,4BAA4B;AAAA,IAC7C;AACA,UAAM,aAAa,KAAK,MAAM,YAAY;AAE1C,QAAI,WAAW,QAAQ,SAAS;AAC/B,YAAM,IAAI,MAAM,iCAAiC;AAAA,IAClD;AAEA,WAAO;AAAA,EACR,SAAS,OAAO;AACf,UAAM,IAAI,MAAM,qBAAqB;AAAA,EACtC;AACD;;;AD1LA,IAAM,iBAAiB,MAAM,QAAQ,QAAQ,IAAI,CAAC;AAElD,eAAe,cAAc;AAC7B,eAAe,WAAW,kBAAkB;AAC5C,eAAe,MAAM,+BAA+B;AACpD,eAAe,KAAK,EAAE,MAAM,QAAQ,GAAG;AACvC,eAAe,QAAQ,EAAE,MAAM,WAAW,GAAG;AAC7C,eAAe,OAAO;AACtB,eAAe,eAAe,IAAI;AAGlC,eAAe,QAAQ,aAAa;AAEpC,eAAe,MAAM;",
+  "names": ["yargs"]
+}

package/build/index.d.ts

@@ -1 +1,25 @@
+export declare class SpiderScanner {
+    private header;
+    private url;
+    private logger;
+    private depth;
+    private concurrency;
+    private retries;
+    private timeout;
+    constructor(url: string, options?: SpiderScannerOptions);
+    private normalizeDomain;
+    private convertRelativeUrlToAbsolute;
+    private isInternalLink;
+    private fetchWithRetries;
+    private extractLinks;
+    crawl(): Promise<Array<string>>;
+}
+
+export declare interface SpiderScannerOptions {
+    depth?: number;
+    concurrency?: number;
+    retries?: number;
+    timeout?: number;
+}
+
 export { }

package/build/index.js

@@ -1,48 +1,35 @@
-#!/usr/bin/env node
-
-// src/index.ts
-import yargs from "yargs";
-import { hideBin } from "yargs/helpers";
-
 // src/modules/spider/index.ts
 import fetch from "isomorphic-fetch";
 import jsdom from "jsdom";
 import UserAgent from "user-agents";
 
-// src/lib/logger.ts
-var Logger = class {
-  moduleName;
-  colors = {
-    error: "\x1B[31m",
-    info: "\x1B[32m",
-    warn: "\x1B[33m",
-    debug: "\x1B[35m",
-    reset: "\x1B[0m",
-    module: "\x1B[46m"
-  };
-  constructor(moduleName) {
-    this.moduleName = moduleName;
-  }
-  formatMessage(level, ...message) {
-    const timestamp = (/* @__PURE__ */ new Date()).toTimeString().split(" ")[0];
-    return `[${level}] ${this.colors[level]}${this.colors.reset}${this.colors[level]}[${timestamp}]${this.colors.reset} ${this.colors.module}[${this.moduleName}]${this.colors.reset} ${this.colors[level]}${message}${this.colors.reset}`;
-  }
-  error(...message) {
-    console.error(this.formatMessage("error", ...message));
-  }
-  info(...message) {
-    console.info(this.formatMessage("info", ...message));
-  }
-  warn(...message) {
-    console.warn(this.formatMessage("warn", ...message));
-  }
-  log(...message) {
-    console.log(this.formatMessage("info", ...message));
-  }
-  debug(...message) {
-    console.debug(this.formatMessage("debug", ...message));
-  }
-};
+// src/utils/index.ts
+import winston from "winston";
+var createLogger = (label) => winston.createLogger({
+  levels: {
+    error: 0,
+    warn: 1,
+    info: 2,
+    http: 3,
+    verbose: 4,
+    debug: 5,
+    silly: 6
+  },
+  format: winston.format.combine(
+    winston.format.label({ label }),
+    winston.format.colorize(),
+    winston.format.timestamp({
+      format: () => {
+        return (/* @__PURE__ */ new Date()).toLocaleString("en-US");
+      }
+    }),
+    winston.format.align(),
+    winston.format.printf(
+      (info) => `\x1B[34m(${info.label})\x1B[0m \x1B[33m${info.timestamp}\x1B[0m [${info.level}]: ${info.message}`
+    )
+  ),
+  transports: [new winston.transports.Console()]
+});
 
 // src/modules/spider/index.ts
 var SpiderScanner = class {
@@ -50,12 +37,26 @@ var SpiderScanner = class {
     "User-Agent": new UserAgent().toString()
   };
   url;
-  logger = new Logger("Spider");
-  constructor(url) {
+  logger = createLogger("SpiderScanner");
+  depth;
+  concurrency;
+  retries;
+  timeout;
+  constructor(url, options = {}) {
+    const {
+      depth = 250,
+      concurrency = 5,
+      retries = 3,
+      timeout = 5e3
+    } = options;
+    this.depth = depth;
+    this.concurrency = concurrency;
+    this.retries = retries;
+    this.timeout = timeout;
     try {
       this.url = new URL(url);
       this.logger.info(
-        `Initialized with URL: ${url} & User-Agent: ${this.header["User-Agent"]}`
+        `Initialized with URL: ${url}, User-Agent: ${this.header["User-Agent"]}`
       );
     } catch (error) {
       if (error instanceof TypeError) {
@@ -66,7 +67,6 @@ var SpiderScanner = class {
       throw error;
     }
   }
-  // Normalize domains (removes 'www.')
   normalizeDomain(domain) {
     return domain.startsWith("www.") ? domain.slice(4) : domain;
   }
@@ -87,20 +87,35 @@ var SpiderScanner = class {
       return false;
     }
   }
-  async fetchUrl(url) {
-    try {
-      this.logger.debug(`Fetching URL: ${url}`);
-      const response = await fetch(url, { headers: this.header });
-      if (!response.ok) {
+  async fetchWithRetries(url, retries) {
+    for (let attempt = 1; attempt <= retries; attempt++) {
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), this.timeout);
+      try {
+        this.logger.debug(`Fetching URL (Attempt ${attempt}): ${url}`);
+        const randomUserAgent = new UserAgent().toString();
+        this.logger.info(`Changing User-Agent to: ${randomUserAgent}`);
+        this.header["User-Agent"] = randomUserAgent;
+        const response = await fetch(url, {
+          headers: this.header,
+          signal: controller.signal,
+          redirect: "follow"
+        });
+        clearTimeout(timeoutId);
+        if (response.ok) {
+          this.logger.info(`Successfully fetched URL: ${url}`);
+          return await response.text();
+        }
         this.logger.warn(`Failed to fetch URL (${response.status}): ${url}`);
-        return null;
+      } catch (error) {
+        if (error.name === "AbortError") {
+          this.logger.warn(`Fetch timed out: ${url}`);
+        } else {
+          this.logger.error(`Error fetching URL: ${url} - ${error}`);
+        }
       }
-      this.logger.info(`Successfully fetched URL: ${url}`);
-      return await response.text();
-    } catch (error) {
-      this.logger.error(`Error fetching URL: ${url} - ${error}`);
-      return null;
     }
+    return null;
   }
   extractLinks(html) {
     const { JSDOM } = jsdom;
@@ -113,33 +128,38 @@ var SpiderScanner = class {
     );
     return internalLinks.map((link) => this.convertRelativeUrlToAbsolute(link));
   }
-  extractForms(html) {
-    const { JSDOM } = jsdom;
-    const dom = new JSDOM(html);
-    const forms = Array.from(dom.window.document.querySelectorAll("form"));
-    this.logger.debug(`Extracted ${forms.length} forms from HTML content`);
-    return forms.map((form, index) => {
-      const fields = Array.from(form.querySelectorAll("input")).map(
-        (input) => ({
-          name: input.name,
-          id: input.id,
-          class: input.className,
-          type: input.type
-        })
-      );
-      return {
-        id: index,
-        url: this.url.href,
-        fields
-      };
-    });
-  }
-  // Main function to scan the website with concurrency support and return both links and forms
-  async crawl(depth = 250, concurrency = 5) {
+  async crawl() {
     const visited = /* @__PURE__ */ new Set();
     const queue = /* @__PURE__ */ new Set([this.url.href]);
     const resultLinks = /* @__PURE__ */ new Set();
-    const resultForms = /* @__PURE__ */ new Set();
+    const assetExtensions = [
+      ".css",
+      ".js",
+      ".png",
+      ".jpg",
+      ".jpeg",
+      ".gif",
+      ".svg",
+      ".ico",
+      ".webp",
+      ".mp4",
+      ".mp3",
+      ".wav",
+      ".avi",
+      ".mov",
+      ".webm",
+      ".pdf",
+      ".doc",
+      ".docx",
+      ".xls",
+      ".xlsx",
+      ".ppt",
+      ".pptx",
+      ".zip",
+      ".rar",
+      ".tar",
+      ".gz"
+    ];
     const fetchAndExtract = async (currentUrl) => {
       if (visited.has(currentUrl)) {
         this.logger.debug(`Skipping already visited URL: ${currentUrl}`);
@@ -147,15 +167,18 @@ var SpiderScanner = class {
       }
       visited.add(currentUrl);
       this.logger.info(`Visiting URL: ${currentUrl}`);
-      const html = await this.fetchUrl(currentUrl);
+      const html = await this.fetchWithRetries(currentUrl, this.retries);
       if (!html) return;
       const links = this.extractLinks(html);
-      const forms = this.extractForms(html);
-      for (const form of forms) {
-        resultForms.add(form);
+      for (const link of links) {
+        if (assetExtensions.some((ext) => link.endsWith(ext))) {
+          this.logger.debug(`Ignoring asset link: ${link}`);
+          continue;
+        }
+        this.logger.debug(`Found link: ${link}`);
       }
       for (const link of links) {
-        if (!visited.has(link) && queue.size < depth) {
+        if (!visited.has(link) && queue.size < this.depth) {
           queue.add(link);
           this.logger.debug(`Added to queue: ${link}`);
         }
@@ -163,91 +186,25 @@ var SpiderScanner = class {
       resultLinks.add(currentUrl);
     };
     const processBatch = async () => {
-      const batch = Array.from(queue).slice(0, concurrency);
+      const batch = Array.from(queue).slice(0, this.concurrency);
       for (const url of batch) {
         queue.delete(url);
       }
       await Promise.allSettled(batch.map((url) => fetchAndExtract(url)));
     };
     this.logger.info(
-      `Starting crawl with depth: ${depth}, concurrency: ${concurrency}`
+      `Starting crawl with depth: ${this.depth}, concurrency: ${this.concurrency}`
     );
-    while (queue.size > 0 && visited.size < depth) {
+    while (queue.size > 0 && visited.size < this.depth) {
       await processBatch();
     }
     this.logger.info(
-      `Crawling completed. Total pages visited: ${resultLinks.size}, Total forms found: ${resultForms.size}`
+      `Crawling completed. Total pages visited: ${resultLinks.size}`
     );
-    return {
-      links: Array.from(resultLinks),
-      forms: Array.from(resultForms)
-    };
+    return Array.from(resultLinks);
   }
 };
-
-// src/index.ts
-var commandHandler = yargs(hideBin(process.argv));
-commandHandler.command(
-  "xss",
-  "Scan for XSS vulnerabilities",
-  {
-    url: {
-      describe: "URL to scan",
-      demandOption: true,
-      type: "string",
-      coerce: (value) => {
-        try {
-          new URL(value);
-          return value;
-        } catch (err) {
-          throw new Error("Invalid URL format");
-        }
-      }
-    },
-    wordlist: {
-      describe: "Path to wordlist file",
-      type: "string"
-    }
-  },
-  (argv) => {
-    console.log("Scanning for XSS vulnerabilities...");
-    console.log(`URL: ${argv.url}`);
-    console.log(`Wordlist: ${argv.wordlist || "Default"}`);
-  }
-);
-commandHandler.command(
-  "spider",
-  "Scan a website for vulnerabilities",
-  {
-    url: {
-      describe: "URL to scan",
-      demandOption: true,
-      type: "string",
-      coerce: (value) => {
-        try {
-          new URL(value);
-          return value;
-        } catch (err) {
-          throw new Error("Invalid URL format");
-        }
-      }
-    }
-  },
-  (argv) => {
-    const spider = new SpiderScanner(argv.url);
-    spider.crawl().then((output) => {
-      console.log(
-        JSON.stringify(
-          {
-            forms: output.forms,
-            links: output.links
-          },
-          null,
-          2
-        )
-      );
-    });
-  }
-);
-commandHandler.parse();
+export {
+  SpiderScanner
+};
 //# sourceMappingURL=index.js.map

package/build/index.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
-  "sources": ["../src/index.ts", "../src/modules/spider/index.ts", "../src/lib/logger.ts"],
-  "sourcesContent": ["#!/usr/bin/env node\n\nimport yargs from \"yargs\";\nimport { hideBin } from \"yargs/helpers\";\nimport { SpiderScanner } from \"./modules\";\n\nconst commandHandler = yargs(hideBin(process.argv));\n\n/**\n * Command to scan for XSS vulnerabilities\n *\n * @param {string} url - URL to scan\n * @param {string} wordlist - Path to wordlist file\n * @returns {void}\n *\n * @example\n * npx sentinel-scanner xss --url https://example.com\n */\ncommandHandler.command(\n\t\"xss\",\n\t\"Scan for XSS vulnerabilities\",\n\t{\n\t\turl: {\n\t\t\tdescribe: \"URL to scan\",\n\t\t\tdemandOption: true,\n\t\t\ttype: \"string\",\n\t\t\tcoerce: (value) => {\n\t\t\t\ttry {\n\t\t\t\t\tnew URL(value);\n\t\t\t\t\treturn value;\n\t\t\t\t} catch (err) {\n\t\t\t\t\tthrow new Error(\"Invalid URL format\");\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t\twordlist: {\n\t\t\tdescribe: \"Path to wordlist file\",\n\t\t\ttype: \"string\",\n\t\t},\n\t},\n\t(argv) => {\n\t\tconsole.log(\"Scanning for XSS vulnerabilities...\");\n\t\tconsole.log(`URL: ${argv.url}`);\n\t\tconsole.log(`Wordlist: ${argv.wordlist || \"Default\"}`);\n\t},\n);\n\n// Command to Spider a website\ncommandHandler.command(\n\t\"spider\",\n\t\"Scan a website for vulnerabilities\",\n\t{\n\t\turl: {\n\t\t\tdescribe: \"URL to scan\",\n\t\t\tdemandOption: true,\n\t\t\ttype: \"string\",\n\t\t\tcoerce: (value) => {\n\t\t\t\ttry {\n\t\t\t\t\tnew URL(value);\n\t\t\t\t\treturn value;\n\t\t\t\t} catch (err) {\n\t\t\t\t\tthrow new Error(\"Invalid URL format\");\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t},\n\t(argv) => {\n\t\tconst spider = new SpiderScanner(argv.url);\n\n\t\tspider.crawl().then((output) => {\n\t\t\tconsole.log(\n\t\t\t\tJSON.stringify(\n\t\t\t\t\t{\n\t\t\t\t\t\tforms: output.forms,\n\t\t\t\t\t\tlinks: output.links,\n\t\t\t\t\t},\n\t\t\t\t\tnull,\n\t\t\t\t\t2,\n\t\t\t\t),\n\t\t\t);\n\t\t});\n\t},\n);\n\n// Parse arguments and handle commands\ncommandHandler.parse();\n", "import fetch from \"isomorphic-fetch\";\nimport jsdom from \"jsdom\";\nimport UserAgent from \"user-agents\";\nimport Logger from \"../../lib/logger\";\n\nexport type FormOutput = {\n\tid: number;\n\turl: string;\n\tfields: Array<{ name: string; id: string; class: string; type: string }>;\n};\n\nexport type CrawlOutput = {\n\tlinks: string[];\n\tforms: FormOutput[];\n};\n\nexport default class SpiderScanner {\n\tprivate header: Record<string, string> = {\n\t\t\"User-Agent\": new UserAgent().toString(),\n\t};\n\tprivate url: URL;\n\tprivate logger = new Logger(\"Spider\");\n\n\tconstructor(url: string) {\n\t\ttry {\n\t\t\tthis.url = new URL(url);\n\t\t\tthis.logger.info(\n\t\t\t\t`Initialized with URL: ${url} & User-Agent: ${this.header[\"User-Agent\"]}`,\n\t\t\t);\n\t\t} catch (error) {\n\t\t\tif (error instanceof TypeError) {\n\t\t\t\tthis.logger.error(\"Invalid URL\");\n\t\t\t\tthrow new Error(\"Invalid URL\");\n\t\t\t}\n\t\t\tthis.logger.error(`Unexpected error in constructor: ${error}`);\n\t\t\tthrow error;\n\t\t}\n\t}\n\n\t// Normalize domains (removes 'www.')\n\tprivate normalizeDomain(domain: string): string {\n\t\treturn domain.startsWith(\"www.\") ? domain.slice(4) : domain;\n\t}\n\n\tprivate convertRelativeUrlToAbsolute(url: string): string {\n\t\treturn new URL(url, this.url.toString()).toString();\n\t}\n\n\tprivate isInternalLink(url: string): boolean {\n\t\ttry {\n\t\t\tconst parsedUrl = new URL(url, this.url.href);\n\t\t\tif (![\"http:\", \"https:\"].includes(parsedUrl.protocol)) {\n\t\t\t\treturn false;\n\t\t\t}\n\t\t\tconst baseDomain = this.normalizeDomain(this.url.hostname);\n\t\t\tconst parsedDomain = this.normalizeDomain(parsedUrl.hostname);\n\t\t\treturn parsedDomain === baseDomain;\n\t\t} catch (error) {\n\t\t\tthis.logger.warn(`Error parsing URL: ${url} - ${error}`);\n\t\t\treturn false;\n\t\t}\n\t}\n\n\tprivate async fetchUrl(url: string): Promise<string | null> {\n\t\ttry {\n\t\t\tthis.logger.debug(`Fetching URL: ${url}`);\n\t\t\tconst response = await fetch(url, { headers: this.header });\n\t\t\tif (!response.ok) {\n\t\t\t\tthis.logger.warn(`Failed to fetch URL (${response.status}): ${url}`);\n\t\t\t\treturn null;\n\t\t\t}\n\t\t\tthis.logger.info(`Successfully fetched URL: ${url}`);\n\t\t\treturn await response.text();\n\t\t} catch (error) {\n\t\t\tthis.logger.error(`Error fetching URL: ${url} - ${error}`);\n\t\t\treturn null;\n\t\t}\n\t}\n\n\tprivate extractLinks(html: string): string[] {\n\t\tconst { JSDOM } = jsdom;\n\t\tconst dom = new JSDOM(html);\n\t\tconst links = Array.from(dom.window.document.querySelectorAll(\"a\"));\n\t\tconst hrefs = links.map((link) => link.href);\n\t\tconst internalLinks = hrefs.filter((href) => this.isInternalLink(href));\n\t\tthis.logger.debug(\n\t\t\t`Extracted ${internalLinks.length} internal links from HTML content`,\n\t\t);\n\t\treturn internalLinks.map((link) => this.convertRelativeUrlToAbsolute(link));\n\t}\n\n\tprivate extractForms(html: string): FormOutput[] {\n\t\tconst { JSDOM } = jsdom;\n\t\tconst dom = new JSDOM(html);\n\t\tconst forms = Array.from(dom.window.document.querySelectorAll(\"form\"));\n\t\tthis.logger.debug(`Extracted ${forms.length} forms from HTML content`);\n\n\t\treturn forms.map((form, index) => {\n\t\t\tconst fields = Array.from(form.querySelectorAll(\"input\")).map(\n\t\t\t\t(input) => ({\n\t\t\t\t\tname: input.name,\n\t\t\t\t\tid: input.id,\n\t\t\t\t\tclass: input.className,\n\t\t\t\t\ttype: input.type,\n\t\t\t\t}),\n\t\t\t);\n\n\t\t\treturn {\n\t\t\t\tid: index,\n\t\t\t\turl: this.url.href,\n\t\t\t\tfields,\n\t\t\t};\n\t\t});\n\t}\n\n\t// Main function to scan the website with concurrency support and return both links and forms\n\tpublic async crawl(depth = 250, concurrency = 5): Promise<CrawlOutput> {\n\t\tconst visited = new Set<string>();\n\t\tconst queue = new Set<string>([this.url.href]);\n\t\tconst resultLinks = new Set<string>();\n\t\tconst resultForms = new Set<FormOutput>();\n\n\t\tconst fetchAndExtract = async (currentUrl: string) => {\n\t\t\tif (visited.has(currentUrl)) {\n\t\t\t\tthis.logger.debug(`Skipping already visited URL: ${currentUrl}`);\n\t\t\t\treturn;\n\t\t\t}\n\t\t\tvisited.add(currentUrl);\n\t\t\tthis.logger.info(`Visiting URL: ${currentUrl}`);\n\n\t\t\tconst html = await this.fetchUrl(currentUrl);\n\t\t\tif (!html) return;\n\n\t\t\t// Extract links and forms\n\t\t\tconst links = this.extractLinks(html);\n\t\t\tconst forms = this.extractForms(html);\n\n\t\t\tfor (const form of forms) {\n\t\t\t\tresultForms.add(form);\n\t\t\t}\n\n\t\t\tfor (const link of links) {\n\t\t\t\tif (!visited.has(link) && queue.size < depth) {\n\t\t\t\t\tqueue.add(link);\n\t\t\t\t\tthis.logger.debug(`Added to queue: ${link}`);\n\t\t\t\t}\n\t\t\t}\n\t\t\tresultLinks.add(currentUrl);\n\t\t};\n\n\t\tconst processBatch = async () => {\n\t\t\tconst batch = Array.from(queue).slice(0, concurrency);\n\t\t\tfor (const url of batch) {\n\t\t\t\tqueue.delete(url);\n\t\t\t}\n\t\t\tawait Promise.allSettled(batch.map((url) => fetchAndExtract(url)));\n\t\t};\n\n\t\tthis.logger.info(\n\t\t\t`Starting crawl with depth: ${depth}, concurrency: ${concurrency}`,\n\t\t);\n\t\twhile (queue.size > 0 && visited.size < depth) {\n\t\t\tawait processBatch();\n\t\t}\n\n\t\tthis.logger.info(\n\t\t\t`Crawling completed. Total pages visited: ${resultLinks.size}, Total forms found: ${resultForms.size}`,\n\t\t);\n\n\t\treturn {\n\t\t\tlinks: Array.from(resultLinks),\n\t\t\tforms: Array.from(resultForms),\n\t\t};\n\t}\n}\n", "export default class Logger {\n\tprivate moduleName: string;\n\tprivate colors = {\n\t\terror: \"\\x1b[31m\",\n\t\tinfo: \"\\x1b[32m\",\n\t\twarn: \"\\x1b[33m\",\n\t\tdebug: \"\\x1b[35m\",\n\t\treset: \"\\x1b[0m\",\n\t\tmodule: \"\\x1b[46m\",\n\t};\n\n\tconstructor(moduleName: string) {\n\t\tthis.moduleName = moduleName;\n\t}\n\n\tprivate formatMessage(\n\t\tlevel: keyof typeof this.colors,\n\t\t...message: string[]\n\t): string {\n\t\tconst timestamp = new Date().toTimeString().split(\" \")[0];\n\t\treturn `[${level}] ${this.colors[level]}${this.colors.reset}${this.colors[level]}[${timestamp}]${this.colors.reset} ${this.colors.module}[${this.moduleName}]${this.colors.reset} ${this.colors[level]}${message}${this.colors.reset}`;\n\t}\n\n\tpublic error(...message: string[]): void {\n\t\tconsole.error(this.formatMessage(\"error\", ...message));\n\t}\n\n\tpublic info(...message: string[]): void {\n\t\tconsole.info(this.formatMessage(\"info\", ...message));\n\t}\n\n\tpublic warn(...message: string[]): void {\n\t\tconsole.warn(this.formatMessage(\"warn\", ...message));\n\t}\n\n\tpublic log(...message: string[]): void {\n\t\tconsole.log(this.formatMessage(\"info\", ...message));\n\t}\n\n\tpublic debug(...message: string[]): void {\n\t\tconsole.debug(this.formatMessage(\"debug\", ...message));\n\t}\n}\n"],
-  "mappings": ";;;AAEA,OAAO,WAAW;AAClB,SAAS,eAAe;;;ACHxB,OAAO,WAAW;AAClB,OAAO,WAAW;AAClB,OAAO,eAAe;;;ACFtB,IAAqB,SAArB,MAA4B;AAAA,EACnB;AAAA,EACA,SAAS;AAAA,IAChB,OAAO;AAAA,IACP,MAAM;AAAA,IACN,MAAM;AAAA,IACN,OAAO;AAAA,IACP,OAAO;AAAA,IACP,QAAQ;AAAA,EACT;AAAA,EAEA,YAAY,YAAoB;AAC/B,SAAK,aAAa;AAAA,EACnB;AAAA,EAEQ,cACP,UACG,SACM;AACT,UAAM,aAAY,oBAAI,KAAK,GAAE,aAAa,EAAE,MAAM,GAAG,EAAE,CAAC;AACxD,WAAO,IAAI,KAAK,KAAK,KAAK,OAAO,KAAK,CAAC,GAAG,KAAK,OAAO,KAAK,GAAG,KAAK,OAAO,KAAK,CAAC,IAAI,SAAS,IAAI,KAAK,OAAO,KAAK,IAAI,KAAK,OAAO,MAAM,IAAI,KAAK,UAAU,IAAI,KAAK,OAAO,KAAK,IAAI,KAAK,OAAO,KAAK,CAAC,GAAG,OAAO,GAAG,KAAK,OAAO,KAAK;AAAA,EACrO;AAAA,EAEO,SAAS,SAAyB;AACxC,YAAQ,MAAM,KAAK,cAAc,SAAS,GAAG,OAAO,CAAC;AAAA,EACtD;AAAA,EAEO,QAAQ,SAAyB;AACvC,YAAQ,KAAK,KAAK,cAAc,QAAQ,GAAG,OAAO,CAAC;AAAA,EACpD;AAAA,EAEO,QAAQ,SAAyB;AACvC,YAAQ,KAAK,KAAK,cAAc,QAAQ,GAAG,OAAO,CAAC;AAAA,EACpD;AAAA,EAEO,OAAO,SAAyB;AACtC,YAAQ,IAAI,KAAK,cAAc,QAAQ,GAAG,OAAO,CAAC;AAAA,EACnD;AAAA,EAEO,SAAS,SAAyB;AACxC,YAAQ,MAAM,KAAK,cAAc,SAAS,GAAG,OAAO,CAAC;AAAA,EACtD;AACD;;;AD1BA,IAAqB,gBAArB,MAAmC;AAAA,EAC1B,SAAiC;AAAA,IACxC,cAAc,IAAI,UAAU,EAAE,SAAS;AAAA,EACxC;AAAA,EACQ;AAAA,EACA,SAAS,IAAI,OAAO,QAAQ;AAAA,EAEpC,YAAY,KAAa;AACxB,QAAI;AACH,WAAK,MAAM,IAAI,IAAI,GAAG;AACtB,WAAK,OAAO;AAAA,QACX,yBAAyB,GAAG,kBAAkB,KAAK,OAAO,YAAY,CAAC;AAAA,MACxE;AAAA,IACD,SAAS,OAAO;AACf,UAAI,iBAAiB,WAAW;AAC/B,aAAK,OAAO,MAAM,aAAa;AAC/B,cAAM,IAAI,MAAM,aAAa;AAAA,MAC9B;AACA,WAAK,OAAO,MAAM,oCAAoC,KAAK,EAAE;AAC7D,YAAM;AAAA,IACP;AAAA,EACD;AAAA;AAAA,EAGQ,gBAAgB,QAAwB;AAC/C,WAAO,OAAO,WAAW,MAAM,IAAI,OAAO,MAAM,CAAC,IAAI;AAAA,EACtD;AAAA,EAEQ,6BAA6B,KAAqB;AACzD,WAAO,IAAI,IAAI,KAAK,KAAK,IAAI,SAAS,CAAC,EAAE,SAAS;AAAA,EACnD;AAAA,EAEQ,eAAe,KAAsB;AAC5C,QAAI;AACH,YAAM,YAAY,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI;AAC5C,UAAI,CAAC,CAAC,SAAS,QAAQ,EAAE,SAAS,UAAU,QAAQ,GAAG;AACtD,eAAO;AAAA,MACR;AACA,YAAM,aAAa,KAAK,gBAAgB,KAAK,IAAI,QAAQ;AACzD,YAAM,eAAe,KAAK,gBAAgB,UAAU,QAAQ;AAC5D,aAAO,iBAAiB;AAAA,IACzB,SAAS,OAAO;AACf,WAAK,OAAO,KAAK,sBAAsB,GAAG,MAAM,KAAK,EAAE;AACvD,aAAO;AAAA,IACR;AAAA,EACD;AAAA,EAEA,MAAc,SAAS,KAAqC;AAC3D,QAAI;AACH,WAAK,OAAO,MAAM,iBAAiB,GAAG,EAAE;AACxC,YAAM,WAAW,MAAM,MAAM,KAAK,EAAE,SAAS,KAAK,OAAO,CAAC;AAC1D,UAAI,CAAC,SAAS,IAAI;AACjB,aAAK,OAAO,KAAK,wBAAwB,SAAS,MAAM,MAAM,GAAG,EAAE;AACnE,eAAO;AAAA,MACR;AACA,WAAK,OAAO,KAAK,6BAA6B,GAAG,EAAE;AACnD,aAAO,MAAM,SAAS,KAAK;AAAA,IAC5B,SAAS,OAAO;AACf,WAAK,OAAO,MAAM,uBAAuB,GAAG,MAAM,KAAK,EAAE;AACzD,aAAO;AAAA,IACR;AAAA,EACD;AAAA,EAEQ,aAAa,MAAwB;AAC5C,UAAM,EAAE,MAAM,IAAI;AAClB,UAAM,MAAM,IAAI,MAAM,IAAI;AAC1B,UAAM,QAAQ,MAAM,KAAK,IAAI,OAAO,SAAS,iBAAiB,GAAG,CAAC;AAClE,UAAM,QAAQ,MAAM,IAAI,CAAC,SAAS,KAAK,IAAI;AAC3C,UAAM,gBAAgB,MAAM,OAAO,CAAC,SAAS,KAAK,eAAe,IAAI,CAAC;AACtE,SAAK,OAAO;AAAA,MACX,aAAa,cAAc,MAAM;AAAA,IAClC;AACA,WAAO,cAAc,IAAI,CAAC,SAAS,KAAK,6BAA6B,IAAI,CAAC;AAAA,EAC3E;AAAA,EAEQ,aAAa,MAA4B;AAChD,UAAM,EAAE,MAAM,IAAI;AAClB,UAAM,MAAM,IAAI,MAAM,IAAI;AAC1B,UAAM,QAAQ,MAAM,KAAK,IAAI,OAAO,SAAS,iBAAiB,MAAM,CAAC;AACrE,SAAK,OAAO,MAAM,aAAa,MAAM,MAAM,0BAA0B;AAErE,WAAO,MAAM,IAAI,CAAC,MAAM,UAAU;AACjC,YAAM,SAAS,MAAM,KAAK,KAAK,iBAAiB,OAAO,CAAC,EAAE;AAAA,QACzD,CAAC,WAAW;AAAA,UACX,MAAM,MAAM;AAAA,UACZ,IAAI,MAAM;AAAA,UACV,OAAO,MAAM;AAAA,UACb,MAAM,MAAM;AAAA,QACb;AAAA,MACD;AAEA,aAAO;AAAA,QACN,IAAI;AAAA,QACJ,KAAK,KAAK,IAAI;AAAA,QACd;AAAA,MACD;AAAA,IACD,CAAC;AAAA,EACF;AAAA;AAAA,EAGA,MAAa,MAAM,QAAQ,KAAK,cAAc,GAAyB;AACtE,UAAM,UAAU,oBAAI,IAAY;AAChC,UAAM,QAAQ,oBAAI,IAAY,CAAC,KAAK,IAAI,IAAI,CAAC;AAC7C,UAAM,cAAc,oBAAI,IAAY;AACpC,UAAM,cAAc,oBAAI,IAAgB;AAExC,UAAM,kBAAkB,OAAO,eAAuB;AACrD,UAAI,QAAQ,IAAI,UAAU,GAAG;AAC5B,aAAK,OAAO,MAAM,iCAAiC,UAAU,EAAE;AAC/D;AAAA,MACD;AACA,cAAQ,IAAI,UAAU;AACtB,WAAK,OAAO,KAAK,iBAAiB,UAAU,EAAE;AAE9C,YAAM,OAAO,MAAM,KAAK,SAAS,UAAU;AAC3C,UAAI,CAAC,KAAM;AAGX,YAAM,QAAQ,KAAK,aAAa,IAAI;AACpC,YAAM,QAAQ,KAAK,aAAa,IAAI;AAEpC,iBAAW,QAAQ,OAAO;AACzB,oBAAY,IAAI,IAAI;AAAA,MACrB;AAEA,iBAAW,QAAQ,OAAO;AACzB,YAAI,CAAC,QAAQ,IAAI,IAAI,KAAK,MAAM,OAAO,OAAO;AAC7C,gBAAM,IAAI,IAAI;AACd,eAAK,OAAO,MAAM,mBAAmB,IAAI,EAAE;AAAA,QAC5C;AAAA,MACD;AACA,kBAAY,IAAI,UAAU;AAAA,IAC3B;AAEA,UAAM,eAAe,YAAY;AAChC,YAAM,QAAQ,MAAM,KAAK,KAAK,EAAE,MAAM,GAAG,WAAW;AACpD,iBAAW,OAAO,OAAO;AACxB,cAAM,OAAO,GAAG;AAAA,MACjB;AACA,YAAM,QAAQ,WAAW,MAAM,IAAI,CAAC,QAAQ,gBAAgB,GAAG,CAAC,CAAC;AAAA,IAClE;AAEA,SAAK,OAAO;AAAA,MACX,8BAA8B,KAAK,kBAAkB,WAAW;AAAA,IACjE;AACA,WAAO,MAAM,OAAO,KAAK,QAAQ,OAAO,OAAO;AAC9C,YAAM,aAAa;AAAA,IACpB;AAEA,SAAK,OAAO;AAAA,MACX,4CAA4C,YAAY,IAAI,wBAAwB,YAAY,IAAI;AAAA,IACrG;AAEA,WAAO;AAAA,MACN,OAAO,MAAM,KAAK,WAAW;AAAA,MAC7B,OAAO,MAAM,KAAK,WAAW;AAAA,IAC9B;AAAA,EACD;AACD;;;ADxKA,IAAM,iBAAiB,MAAM,QAAQ,QAAQ,IAAI,CAAC;AAYlD,eAAe;AAAA,EACd;AAAA,EACA;AAAA,EACA;AAAA,IACC,KAAK;AAAA,MACJ,UAAU;AAAA,MACV,cAAc;AAAA,MACd,MAAM;AAAA,MACN,QAAQ,CAAC,UAAU;AAClB,YAAI;AACH,cAAI,IAAI,KAAK;AACb,iBAAO;AAAA,QACR,SAAS,KAAK;AACb,gBAAM,IAAI,MAAM,oBAAoB;AAAA,QACrC;AAAA,MACD;AAAA,IACD;AAAA,IACA,UAAU;AAAA,MACT,UAAU;AAAA,MACV,MAAM;AAAA,IACP;AAAA,EACD;AAAA,EACA,CAAC,SAAS;AACT,YAAQ,IAAI,qCAAqC;AACjD,YAAQ,IAAI,QAAQ,KAAK,GAAG,EAAE;AAC9B,YAAQ,IAAI,aAAa,KAAK,YAAY,SAAS,EAAE;AAAA,EACtD;AACD;AAGA,eAAe;AAAA,EACd;AAAA,EACA;AAAA,EACA;AAAA,IACC,KAAK;AAAA,MACJ,UAAU;AAAA,MACV,cAAc;AAAA,MACd,MAAM;AAAA,MACN,QAAQ,CAAC,UAAU;AAClB,YAAI;AACH,cAAI,IAAI,KAAK;AACb,iBAAO;AAAA,QACR,SAAS,KAAK;AACb,gBAAM,IAAI,MAAM,oBAAoB;AAAA,QACrC;AAAA,MACD;AAAA,IACD;AAAA,EACD;AAAA,EACA,CAAC,SAAS;AACT,UAAM,SAAS,IAAI,cAAc,KAAK,GAAG;AAEzC,WAAO,MAAM,EAAE,KAAK,CAAC,WAAW;AAC/B,cAAQ;AAAA,QACP,KAAK;AAAA,UACJ;AAAA,YACC,OAAO,OAAO;AAAA,YACd,OAAO,OAAO;AAAA,UACf;AAAA,UACA;AAAA,UACA;AAAA,QACD;AAAA,MACD;AAAA,IACD,CAAC;AAAA,EACF;AACD;AAGA,eAAe,MAAM;",
+  "sources": ["../src/modules/spider/index.ts", "../src/utils/index.ts"],
+  "sourcesContent": ["import fetch from \"isomorphic-fetch\";\nimport jsdom from \"jsdom\";\nimport UserAgent from \"user-agents\";\nimport { createLogger } from \"../../utils\";\n\nexport interface SpiderScannerOptions {\n\tdepth?: number;\n\tconcurrency?: number;\n\tretries?: number;\n\ttimeout?: number;\n}\n\nexport default class SpiderScanner {\n\tprivate header: Record<string, string> = {\n\t\t\"User-Agent\": new UserAgent().toString(),\n\t};\n\tprivate url: URL;\n\tprivate logger = createLogger(\"SpiderScanner\");\n\n\tprivate depth: number;\n\tprivate concurrency: number;\n\tprivate retries: number;\n\tprivate timeout: number;\n\n\tconstructor(url: string, options: SpiderScannerOptions = {}) {\n\t\tconst {\n\t\t\tdepth = 250,\n\t\t\tconcurrency = 5,\n\t\t\tretries = 3,\n\t\t\ttimeout = 5000,\n\t\t} = options;\n\t\tthis.depth = depth;\n\t\tthis.concurrency = concurrency;\n\t\tthis.retries = retries;\n\t\tthis.timeout = timeout;\n\n\t\ttry {\n\t\t\tthis.url = new URL(url);\n\t\t\tthis.logger.info(\n\t\t\t\t`Initialized with URL: ${url}, User-Agent: ${this.header[\"User-Agent\"]}`,\n\t\t\t);\n\t\t} catch (error) {\n\t\t\tif (error instanceof TypeError) {\n\t\t\t\tthis.logger.error(\"Invalid URL\");\n\t\t\t\tthrow new Error(\"Invalid URL\");\n\t\t\t}\n\t\t\tthis.logger.error(`Unexpected error in constructor: ${error}`);\n\t\t\tthrow error;\n\t\t}\n\t}\n\n\tprivate normalizeDomain(domain: string): string {\n\t\treturn domain.startsWith(\"www.\") ? domain.slice(4) : domain;\n\t}\n\n\tprivate convertRelativeUrlToAbsolute(url: string): string {\n\t\treturn new URL(url, this.url.toString()).toString();\n\t}\n\n\tprivate isInternalLink(url: string): boolean {\n\t\ttry {\n\t\t\tconst parsedUrl = new URL(url, this.url.href);\n\t\t\tif (![\"http:\", \"https:\"].includes(parsedUrl.protocol)) {\n\t\t\t\treturn false;\n\t\t\t}\n\t\t\tconst baseDomain = this.normalizeDomain(this.url.hostname);\n\t\t\tconst parsedDomain = this.normalizeDomain(parsedUrl.hostname);\n\t\t\treturn parsedDomain === baseDomain;\n\t\t} catch (error) {\n\t\t\tthis.logger.warn(`Error parsing URL: ${url} - ${error}`);\n\t\t\treturn false;\n\t\t}\n\t}\n\n\tprivate async fetchWithRetries(\n\t\turl: string,\n\t\tretries: number,\n\t): Promise<string | null> {\n\t\tfor (let attempt = 1; attempt <= retries; attempt++) {\n\t\t\tconst controller = new AbortController();\n\t\t\tconst timeoutId = setTimeout(() => controller.abort(), this.timeout);\n\n\t\t\ttry {\n\t\t\t\tthis.logger.debug(`Fetching URL (Attempt ${attempt}): ${url}`);\n\t\t\t\tconst randomUserAgent = new UserAgent().toString();\n\t\t\t\tthis.logger.info(`Changing User-Agent to: ${randomUserAgent}`);\n\t\t\t\tthis.header[\"User-Agent\"] = randomUserAgent;\n\t\t\t\tconst response = await fetch(url, {\n\t\t\t\t\theaders: this.header,\n\t\t\t\t\tsignal: controller.signal,\n\t\t\t\t\tredirect: \"follow\",\n\t\t\t\t});\n\n\t\t\t\tclearTimeout(timeoutId);\n\n\t\t\t\tif (response.ok) {\n\t\t\t\t\tthis.logger.info(`Successfully fetched URL: ${url}`);\n\t\t\t\t\treturn await response.text();\n\t\t\t\t}\n\n\t\t\t\tthis.logger.warn(`Failed to fetch URL (${response.status}): ${url}`);\n\t\t\t} catch (error) {\n\t\t\t\tif ((error as Error).name === \"AbortError\") {\n\t\t\t\t\tthis.logger.warn(`Fetch timed out: ${url}`);\n\t\t\t\t} else {\n\t\t\t\t\tthis.logger.error(`Error fetching URL: ${url} - ${error}`);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn null;\n\t}\n\n\tprivate extractLinks(html: string): string[] {\n\t\tconst { JSDOM } = jsdom;\n\t\tconst dom = new JSDOM(html);\n\t\tconst links = Array.from(dom.window.document.querySelectorAll(\"a\"));\n\t\tconst hrefs = links.map((link) => link.href);\n\t\tconst internalLinks = hrefs.filter((href) => this.isInternalLink(href));\n\t\tthis.logger.debug(\n\t\t\t`Extracted ${internalLinks.length} internal links from HTML content`,\n\t\t);\n\t\treturn internalLinks.map((link) => this.convertRelativeUrlToAbsolute(link));\n\t}\n\n\tpublic async crawl(): Promise<Array<string>> {\n\t\tconst visited = new Set<string>();\n\t\tconst queue = new Set<string>([this.url.href]);\n\t\tconst resultLinks = new Set<string>();\n\n\t\t// Assets to ignore\n\t\tconst assetExtensions = [\n\t\t\t\".css\",\n\t\t\t\".js\",\n\t\t\t\".png\",\n\t\t\t\".jpg\",\n\t\t\t\".jpeg\",\n\t\t\t\".gif\",\n\t\t\t\".svg\",\n\t\t\t\".ico\",\n\t\t\t\".webp\",\n\t\t\t\".mp4\",\n\t\t\t\".mp3\",\n\t\t\t\".wav\",\n\t\t\t\".avi\",\n\t\t\t\".mov\",\n\t\t\t\".webm\",\n\t\t\t\".pdf\",\n\t\t\t\".doc\",\n\t\t\t\".docx\",\n\t\t\t\".xls\",\n\t\t\t\".xlsx\",\n\t\t\t\".ppt\",\n\t\t\t\".pptx\",\n\t\t\t\".zip\",\n\t\t\t\".rar\",\n\t\t\t\".tar\",\n\t\t\t\".gz\",\n\t\t];\n\n\t\tconst fetchAndExtract = async (currentUrl: string) => {\n\t\t\tif (visited.has(currentUrl)) {\n\t\t\t\tthis.logger.debug(`Skipping already visited URL: ${currentUrl}`);\n\t\t\t\treturn;\n\t\t\t}\n\t\t\tvisited.add(currentUrl);\n\t\t\tthis.logger.info(`Visiting URL: ${currentUrl}`);\n\n\t\t\tconst html = await this.fetchWithRetries(currentUrl, this.retries);\n\t\t\tif (!html) return;\n\n\t\t\tconst links = this.extractLinks(html);\n\n\t\t\t// Filter out asset links\n\t\t\tfor (const link of links) {\n\t\t\t\tif (assetExtensions.some((ext) => link.endsWith(ext))) {\n\t\t\t\t\tthis.logger.debug(`Ignoring asset link: ${link}`);\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tthis.logger.debug(`Found link: ${link}`);\n\t\t\t}\n\n\t\t\tfor (const link of links) {\n\t\t\t\tif (!visited.has(link) && queue.size < this.depth) {\n\t\t\t\t\tqueue.add(link);\n\t\t\t\t\tthis.logger.debug(`Added to queue: ${link}`);\n\t\t\t\t}\n\t\t\t}\n\t\t\tresultLinks.add(currentUrl);\n\t\t};\n\n\t\tconst processBatch = async () => {\n\t\t\tconst batch = Array.from(queue).slice(0, this.concurrency);\n\t\t\tfor (const url of batch) {\n\t\t\t\tqueue.delete(url);\n\t\t\t}\n\t\t\tawait Promise.allSettled(batch.map((url) => fetchAndExtract(url)));\n\t\t};\n\n\t\tthis.logger.info(\n\t\t\t`Starting crawl with depth: ${this.depth}, concurrency: ${this.concurrency}`,\n\t\t);\n\t\twhile (queue.size > 0 && visited.size < this.depth) {\n\t\t\tawait processBatch();\n\t\t}\n\n\t\tthis.logger.info(\n\t\t\t`Crawling completed. Total pages visited: ${resultLinks.size}`,\n\t\t);\n\n\t\treturn Array.from(resultLinks);\n\t}\n}\n", "import winston from \"winston\";\n\nexport const createLogger = (label: string) =>\n\twinston.createLogger({\n\t\tlevels: {\n\t\t\terror: 0,\n\t\t\twarn: 1,\n\t\t\tinfo: 2,\n\t\t\thttp: 3,\n\t\t\tverbose: 4,\n\t\t\tdebug: 5,\n\t\t\tsilly: 6,\n\t\t},\n\t\tformat: winston.format.combine(\n\t\t\twinston.format.label({ label }),\n\t\t\twinston.format.colorize(),\n\t\t\twinston.format.timestamp({\n\t\t\t\tformat: () => {\n\t\t\t\t\treturn new Date().toLocaleString(\"en-US\");\n\t\t\t\t},\n\t\t\t}),\n\t\t\twinston.format.align(),\n\t\t\twinston.format.printf(\n\t\t\t\t(info) =>\n\t\t\t\t\t`\\x1b[34m(${info.label})\\x1b[0m \\x1b[33m${info.timestamp}\\x1b[0m [${info.level}]: ${info.message}`,\n\t\t\t),\n\t\t),\n\t\ttransports: [new winston.transports.Console()],\n\t});\n"],
+  "mappings": ";AAAA,OAAO,WAAW;AAClB,OAAO,WAAW;AAClB,OAAO,eAAe;;;ACFtB,OAAO,aAAa;AAEb,IAAM,eAAe,CAAC,UAC5B,QAAQ,aAAa;AAAA,EACpB,QAAQ;AAAA,IACP,OAAO;AAAA,IACP,MAAM;AAAA,IACN,MAAM;AAAA,IACN,MAAM;AAAA,IACN,SAAS;AAAA,IACT,OAAO;AAAA,IACP,OAAO;AAAA,EACR;AAAA,EACA,QAAQ,QAAQ,OAAO;AAAA,IACtB,QAAQ,OAAO,MAAM,EAAE,MAAM,CAAC;AAAA,IAC9B,QAAQ,OAAO,SAAS;AAAA,IACxB,QAAQ,OAAO,UAAU;AAAA,MACxB,QAAQ,MAAM;AACb,gBAAO,oBAAI,KAAK,GAAE,eAAe,OAAO;AAAA,MACzC;AAAA,IACD,CAAC;AAAA,IACD,QAAQ,OAAO,MAAM;AAAA,IACrB,QAAQ,OAAO;AAAA,MACd,CAAC,SACA,YAAY,KAAK,KAAK,oBAAoB,KAAK,SAAS,YAAY,KAAK,KAAK,MAAM,KAAK,OAAO;AAAA,IAClG;AAAA,EACD;AAAA,EACA,YAAY,CAAC,IAAI,QAAQ,WAAW,QAAQ,CAAC;AAC9C,CAAC;;;ADhBF,IAAqB,gBAArB,MAAmC;AAAA,EAC1B,SAAiC;AAAA,IACxC,cAAc,IAAI,UAAU,EAAE,SAAS;AAAA,EACxC;AAAA,EACQ;AAAA,EACA,SAAS,aAAa,eAAe;AAAA,EAErC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAER,YAAY,KAAa,UAAgC,CAAC,GAAG;AAC5D,UAAM;AAAA,MACL,QAAQ;AAAA,MACR,cAAc;AAAA,MACd,UAAU;AAAA,MACV,UAAU;AAAA,IACX,IAAI;AACJ,SAAK,QAAQ;AACb,SAAK,cAAc;AACnB,SAAK,UAAU;AACf,SAAK,UAAU;AAEf,QAAI;AACH,WAAK,MAAM,IAAI,IAAI,GAAG;AACtB,WAAK,OAAO;AAAA,QACX,yBAAyB,GAAG,iBAAiB,KAAK,OAAO,YAAY,CAAC;AAAA,MACvE;AAAA,IACD,SAAS,OAAO;AACf,UAAI,iBAAiB,WAAW;AAC/B,aAAK,OAAO,MAAM,aAAa;AAC/B,cAAM,IAAI,MAAM,aAAa;AAAA,MAC9B;AACA,WAAK,OAAO,MAAM,oCAAoC,KAAK,EAAE;AAC7D,YAAM;AAAA,IACP;AAAA,EACD;AAAA,EAEQ,gBAAgB,QAAwB;AAC/C,WAAO,OAAO,WAAW,MAAM,IAAI,OAAO,MAAM,CAAC,IAAI;AAAA,EACtD;AAAA,EAEQ,6BAA6B,KAAqB;AACzD,WAAO,IAAI,IAAI,KAAK,KAAK,IAAI,SAAS,CAAC,EAAE,SAAS;AAAA,EACnD;AAAA,EAEQ,eAAe,KAAsB;AAC5C,QAAI;AACH,YAAM,YAAY,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI;AAC5C,UAAI,CAAC,CAAC,SAAS,QAAQ,EAAE,SAAS,UAAU,QAAQ,GAAG;AACtD,eAAO;AAAA,MACR;AACA,YAAM,aAAa,KAAK,gBAAgB,KAAK,IAAI,QAAQ;AACzD,YAAM,eAAe,KAAK,gBAAgB,UAAU,QAAQ;AAC5D,aAAO,iBAAiB;AAAA,IACzB,SAAS,OAAO;AACf,WAAK,OAAO,KAAK,sBAAsB,GAAG,MAAM,KAAK,EAAE;AACvD,aAAO;AAAA,IACR;AAAA,EACD;AAAA,EAEA,MAAc,iBACb,KACA,SACyB;AACzB,aAAS,UAAU,GAAG,WAAW,SAAS,WAAW;AACpD,YAAM,aAAa,IAAI,gBAAgB;AACvC,YAAM,YAAY,WAAW,MAAM,WAAW,MAAM,GAAG,KAAK,OAAO;AAEnE,UAAI;AACH,aAAK,OAAO,MAAM,yBAAyB,OAAO,MAAM,GAAG,EAAE;AAC7D,cAAM,kBAAkB,IAAI,UAAU,EAAE,SAAS;AACjD,aAAK,OAAO,KAAK,2BAA2B,eAAe,EAAE;AAC7D,aAAK,OAAO,YAAY,IAAI;AAC5B,cAAM,WAAW,MAAM,MAAM,KAAK;AAAA,UACjC,SAAS,KAAK;AAAA,UACd,QAAQ,WAAW;AAAA,UACnB,UAAU;AAAA,QACX,CAAC;AAED,qBAAa,SAAS;AAEtB,YAAI,SAAS,IAAI;AAChB,eAAK,OAAO,KAAK,6BAA6B,GAAG,EAAE;AACnD,iBAAO,MAAM,SAAS,KAAK;AAAA,QAC5B;AAEA,aAAK,OAAO,KAAK,wBAAwB,SAAS,MAAM,MAAM,GAAG,EAAE;AAAA,MACpE,SAAS,OAAO;AACf,YAAK,MAAgB,SAAS,cAAc;AAC3C,eAAK,OAAO,KAAK,oBAAoB,GAAG,EAAE;AAAA,QAC3C,OAAO;AACN,eAAK,OAAO,MAAM,uBAAuB,GAAG,MAAM,KAAK,EAAE;AAAA,QAC1D;AAAA,MACD;AAAA,IACD;AACA,WAAO;AAAA,EACR;AAAA,EAEQ,aAAa,MAAwB;AAC5C,UAAM,EAAE,MAAM,IAAI;AAClB,UAAM,MAAM,IAAI,MAAM,IAAI;AAC1B,UAAM,QAAQ,MAAM,KAAK,IAAI,OAAO,SAAS,iBAAiB,GAAG,CAAC;AAClE,UAAM,QAAQ,MAAM,IAAI,CAAC,SAAS,KAAK,IAAI;AAC3C,UAAM,gBAAgB,MAAM,OAAO,CAAC,SAAS,KAAK,eAAe,IAAI,CAAC;AACtE,SAAK,OAAO;AAAA,MACX,aAAa,cAAc,MAAM;AAAA,IAClC;AACA,WAAO,cAAc,IAAI,CAAC,SAAS,KAAK,6BAA6B,IAAI,CAAC;AAAA,EAC3E;AAAA,EAEA,MAAa,QAAgC;AAC5C,UAAM,UAAU,oBAAI,IAAY;AAChC,UAAM,QAAQ,oBAAI,IAAY,CAAC,KAAK,IAAI,IAAI,CAAC;AAC7C,UAAM,cAAc,oBAAI,IAAY;AAGpC,UAAM,kBAAkB;AAAA,MACvB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAEA,UAAM,kBAAkB,OAAO,eAAuB;AACrD,UAAI,QAAQ,IAAI,UAAU,GAAG;AAC5B,aAAK,OAAO,MAAM,iCAAiC,UAAU,EAAE;AAC/D;AAAA,MACD;AACA,cAAQ,IAAI,UAAU;AACtB,WAAK,OAAO,KAAK,iBAAiB,UAAU,EAAE;AAE9C,YAAM,OAAO,MAAM,KAAK,iBAAiB,YAAY,KAAK,OAAO;AACjE,UAAI,CAAC,KAAM;AAEX,YAAM,QAAQ,KAAK,aAAa,IAAI;AAGpC,iBAAW,QAAQ,OAAO;AACzB,YAAI,gBAAgB,KAAK,CAAC,QAAQ,KAAK,SAAS,GAAG,CAAC,GAAG;AACtD,eAAK,OAAO,MAAM,wBAAwB,IAAI,EAAE;AAChD;AAAA,QACD;AACA,aAAK,OAAO,MAAM,eAAe,IAAI,EAAE;AAAA,MACxC;AAEA,iBAAW,QAAQ,OAAO;AACzB,YAAI,CAAC,QAAQ,IAAI,IAAI,KAAK,MAAM,OAAO,KAAK,OAAO;AAClD,gBAAM,IAAI,IAAI;AACd,eAAK,OAAO,MAAM,mBAAmB,IAAI,EAAE;AAAA,QAC5C;AAAA,MACD;AACA,kBAAY,IAAI,UAAU;AAAA,IAC3B;AAEA,UAAM,eAAe,YAAY;AAChC,YAAM,QAAQ,MAAM,KAAK,KAAK,EAAE,MAAM,GAAG,KAAK,WAAW;AACzD,iBAAW,OAAO,OAAO;AACxB,cAAM,OAAO,GAAG;AAAA,MACjB;AACA,YAAM,QAAQ,WAAW,MAAM,IAAI,CAAC,QAAQ,gBAAgB,GAAG,CAAC,CAAC;AAAA,IAClE;AAEA,SAAK,OAAO;AAAA,MACX,8BAA8B,KAAK,KAAK,kBAAkB,KAAK,WAAW;AAAA,IAC3E;AACA,WAAO,MAAM,OAAO,KAAK,QAAQ,OAAO,KAAK,OAAO;AACnD,YAAM,aAAa;AAAA,IACpB;AAEA,SAAK,OAAO;AAAA,MACX,4CAA4C,YAAY,IAAI;AAAA,IAC7D;AAEA,WAAO,MAAM,KAAK,WAAW;AAAA,EAC9B;AACD;",
   "names": []
 }

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "sentinel-scanner",
   "description": "[WIP] An open-source web app vulnerability scanner developed by Rebackk.",
-  "version": "1.1.0-alpha.1",
+  "version": "1.1.0",
   "exports": "./build/index.js",
   "types": "./build/index.d.ts",
-  "bin": "./build/index.js",
+  "bin": "./build/bin.js",
   "type": "module",
   "main": "./build/index.js",
   "license": "Apache-2.0",
@@ -28,10 +28,9 @@
     "test": "node --disable-warning=ExperimentalWarning --experimental-strip-types ./scripts/test.ts test",
     "test:watch": "node --disable-warning=ExperimentalWarning --experimental-strip-types ./scripts/test.ts test:watch",
     "test:coverage": "node --disable-warning=ExperimentalWarning --experimental-strip-types ./scripts/test.ts test:coverage",
-    "spell:check": "cspell \"{README.md,CODE_OF_CONDUCT.md,CONTRIBUTING.md,.github/*.md,src/**/*.ts}\"",
+    "spell:check": "cspell \"{DISCLAIMER.md,README.md,CODE_OF_CONDUCT.md,CONTRIBUTING.md,.github/*.md,src/**/*.ts}\"",
     "cz": "cz",
     "semantic-release": "semantic-release",
-    "postinstall": "prisma generate",
     "prepare": "husky"
   },
   "devDependencies": {
@@ -55,10 +54,10 @@
     "cz-conventional-changelog": "^3.3.0",
     "esbuild": "^0.23.1",
     "husky": "^9.1.6",
-    "prisma": "^5.22.0",
     "rimraf": "^6.0.1",
     "semantic-release": "^24.2.0",
-    "typescript": "^5.6.3"
+    "typescript": "^5.4.2",
+    "winston": "^3.17.0"
   },
   "config": {
     "commitizen": {
@@ -66,10 +65,9 @@
     }
   },
   "dependencies": {
-    "@prisma/client": "^5.22.0",
     "isomorphic-fetch": "^3.0.0",
     "jsdom": "^25.0.1",
-    "user-agents": "^1.1.353",
+    "user-agents": "^1.1.359",
     "yargs": "^17.7.2"
   }
 }

package/scripts/build.ts

@@ -62,7 +62,10 @@ async function build(): Promise<void> {
     sourcemap: true,
     external: [],
     bundle: true,
-    entryPoints: [path.join(srcPath, "index.ts")],
+    entryPoints: [
+      path.join(srcPath, "index.ts"),
+      path.join(srcPath, "bin.ts"),
+    ],
     outdir: buildPath,
     packages: "external",
   });

package/src/bin.ts

@@ -0,0 +1,20 @@
+#!/usr/bin/env node --no-warnings
+
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { spiderCommand } from "./commands/spider";
+
+const commandHandler = yargs(hideBin(process.argv));
+
+commandHandler.demandCommand();
+commandHandler.scriptName("sentinel-scanner");
+commandHandler.usage("Usage: $0 <command> [options]");
+commandHandler.help().alias("help", "h");
+commandHandler.version().alias("version", "v");
+commandHandler.strict();
+commandHandler.showHelpOnFail(true);
+
+// Handle Commands
+commandHandler.command(spiderCommand);
+
+commandHandler.parse();