sentinel-scanner 1.0.1 → 1.1.0-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../src/bin.ts", "../src/commands/spider.ts", "../src/modules/spider/index.ts", "../src/utils/index.ts"],
4
+ "sourcesContent": ["#!/usr/bin/env node --no-warnings\n\nimport yargs from \"yargs\";\nimport { hideBin } from \"yargs/helpers\";\nimport { spiderCommand } from \"./commands/spider\";\n\nconst commandHandler = yargs(hideBin(process.argv));\n\ncommandHandler.demandCommand();\ncommandHandler.scriptName(\"sentinel-scanner\");\ncommandHandler.usage(\"Usage: $0 <command> [options]\");\ncommandHandler.help().alias(\"help\", \"h\");\ncommandHandler.version().alias(\"version\", \"v\");\ncommandHandler.strict();\ncommandHandler.showHelpOnFail(true);\n\n// Handle Commands\ncommandHandler.command(spiderCommand);\n\ncommandHandler.parse();\n", "import fs from \"node:fs\";\nimport path from \"node:path\";\nimport type { ArgumentsCamelCase, CommandModule } from \"yargs\";\nimport SpiderScanner from \"../modules/spider\";\nimport { createLogger } from \"../utils\";\n\nexport type SpiderScannerCLIOptions = {\n\turl: string;\n\tdepth?: number;\n\toutput?: string;\n\tconcurrency?: number;\n\ttimeout?: number;\n\tretries?: number;\n};\n\nconst cliLogger = createLogger(\"CLI\");\n\nexport const spiderCommand: CommandModule = {\n\tcommand: \"spider\",\n\tdescribe:\n\t\t\"Crawl a website and get an array of URLs which are internal to the website\",\n\tbuilder: (yargs) => {\n\t\treturn yargs\n\t\t\t.option(\"url\", {\n\t\t\t\talias: \"u\",\n\t\t\t\ttype: \"string\",\n\t\t\t\tdescription: \"The URL of the website to scan\",\n\t\t\t\tdemandOption: true,\n\t\t\t\tcoerce: (url) => {\n\t\t\t\t\ttry {\n\t\t\t\t\t\tnew URL(url);\n\n\t\t\t\t\t\treturn url;\n\t\t\t\t\t} catch (error) {\n\t\t\t\t\t\tthrow new Error(`Invalid URL: ${url}`);\n\t\t\t\t\t}\n\t\t\t\t},\n\t\t\t})\n\t\t\t.option(\"depth\", {\n\t\t\t\talias: \"d\",\n\t\t\t\ttype: \"number\",\n\t\t\t\tdescription: \"The maximum depth to crawl\",\n\t\t\t\tdefault: 250,\n\t\t\t\tcoerce: (depth) => {\n\t\t\t\t\tif (depth < 0) {\n\t\t\t\t\t\tthrow new Error(\"Depth must be a positive number\");\n\t\t\t\t\t}\n\n\t\t\t\t\tif (depth > 250) {\n\t\t\t\t\t\tthrow new Error(\"Depth must be less than 250\");\n\t\t\t\t\t}\n\n\t\t\t\t\treturn depth;\n\t\t\t\t},\n\t\t\t})\n\t\t\t.option(\"output\", {\n\t\t\t\talias: \"o\",\n\t\t\t\ttype: \"string\",\n\t\t\t\tdescription:\n\t\t\t\t\t\"The output file to write the results to. Must be a JSON file\",\n\t\t\t\tcoerce: (output) => {\n\t\t\t\t\ttry {\n\t\t\t\t\t\t// Should throw an error if the path is invalid\n\t\t\t\t\t\t// Should Be A JSON File\n\t\t\t\t\t\tconst resolvedPath = path.resolve(output);\n\t\t\t\t\t\tconst parsedPath = path.parse(resolvedPath);\n\n\t\t\t\t\t\tif (parsedPath.ext !== \".json\") {\n\t\t\t\t\t\t\tthrow new Error(\"Output file must be a JSON file\");\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tif (fs.existsSync(resolvedPath)) {\n\t\t\t\t\t\t\tthrow new Error(\"Output file already exists\");\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\treturn resolvedPath;\n\t\t\t\t\t} catch (error) {\n\t\t\t\t\t\tthrow new Error(`Invalid output file: ${output}`);\n\t\t\t\t\t}\n\t\t\t\t},\n\t\t\t\tdefault: getDefaultFilePath(),\n\t\t\t})\n\t\t\t.option(\"concurrency\", {\n\t\t\t\talias: \"c\",\n\t\t\t\ttype: \"number\",\n\t\t\t\tdescription: \"The number of concurrent requests to make\",\n\t\t\t\tdefault: 10,\n\t\t\t\tcoerce: (concurrency) => {\n\t\t\t\t\tif (concurrency < 1) {\n\t\t\t\t\t\tthrow new Error(\"Concurrency must be a positive number\");\n\t\t\t\t\t}\n\n\t\t\t\t\tif (concurrency > 20) {\n\t\t\t\t\t\tthrow new Error(\"Concurrency must be less than 20\");\n\t\t\t\t\t}\n\n\t\t\t\t\treturn concurrency;\n\t\t\t\t},\n\t\t\t})\n\t\t\t.option(\"timeout\", {\n\t\t\t\talias: \"t\",\n\t\t\t\ttype: \"number\",\n\t\t\t\tdescription: \"The timeout for each request in milliseconds\",\n\t\t\t\tdefault: 5000,\n\t\t\t\tcoerce: (timeout) => {\n\t\t\t\t\tif (timeout < 0) {\n\t\t\t\t\t\tthrow new Error(\"Timeout must be a positive number\");\n\t\t\t\t\t}\n\n\t\t\t\t\tif (timeout > 25_000) {\n\t\t\t\t\t\tthrow new Error(\"Timeout must be less than 25,000\");\n\t\t\t\t\t}\n\n\t\t\t\t\treturn timeout;\n\t\t\t\t},\n\t\t\t})\n\t\t\t.option(\"retries\", {\n\t\t\t\talias: \"r\",\n\t\t\t\ttype: \"number\",\n\t\t\t\tdescription: \"The number of retries for each request\",\n\t\t\t\tdefault: 3,\n\t\t\t\tcoerce: (retries) => {\n\t\t\t\t\tif (retries < 0) {\n\t\t\t\t\t\tthrow new Error(\"Retries must be a positive number\");\n\t\t\t\t\t}\n\n\t\t\t\t\tif (retries > 10) {\n\t\t\t\t\t\tthrow new Error(\"Retries must be less than 10\");\n\t\t\t\t\t}\n\n\t\t\t\t\treturn retries;\n\t\t\t\t},\n\t\t\t});\n\t},\n\thandler: async (args) => {\n\t\ttry {\n\t\t\tconst argData = args as ArgumentsCamelCase<SpiderScannerCLIOptions>;\n\n\t\t\tconst scanner = new SpiderScanner(argData.url, {\n\t\t\t\tdepth: argData.depth ?? 250,\n\t\t\t\tconcurrency: argData.concurrency ?? 10,\n\t\t\t\ttimeout: argData.timeout ?? 5000,\n\t\t\t\tretries: argData.retries ?? 3,\n\t\t\t});\n\n\t\t\tcliLogger.info(\"Starting to crawl website\");\n\n\t\t\tconst results = await scanner.crawl();\n\n\t\t\tif (argData.output) {\n\t\t\t\tfs.writeFileSync(argData.output, JSON.stringify(results, null, 2));\n\t\t\t\tcliLogger.info(`Results written to ${argData.output}`);\n\t\t\t} else {\n\t\t\t\tconst resolvedPath = getDefaultFilePath();\n\t\t\t\tfs.writeFileSync(resolvedPath, JSON.stringify(results, null, 2));\n\t\t\t\tcliLogger.info(`Results written to ${resolvedPath}`);\n\t\t\t}\n\t\t} catch (error) {\n\t\t\tif (error instanceof Error) {\n\t\t\t\tcliLogger.error(error.message);\n\t\t\t}\n\t\t\tcliLogger.error(\"Failed to run spider command\");\n\t\t\tprocess.exit(1);\n\t\t}\n\t},\n};\n\nconst getDefaultFilePath = () => {\n\ttry {\n\t\tconst resolvedDir = path.resolve(\"sentinel_output\");\n\t\t// Check If Directory Exists\n\t\tif (!fs.existsSync(resolvedDir)) {\n\t\t\tfs.mkdirSync(resolvedDir);\n\t\t}\n\n\t\tconst resolvedPath = path.resolve(\n\t\t\t`sentinel_output/spider_${Date.now()}.json`,\n\t\t);\n\t\t// Check If File Exists\n\t\tif (fs.existsSync(resolvedPath)) {\n\t\t\tthrow new Error(\"Output file already exists\");\n\t\t}\n\t\tconst parsedPath = path.parse(resolvedPath);\n\n\t\tif (parsedPath.ext !== \".json\") {\n\t\t\tthrow new Error(\"Output file must be a JSON file\");\n\t\t}\n\n\t\treturn resolvedPath;\n\t} catch (error) {\n\t\tthrow new Error(\"Invalid output file\");\n\t}\n};\n", "import fetch from \"isomorphic-fetch\";\nimport jsdom from \"jsdom\";\nimport UserAgent from \"user-agents\";\nimport { createLogger } from \"../../utils\";\n\nexport interface SpiderScannerOptions {\n\tdepth?: number;\n\tconcurrency?: number;\n\tretries?: number;\n\ttimeout?: number;\n}\n\nexport default class SpiderScanner {\n\tprivate header: Record<string, string> = {\n\t\t\"User-Agent\": new UserAgent().toString(),\n\t};\n\tprivate url: URL;\n\tprivate logger = createLogger(\"SpiderScanner\");\n\n\tprivate depth: number;\n\tprivate concurrency: number;\n\tprivate retries: number;\n\tprivate timeout: number;\n\n\tconstructor(url: string, options: SpiderScannerOptions = {}) {\n\t\tconst {\n\t\t\tdepth = 250,\n\t\t\tconcurrency = 5,\n\t\t\tretries = 3,\n\t\t\ttimeout = 5000,\n\t\t} = options;\n\t\tthis.depth = depth;\n\t\tthis.concurrency = concurrency;\n\t\tthis.retries = retries;\n\t\tthis.timeout = timeout;\n\n\t\ttry {\n\t\t\tthis.url = new URL(url);\n\t\t\tthis.logger.info(\n\t\t\t\t`Initialized with URL: ${url}, User-Agent: ${this.header[\"User-Agent\"]}`,\n\t\t\t);\n\t\t} catch (error) {\n\t\t\tif (error instanceof TypeError) {\n\t\t\t\tthis.logger.error(\"Invalid URL\");\n\t\t\t\tthrow new Error(\"Invalid URL\");\n\t\t\t}\n\t\t\tthis.logger.error(`Unexpected error in constructor: ${error}`);\n\t\t\tthrow error;\n\t\t}\n\t}\n\n\tprivate normalizeDomain(domain: string): string {\n\t\treturn domain.startsWith(\"www.\") ? domain.slice(4) : domain;\n\t}\n\n\tprivate convertRelativeUrlToAbsolute(url: string): string {\n\t\treturn new URL(url, this.url.toString()).toString();\n\t}\n\n\tprivate isInternalLink(url: string): boolean {\n\t\ttry {\n\t\t\tconst parsedUrl = new URL(url, this.url.href);\n\t\t\tif (![\"http:\", \"https:\"].includes(parsedUrl.protocol)) {\n\t\t\t\treturn false;\n\t\t\t}\n\t\t\tconst baseDomain = this.normalizeDomain(this.url.hostname);\n\t\t\tconst parsedDomain = this.normalizeDomain(parsedUrl.hostname);\n\t\t\treturn parsedDomain === baseDomain;\n\t\t} catch (error) {\n\t\t\tthis.logger.warn(`Error parsing URL: ${url} - ${error}`);\n\t\t\treturn false;\n\t\t}\n\t}\n\n\tprivate async fetchWithRetries(\n\t\turl: string,\n\t\tretries: number,\n\t): Promise<string | null> {\n\t\tfor (let attempt = 1; attempt <= retries; attempt++) {\n\t\t\tconst controller = new AbortController();\n\t\t\tconst timeoutId = setTimeout(() => controller.abort(), this.timeout);\n\n\t\t\ttry {\n\t\t\t\tthis.logger.debug(`Fetching URL (Attempt ${attempt}): ${url}`);\n\t\t\t\tconst randomUserAgent = new UserAgent().toString();\n\t\t\t\tthis.logger.info(`Changing User-Agent to: ${randomUserAgent}`);\n\t\t\t\tthis.header[\"User-Agent\"] = randomUserAgent;\n\t\t\t\tconst response = await fetch(url, {\n\t\t\t\t\theaders: this.header,\n\t\t\t\t\tsignal: controller.signal,\n\t\t\t\t\tredirect: \"follow\",\n\t\t\t\t});\n\n\t\t\t\tclearTimeout(timeoutId);\n\n\t\t\t\tif (response.ok) {\n\t\t\t\t\tthis.logger.info(`Successfully fetched URL: ${url}`);\n\t\t\t\t\treturn await response.text();\n\t\t\t\t}\n\n\t\t\t\tthis.logger.warn(`Failed to fetch URL (${response.status}): ${url}`);\n\t\t\t} catch (error) {\n\t\t\t\tif ((error as Error).name === \"AbortError\") {\n\t\t\t\t\tthis.logger.warn(`Fetch timed out: ${url}`);\n\t\t\t\t} else {\n\t\t\t\t\tthis.logger.error(`Error fetching URL: ${url} - ${error}`);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn null;\n\t}\n\n\tprivate extractLinks(html: string): string[] {\n\t\tconst { JSDOM } = jsdom;\n\t\tconst dom = new JSDOM(html);\n\t\tconst links = Array.from(dom.window.document.querySelectorAll(\"a\"));\n\t\tconst hrefs = links.map((link) => link.href);\n\t\tconst internalLinks = hrefs.filter((href) => this.isInternalLink(href));\n\t\tthis.logger.debug(\n\t\t\t`Extracted ${internalLinks.length} internal links from HTML content`,\n\t\t);\n\t\treturn internalLinks.map((link) => this.convertRelativeUrlToAbsolute(link));\n\t}\n\n\tpublic async crawl(): Promise<Array<string>> {\n\t\tconst visited = new Set<string>();\n\t\tconst queue = new Set<string>([this.url.href]);\n\t\tconst resultLinks = new Set<string>();\n\n\t\t// Assets to ignore\n\t\tconst assetExtensions = [\n\t\t\t\".css\",\n\t\t\t\".js\",\n\t\t\t\".png\",\n\t\t\t\".jpg\",\n\t\t\t\".jpeg\",\n\t\t\t\".gif\",\n\t\t\t\".svg\",\n\t\t\t\".ico\",\n\t\t\t\".webp\",\n\t\t\t\".mp4\",\n\t\t\t\".mp3\",\n\t\t\t\".wav\",\n\t\t\t\".avi\",\n\t\t\t\".mov\",\n\t\t\t\".webm\",\n\t\t\t\".pdf\",\n\t\t\t\".doc\",\n\t\t\t\".docx\",\n\t\t\t\".xls\",\n\t\t\t\".xlsx\",\n\t\t\t\".ppt\",\n\t\t\t\".pptx\",\n\t\t\t\".zip\",\n\t\t\t\".rar\",\n\t\t\t\".tar\",\n\t\t\t\".gz\",\n\t\t];\n\n\t\tconst fetchAndExtract = async (currentUrl: string) => {\n\t\t\tif (visited.has(currentUrl)) {\n\t\t\t\tthis.logger.debug(`Skipping already visited URL: ${currentUrl}`);\n\t\t\t\treturn;\n\t\t\t}\n\t\t\tvisited.add(currentUrl);\n\t\t\tthis.logger.info(`Visiting URL: ${currentUrl}`);\n\n\t\t\tconst html = await this.fetchWithRetries(currentUrl, this.retries);\n\t\t\tif (!html) return;\n\n\t\t\tconst links = this.extractLinks(html);\n\n\t\t\t// Filter out asset links\n\t\t\tfor (const link of links) {\n\t\t\t\tif (assetExtensions.some((ext) => link.endsWith(ext))) {\n\t\t\t\t\tthis.logger.debug(`Ignoring asset link: ${link}`);\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tthis.logger.debug(`Found link: ${link}`);\n\t\t\t}\n\n\t\t\tfor (const link of links) {\n\t\t\t\tif (!visited.has(link) && queue.size < this.depth) {\n\t\t\t\t\tqueue.add(link);\n\t\t\t\t\tthis.logger.debug(`Added to queue: ${link}`);\n\t\t\t\t}\n\t\t\t}\n\t\t\tresultLinks.add(currentUrl);\n\t\t};\n\n\t\tconst processBatch = async () => {\n\t\t\tconst batch = Array.from(queue).slice(0, this.concurrency);\n\t\t\tfor (const url of batch) {\n\t\t\t\tqueue.delete(url);\n\t\t\t}\n\t\t\tawait Promise.allSettled(batch.map((url) => fetchAndExtract(url)));\n\t\t};\n\n\t\tthis.logger.info(\n\t\t\t`Starting crawl with depth: ${this.depth}, concurrency: ${this.concurrency}`,\n\t\t);\n\t\twhile (queue.size > 0 && visited.size < this.depth) {\n\t\t\tawait processBatch();\n\t\t}\n\n\t\tthis.logger.info(\n\t\t\t`Crawling completed. Total pages visited: ${resultLinks.size}`,\n\t\t);\n\n\t\treturn Array.from(resultLinks);\n\t}\n}\n", "import winston from \"winston\";\n\nexport const createLogger = (label: string) =>\n\twinston.createLogger({\n\t\tlevels: {\n\t\t\terror: 0,\n\t\t\twarn: 1,\n\t\t\tinfo: 2,\n\t\t\thttp: 3,\n\t\t\tverbose: 4,\n\t\t\tdebug: 5,\n\t\t\tsilly: 6,\n\t\t},\n\t\tformat: winston.format.combine(\n\t\t\twinston.format.label({ label }),\n\t\t\twinston.format.colorize(),\n\t\t\twinston.format.timestamp({\n\t\t\t\tformat: () => {\n\t\t\t\t\treturn new Date().toLocaleString(\"en-US\");\n\t\t\t\t},\n\t\t\t}),\n\t\t\twinston.format.align(),\n\t\t\twinston.format.printf(\n\t\t\t\t(info) =>\n\t\t\t\t\t`\\x1b[34m(${info.label})\\x1b[0m \\x1b[33m${info.timestamp}\\x1b[0m [${info.level}]: ${info.message}`,\n\t\t\t),\n\t\t),\n\t\ttransports: [new winston.transports.Console()],\n\t});\n"],
5
+ "mappings": ";;;AAEA,OAAO,WAAW;AAClB,SAAS,eAAe;;;ACHxB,OAAO,QAAQ;AACf,OAAO,UAAU;;;ACDjB,OAAO,WAAW;AAClB,OAAO,WAAW;AAClB,OAAO,eAAe;;;ACFtB,OAAO,aAAa;AAEb,IAAM,eAAe,CAAC,UAC5B,QAAQ,aAAa;AAAA,EACpB,QAAQ;AAAA,IACP,OAAO;AAAA,IACP,MAAM;AAAA,IACN,MAAM;AAAA,IACN,MAAM;AAAA,IACN,SAAS;AAAA,IACT,OAAO;AAAA,IACP,OAAO;AAAA,EACR;AAAA,EACA,QAAQ,QAAQ,OAAO;AAAA,IACtB,QAAQ,OAAO,MAAM,EAAE,MAAM,CAAC;AAAA,IAC9B,QAAQ,OAAO,SAAS;AAAA,IACxB,QAAQ,OAAO,UAAU;AAAA,MACxB,QAAQ,MAAM;AACb,gBAAO,oBAAI,KAAK,GAAE,eAAe,OAAO;AAAA,MACzC;AAAA,IACD,CAAC;AAAA,IACD,QAAQ,OAAO,MAAM;AAAA,IACrB,QAAQ,OAAO;AAAA,MACd,CAAC,SACA,YAAY,KAAK,KAAK,oBAAoB,KAAK,SAAS,YAAY,KAAK,KAAK,MAAM,KAAK,OAAO;AAAA,IAClG;AAAA,EACD;AAAA,EACA,YAAY,CAAC,IAAI,QAAQ,WAAW,QAAQ,CAAC;AAC9C,CAAC;;;ADhBF,IAAqB,gBAArB,MAAmC;AAAA,EAC1B,SAAiC;AAAA,IACxC,cAAc,IAAI,UAAU,EAAE,SAAS;AAAA,EACxC;AAAA,EACQ;AAAA,EACA,SAAS,aAAa,eAAe;AAAA,EAErC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAER,YAAY,KAAa,UAAgC,CAAC,GAAG;AAC5D,UAAM;AAAA,MACL,QAAQ;AAAA,MACR,cAAc;AAAA,MACd,UAAU;AAAA,MACV,UAAU;AAAA,IACX,IAAI;AACJ,SAAK,QAAQ;AACb,SAAK,cAAc;AACnB,SAAK,UAAU;AACf,SAAK,UAAU;AAEf,QAAI;AACH,WAAK,MAAM,IAAI,IAAI,GAAG;AACtB,WAAK,OAAO;AAAA,QACX,yBAAyB,GAAG,iBAAiB,KAAK,OAAO,YAAY,CAAC;AAAA,MACvE;AAAA,IACD,SAAS,OAAO;AACf,UAAI,iBAAiB,WAAW;AAC/B,aAAK,OAAO,MAAM,aAAa;AAC/B,cAAM,IAAI,MAAM,aAAa;AAAA,MAC9B;AACA,WAAK,OAAO,MAAM,oCAAoC,KAAK,EAAE;AAC7D,YAAM;AAAA,IACP;AAAA,EACD;AAAA,EAEQ,gBAAgB,QAAwB;AAC/C,WAAO,OAAO,WAAW,MAAM,IAAI,OAAO,MAAM,CAAC,IAAI;AAAA,EACtD;AAAA,EAEQ,6BAA6B,KAAqB;AACzD,WAAO,IAAI,IAAI,KAAK,KAAK,IAAI,SAAS,CAAC,EAAE,SAAS;AAAA,EACnD;AAAA,EAEQ,eAAe,KAAsB;AAC5C,QAAI;AACH,YAAM,YAAY,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI;AAC5C,UAAI,CAAC,CAAC,SAAS,QAAQ,EAAE,SAAS,UAAU,QAAQ,GAAG;AACtD,eAAO;AAAA,MACR;AACA,YAAM,aAAa,KAAK,gBAAgB,KAAK,IAAI,QAAQ;AACzD,YAAM,eAAe,KAAK,gBAAgB,UAAU,QAAQ;AAC5D,aAAO,iBAAiB;AAAA,IACzB,SAAS,OAAO;AACf,WAAK,OAAO,KAAK,sBAAsB,GAAG,MAAM,KAAK,EAAE;AACvD,aAAO;AAAA,IACR;AAAA,EACD;AAAA,EAEA,MAAc,iBACb,KACA,SACyB;AACzB,aAAS,UAAU,GAAG,WAAW,SAAS,WAAW;AACpD,YAAM,aAAa,IAAI,gBAAgB;AACvC,YAAM,YAAY,WAAW,MAAM,WAAW,MAAM,GAAG,KAAK,OAAO;AAEnE,UAAI;AACH,aAAK,OAAO,MAAM,yBAAyB,OAAO,MAAM,GAAG,EAAE;AAC7D,cAAM,kBAAkB,IAAI,UAAU,EAAE,SAAS;AACjD,aAAK,OAAO,KAAK,2BAA2B,eAAe,EAAE;AAC7D,aAAK,OAAO,YAAY,IAAI;AAC5B,cAAM,WAAW,MAAM,MAAM,KAAK;AAAA,UACjC,SAAS,KAAK;AAAA,UACd,QAAQ,WAAW;AAAA,UACnB,UAAU;AAAA,QACX,CAAC;AAED,qBAAa,SAAS;AAEtB,YAAI,SAAS,IAAI;AAChB,eAAK,OAAO,KAAK,6BAA6B,GAAG,EAAE;AACnD,iBAAO,MAAM,SAAS,KAAK;AAAA,QAC5B;AAEA,aAAK,OAAO,KAAK,wBAAwB,SAAS,MAAM,MAAM,GAAG,EAAE;AAAA,MACpE,SAAS,OAAO;AACf,YAAK,MAAgB,SAAS,cAAc;AAC3C,eAAK,OAAO,KAAK,oBAAoB,GAAG,EAAE;AAAA,QAC3C,OAAO;AACN,eAAK,OAAO,MAAM,uBAAuB,GAAG,MAAM,KAAK,EAAE;AAAA,QAC1D;AAAA,MACD;AAAA,IACD;AACA,WAAO;AAAA,EACR;AAAA,EAEQ,aAAa,MAAwB;AAC5C,UAAM,EAAE,MAAM,IAAI;AAClB,UAAM,MAAM,IAAI,MAAM,IAAI;AAC1B,UAAM,QAAQ,MAAM,KAAK,IAAI,OAAO,SAAS,iBAAiB,GAAG,CAAC;AAClE,UAAM,QAAQ,MAAM,IAAI,CAAC,SAAS,KAAK,IAAI;AAC3C,UAAM,gBAAgB,MAAM,OAAO,CAAC,SAAS,KAAK,eAAe,IAAI,CAAC;AACtE,SAAK,OAAO;AAAA,MACX,aAAa,cAAc,MAAM;AAAA,IAClC;AACA,WAAO,cAAc,IAAI,CAAC,SAAS,KAAK,6BAA6B,IAAI,CAAC;AAAA,EAC3E;AAAA,EAEA,MAAa,QAAgC;AAC5C,UAAM,UAAU,oBAAI,IAAY;AAChC,UAAM,QAAQ,oBAAI,IAAY,CAAC,KAAK,IAAI,IAAI,CAAC;AAC7C,UAAM,cAAc,oBAAI,IAAY;AAGpC,UAAM,kBAAkB;AAAA,MACvB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACD;AAEA,UAAM,kBAAkB,OAAO,eAAuB;AACrD,UAAI,QAAQ,IAAI,UAAU,GAAG;AAC5B,aAAK,OAAO,MAAM,iCAAiC,UAAU,EAAE;AAC/D;AAAA,MACD;AACA,cAAQ,IAAI,UAAU;AACtB,WAAK,OAAO,KAAK,iBAAiB,UAAU,EAAE;AAE9C,YAAM,OAAO,MAAM,KAAK,iBAAiB,YAAY,KAAK,OAAO;AACjE,UAAI,CAAC,KAAM;AAEX,YAAM,QAAQ,KAAK,aAAa,IAAI;AAGpC,iBAAW,QAAQ,OAAO;AACzB,YAAI,gBAAgB,KAAK,CAAC,QAAQ,KAAK,SAAS,GAAG,CAAC,GAAG;AACtD,eAAK,OAAO,MAAM,wBAAwB,IAAI,EAAE;AAChD;AAAA,QACD;AACA,aAAK,OAAO,MAAM,eAAe,IAAI,EAAE;AAAA,MACxC;AAEA,iBAAW,QAAQ,OAAO;AACzB,YAAI,CAAC,QAAQ,IAAI,IAAI,KAAK,MAAM,OAAO,KAAK,OAAO;AAClD,gBAAM,IAAI,IAAI;AACd,eAAK,OAAO,MAAM,mBAAmB,IAAI,EAAE;AAAA,QAC5C;AAAA,MACD;AACA,kBAAY,IAAI,UAAU;AAAA,IAC3B;AAEA,UAAM,eAAe,YAAY;AAChC,YAAM,QAAQ,MAAM,KAAK,KAAK,EAAE,MAAM,GAAG,KAAK,WAAW;AACzD,iBAAW,OAAO,OAAO;AACxB,cAAM,OAAO,GAAG;AAAA,MACjB;AACA,YAAM,QAAQ,WAAW,MAAM,IAAI,CAAC,QAAQ,gBAAgB,GAAG,CAAC,CAAC;AAAA,IAClE;AAEA,SAAK,OAAO;AAAA,MACX,8BAA8B,KAAK,KAAK,kBAAkB,KAAK,WAAW;AAAA,IAC3E;AACA,WAAO,MAAM,OAAO,KAAK,QAAQ,OAAO,KAAK,OAAO;AACnD,YAAM,aAAa;AAAA,IACpB;AAEA,SAAK,OAAO;AAAA,MACX,4CAA4C,YAAY,IAAI;AAAA,IAC7D;AAEA,WAAO,MAAM,KAAK,WAAW;AAAA,EAC9B;AACD;;;ADpMA,IAAM,YAAY,aAAa,KAAK;AAE7B,IAAM,gBAA+B;AAAA,EAC3C,SAAS;AAAA,EACT,UACC;AAAA,EACD,SAAS,CAACA,WAAU;AACnB,WAAOA,OACL,OAAO,OAAO;AAAA,MACd,OAAO;AAAA,MACP,MAAM;AAAA,MACN,aAAa;AAAA,MACb,cAAc;AAAA,MACd,QAAQ,CAAC,QAAQ;AAChB,YAAI;AACH,cAAI,IAAI,GAAG;AAEX,iBAAO;AAAA,QACR,SAAS,OAAO;AACf,gBAAM,IAAI,MAAM,gBAAgB,GAAG,EAAE;AAAA,QACtC;AAAA,MACD;AAAA,IACD,CAAC,EACA,OAAO,SAAS;AAAA,MAChB,OAAO;AAAA,MACP,MAAM;AAAA,MACN,aAAa;AAAA,MACb,SAAS;AAAA,MACT,QAAQ,CAAC,UAAU;AAClB,YAAI,QAAQ,GAAG;AACd,gBAAM,IAAI,MAAM,iCAAiC;AAAA,QAClD;AAEA,YAAI,QAAQ,KAAK;AAChB,gBAAM,IAAI,MAAM,6BAA6B;AAAA,QAC9C;AAEA,eAAO;AAAA,MACR;AAAA,IACD,CAAC,EACA,OAAO,UAAU;AAAA,MACjB,OAAO;AAAA,MACP,MAAM;AAAA,MACN,aACC;AAAA,MACD,QAAQ,CAAC,WAAW;AACnB,YAAI;AAGH,gBAAM,eAAe,KAAK,QAAQ,MAAM;AACxC,gBAAM,aAAa,KAAK,MAAM,YAAY;AAE1C,cAAI,WAAW,QAAQ,SAAS;AAC/B,kBAAM,IAAI,MAAM,iCAAiC;AAAA,UAClD;AAEA,cAAI,GAAG,WAAW,YAAY,GAAG;AAChC,kBAAM,IAAI,MAAM,4BAA4B;AAAA,UAC7C;AAEA,iBAAO;AAAA,QACR,SAAS,OAAO;AACf,gBAAM,IAAI,MAAM,wBAAwB,MAAM,EAAE;AAAA,QACjD;AAAA,MACD;AAAA,MACA,SAAS,mBAAmB;AAAA,IAC7B,CAAC,EACA,OAAO,eAAe;AAAA,MACtB,OAAO;AAAA,MACP,MAAM;AAAA,MACN,aAAa;AAAA,MACb,SAAS;AAAA,MACT,QAAQ,CAAC,gBAAgB;AACxB,YAAI,cAAc,GAAG;AACpB,gBAAM,IAAI,MAAM,uCAAuC;AAAA,QACxD;AAEA,YAAI,cAAc,IAAI;AACrB,gBAAM,IAAI,MAAM,kCAAkC;AAAA,QACnD;AAEA,eAAO;AAAA,MACR;AAAA,IACD,CAAC,EACA,OAAO,WAAW;AAAA,MAClB,OAAO;AAAA,MACP,MAAM;AAAA,MACN,aAAa;AAAA,MACb,SAAS;AAAA,MACT,QAAQ,CAAC,YAAY;AACpB,YAAI,UAAU,GAAG;AAChB,gBAAM,IAAI,MAAM,mCAAmC;AAAA,QACpD;AAEA,YAAI,UAAU,MAAQ;AACrB,gBAAM,IAAI,MAAM,kCAAkC;AAAA,QACnD;AAEA,eAAO;AAAA,MACR;AAAA,IACD,CAAC,EACA,OAAO,WAAW;AAAA,MAClB,OAAO;AAAA,MACP,MAAM;AAAA,MACN,aAAa;AAAA,MACb,SAAS;AAAA,MACT,QAAQ,CAAC,YAAY;AACpB,YAAI,UAAU,GAAG;AAChB,gBAAM,IAAI,MAAM,mCAAmC;AAAA,QACpD;AAEA,YAAI,UAAU,IAAI;AACjB,gBAAM,IAAI,MAAM,8BAA8B;AAAA,QAC/C;AAEA,eAAO;AAAA,MACR;AAAA,IACD,CAAC;AAAA,EACH;AAAA,EACA,SAAS,OAAO,SAAS;AACxB,QAAI;AACH,YAAM,UAAU;AAEhB,YAAM,UAAU,IAAI,cAAc,QAAQ,KAAK;AAAA,QAC9C,OAAO,QAAQ,SAAS;AAAA,QACxB,aAAa,QAAQ,eAAe;AAAA,QACpC,SAAS,QAAQ,WAAW;AAAA,QAC5B,SAAS,QAAQ,WAAW;AAAA,MAC7B,CAAC;AAED,gBAAU,KAAK,2BAA2B;AAE1C,YAAM,UAAU,MAAM,QAAQ,MAAM;AAEpC,UAAI,QAAQ,QAAQ;AACnB,WAAG,cAAc,QAAQ,QAAQ,KAAK,UAAU,SAAS,MAAM,CAAC,CAAC;AACjE,kBAAU,KAAK,sBAAsB,QAAQ,MAAM,EAAE;AAAA,MACtD,OAAO;AACN,cAAM,eAAe,mBAAmB;AACxC,WAAG,cAAc,cAAc,KAAK,UAAU,SAAS,MAAM,CAAC,CAAC;AAC/D,kBAAU,KAAK,sBAAsB,YAAY,EAAE;AAAA,MACpD;AAAA,IACD,SAAS,OAAO;AACf,UAAI,iBAAiB,OAAO;AAC3B,kBAAU,MAAM,MAAM,OAAO;AAAA,MAC9B;AACA,gBAAU,MAAM,8BAA8B;AAC9C,cAAQ,KAAK,CAAC;AAAA,IACf;AAAA,EACD;AACD;AAEA,IAAM,qBAAqB,MAAM;AAChC,MAAI;AACH,UAAM,cAAc,KAAK,QAAQ,iBAAiB;AAElD,QAAI,CAAC,GAAG,WAAW,WAAW,GAAG;AAChC,SAAG,UAAU,WAAW;AAAA,IACzB;AAEA,UAAM,eAAe,KAAK;AAAA,MACzB,0BAA0B,KAAK,IAAI,CAAC;AAAA,IACrC;AAEA,QAAI,GAAG,WAAW,YAAY,GAAG;AAChC,YAAM,IAAI,MAAM,4BAA4B;AAAA,IAC7C;AACA,UAAM,aAAa,KAAK,MAAM,YAAY;AAE1C,QAAI,WAAW,QAAQ,SAAS;AAC/B,YAAM,IAAI,MAAM,iCAAiC;AAAA,IAClD;AAEA,WAAO;AAAA,EACR,SAAS,OAAO;AACf,UAAM,IAAI,MAAM,qBAAqB;AAAA,EACtC;AACD;;;AD1LA,IAAM,iBAAiB,MAAM,QAAQ,QAAQ,IAAI,CAAC;AAElD,eAAe,cAAc;AAC7B,eAAe,WAAW,kBAAkB;AAC5C,eAAe,MAAM,+BAA+B;AACpD,eAAe,KAAK,EAAE,MAAM,QAAQ,GAAG;AACvC,eAAe,QAAQ,EAAE,MAAM,WAAW,GAAG;AAC7C,eAAe,OAAO;AACtB,eAAe,eAAe,IAAI;AAGlC,eAAe,QAAQ,aAAa;AAEpC,eAAe,MAAM;",
6
+ "names": ["yargs"]
7
+ }
package/build/index.d.ts CHANGED
@@ -1 +1,25 @@
1
+ export declare class SpiderScanner {
2
+ private header;
3
+ private url;
4
+ private logger;
5
+ private depth;
6
+ private concurrency;
7
+ private retries;
8
+ private timeout;
9
+ constructor(url: string, options?: SpiderScannerOptions);
10
+ private normalizeDomain;
11
+ private convertRelativeUrlToAbsolute;
12
+ private isInternalLink;
13
+ private fetchWithRetries;
14
+ private extractLinks;
15
+ crawl(): Promise<Array<string>>;
16
+ }
17
+
18
+ export declare interface SpiderScannerOptions {
19
+ depth?: number;
20
+ concurrency?: number;
21
+ retries?: number;
22
+ timeout?: number;
23
+ }
24
+
1
25
  export { }