retire 4.4.2 → 5.0.0

package/CHANGELOG.md

@@ -1,12 +1,17 @@
 # Changelog
 
+## [5.0.0-beta.1]
+
+### Added
+
+- Deepscan option with `--deep` where retire.js will also try to detect libraries using JavaScript AST (slower)
+
 ## [4.4.2]
 
 ### Fix
 
 - Fix matching to include all matches
 
-
 ## [4.4.1]
 
 ### Chore

package/README.md

@@ -1,23 +1,20 @@
 Command line scanner looking for use of known vulnerable js files and node modules in web projects and/or node projects.
 
-Install
--------
+## Install
 
     npm install -g retire
 
+## Usage
 
-Usage
------
-
-````
+```
 Usage: retire [options]
 
 Options:
   -V, --version            output the version number
   -v, --verbose            Show identified files (by default only vulnerable files are shown)
   -c, --nocache            Don't use local cache
-  --jspath <path>          Folder to scan for javascript files
-  --path <path>            Folder to scan for both
+  --jspath <path>          Folder to scan for javascript files (deprecated)
+  --path <path>            Folder to scan for javascript files
   --jsrepo <path|url>      Local or internal version of repo. Can be multiple comma separated. Default: 'central')
   --cachedir <path>        Path to use for local cache instead of /tmp/.retire-cache
   --proxy <url>            Proxy url (http://some.host:8080)
@@ -33,23 +30,25 @@ Options:
   --ext <extensions>       Comma separated list of file extensions for JavaScript files. The default is "js"
   --cacert <path>          Use the specified certificate file to verify the peer used for fetching remote jsrepo/noderepo files
   --includeOsv             Include OSV advisories in the output
+  --deep                   Deep scan (slower and experimental)
   -h, --help               display help for command
-````
+```
 
 The `depcheck` output format mimics the output of OWASP Dependency Check, but lacks some information compared to OWASP Dependency Check, because that information is not in the repo.
 The `cyclonedx` output format is based on based on the https://github.com/CycloneDX spec.
 
-.retireignore
--------------
-````
+## .retireignore
+
+```
 @qs                                                             # ignore this module regardless of location
 node_modules/connect/node_modules/body-parser/node_modules/qs   # ignore specific path
-````
+```
+
 Due to a bug in ignore resolving, please upgrade to >= 1.1.3
 
-.retireignore.json
-------------------
-````
+## .retireignore.json
+
+```
 [
 	{
 		"component": "jquery",
@@ -67,8 +66,8 @@ Due to a bug in ignore resolving, please upgrade to >= 1.1.3
 	}
 
 ]
-````
+```
+
+## Source code / Reporting an issue
 
-Source code / Reporting an issue
---------------------------------
 The source code and issue tracker can be found at [https://github.com/RetireJS/retire.js](https://github.com/RetireJS/retire.js)

package/lib/cli.js

@@ -71,12 +71,15 @@ const prg = commander_1.program
     .option('--ext <extensions>', 'Comma separated list of file extensions for JavaScript files. The default is "js"')
     .option('--cacert <path>', 'Use the specified certificate file to verify the peer used for fetching remote jsrepo/noderepo files')
     .option('--includeOsv', 'Include OSV advisories in the output')
+    .option('--deep', 'Deep scan (slower and experimental)')
     .parse()
     .opts();
 const colorwarn = prg.colors ? ansi_colors_1.default.red : (x) => x;
 const jsrepolocation = (prg.jsrepo ?? "'central'")
     .split(',')
-    .map((x) => x === "'central'" ? 'https://raw.githubusercontent.com/RetireJS/retire.js/master/repository/jsrepository.json' : x);
+    .map((x) => x === "'central'"
+    ? 'https://raw.githubusercontent.com/RetireJS/retire.js/master/repository/jsrepository-v2.json'
+    : x);
 const ignorefile = prg.ignoreFile ?? defaultIgnoreFiles.filter((x) => fs_1.default.existsSync(x))[0];
 const scanpath = prg.path ?? prg.jspath ?? '.';
 const log = reporting.open({
@@ -108,6 +111,7 @@ const config = {
     includeOsv: !!prg.includeOsv,
     verbose: !!prg.verbose,
     proxy: prg.proxy,
+    deep: !!prg.deep,
 };
 log.info(`retire.js v${retire.version}`);
 function exitWithError(msg) {

package/lib/deepscan.d.ts

@@ -0,0 +1,2 @@
+import { Component, Repository } from './types';
+export declare function deepScan(content: string, repo: Repository): Component[];

package/lib/deepscan.js

@@ -0,0 +1,37 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.deepScan = void 0;
+const astronomical_1 = require("astronomical");
+function deepScan(content, repo) {
+    const astQueries = {};
+    const backMap = {};
+    Object.entries(repo).forEach(([name, data]) => {
+        data.extractors.ast?.forEach((query, i) => {
+            astQueries[`${name}_${i}`] = query;
+            backMap[`${name}_${i}`] = name;
+        });
+    });
+    const results = (0, astronomical_1.multiQuery)(content, astQueries);
+    const detected = [];
+    Object.entries(results).forEach(([key, value]) => {
+        value.forEach((match) => {
+            const component = backMap[key];
+            if (typeof match !== 'string')
+                return;
+            detected.push({
+                version: match,
+                component: component,
+                npmname: repo[component].npmname,
+                basePurl: repo[component].basePurl,
+                detection: 'ast',
+            });
+        });
+    });
+    return detected.reduce((acc, cur) => {
+        if (acc.some((c) => c.component === cur.component && c.version === cur.version))
+            return acc;
+        acc.push(cur);
+        return acc;
+    }, []);
+}
+exports.deepScan = deepScan;

package/lib/repo.js

@@ -144,6 +144,7 @@ function validateRepository(repo, replacer) {
             filecontent: z.array(regexValidator).optional(),
             filecontentreplace: z.array(replaceValidator).optional(),
             hashes: z.record(z.string().regex(/^[a-f0-9]+$/i), versionValidator).optional(),
+            ast: z.array(z.string()).optional(),
         })
             .strict(),
     })

package/lib/retire.js

@@ -4,7 +4,7 @@
  */
 
 var exports = exports || {};
-exports.version = '4.4.2';
+exports.version = '5.0.0';
 
 function isDefined(o) {
   return typeof o !== 'undefined';
@@ -26,7 +26,7 @@ function scan(data, extractor, repo, matcher = simpleMatch) {
     if (!isDefined(extractors)) continue;
     for (var i in extractors) {
       var matches = matcher(extractors[i], data);
-      matches.forEach(match => {
+      matches.forEach((match) => {
         match = match.replace(/(\.|-)min$/, '');
         detected.push({
           version: match,
@@ -42,20 +42,20 @@ function scan(data, extractor, repo, matcher = simpleMatch) {
 }
 
 function simpleMatch(regex, data) {
-  var re = new RegExp(regex, "g");
+  var re = new RegExp(regex, 'g');
   const result = [];
   let match;
-  while (match = re.exec(data)) {
+  while ((match = re.exec(data))) {
     result.push(match[1]);
   }
   return result;
 }
 function replacementMatch(regex, data) {
   var ar = /^\/(.*[^\\])\/([^\/]+)\/$/.exec(regex);
-  var re = new RegExp(ar[1], "g");
+  var re = new RegExp(ar[1], 'g');
   const result = [];
   let match;
-  while(match = re.exec(data)) {
+  while ((match = re.exec(data))) {
     var ver = null;
     if (match) {
       ver = match[0].replace(new RegExp(ar[1]), ar[2]);

package/lib/scanner.js

@@ -30,6 +30,7 @@ const fs = __importStar(require("fs"));
 const crypto = __importStar(require("crypto"));
 const path = __importStar(require("path"));
 const depsdev_1 = require("./depsdev");
+const deepscan_1 = require("./deepscan");
 const events = new events_1.EventEmitter();
 const hash = {
     sha1: (data) => {
@@ -128,7 +129,11 @@ function scanJsFile(file, repo, options) {
     }
     let results = retire.scanFileName(file, repo);
     if (!results || results.length === 0) {
-        results = retire.scanFileContent(fs.readFileSync(file, 'utf-8'), repo, hash);
+        const content = fs.readFileSync(file, 'utf-8');
+        results = retire.scanFileContent(content, repo, hash);
+        if (options.deep) {
+            results = results.concat((0, deepscan_1.deepScan)(content, repo));
+        }
     }
     emitResults({ file: file, results: results }, options);
 }

package/lib/types.d.ts

@@ -12,6 +12,7 @@ export type Repository = Record<string, {
         filecontent?: string[];
         filecontentreplace?: string[];
         hashes?: Record<string, string>;
+        ast?: string[];
     };
 }>;
 export type Vulnerability = {
@@ -34,6 +35,7 @@ export type Component = {
     npmname?: string;
     version: string;
     vulnerabilities?: Vulnerability[];
+    detection?: string;
 };
 export type Finding = {
     results: Component[];
@@ -86,5 +88,6 @@ export type Options = {
     path: string;
     exitwith: number;
     includeOsv?: boolean;
+    deep?: boolean;
 };
 export {};

package/package.json

@@ -2,7 +2,7 @@
   "author": "Erlend Oftedal <erlend@oftedal.no>",
   "name": "retire",
   "description": "Retire is a tool for detecting use of vulnerable libraries",
-  "version": "4.4.2",
+  "version": "5.0.0",
   "license": "Apache-2.0",
   "repository": {
     "type": "git",
@@ -14,6 +14,7 @@
   "main": "./lib/retire.js",
   "dependencies": {
     "ansi-colors": "^4.1.1",
+    "astronomical": "^1.0.0",
     "commander": "^10.0.1",
     "proxy-agent": "^6.4.0",
     "uuid": "^9.0.1",
@@ -23,9 +24,9 @@
   "devDependencies": {
     "@types/node": "^18.13.0",
     "@types/uuid": "^9.0.0",
-    "@typescript-eslint/eslint-plugin": "^6.11.0",
-    "@typescript-eslint/parser": "^6.11.0",
-    "chai": "^4.3.4",
+    "@typescript-eslint/eslint-plugin": "^7.1.1",
+    "@typescript-eslint/parser": "^7.1.1",
+    "chai": "^4.x.x",
     "eslint": "^8.34.0",
     "eslint-config-prettier": "^9.0.0",
     "jsonschema": "^1.4.1",