v8r 5.0.0 → 5.1.0

package/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 📦 [5.1.0](https://www.npmjs.com/package/v8r/v/5.1.0) - 2025-07-20
+
+* v8r now pre-warms the cache and fetches schemas in parallel.
+  This will improve decrease total run time for any run that involves fetching
+  more than one remote schema, or involves a schema with remote `$ref`s.
+* Improve handling of empty yaml files.
+
 ## 📦 [5.0.0](https://www.npmjs.com/package/v8r/v/5.0.0) - 2025-05-10
 
 Following on from the deprecations in version 4.4.0,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "v8r",
-  "version": "5.0.0",
+  "version": "5.1.0",
   "description": "A command-line JSON, YAML and TOML validator that's on your wavelength",
   "scripts": {
     "test": "V8R_CACHE_NAME=v8r-test c8 --reporter=text mocha \"src/**/*.spec.js\"",
@@ -42,14 +42,16 @@
     "js-yaml": "^4.0.0",
     "json5": "^2.2.0",
     "minimatch": "^10.0.0",
+    "p-limit": "^6.2.0",
+    "p-mutex": "^1.0.0",
     "smol-toml": "^1.0.1",
-    "yargs": "^17.0.1"
+    "yargs": "^18.0.0"
   },
   "devDependencies": {
     "c8": "^10.1.2",
     "eslint": "^9.9.0",
     "eslint-config-prettier": "^10.1.2",
-    "eslint-plugin-jsdoc": "^50.2.2",
+    "eslint-plugin-jsdoc": "^51.4.1",
     "eslint-plugin-mocha": "^11.0.0",
     "eslint-plugin-prettier": "^5.0.0",
     "mocha": "^11.0.1",

package/src/bootstrap.js

@@ -53,6 +53,8 @@ function mergeConfigs(args, config) {
   if (config.filepath) {
     mergedConfig.configFileRelativePath = getRelativeFilePath(config);
   }
+  // hard-coded - this can't be set via CLI or config file
+  mergedConfig.cachePrewarm = true;
   return mergedConfig;
 }
 

package/src/cache-prewarm.js

@@ -0,0 +1,87 @@
+import isUrl from "is-url";
+import pLimit from "p-limit";
+import { getCatalogs, getMatchForFilename } from "./catalogs.js";
+
+const limit = pLimit(10);
+
+async function fetch(location, cache) {
+  return await cache.fetch(location, false);
+}
+
+function fetchWithLimit(url, cache) {
+  return limit(() => fetch(url, cache));
+}
+
+function normalizeUrl(ref) {
+  try {
+    const url = new URL(ref);
+    url.hash = "";
+    return url.toString();
+  } catch {
+    return null;
+  }
+}
+
+function getRemoteRefs(node) {
+  let refs = [];
+  if (Array.isArray(node)) {
+    for (const v of node) {
+      if (typeof v === "object" && v !== null) {
+        refs = refs.concat(getRemoteRefs(v));
+      }
+    }
+  } else if (typeof node === "object" && node !== null) {
+    for (const [k, v] of Object.entries(node)) {
+      if (k === "$ref" && typeof v === "string") {
+        const resolved = normalizeUrl(v);
+        if (resolved && isUrl(resolved)) {
+          refs.push(resolved);
+        }
+      }
+      if (typeof v === "object" && v !== null) {
+        refs = refs.concat(getRemoteRefs(v));
+      }
+    }
+  }
+  return Array.from(new Set(refs));
+}
+
+async function fetchAndRecurse(url, cache) {
+  const schema = await fetchWithLimit(url, cache);
+  const refs = getRemoteRefs(schema).filter(
+    (ref) => cache.get(ref) === undefined,
+  );
+  await Promise.all(refs.map((ref) => fetchAndRecurse(ref, cache)));
+}
+
+async function prewarmSchemaCache(filenames, config, cache) {
+  const catalogs = getCatalogs(config);
+  const schemaLocations = new Set();
+
+  for (const filename of filenames) {
+    let catalogMatch;
+    try {
+      catalogMatch = config.schema
+        ? {}
+        : await getMatchForFilename(catalogs, filename, "debug", cache);
+    } catch {
+      catalogMatch = {};
+    }
+    const schemaLocation = config.schema || catalogMatch.location;
+
+    if (schemaLocation) {
+      schemaLocations.add(schemaLocation);
+    }
+    cache.resetCounters();
+  }
+
+  await Promise.all(
+    Array.from(schemaLocations)
+      .filter((schemaLocation) => isUrl(schemaLocation))
+      .map((url) => fetchAndRecurse(url, cache)),
+  );
+  cache.persist();
+  cache.resetCounters();
+}
+
+export { getRemoteRefs, prewarmSchemaCache, normalizeUrl };

package/src/cache.js

@@ -1,4 +1,5 @@
 import got from "got";
+import Mutex from "p-mutex";
 import logger from "./logger.js";
 import { parseSchema } from "./parser.js";
 
@@ -7,39 +8,53 @@ class Cache {
     this.cache = flatCache;
     this.ttl = this.cache._cache.ttl || 0;
     this.callCounter = {};
+    this.locks = {};
     this.callLimit = 10;
     if (this.ttl === 0) {
       this.cache.clear();
     }
   }
 
-  limitDepth(url) {
+  getMutex(url) {
+    if (!(url in this.locks)) {
+      this.locks[url] = new Mutex();
+    }
+    return this.locks[url];
+  }
+
+  async limitDepth(url) {
     /*
     It is possible to create cyclic dependencies with external references
-    in JSON schema. Ajv doesn't detect this when resolving external references,
+    in JSON schema.
+    We try to mitigate this issue during cache pre-warming.
+    Ajv doesn't detect this when resolving external references,
     so we keep a count of how many times we've called the same URL.
     If we are calling the same URL over and over we've probably hit a circular
     external reference and we need to break the loop.
     */
-    if (url in this.callCounter) {
-      this.callCounter[url]++;
-    } else {
-      this.callCounter[url] = 1;
-    }
-    if (this.callCounter[url] > this.callLimit) {
-      throw new Error(
-        `Called ${url} >${this.callLimit} times. Possible circular reference.`,
-      );
-    }
+    const mutex = this.getMutex(url);
+
+    await mutex.withLock(async () => {
+      if (url in this.callCounter) {
+        this.callCounter[url]++;
+      } else {
+        this.callCounter[url] = 1;
+      }
+      if (this.callCounter[url] > this.callLimit) {
+        throw new Error(
+          `Called ${url} >${this.callLimit} times. Possible circular reference.`,
+        );
+      }
+    });
   }
 
   resetCounters() {
     this.callCounter = {};
   }
 
-  async fetch(url) {
-    this.limitDepth(url);
-    const cachedResponse = this.cache.getKey(url);
+  async fetch(url, persist = true) {
+    await this.limitDepth(url);
+    const cachedResponse = this.cache.get(url);
     if (cachedResponse !== undefined) {
       logger.debug(`Cache hit: using cached response from ${url}`);
       return cachedResponse.body;
@@ -50,8 +65,10 @@ class Cache {
       const resp = await got(url);
       const parsedBody = parseSchema(resp.body, url);
       if (this.ttl > 0) {
-        this.cache.setKey(url, { body: parsedBody });
-        this.cache.save(true);
+        this.cache.set(url, { body: parsedBody });
+        if (persist) {
+          this.cache.save(true);
+        }
       }
       return parsedBody;
     } catch (error) {
@@ -61,6 +78,14 @@ class Cache {
       throw new Error(`Failed fetching ${url}`);
     }
   }
+
+  persist() {
+    this.cache.save(true);
+  }
+
+  get(key) {
+    return this.cache.get(key);
+  }
 }
 
 export { Cache };

package/src/catalogs.js

@@ -74,7 +74,7 @@ function getMultipleMatchesLogMessage(matches) {
     .join("\n");
 }
 
-async function getMatchForFilename(catalogs, filename, cache) {
+async function getMatchForFilename(catalogs, filename, logLevel, cache) {
   for (const [i, rec] of catalogs.entries()) {
     const catalogLocation = rec.location;
     const catalog =
@@ -107,7 +107,7 @@ async function getMatchForFilename(catalogs, filename, cache) {
       (matches.length === 1 && matches[0].versions == null) ||
       (matches.length === 1 && Object.keys(matches[0].versions).length === 1)
     ) {
-      logger.info(`Found schema in ${catalogLocation} ...`);
+      logger[logLevel](`Found schema in ${catalogLocation} ...`);
       return coerceMatch(matches[0]); // Exactly one match found. We're done.
     }
 
@@ -122,7 +122,7 @@ async function getMatchForFilename(catalogs, filename, cache) {
     ) {
       // We found >1 matches in the same catalog. This is always a hard error.
       const matchesLog = getMultipleMatchesLogMessage(matches);
-      logger.info(
+      logger[logLevel](
         `Found multiple possible matches for ${filename}. Possible matches:\n\n${matchesLog}`,
       );
       throw new Error(

package/src/cli.js

@@ -12,6 +12,7 @@ import { getFromUrlOrFile } from "./io.js";
 import logger from "./logger.js";
 import { getDocumentLocation } from "./output-formatters.js";
 import { parseFile } from "./parser.js";
+import { prewarmSchemaCache } from "./cache-prewarm.js";
 
 const EXIT = {
   VALID: 0,
@@ -92,7 +93,7 @@ async function validateFile(filename, config, plugins, cache) {
     const catalogs = getCatalogs(config);
     const catalogMatch = config.schema
       ? {}
-      : await getMatchForFilename(catalogs, filename, cache);
+      : await getMatchForFilename(catalogs, filename, "info", cache);
     schemaLocation = config.schema || catalogMatch.location;
     schema = await getFromUrlOrFile(schemaLocation, cache);
     logger.info(
@@ -182,6 +183,12 @@ function Validator() {
     const ttl = secondsToMilliseconds(config.cacheTtl || 0);
     const cache = new Cache(getFlatCache(ttl));
 
+    if (config.cachePrewarm && ttl > 5000) {
+      logger.info("Pre-warming the cache");
+      await prewarmSchemaCache(filenames, config, cache);
+      logger.debug("Cache pre-warming complete");
+    }
+
     let results = [];
     for (const filename of filenames) {
       const fileResults = await validateFile(filename, config, plugins, cache);

package/src/parser.js

@@ -10,6 +10,9 @@ function parseFile(plugins, contents, filename, parser) {
       const maybeDocuments = Array.isArray(parsedFile)
         ? parsedFile
         : [parsedFile];
+      if (maybeDocuments.length === 0) {
+        throw new Error(`No documents to validate found in ${filename}`);
+      }
       for (const doc of maybeDocuments) {
         if (!(doc instanceof Document)) {
           throw new Error(

package/src/test-helpers.js

@@ -1,4 +1,3 @@
-import { clearCacheById } from "flat-cache";
 import logger from "./logger.js";
 
 const origWriteOut = logger.writeOut;
@@ -7,7 +6,6 @@ const testCacheName = process.env.V8R_CACHE_NAME;
 const env = process.env;
 
 function setUp() {
-  clearCacheById(testCacheName);
   logger.resetStdout();
   logger.resetStderr();
   logger.writeOut = function () {};
@@ -16,7 +14,6 @@ function setUp() {
 }
 
 function tearDown() {
-  clearCacheById(testCacheName);
   logger.resetStdout();
   logger.resetStderr();
   logger.writeOut = origWriteOut;