v8r 0.5.0 → 0.8.0

package/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Changelog
 
+## 📦 [0.8.0](https://www.npmjs.com/package/v8r/v/0.9.0) - 2021-12-25
+
+* Switch from CommonJS to ESModules internally
+* Requires node `^12.20.0 || ^14.13.1 || >=15.0.0`
+
+## 📦 [0.7.0](https://www.npmjs.com/package/v8r/v/0.7.0) - 2021-11-30
+
+* Upgrade to ajv 8 internally
+  Adds compatibility for JSON Schema draft 2019-09 and draft 2020-12
+* Docs/logging improvements to clarify behaviour of `--catalogs` param
+
+## 📦 [0.6.1](https://www.npmjs.com/package/v8r/v/0.6.1) - 2021-08-06
+
+* Refactor cache module to remove global state
+
+## 📦 [0.6.0](https://www.npmjs.com/package/v8r/v/0.6.0) - 2021-07-28
+
+* Add the ability to search custom schema catalogs using the `--catalogs` param
+
 ## 📦 [0.5.0](https://www.npmjs.com/package/v8r/v/0.5.0) - 2021-01-13
 
 * Allow validation against a local schema

package/README.md

@@ -1,10 +1,10 @@
 # v8r
 
-![Build status](https://github.com/chris48s/v8r/workflows/Run%20tests/badge.svg?branch=main)
-[![codecov](https://codecov.io/gh/chris48s/v8r/branch/main/graph/badge.svg?token=KL998A5CJH)](https://codecov.io/gh/chris48s/v8r)
-![NPM version](https://img.shields.io/npm/v/v8r.svg)
-![License](https://img.shields.io/npm/l/v8r.svg)
-![Node Compatibility](https://img.shields.io/node/v/v8r.svg)
+![build](https://raw.githubusercontent.com/chris48s/v8r/badges/.badges/main/build-status.svg)
+![coverage](https://raw.githubusercontent.com/chris48s/v8r/badges/.badges/main/coverage.svg)
+![version](https://raw.githubusercontent.com/chris48s/v8r/badges/.badges/main/package-version.svg)
+![license](https://raw.githubusercontent.com/chris48s/v8r/badges/.badges/main/package-license.svg)
+![node](https://raw.githubusercontent.com/chris48s/v8r/badges/.badges/main/package-node-version.svg)
 
 A command-line JSON and YAML validator that's on your wavelength.
 
@@ -50,10 +50,25 @@ Validating action.yml against schema from https://json.schemastore.org/github-ac
 $ v8r feature.geojson
 ❌ Could not find a schema to validate feature.geojson
 
-# ..you can specify one
+# ..you can specify a schema
 $ v8r feature.geojson -s https://json.schemastore.org/geojson
 Validating feature.geojson against schema from https://json.schemastore.org/geojson ...
 ✅ feature.geojson is valid
+
+# ..or use a custom catalog
+# v8r will search any custom catalogs before falling back to Schema Store
+$ cat > my-catalog.json <<EOF
+{ "\$schema": "https://json.schemastore.org/schema-catalog.json",
+  "version": 1,
+  "schemas": [ { "name": "geojson",
+                 "description": "geojson",
+                 "url": "https://json.schemastore.org/geojson.json",
+                 "fileMatch": ["*.geojson"] } ] }
+EOF
+$ v8r feature.geojson -c my-catalog.json
+ℹ️ Found schema in my-catalog.json ...
+Validating feature.geojson against schema from https://json.schemastore.org/geojson ...
+✅ feature.geojson is valid
 ```
 
 ## Exit codes
@@ -89,7 +104,13 @@ Validating feature.geojson against schema from https://json.schemastore.org/geoj
 
 ### ❓ What JSON schema versions are supported?
 
-💡 `v8r` works with JSON schema draft-04, draft-06 and draft-07.
+💡 `v8r` works with JSON schema drafts:
+
+* draft-04
+* draft-06
+* draft-07
+* draft 2019-09
+* draft 2020-12
 
 ### ❓ Will 100% of the schemas on schemastore.org work with this tool?
 

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "v8r",
-  "version": "0.5.0",
+  "version": "0.8.0",
   "description": "A command-line JSON and YAML validator that's on your wavelength",
   "scripts": {
-    "test": "V8R_CACHE_NAME=v8r-test nyc --reporter=text mocha \"src/**/*.spec.js\"",
+    "test": "V8R_CACHE_NAME=v8r-test c8 --reporter=text mocha \"src/**/*.spec.js\"",
     "lint": "eslint \"src/**/*.js\"",
-    "coverage": "nyc report --reporter=text-lcov > coverage.lcov",
+    "coverage": "c8 report --reporter=cobertura",
     "prettier": "prettier --write \"**/*.js\"",
     "prettier:check": "prettier --check \"**/*.js\"",
     "v8r": "src/index.js"
@@ -14,6 +14,7 @@
     "v8r": "src/index.js"
   },
   "main": "src/index.js",
+  "exports": "./src/index.js",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/chris48s/v8r.git"
@@ -22,29 +23,32 @@
   "author": "chris48s",
   "license": "MIT",
   "dependencies": {
-    "ajv": "^6.12.6",
+    "ajv": "^8.8.2",
+    "ajv-draft-04": "^1.0.0",
+    "ajv-formats": "^2.1.1",
     "flat-cache": "^3.0.4",
     "got": "^11.8.0",
     "is-url": "^1.2.4",
     "js-yaml": "^4.0.0",
     "minimatch": "^3.0.4",
-    "yargs": "^16.1.0"
+    "yargs": "^17.0.1"
   },
   "devDependencies": {
+    "c8": "^7.10.0",
     "chai": "^4.2.0",
     "chai-as-promised": "^7.1.1",
-    "eslint": "^7.16.0",
-    "eslint-config-prettier": "^7.1.0",
-    "eslint-plugin-mocha": "^8.0.0",
-    "eslint-plugin-prettier": "^3.3.0",
-    "mocha": "^8.2.1",
+    "eslint": "^8.0.1",
+    "eslint-config-prettier": "^8.1.0",
+    "eslint-plugin-mocha": "^9.0.0",
+    "eslint-plugin-prettier": "^4.0.0",
+    "mocha": "^9.0.0",
     "nock": "^13.0.4",
-    "nyc": "^15.1.0",
     "prettier": "^2.1.2"
   },
   "engines": {
-    "node": ">= 12"
+    "node": "^12.20.0 || ^14.13.1 || >=15.0.0"
   },
+  "type": "module",
   "keywords": [
     "JSON",
     "YAML",

package/src/ajv.js

@@ -0,0 +1,69 @@
+import { createRequire } from "module";
+// TODO: once JSON modules is stable these requires could become imports
+// https://nodejs.org/api/esm.html#esm_experimental_json_modules
+const require = createRequire(import.meta.url);
+
+import AjvDraft4 from "ajv-draft-04";
+import Ajv from "ajv";
+import Ajv2019 from "ajv/dist/2019.js";
+import Ajv2020 from "ajv/dist/2020.js";
+import addFormats from "ajv-formats";
+
+function _ajvFactory(schema, cache) {
+  const resolver = (url) => cache.fetch(url);
+  const opts = { loadSchema: resolver, strict: "log" };
+
+  if (
+    typeof schema["$schema"] === "string" ||
+    schema["$schema"] instanceof String
+  ) {
+    if (schema["$schema"].includes("json-schema.org/draft-04/schema")) {
+      opts.schemaId = "auto";
+      return new AjvDraft4(opts);
+    } else if (schema["$schema"].includes("json-schema.org/draft-06/schema")) {
+      const ajvDraft06 = new Ajv(opts);
+      ajvDraft06.addMetaSchema(
+        require("ajv/lib/refs/json-schema-draft-06.json")
+      );
+      return ajvDraft06;
+    } else if (schema["$schema"].includes("json-schema.org/draft-07/schema")) {
+      return new Ajv(opts);
+    } else if (
+      schema["$schema"].includes("json-schema.org/draft/2019-09/schema")
+    ) {
+      return new Ajv2019(opts);
+    } else if (
+      schema["$schema"].includes("json-schema.org/draft/2020-12/schema")
+    ) {
+      return new Ajv2020(opts);
+    }
+  }
+
+  // hedge our bets as best we can
+  const ajv = new Ajv(opts);
+  ajv.addMetaSchema(require("ajv/lib/refs/json-schema-draft-06.json"));
+  return ajv;
+
+  /* TODO:
+  const ajv = new Ajv2019(opts);
+  ajv.addMetaSchema(require("ajv/dist/refs/json-schema-draft-07.json"));
+  return ajv
+
+  might also be an equally valid fallback here
+  */
+}
+
+async function validate(data, schema, cache) {
+  const ajv = _ajvFactory(schema, cache);
+  addFormats(ajv);
+  const validateFn = await ajv.compileAsync(schema);
+  const valid = validateFn(data);
+  if (!valid) {
+    console.log("\nErrors:");
+    console.log(validateFn.errors);
+    console.log("");
+  }
+  return valid;
+}
+
+export { _ajvFactory, validate };

package/src/cache.js

@@ -1,66 +1,73 @@
-"use strict";
+import got from "got";
 
-const got = require("got");
-const callCounter = {};
-const callLimit = 10;
-
-function expire(cache, ttl) {
-  Object.entries(cache.all()).forEach(function ([url, cachedResponse]) {
-    if (!("timestamp" in cachedResponse) || !("body" in cachedResponse)) {
-      console.debug(`ℹ️ Cache error: deleting malformed response`);
-      cache.removeKey(url);
-    } else if (Date.now() > cachedResponse.timestamp + ttl) {
-      console.debug(`ℹ️ Cache stale: deleting cached response from ${url}`);
-      cache.removeKey(url);
-    }
-    cache.save(true);
-  });
-}
-
-function limitDepth(url) {
-  /*
-  It is possible to create cyclic dependencies with external references
-  in JSON schema. Ajv doesn't detect this when resolving external references,
-  so we keep a count of how many times we've called the same URL.
-  If we are calling the same URL over and over we've probably hit a circular
-  external reference and we need to break the loop.
-  */
-  if (url in callCounter) {
-    callCounter[url]++;
-  } else {
-    callCounter[url] = 1;
+class Cache {
+  constructor(flatCache, ttl) {
+    this.cache = flatCache;
+    this.ttl = ttl;
+    this.callCounter = {};
+    this.callLimit = 10;
   }
-  if (callCounter[url] > callLimit) {
-    throw new Error(
-      `❌ Called ${url} ${callLimit} times. Possible circular reference.`
+
+  expire() {
+    Object.entries(this.cache.all()).forEach(
+      function ([url, cachedResponse]) {
+        if (!("timestamp" in cachedResponse) || !("body" in cachedResponse)) {
+          console.debug(`ℹ️ Cache error: deleting malformed response`);
+          this.cache.removeKey(url);
+        } else if (Date.now() > cachedResponse.timestamp + this.ttl) {
+          console.debug(`ℹ️ Cache stale: deleting cached response from ${url}`);
+          this.cache.removeKey(url);
+        }
+        this.cache.save(true);
+      }.bind(this)
     );
   }
-}
 
-async function cachedFetch(url, cache, ttl) {
-  limitDepth(url);
-  expire(cache, ttl);
-  const cachedResponse = cache.getKey(url);
-  if (cachedResponse !== undefined) {
-    console.debug(`ℹ️ Cache hit: using cached response from ${url}`);
-    return cachedResponse.body;
+  limitDepth(url) {
+    /*
+    It is possible to create cyclic dependencies with external references
+    in JSON schema. Ajv doesn't detect this when resolving external references,
+    so we keep a count of how many times we've called the same URL.
+    If we are calling the same URL over and over we've probably hit a circular
+    external reference and we need to break the loop.
+    */
+    if (url in this.callCounter) {
+      this.callCounter[url]++;
+    } else {
+      this.callCounter[url] = 1;
+    }
+    if (this.callCounter[url] > this.callLimit) {
+      throw new Error(
+        `❌ Called ${url} >${this.callLimit} times. Possible circular reference.`
+      );
+    }
   }
 
-  try {
-    console.debug(`ℹ️ Cache miss: calling ${url}`);
-    const resp = await got(url);
-    const parsedBody = JSON.parse(resp.body);
-    if (ttl > 0) {
-      cache.setKey(url, { timestamp: Date.now(), body: parsedBody });
-      cache.save(true);
+  async fetch(url) {
+    this.limitDepth(url);
+    this.expire();
+    const cachedResponse = this.cache.getKey(url);
+    if (cachedResponse !== undefined) {
+      console.debug(`ℹ️ Cache hit: using cached response from ${url}`);
+      return cachedResponse.body;
     }
-    return parsedBody;
-  } catch (error) {
-    if (error.response) {
-      throw new Error(`❌ Failed fetching ${url}\n${error.response.body}`);
+
+    try {
+      console.debug(`ℹ️ Cache miss: calling ${url}`);
+      const resp = await got(url);
+      const parsedBody = JSON.parse(resp.body);
+      if (this.ttl > 0) {
+        this.cache.setKey(url, { timestamp: Date.now(), body: parsedBody });
+        this.cache.save(true);
+      }
+      return parsedBody;
+    } catch (error) {
+      if (error.response) {
+        throw new Error(`❌ Failed fetching ${url}\n${error.response.body}`);
+      }
+      throw new Error(`❌ Failed fetching ${url}`);
     }
-    throw new Error(`❌ Failed fetching ${url}`);
   }
 }
 
-module.exports = { cachedFetch, expire };
+export { Cache };

package/src/cli.js

@@ -1,25 +1,70 @@
-"use strict";
-
-const Ajv = require("ajv");
-const flatCache = require("flat-cache");
-const fs = require("fs");
-const isUrl = require("is-url");
-const minimatch = require("minimatch");
-const os = require("os");
-const path = require("path");
-const yaml = require("js-yaml");
-const yargs = require("yargs/yargs");
-const { hideBin } = require("yargs/helpers");
-const { cachedFetch } = require("./cache.js");
-const logging = require("./logging.js");
+import flatCache from "flat-cache";
+import fs from "fs";
+import isUrl from "is-url";
+import minimatch from "minimatch";
+import os from "os";
+import path from "path";
+import yaml from "js-yaml";
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { validate } from "./ajv.js";
+import { Cache } from "./cache.js";
+import logging from "./logging.js";
 
 const SCHEMASTORE_CATALOG_URL =
   "https://www.schemastore.org/api/json/catalog.json";
+
+const SCHEMASTORE_CATALOG_SCHEMA_URL =
+  "https://json.schemastore.org/schema-catalog.json";
+
 const CACHE_DIR = path.join(os.tmpdir(), "flat-cache");
 
-async function getSchemaUrlForFilename(filename, cache, ttl) {
-  const { schemas } = await cachedFetch(SCHEMASTORE_CATALOG_URL, cache, ttl);
+async function getFromUrlOrFile(location, cache) {
+  return isUrl(location)
+    ? await cache.fetch(location)
+    : JSON.parse(fs.readFileSync(location, "utf8").toString());
+}
+
+async function getSchemaUrlForFilename(catalogs, filename, cache) {
+  for (const [i, catalogLocation] of catalogs.entries()) {
+    const catalog = await getFromUrlOrFile(catalogLocation, cache);
+    const catalogSchema = await getFromUrlOrFile(
+      SCHEMASTORE_CATALOG_SCHEMA_URL,
+      cache
+    );
+
+    // Validate the catalog
+    const valid = await validate(catalog, catalogSchema, cache);
+    if (!valid || catalog.schemas === undefined) {
+      throw new Error(`❌ Malformed catalog at ${catalogLocation}`);
+    }
+
+    const { schemas } = catalog;
+    const matches = getSchemaMatchesForFilename(schemas, filename);
+    console.debug(`ℹ️ Searching for schema in ${catalogLocation} ...`);
+    if (matches.length === 1) {
+      console.log(`ℹ️ Found schema in ${catalogLocation} ...`);
+      return matches[0].url; // Match found. We're done.
+    }
+    if (matches.length === 0 && i < catalogs.length - 1) {
+      continue; // No match found. Try the next catalog in the array.
+    }
+    if (matches.length > 1) {
+      // We found >1 matches in the same catalog. This is always a hard error.
+      console.log(
+        `Found multiple possible schemas for ${filename}. Possible matches:`
+      );
+      matches.forEach(function (match) {
+        console.log(`${match.description}: ${match.url}`);
+      });
+    }
+    // Either we found >1 matches in the same catalog or we found 0 matches
+    // in the last catalog and there are no more catalogs left to try.
+    throw new Error(`❌ Could not find a schema to validate ${filename}`);
+  }
+}
 
+function getSchemaMatchesForFilename(schemas, filename) {
   const matches = [];
   schemas.forEach(function (schema) {
     if ("fileMatch" in schema) {
@@ -35,33 +80,7 @@ async function getSchemaUrlForFilename(filename, cache, ttl) {
       }
     }
   });
-
-  if (matches.length === 1) {
-    return matches[0].url;
-  }
-  if (matches.length > 1) {
-    console.log(
-      `Found multiple possible schemas for ${filename}. Possible matches:`
-    );
-    matches.forEach(function (match) {
-      console.log(`${match.description}: ${match.url}`);
-    });
-  }
-  throw new Error(`❌ Could not find a schema to validate ${filename}`);
-}
-
-async function validate(data, schema, resolver) {
-  const ajv = new Ajv({ schemaId: "auto", loadSchema: resolver });
-  ajv.addMetaSchema(require("ajv/lib/refs/json-schema-draft-04.json"));
-  ajv.addMetaSchema(require("ajv/lib/refs/json-schema-draft-06.json"));
-  const validate = await ajv.compileAsync(schema);
-  const valid = validate(data);
-  if (!valid) {
-    console.log("\nErrors:");
-    console.log(validate.errors);
-    console.log("");
-  }
-  return valid;
+  return matches;
 }
 
 function parseFile(contents, format) {
@@ -82,7 +101,7 @@ function secondsToMilliseconds(seconds) {
   return seconds * 1000;
 }
 
-function getCache() {
+function getFlatCache() {
   if (process.env.V8R_CACHE_NAME) {
     return flatCache.load(process.env.V8R_CACHE_NAME);
   }
@@ -90,35 +109,29 @@ function getCache() {
 }
 
 function Validator() {
-  const cache = getCache();
-
   return async function (args) {
     const filename = args.filename;
     const ttl = secondsToMilliseconds(args.cacheTtl || 0);
+    const cache = new Cache(getFlatCache(), ttl);
 
     const data = parseFile(
       fs.readFileSync(filename, "utf8").toString(),
       path.extname(filename)
     );
+
     const schemaLocation =
-      args.schema || (await getSchemaUrlForFilename(filename, cache, ttl));
-    const schema = isUrl(schemaLocation)
-      ? await cachedFetch(schemaLocation, cache, ttl)
-      : JSON.parse(fs.readFileSync(schemaLocation, "utf8").toString());
-    if (
-      "$schema" in schema &&
-      schema.$schema.includes("json-schema.org/draft/2019-09/schema")
-    ) {
-      throw new Error(`❌ Unsupported JSON schema version ${schema.$schema}`);
-    }
+      args.schema ||
+      (await getSchemaUrlForFilename(
+        (args.catalogs || []).concat([SCHEMASTORE_CATALOG_URL]),
+        filename,
+        cache
+      ));
+    const schema = await getFromUrlOrFile(schemaLocation, cache);
     console.log(
       `Validating ${filename} against schema from ${schemaLocation} ...`
     );
 
-    const resolver = function (url) {
-      return cachedFetch(url, cache, ttl);
-    };
-    const valid = await validate(data, schema, resolver);
+    const valid = await validate(data, schema, cache);
     if (valid) {
       console.log(`✅ ${filename} is valid`);
     } else {
@@ -169,6 +182,14 @@ function parseArgs(argv) {
       describe:
         "Local path or URL of schema to validate file against. If not supplied, we will attempt to find an appropriate schema on schemastore.org using the filename",
     })
+    .option("catalogs", {
+      type: "string",
+      alias: "c",
+      array: true,
+      describe:
+        "Local path or URL of custom catalogs to use prior to schemastore.org",
+    })
+    .conflicts("schema", "catalogs")
     .option("ignore-errors", {
       type: "boolean",
       default: false,
@@ -183,4 +204,4 @@ function parseArgs(argv) {
     }).argv;
 }
 
-module.exports = { cli, parseArgs };
+export { cli, parseArgs };

package/src/index.js

@@ -1,8 +1,6 @@
 #!/usr/bin/env node
 
-"use strict";
-
-const { cli, parseArgs } = require("./cli.js");
+import { cli, parseArgs } from "./cli.js";
 
 (async () => {
   const exitCode = await cli(parseArgs(process.argv));

package/src/logging.js

@@ -1,5 +1,3 @@
-"use strict";
-
 const origWarn = console.warn;
 const origInfo = console.info;
 const origDebug = console.debug;
@@ -19,4 +17,5 @@ function cleanup() {
   console.debug = origDebug;
 }
 
-module.exports = { cleanup, init };
+const logging = { cleanup, init };
+export default logging;