schemastore-to-typescript 0.2.12

package/package.json

@@ -0,0 +1,80 @@
+{
+  "name": "schemastore-to-typescript",
+  "version": "0.2.12",
+  "description": "Fetch JSON schemas from the JSON Schema Store catalog and compile them to TypeScript typings",
+  "keywords": [
+    "cli",
+    "json-schema",
+    "schemastore",
+    "typescript"
+  ],
+  "homepage": "https://github.com/langri-sha/schemastore-to-typescript#readme",
+  "bugs": {
+    "url": "https://github.com/langri-sha/schemastore-to-typescript/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/langri-sha/schemastore-to-typescript.git"
+  },
+  "license": "MIT",
+  "author": {
+    "name": "Filip Dupanović",
+    "email": "filip.dupanovic@gmail.com",
+    "url": "https://langri-sha.com",
+    "organization": false
+  },
+  "type": "module",
+  "main": "dist/index.js",
+  "bin": {
+    "schemastore-to-typescript": "dist/cli.js"
+  },
+  "scripts": {
+    "change": "pnpm exec projen change",
+    "default": "pnpm exec projen default",
+    "prepare": "husky",
+    "prepublishOnly": "rm -rf dist; tsc --project tsconfig.build.json",
+    "renovate": "pnpm exec projen renovate",
+    "test": "vitest"
+  },
+  "dependencies": {
+    "commander": "15.0.0",
+    "debug": "4.4.3",
+    "env-paths": "4.0.0",
+    "es-main": "1.4.0",
+    "got": "16.0.0",
+    "json-schema-to-typescript": "16.0.0",
+    "keyv": "5.6.0",
+    "keyv-file": "5.3.5"
+  },
+  "devDependencies": {
+    "@langri-sha/eslint-config": "0.9.15",
+    "@langri-sha/lint-staged": "0.9.7",
+    "@langri-sha/prettier": "0.4.7",
+    "@langri-sha/projen-project": "0.31.0",
+    "@langri-sha/tsconfig": "1.0.1",
+    "@langri-sha/vitest": "0.2.0",
+    "@types/debug": "4.1.13",
+    "@types/node": "24.13.6",
+    "beachball": "2.66.0",
+    "eslint": "10.11.0",
+    "husky": "9.1.7",
+    "lint-staged": "17.5.1",
+    "prettier": "3.9.8",
+    "projen": "0.86.5",
+    "tsx": "4.23.13",
+    "typescript": "5.9.3",
+    "vitest": "5.0.1"
+  },
+  "packageManager": "pnpm@12.5.1",
+  "devEngines": {
+    "runtime": {
+      "name": "node",
+      "version": ">= 24.16.0"
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "//": "~~ Generated by projen. To modify, edit .projenrc.ts and run \"pnpm exec projen\".",
+  "types": "dist/index.d.ts"
+}

package/readme.md

@@ -0,0 +1,52 @@
+# schemastore-to-typescript
+
+Fetch [JSON Schema] from the [JSON Schema Store] catalog and compile [TypeScript
+typings].
+
+The tool first fetches the schema catalog from the JSON Schema Store API,
+searches for the requested schema by name (case-insensitive), and then downloads
+and compiles the schema to TypeScript definitions.
+
+## Features
+
+- uses [`json-schema-to-typescript`] for TypeScript generation
+- fetches schemas from the [JSON Schema Store catalog API]
+- case-insensitive schema name matching
+- offline cache using [`got`] for both catalog and schema requests
+
+## Usage
+
+```sh
+npm install -D schemastore-to-typescript
+```
+
+Name a schema as it appears in the catalog, and optionally where to write its
+typings — `<schema>.d.ts` in the working directory by default:
+
+```sh
+schemastore-to-typescript renovate src/renovate.ts
+schemastore-to-typescript 'cargo manifest' src/cargo.ts
+```
+
+Requests are cached in the user cache directory, such as
+`~/.cache/schemastore-to-typescript-nodejs` on Linux. Pass `--no-cache` to skip
+it.
+
+The same is available as a function, which resolves to the compiled module and
+takes `false` as its second argument to skip the cache:
+
+```ts
+import { compile } from 'schemastore-to-typescript'
+
+const typings = await compile('swcrc')
+```
+
+[`got`]: https://www.npmjs.com/package/got
+[`json-schema-to-typescript`]:
+  https://www.npmjs.com/package/json-schema-to-typescript
+[json schema store]: https://www.schemastore.org/
+[json schema store catalog api]:
+  https://www.schemastore.org/api/json/catalog.json
+[json schema]: https://json-schema.org/
+[typescript typings]:
+  https://www.typescriptlang.org/docs/handbook/declaration-files/templates/module-d-ts.html

package/src/cli.ts

@@ -0,0 +1,43 @@
+#!/usr/bin/env node
+import * as fs from 'node:fs/promises'
+import * as path from 'node:path'
+
+import { Command } from 'commander'
+import createDebug from 'debug'
+import esMain from 'es-main'
+
+import { compile } from './index.js'
+
+const debug = createDebug('schema-store-to-typescript')
+
+export const program: Command = new Command()
+  .name('schemastore-to-typescript')
+  .description('Compile a JSON schema store schema to TypeScript definitions')
+  .argument('<schema>', 'JSON schema store schema')
+  .argument('[output]', 'Path to output the TypeScript definition')
+  .option('-n, --no-cache', 'Skip caching requests')
+  .showHelpAfterError()
+  .action(async (schema, output, { cache }) => {
+    output ??= `${schema}.d.ts`
+
+    const compiled = await compile(schema, cache)
+    await fs.writeFile(
+      // Using `process.cwd()` as escape hatch for `process.chdir()` not being
+      // available in workers. See: https://github.com/vitest-dev/vitest/issues/1436.
+      path.resolve(process.cwd(), output),
+      compiled,
+    )
+  })
+
+if (esMain(import.meta)) {
+  debug('Parsing arguments', process.argv)
+  program
+    .parseAsync(process.argv)
+    .then(() => {
+      process.exit(0)
+    })
+    .catch((error) => {
+      console.error(error)
+      process.exit(1)
+    })
+}

package/src/index.ts

@@ -0,0 +1,256 @@
+import * as path from 'node:path'
+
+import createDebug from 'debug'
+import envPaths from 'env-paths'
+import got, { HTTPError } from 'got'
+import {
+  type JSONSchema,
+  compile as compileSource,
+} from 'json-schema-to-typescript'
+import Keyv from 'keyv'
+import { KeyvFile } from 'keyv-file'
+
+import { normalizeSchema } from './normalize.js'
+
+const debug = createDebug('schema-store-to-typescript')
+const paths = envPaths('schemastore-to-typescript')
+
+const store = new KeyvFile({
+  filename: path.join(paths.cache, 'requests.json'),
+  writeDelay: 0,
+})
+
+const keyv = new Keyv({ store })
+
+// Both are declared private in keyv-file's typings, but flushing the cache
+// needs them — see `flushCache`.
+const flushable = store as unknown as {
+  save(): Promise<unknown>
+  saveToDisk(): Promise<unknown>
+}
+
+/**
+ * `keyv-file@5.3.5` coalesces a `set` onto an in-flight save: `saveToDisk`
+ * snapshots the store synchronously at entry, then awaits `mkdir` and
+ * `writeFile`, and only clears the in-flight promise afterwards. A `set`
+ * landing in that window gets that promise back — it resolves, but the value
+ * was never in the snapshot and no further save is scheduled, so it stays in
+ * memory for the rest of the process.
+ *
+ * `compile` writes the catalog and then the schema straight into that window,
+ * which loses the schema. `writeDelay` is no help: the window is the write
+ * itself, not the timer.
+ *
+ * So settle whatever is in flight first — its `writeFile` would otherwise race
+ * ours and could land last with the older snapshot — and then write the
+ * current one unconditionally.
+ */
+const flushCache = async () => {
+  await flushable.save()
+  await flushable.saveToDisk()
+}
+
+interface CatalogSchema {
+  name: string
+  description: string
+  fileMatch?: string[]
+  url: string
+}
+
+interface Catalog {
+  $schema: string
+  version: number
+  schemas: CatalogSchema[]
+}
+
+// Holds literal instance data rather than nested schemas, so it must not be
+// walked as if it were part of the schema shape.
+const nonSchemaKeys = new Set(['default', 'examples', 'const', 'enum'])
+
+/**
+ * Some published schemas (e.g. Renovate's `registryAliases`) combine a
+ * `$ref` with a concrete `additionalProperties` schema on the same node — a
+ * JSON Schema anti-pattern that `json-schema-to-typescript` compiles into an
+ * interface whose own properties conflict with its index signature,
+ * producing uncompilable TypeScript (TS2411). `additionalProperties` is the
+ * more specific, deliberate declaration, so drop the `$ref` in favor of it.
+ *
+ * A node declaring `additionalProperties` at all compiles to an interface
+ * carrying a string index signature, and an optional member cannot sit beside
+ * one: its `undefined` is not assignable to the index type, which is TS2411
+ * again (Cargo's `[features]`, `[patch]` and `[lints]`). Where the index type
+ * is `unknown` the compiler does allow it, but the named members are then
+ * advisory — the same index signature accepts them either way, and following
+ * one is how Cargo's `package.metadata` reaches `quikrun`, a mutually
+ * recursive schema that overflows the compiler's stack and takes the whole
+ * manifest down with it.
+ *
+ * So optional members are dropped in favour of `additionalProperties`, which
+ * is the general contract they were there to narrow. Required ones stay: they
+ * are a promise the type has to keep.
+ */
+const sanitizeSchema = <T>(node: T): T => {
+  if (Array.isArray(node)) {
+    return node.map(sanitizeSchema) as unknown as T
+  }
+
+  if (node === null || typeof node !== 'object') {
+    return node
+  }
+
+  const sanitized: Record<string, unknown> = {}
+
+  for (const [key, value] of Object.entries(node as Record<string, unknown>)) {
+    sanitized[key] = nonSchemaKeys.has(key) ? value : sanitizeSchema(value)
+  }
+
+  if (
+    typeof sanitized.$ref === 'string' &&
+    typeof sanitized.additionalProperties === 'object' &&
+    sanitized.additionalProperties !== null
+  ) {
+    delete sanitized.$ref
+  }
+
+  if (
+    sanitized.additionalProperties !== undefined &&
+    sanitized.additionalProperties !== false &&
+    typeof sanitized.properties === 'object' &&
+    sanitized.properties !== null
+  ) {
+    const required = new Set(
+      Array.isArray(sanitized.required) ? sanitized.required : [],
+    )
+
+    const properties = Object.fromEntries(
+      Object.entries(sanitized.properties).filter(([property]) =>
+        required.has(property),
+      ),
+    )
+
+    if (Object.keys(properties).length > 0) {
+      sanitized.properties = properties
+    } else {
+      delete sanitized.properties
+    }
+  }
+
+  return sanitized as T
+}
+
+export const compile = async (
+  name: string,
+  cache: boolean = true,
+): Promise<string> => {
+  const catalogUrl = 'https://www.schemastore.org/api/json/catalog.json'
+
+  let catalog: Catalog
+  let schema: JSONSchema
+
+  const fetchWithRetry = async <T>(
+    url: string,
+    retryOnCacheError: boolean = true,
+  ): Promise<T> => {
+    try {
+      return await got(url, {
+        cache: cache ? keyv : undefined,
+        headers: {
+          accept: 'application/json',
+        },
+        decompress: true,
+      }).json<T>()
+    } catch (e) {
+      debug(`Error fetching ${url}:`, e)
+
+      if (
+        retryOnCacheError &&
+        cache &&
+        e instanceof Error &&
+        (e.message.includes('Unexpected token') ||
+          e.message.includes('is not valid JSON'))
+      ) {
+        debug(`Clearing cache for ${url} and retrying...`)
+        try {
+          await keyv.delete(`GET:${url}`)
+        } catch (clearError) {
+          debug(`Failed to clear cache:`, clearError)
+        }
+
+        return fetchWithRetry<T>(url, false)
+      }
+
+      throw e
+    }
+  }
+
+  try {
+    // First, fetch the catalog
+    catalog = await fetchWithRetry<Catalog>(catalogUrl)
+  } catch (e) {
+    debug(e)
+
+    if (e instanceof HTTPError) {
+      throw new Error(
+        `Couldn't retrieve catalog from ${catalogUrl}. [${e.code}]`,
+        { cause: e },
+      )
+    }
+
+    throw e
+  }
+
+  const schemaMap = new Map<string, CatalogSchema>()
+  for (const schema of catalog.schemas) {
+    schemaMap.set(schema.name.toLowerCase(), schema)
+  }
+
+  const catalogSchema = schemaMap.get(name.toLowerCase())
+
+  if (!catalogSchema) {
+    throw new Error(
+      `Couldn't find schema ${name} in the catalog. Are you sure it exists?`,
+    )
+  }
+
+  try {
+    // Fetch the actual schema from the found URL
+    debug(`Fetching schema from: ${catalogSchema.url}`)
+
+    schema = await fetchWithRetry<JSONSchema>(catalogSchema.url)
+
+    if (typeof schema === 'object' && schema !== null) {
+      schema.title = catalogSchema.name
+    }
+  } catch (e) {
+    debug(e)
+
+    if (e instanceof HTTPError) {
+      if (e.response.statusCode === 404) {
+        throw new Error(
+          `Couldn't find schema ${name} at ${catalogSchema.url}. Are you sure it exists?`,
+          { cause: e },
+        )
+      }
+
+      throw new Error(
+        `Couldn't retrieve schema from ${catalogSchema.url}. [${e.code}] Status: ${e.response.statusCode}`,
+        { cause: e },
+      )
+    }
+
+    if (e instanceof Error && e.message.includes('is not valid JSON')) {
+      throw new Error(
+        `Failed to parse JSON from ${catalogSchema.url}. The server might be returning invalid or compressed content. Error: ${e.message}`,
+        { cause: e },
+      )
+    }
+
+    throw e
+  }
+
+  if (cache) {
+    await flushCache()
+  }
+
+  return compileSource(sanitizeSchema(normalizeSchema(schema)), name)
+}

package/src/normalize.ts

@@ -0,0 +1,274 @@
+import type { JSONSchema } from 'json-schema-to-typescript'
+
+type Node = Record<string, unknown>
+
+// Maps the JSON pointer of a schema node to the `definitions` name each of its
+// `$defs` members moves to.
+type Renames = Map<string, Map<string, string>>
+
+const isObject = (value: unknown): value is Node =>
+  typeof value === 'object' && value !== null && !Array.isArray(value)
+
+// Hold literal instance data rather than nested schemas, so they must not be
+// walked as if they were part of the schema shape.
+const literalKeys = new Set(['const', 'default', 'enum', 'examples'])
+
+// Keyed by name rather than by keyword: a member of one of these called
+// `$defs` is a subschema that happens to be named `$defs`, not a definition
+// block.
+const namedSchemaKeys = new Set([
+  '$defs',
+  'definitions',
+  'dependencies',
+  'dependentSchemas',
+  'patternProperties',
+  'properties',
+])
+
+const escapeToken = (name: string) =>
+  name.replaceAll('~', '~0').replaceAll('/', '~1')
+
+const unescapeToken = (token: string) =>
+  token.replaceAll('~1', '/').replaceAll('~0', '~')
+
+const join = (path: string, token: string) =>
+  path === '' ? token : `${path}/${token}`
+
+const resolvePointer = (document: Node, ref: string): unknown => {
+  if (ref === '#') {
+    return document
+  }
+
+  if (!ref.startsWith('#/')) {
+    return undefined
+  }
+
+  let node: unknown = document
+
+  for (const token of ref.slice(2).split('/')) {
+    if (Array.isArray(node)) {
+      node = node[Number(token)]
+    } else if (isObject(node)) {
+      node = node[unescapeToken(token)]
+    } else {
+      return undefined
+    }
+  }
+
+  return node
+}
+
+const collectRenames = (
+  node: unknown,
+  path: string,
+  inSchema: boolean,
+  renames: Renames,
+): void => {
+  if (Array.isArray(node)) {
+    for (const [index, item] of node.entries()) {
+      collectRenames(item, join(path, String(index)), inSchema, renames)
+    }
+
+    return
+  }
+
+  if (!isObject(node)) {
+    return
+  }
+
+  if (!inSchema) {
+    for (const [name, value] of Object.entries(node)) {
+      collectRenames(value, join(path, escapeToken(name)), true, renames)
+    }
+
+    return
+  }
+
+  if (isObject(node.$defs)) {
+    const taken = new Set(
+      isObject(node.definitions) ? Object.keys(node.definitions) : [],
+    )
+    const mapping = new Map<string, string>()
+
+    for (const name of Object.keys(node.$defs)) {
+      let target = name
+
+      for (let suffix = 2; taken.has(target); suffix++) {
+        target = `${name}_${suffix}`
+      }
+
+      taken.add(target)
+      mapping.set(name, target)
+    }
+
+    renames.set(path, mapping)
+  }
+
+  for (const [key, value] of Object.entries(node)) {
+    if (literalKeys.has(key)) {
+      continue
+    }
+
+    collectRenames(
+      value,
+      join(path, escapeToken(key)),
+      !namedSchemaKeys.has(key),
+      renames,
+    )
+  }
+}
+
+const rewriteRef = (ref: string, renames: Renames): string => {
+  if (!ref.startsWith('#/')) {
+    return ref
+  }
+
+  const tokens = ref.slice(2).split('/')
+  const rewritten: string[] = []
+  let path = ''
+
+  for (let index = 0; index < tokens.length; index++) {
+    const token = tokens[index]
+    const name = tokens[index + 1]
+    const mapping = token === '$defs' ? renames.get(path) : undefined
+    const target =
+      mapping && name !== undefined
+        ? mapping.get(unescapeToken(name))
+        : undefined
+
+    if (target === undefined) {
+      rewritten.push(token)
+      path = join(path, token)
+      continue
+    }
+
+    rewritten.push('definitions', escapeToken(target))
+    path = join(join(path, token), name as string)
+    index++
+  }
+
+  return `#/${rewritten.join('/')}`
+}
+
+const applyRenames = (
+  node: unknown,
+  path: string,
+  inSchema: boolean,
+  renames: Renames,
+): unknown => {
+  if (Array.isArray(node)) {
+    return node.map((item, index) =>
+      applyRenames(item, join(path, String(index)), inSchema, renames),
+    )
+  }
+
+  if (!isObject(node)) {
+    return node
+  }
+
+  if (!inSchema) {
+    return Object.fromEntries(
+      Object.entries(node).map(([name, value]) => [
+        name,
+        applyRenames(value, join(path, escapeToken(name)), true, renames),
+      ]),
+    )
+  }
+
+  const mapping = renames.get(path)
+  const renamed: Node = {}
+
+  for (const [key, value] of Object.entries(node)) {
+    if (literalKeys.has(key)) {
+      renamed[key] = value
+      continue
+    }
+
+    if (key === '$ref' && typeof value === 'string') {
+      renamed[key] = rewriteRef(value, renames)
+      continue
+    }
+
+    if (key === '$defs' && mapping) {
+      continue
+    }
+
+    renamed[key] = applyRenames(
+      value,
+      join(path, escapeToken(key)),
+      !namedSchemaKeys.has(key),
+      renames,
+    )
+  }
+
+  if (mapping) {
+    const defs = node.$defs as Node
+    const definitions: Node = isObject(renamed.definitions)
+      ? renamed.definitions
+      : {}
+
+    for (const [name, target] of mapping) {
+      definitions[target] = applyRenames(
+        defs[name],
+        join(join(path, '$defs'), escapeToken(name)),
+        true,
+        renames,
+      )
+    }
+
+    renamed.definitions = definitions
+  }
+
+  return renamed
+}
+
+/**
+ * A document whose root is `{$ref}` describes the referenced definition, with
+ * any sibling keyword layered on top of it — `title` above all, which is what
+ * names the exported type. The definition stays where it is, so pointers into
+ * it keep resolving; only a copy is hoisted.
+ */
+const inlineRootRef = (document: Node): Node => {
+  let root = document
+  const seen = new Set<string>()
+
+  while (typeof root.$ref === 'string' && !seen.has(root.$ref)) {
+    const target = resolvePointer(document, root.$ref)
+
+    if (!isObject(target) || target === root) {
+      break
+    }
+
+    seen.add(root.$ref)
+
+    const siblings = { ...root }
+    delete siblings.$ref
+
+    root = { ...target, ...siblings }
+  }
+
+  return root
+}
+
+/**
+ * `json-schema-to-typescript` resolves references against draft-07's
+ * vocabulary: it only knows `definitions`, and it expects the root to be a
+ * schema rather than a reference to one. Draft 2020-12 documents that spell
+ * their definitions `$defs` or that point the root at one therefore fail to
+ * resolve, with `Refs should have been resolved by the resolver!`.
+ *
+ * Restate them in the shape it does resolve, without disturbing documents that
+ * are already in it.
+ */
+export const normalizeSchema = (schema: JSONSchema): JSONSchema => {
+  if (!isObject(schema)) {
+    return schema
+  }
+
+  const renames: Renames = new Map()
+  collectRenames(schema, '', true, renames)
+
+  return inlineRootRef(
+    applyRenames(schema, '', true, renames) as Node,
+  ) as JSONSchema
+}