shiki-codegen 1.27.0

package/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2021 Pine Wu
+Copyright (c) 2023 Anthony Fu <https://github.com/antfu>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,9 @@
+# @shikijs/codegen
+
+Codegen for fine-grained Shiki bundles.
+
+[Documentation](https://shiki.style/packages/codegen)
+
+## License
+
+MIT

package/bin.mjs

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+import('./dist/cli.mjs')

package/dist/cli.d.mts

@@ -0,0 +1,2 @@
+
+export {  }

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+
+export {  }

package/dist/cli.mjs

@@ -0,0 +1,33 @@
+import fs from 'node:fs/promises';
+import { resolve, dirname } from 'node:path';
+import { cac } from 'cac';
+import { codegen } from './index.mjs';
+import 'shiki/bundle/full';
+
+const cli = cac("shiki-codegen");
+cli.command("[path]", "Generate shiki bundle module").option("--themes <themes>", "Themes to include").option("--langs <langs>", "Languages to include").option("--engine <engine>", "Engine to use", { default: "javascript" }).option("--precompiled", "Use precompiled languages", { default: false }).option("--shorthands", "Generate shorthands", { default: true }).option("--format", "Use prettier to format", { default: true }).action(async (path, options) => {
+  const output = resolve(path || "shiki.bundle.ts");
+  const langs = (Array.isArray(options.langs) ? options.langs.join(",") : options.langs || "").split(",").map((lang) => lang.trim()).filter(Boolean);
+  const themes = (Array.isArray(options.themes) ? options.themes.join(",") : options.themes || "").split(",").map((theme) => theme.trim()).filter(Boolean);
+  const isTypeScript = !!output.match(/\.[cm]?ts$/i);
+  if (!themes.length) {
+    throw new Error("No themes specified, use --themes=theme-name to specify themes");
+  }
+  if (!langs.length) {
+    throw new Error("No langs specified, use --langs=lang-name to specify langs");
+  }
+  const { code } = await codegen({
+    langs,
+    themes,
+    engine: options.engine,
+    precompiled: options.precompiled,
+    shorthands: options.shorthands,
+    format: options.format,
+    typescript: isTypeScript
+  });
+  await fs.mkdir(dirname(output), { recursive: true });
+  await fs.writeFile(output, code, "utf-8");
+  console.log(`Generated bundle to ${output}`);
+});
+cli.help();
+cli.parse();

package/dist/index.d.mts

@@ -0,0 +1,50 @@
+import { Options } from 'prettier';
+import { BundledLanguage, BundledTheme } from 'shiki';
+
+interface ShikiCodegenOptions {
+    /**
+     * The header to add to the generated code.
+     *
+     * @default '/* Generate by @shikijs/codegen *\/'
+     */
+    header?: string;
+    /**
+     * The languages to bundle.
+     */
+    langs: readonly BundledLanguage[];
+    /**
+     * The themes to bundle.
+     */
+    themes: readonly BundledTheme[];
+    /**
+     * The engine to use for syntax highlighting.
+     */
+    engine: 'oniguruma' | 'javascript' | 'javascript-raw';
+    /**
+     * Use precompiled grammars.
+     * Only available when `engine` is set to `javascript` or `javascript-raw`.
+     */
+    precompiled?: boolean;
+    /**
+     * Whether to generate TypeScript code.
+     *
+     * @default true
+     */
+    typescript?: boolean;
+    /**
+     * Generate shorthands for the highlighter.
+     *
+     * @default true
+     */
+    shorthands?: boolean;
+    /**
+     * Use Prettier to format the generated code.
+     */
+    format?: boolean | Options;
+}
+interface ShikiCodegenResult {
+    code: string;
+}
+declare function codegen(options: ShikiCodegenOptions): Promise<ShikiCodegenResult>;
+
+export { type ShikiCodegenOptions, type ShikiCodegenResult, codegen };

package/dist/index.d.ts

@@ -0,0 +1,50 @@
+import { Options } from 'prettier';
+import { BundledLanguage, BundledTheme } from 'shiki';
+
+interface ShikiCodegenOptions {
+    /**
+     * The header to add to the generated code.
+     *
+     * @default '/* Generate by @shikijs/codegen *\/'
+     */
+    header?: string;
+    /**
+     * The languages to bundle.
+     */
+    langs: readonly BundledLanguage[];
+    /**
+     * The themes to bundle.
+     */
+    themes: readonly BundledTheme[];
+    /**
+     * The engine to use for syntax highlighting.
+     */
+    engine: 'oniguruma' | 'javascript' | 'javascript-raw';
+    /**
+     * Use precompiled grammars.
+     * Only available when `engine` is set to `javascript` or `javascript-raw`.
+     */
+    precompiled?: boolean;
+    /**
+     * Whether to generate TypeScript code.
+     *
+     * @default true
+     */
+    typescript?: boolean;
+    /**
+     * Generate shorthands for the highlighter.
+     *
+     * @default true
+     */
+    shorthands?: boolean;
+    /**
+     * Use Prettier to format the generated code.
+     */
+    format?: boolean | Options;
+}
+interface ShikiCodegenResult {
+    code: string;
+}
+declare function codegen(options: ShikiCodegenOptions): Promise<ShikiCodegenResult>;
+
+export { type ShikiCodegenOptions, type ShikiCodegenResult, codegen };

package/dist/index.mjs

@@ -0,0 +1,152 @@
+import { bundledLanguagesInfo, bundledThemesInfo } from 'shiki/bundle/full';
+
+async function codegen(options) {
+  const {
+    header = "/* Generate by @shikijs/codegen */",
+    typescript = true,
+    precompiled = false,
+    format: _format = true,
+    shorthands = true
+  } = options;
+  const ts = (code2) => typescript ? code2 : "";
+  if (precompiled && options.engine !== "javascript" && options.engine !== "javascript-raw")
+    throw new Error("Precompiled grammars are only available when using the JavaScript engine");
+  const langs = options.langs.map((lang) => {
+    const info = bundledLanguagesInfo.find((i) => i.id === lang || i.aliases?.includes(lang));
+    if (!info)
+      throw new Error(`Language ${lang} not found`);
+    return info;
+  });
+  const themes = options.themes.map((theme) => {
+    const info = bundledThemesInfo.find((i) => i.id === theme);
+    if (!info)
+      throw new Error(`Theme ${theme} not found`);
+    return info;
+  });
+  const langsCode = `{
+${langs.flatMap((lang) => {
+    const ids = [lang.id, ...lang.aliases || []];
+    return ids.map((id) => {
+      return `${JSON.stringify(id)}: () => import('@shikijs/${precompiled ? "langs-precompiled" : "langs"}/${lang.id}'),
+`;
+    });
+  }).join("")}}`;
+  const themesCode = `{
+${themes.map((theme) => {
+    return `${JSON.stringify(theme.id)}: () => import('@shikijs/themes/${theme.id}'),
+`;
+  }).join("")}}`;
+  const typeImports = {
+    "@shikijs/types": ["HighlighterGeneric", "DynamicImportThemeRegistration", "DynamicImportLanguageRegistration"]
+  };
+  const imports = {
+    "@shikijs/core": ["createdBundledHighlighter"]
+  };
+  const lines = [
+    ""
+  ];
+  const exports = [];
+  const typeExports = [];
+  if (typescript) {
+    lines.push(
+      "",
+      `type BundledLanguage = ${langs.flatMap((lang) => [lang.id, ...lang.aliases || []]).map((lang) => `'${lang}'`).join(" | ")}`,
+      `type BundledTheme = ${themes.map((theme) => `'${theme.id}'`).join(" | ")}`,
+      `type Highlighter = HighlighterGeneric<BundledLanguage, BundledTheme>`,
+      ""
+    );
+    typeExports.push("BundledLanguage", "BundledTheme", "Highlighter");
+  }
+  lines.push(
+    "",
+    `const bundledLanguages = ${langsCode}${ts(" as Record<BundledLanguage, DynamicImportLanguageRegistration>")}`,
+    "",
+    `const bundledThemes = ${themesCode}${ts(" as Record<BundledTheme, DynamicImportThemeRegistration>")}`,
+    ""
+  );
+  exports.push("bundledLanguages", "bundledThemes");
+  let engine;
+  if (options.engine === "javascript") {
+    imports["@shikijs/engine-javascript"] = ["createJavaScriptRegexEngine"];
+    engine = "createJavaScriptRegexEngine()";
+  } else if (options.engine === "javascript-raw") {
+    imports["@shikijs/engine-javascript/raw"] = ["createJavaScriptRawEngine"];
+    engine = "createJavaScriptRawEngine()";
+  } else {
+    imports["@shikijs/engine-oniguruma"] = ["createOnigurumaEngine"];
+    engine = "createOnigurumaEngine(import('shiki/wasm'))";
+  }
+  lines.push(
+    "",
+    `const createHighlighter = /* @__PURE__ */ createdBundledHighlighter${ts("<BundledLanguage, BundledTheme>")}({`,
+    `  langs: bundledLanguages,`,
+    `  themes: bundledThemes,`,
+    `  engine: () => ${engine}`,
+    `})`
+  );
+  exports.push("createHighlighter");
+  if (shorthands) {
+    imports["@shikijs/core"].push("createSingletonShorthands");
+    const shorthandFunctions = [
+      "codeToHtml",
+      "codeToHast",
+      "codeToTokensBase",
+      "codeToTokens",
+      "codeToTokensWithThemes",
+      "getSingletonHighlighter",
+      "getLastGrammarState"
+    ];
+    lines.push(
+      "",
+      `const { ${shorthandFunctions.join(", ")} } = /* @__PURE__ */ createSingletonShorthands${ts("<BundledLanguage,BundledTheme>")}(createHighlighter)`
+    );
+    exports.push(
+      ...shorthandFunctions
+    );
+  }
+  lines.unshift(
+    Object.entries(imports).map(
+      ([module, imports2]) => {
+        return `import { ${imports2.sort().join(", ")} } from '${module}'`;
+      }
+    ).join("\n")
+  );
+  if (typescript) {
+    lines.unshift(
+      Object.entries(typeImports).map(
+        ([module, types]) => {
+          return `import type { ${types.sort().join(", ")} } from '${module}'`;
+        }
+      ).join("\n")
+    );
+  }
+  lines.push(
+    "",
+    `export { ${exports.sort().join(", ")} }`
+  );
+  if (typescript) {
+    lines.push(
+      `export type { ${typeExports.sort().join(", ")} }`
+    );
+  }
+  lines.unshift(header);
+  let code = lines.join("\n");
+  if (_format) {
+    const { format } = await import('prettier');
+    const prettierOptions = {
+      parser: typescript ? "typescript" : "babel",
+      semi: false,
+      tabWidth: 2,
+      useTabs: false,
+      singleQuote: true,
+      trailingComma: "all",
+      ..._format === true ? {} : _format
+    };
+    code = await format(code, prettierOptions);
+  }
+  return {
+    code
+  };
+}
+
+export { codegen };

package/package.json

@@ -0,0 +1,50 @@
+{
+  "name": "shiki-codegen",
+  "type": "module",
+  "version": "1.27.0",
+  "description": "Codegen for fine-grained Shiki bundles.",
+  "author": "Anthony Fu <anthonyfu117@hotmail.com>",
+  "license": "MIT",
+  "homepage": "https://github.com/shikijs/shiki#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/shikijs/shiki.git",
+    "directory": "packages/codegen"
+  },
+  "bugs": "https://github.com/shikijs/shiki/issues",
+  "keywords": [
+    "shiki",
+    "codegen"
+  ],
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "default": "./dist/index.mjs"
+    },
+    "./cli": {
+      "types": "./dist/cli.d.mts",
+      "default": "./dist/cli.mjs"
+    }
+  },
+  "main": "./dist/index.mjs",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
+  "bin": {
+    "shiki-codegen": "bin.mjs"
+  },
+  "files": [
+    "bin.mjs",
+    "dist"
+  ],
+  "dependencies": {
+    "cac": "^6.7.14",
+    "prettier": "^3.4.2",
+    "shiki": "1.27.0"
+  },
+  "scripts": {
+    "build": "unbuild",
+    "dev": "unbuild --stub",
+    "test": "vitest"
+  }
+}