repl-sdk 0.0.0 → 1.0.0

package/src/compilers/markdown/build-compiler.js

@@ -0,0 +1,151 @@
+/**
+ * @typedef {import('unified').Plugin} UPlugin
+ */
+import rehypeRaw from 'rehype-raw';
+import rehypeStringify from 'rehype-stringify';
+import remarkGfm from 'remark-gfm';
+import remarkParse from 'remark-parse';
+import remarkRehype from 'remark-rehype';
+import { unified } from 'unified';
+import { visit } from 'unist-util-visit';
+
+import { GLIMDOWN_PREVIEW, GLIMDOWN_RENDER } from './const.js';
+import { headingId } from './heading-id.js';
+import { liveCodeExtraction } from './live-code-extraction.js';
+import { sanitizeForGlimmer } from './sanitize-for-glimmer.js';
+
+/**
+ * @param {import('./types').InternalOptions} options
+ *
+ * @returns {import('unified').Processor<import('hast').Root>}
+ */
+export function buildCompiler(options) {
+  let compiler = unified().use(remarkParse).use(remarkGfm, { singleTilde: true }).use(headingId);
+
+  /**
+   * If this were "use"d after `remarkRehype`,
+   * remark is gone, and folks would need to work with rehype trees
+   */
+  if (options.remarkPlugins) {
+    options.remarkPlugins.forEach((plugin) => {
+      // Arrays are how plugins are passed options (for some reason?)
+      // why not just invoke the the function?
+      if (Array.isArray(plugin)) {
+        // @ts-ignore - unified processor types are complex and change as plugins are added
+        compiler = compiler.use(plugin[0], ...plugin.slice(1));
+      } else {
+        // @ts-ignore - unified processor types are complex and change as plugins are added
+        compiler = compiler.use(plugin);
+      }
+    });
+  }
+
+  // TODO: we only want to do this when we have pre > code.
+  //       code can exist inline.
+  // @ts-ignore - unified processor types are complex and change as plugins are added
+  compiler = compiler.use(liveCodeExtraction, {
+    code: {
+      classList: ['repl-sdk__snippet'],
+      ...options.code,
+    },
+    demo: {
+      classList: ['repl-sdk__demo'],
+      ...options.code,
+    },
+    isLive: options.isLive,
+    isPreview: options.isPreview,
+    isBelow: options.isBelow,
+    needsLive: options.needsLive,
+    ALLOWED_FORMATS: options.ALLOWED_FORMATS,
+    getFlavorFromMeta: options.getFlavorFromMeta,
+  });
+
+  // .use(() => (tree) => visit(tree, (node) => console.log('i', node)))
+  // remark rehype is needed to convert markdown to HTML
+  // However, it also changes all the nodes, so we need another pass
+  // to make sure our Glimmer-aware nodes are in tact
+  // @ts-ignore - unified processor types are complex and change as plugins are added
+  compiler = compiler.use(remarkRehype, { allowDangerousHtml: true });
+
+  // Convert invocables to raw format, so Glimmer can invoke them
+  // @ts-ignore - unified processor types are complex and change as plugins are added
+  compiler = compiler.use(() => (/** @type {unknown} */ tree) => {
+    visit(/** @type {import('hast').Root} */ (tree), function (node) {
+      // We rely on an implicit transformation of data.hProperties => properties
+      const nodeObj = /** @type {Record<string, unknown>} */ (/** @type {unknown} */ (node));
+      const properties = /** @type {Record<string, unknown>} */ (
+        typeof node === 'object' && node !== null && 'properties' in node ? node.properties : {}
+      );
+
+      if (properties?.[/** @type {string} */ (/** @type {unknown} */ (GLIMDOWN_PREVIEW))]) {
+        return 'skip';
+      }
+
+      if (nodeObj.type === 'element' || ('tagName' in nodeObj && nodeObj.tagName === 'code')) {
+        if (properties?.[/** @type {string} */ (/** @type {unknown} */ (GLIMDOWN_RENDER))]) {
+          nodeObj.type = 'glimmer_raw';
+
+          return;
+        }
+
+        return 'skip';
+      }
+
+      if (nodeObj.type === 'text' || nodeObj.type === 'raw') {
+        // definitively not the better way, but this is supposed to detect "glimmer" nodes
+        if (
+          'value' in nodeObj &&
+          typeof nodeObj.value === 'string' &&
+          nodeObj.value.match(/<\/?[_A-Z:0-9].*>/g)
+        ) {
+          nodeObj.type = 'glimmer_raw';
+        }
+
+        nodeObj.type = 'glimmer_raw';
+
+        return 'skip';
+      }
+
+      return;
+    });
+  });
+
+  if (options.rehypePlugins) {
+    options.rehypePlugins.forEach((plugin) => {
+      // Arrays are how plugins are passed options (for some reason?)
+      // why not just invoke the the function?
+      if (Array.isArray(plugin)) {
+        // @ts-ignore - unified processor types are complex and change as plugins are added
+        compiler = compiler.use(plugin[0], ...plugin.slice(1));
+      } else {
+        // @ts-ignore - unified processor types are complex and change as plugins are added
+        compiler = compiler.use(plugin);
+      }
+    });
+  }
+
+  // @ts-ignore - unified processor types are complex and change as plugins are added
+  compiler = compiler
+    .use(rehypeRaw, { passThrough: ['glimmer_raw', 'raw'] })
+    .use(() => (/** @type {unknown} */ tree) => {
+      visit(/** @type {import('hast').Root} */ (tree), 'glimmer_raw', (node) => {
+        /** @type {Record<string, unknown>} */ (node).type = 'raw';
+      });
+    });
+
+  // @ts-ignore - unified processor types are complex and change as plugins are added
+  compiler = compiler.use(sanitizeForGlimmer);
+
+  // Finally convert to string! oofta!
+  // @ts-ignore - unified processor types are complex and change as plugins are added
+  compiler = compiler.use(rehypeStringify, {
+    collapseEmptyAttributes: true,
+    closeSelfClosing: true,
+    allowParseErrors: true,
+    allowDangerousCharacters: true,
+    allowDangerousHtml: true,
+  });
+
+  // @ts-ignore
+  return compiler;
+}

package/src/compilers/markdown/const.js

@@ -0,0 +1,2 @@
+export const GLIMDOWN_PREVIEW = Symbol('__GLIMDOWN_PREVIEW__');
+export const GLIMDOWN_RENDER = Symbol('__GLIMDOWN_RENDER__');

package/src/compilers/markdown/heading-id.js

@@ -0,0 +1,75 @@
+import { kebabCase } from 'change-case';
+import { visit } from 'unist-util-visit';
+
+/**
+ * @param {import('mdast').PhrasingContent[]} children
+ * @return {string}
+ */
+function getDefaultId(children) {
+  return formatDefaultId(extractText(children));
+}
+
+/**
+ * @param {import('mdast').PhrasingContent[]} children
+ * @return {string}
+ */
+function extractText(children) {
+  return children
+    .map(
+      /**
+       * @param {any} child
+       */
+      (child) => {
+        const isEmpty = !child.value?.trim();
+
+        if (!isEmpty) {
+          return child.value;
+        } else if (child.children && child.children.length > 0) {
+          return extractText(child.children);
+        } else {
+          return '';
+        }
+      }
+    )
+    .join(' ');
+}
+
+/**
+ * @param {string} value
+ */
+function formatDefaultId(value) {
+  return kebabCase(value.replaceAll(/\\s+/g, ' ').trim());
+}
+
+/**
+ * @param {import('mdast').Heading} node
+ * @param {string} id
+ */
+function setNodeId(node, id) {
+  if (!node.data) node.data = {};
+  if (!node.data.hProperties) node.data.hProperties = {};
+
+  /** @type {any} */ (node.data).id = node.data.hProperties.id = id;
+}
+
+export function headingId(options = { defaults: false }) {
+  /**
+   * @param {import('mdast').Root} node
+   */
+  return function (node) {
+    visit(node, 'heading', (node) => {
+      const lastChild = node.children[node.children.length - 1];
+
+      if (lastChild && lastChild.type === 'text') {
+        const string = lastChild.value.replace(/ +$/, '');
+        const matched = string.match(/ {#([^]+?)}$/);
+
+        if (matched) {
+          return;
+        }
+      }
+
+      setNodeId(node, getDefaultId(node.children));
+    });
+  };
+}

package/src/compilers/markdown/live-code-extraction.js

@@ -0,0 +1,198 @@
+import { visit } from 'unist-util-visit';
+
+import { nextId } from '../../utils.js';
+import { GLIMDOWN_PREVIEW, GLIMDOWN_RENDER } from './const.js';
+
+/**
+ * Swaps live codeblocks with placeholders that the compiler can then
+ * use to insert compiled-from-other-sources' code into those placeholders.
+ *
+ * @type {import('unified').Plugin<[
+ *   {
+ *     demo: {
+ *       classList: string[]
+ *     },
+ *     code: {
+ *       classList: string[]
+ *     },
+ *     isLive: (meta: string, lang: string) => boolean,
+ *     ALLOWED_FORMATS: string[],
+ *     isPreview: (meta: string) => boolean,
+ *     isBelow: (meta: string) => boolean,
+ *     needsLive: (lang: string) => boolean
+ *     getFlavorFromMeta: (meta: string, lang: string) => string | undefined
+ *   },
+ * ], import('mdast').Root>}
+ */
+export function liveCodeExtraction(options) {
+  const { code, demo, isLive, ALLOWED_FORMATS, isPreview, isBelow, needsLive, getFlavorFromMeta } =
+    options;
+  let { classList: snippetClasses } = code || {};
+  let { classList: demoClasses } = demo || {};
+
+  snippetClasses ??= [];
+  demoClasses ??= [];
+
+  /**
+   * @param {import('mdast').Code} node
+   */
+  function isRelevantCode(node) {
+    if (node.type !== 'code') return false;
+
+    let { meta, lang } = node;
+
+    if (!lang) {
+      return false;
+    }
+
+    meta = meta?.trim() ?? '';
+
+    if (!isLive(meta, lang)) {
+      return false;
+    }
+
+    if (!ALLOWED_FORMATS.includes(lang)) {
+      return false;
+    }
+
+    return true;
+  }
+
+  /**
+   * @param {import('mdast').Code} code
+   * @param {string[]} [classes]
+   */
+  function enhance(code, classes = []) {
+    code.data ??= {};
+    code.data['hProperties'] ??= {};
+    // This is secret-to-us-only API, so we don't really care about the type
+    code.data['hProperties'][/** @type {string} */ (/** @type {unknown} */ (GLIMDOWN_PREVIEW))] =
+      true;
+
+    return {
+      data: {
+        hProperties: { className: classes, 'data-repl-output': true },
+      },
+      type: 'div',
+      hProperties: { className: classes },
+      children: [code],
+    };
+  }
+
+  /**
+   * @template T
+   * @param {T[]} array
+   * @param {number} index
+   * @param {T[]} replacement
+   */
+  function flatReplaceAt(array, index, replacement) {
+    array.splice(index, 1, ...replacement);
+  }
+
+  // because we mutate the tree as we iterate,
+  // we need to make sure we don't loop forever
+  const seen = new Set();
+
+  return function transformer(tree, file) {
+    visit(tree, ['code'], function (node, index, parent) {
+      if (parent === null || parent === undefined) return;
+      if (index === null || index === undefined) return;
+      if (node.type !== 'code') return;
+
+      /** @type {import('mdast').Code} */
+      const codeNode = node;
+
+      const isRelevant = isRelevantCode(codeNode);
+
+      if (!isRelevant) {
+        const enhanced = enhance(codeNode, snippetClasses);
+
+        /** @type {unknown[]} */ (parent.children)[index] = /** @type {unknown} */ (enhanced);
+
+        return 'skip';
+      }
+
+      if (seen.has(codeNode)) {
+        return 'skip';
+      }
+
+      seen.add(codeNode);
+
+      const { meta, lang, value } = codeNode;
+
+      if (!lang) {
+        return 'skip';
+      }
+
+      /**
+       * Sometimes, meta is not required,
+       * like with the `mermaid` language
+       *
+       * NOTE: that if a flavor is required, meta is present
+       *       if the flavor is provided
+       */
+      if (!meta) {
+        if (needsLive(lang)) {
+          return 'skip';
+        }
+      }
+
+      file.data.liveCode ??= [];
+
+      const code = value.trim();
+      const id = nextId();
+
+      const invokeNode = /** @type {import('mdast').Html} */ ({
+        type: 'html',
+        data: {
+          hProperties: {
+            [/** @type {string} */ (/** @type {unknown} */ (GLIMDOWN_RENDER))]: true,
+          },
+        },
+        value: `<div id="${id}" class="${demoClasses}"></div>`,
+      });
+
+      const wrapper = enhance(codeNode, snippetClasses);
+
+      /** @type {unknown[]} */ (file.data.liveCode).push({
+        format: lang,
+        flavor: getFlavorFromMeta(meta ?? '', lang),
+        code,
+        placeholderId: id,
+        meta,
+      });
+
+      const live = isLive(meta || '', lang);
+      const preview = isPreview(meta || '');
+      const below = isBelow(meta || '');
+
+      if (live && preview && below) {
+        flatReplaceAt(/** @type {unknown[]} */ (parent.children), index, [
+          /** @type {unknown} */ (wrapper),
+          /** @type {unknown} */ (invokeNode),
+        ]);
+
+        return 'skip';
+      }
+
+      if (live && preview) {
+        flatReplaceAt(/** @type {unknown[]} */ (parent.children), index, [
+          /** @type {unknown} */ (invokeNode),
+          /** @type {unknown} */ (wrapper),
+        ]);
+
+        return 'skip';
+      }
+
+      if (live) {
+        /** @type {unknown[]} */ (parent.children)[index] = /** @type {unknown} */ (invokeNode);
+
+        return 'skip';
+      }
+
+      /** @type {unknown[]} */ (parent.children)[index] = /** @type {unknown} */ (wrapper);
+
+      return;
+    });
+  };
+}

package/src/compilers/markdown/parse.js

@@ -0,0 +1,22 @@
+/**
+ * @typedef {import('unified').Plugin} UPlugin
+ */
+import { buildCompiler } from './build-compiler.js';
+
+/**
+ * @param {string} input
+ * @param {import('./types').InternalOptions} options
+ *
+ * @returns {Promise<{ text: string; codeBlocks: { lang: string; format: string; code: string; name: string }[] }>}
+ */
+export async function parseMarkdown(input, options) {
+  const markdownCompiler = buildCompiler(options);
+  const processed = await markdownCompiler.process(input);
+  const liveCode = /** @type {{ lang: string; format: string; code: string; name: string }[]} */ (
+    processed.data.liveCode || []
+  );
+  // @ts-ignore - processed is typed as unknown due to unified processor complexity
+  const templateOnly = processed.toString();
+
+  return { text: templateOnly, codeBlocks: liveCode };
+}