react-native-marcus 0.1.1 → 0.2.0

package/src/metro.ts

@@ -0,0 +1,238 @@
+/**
+ * Metro configuration for drawing maths.
+ *
+ * Two things stand between a Metro bundle and a working `MarkdownText` with
+ * `math` on, and `withMath` is both of them:
+ *
+ * 1. **The maths backend is swapped in.** `src/math/index.ts` is a stub that
+ *    draws nothing, and nothing in the library ever imports the real one --
+ *    otherwise every application that installed this package would carry
+ *    MathJax, whether or not it has an equation in it. This resolver points
+ *    that import at `src/math/mathjax.ts` instead, which is what opting in
+ *    means.
+ * 2. **MathJax's own imports resolve.** It addresses its modules through
+ *    Node's *subpath imports* -- the `imports` field in its `package.json`,
+ *    which maps `#js/*`, `#default-font/*` and four others onto real files.
+ *    Metro does not implement that field, so every internal import fails and
+ *    the bundle never builds. There is nothing to fix in MathJax: the field is
+ *    standard and Metro has not caught up. The mapping is read out of MathJax's
+ *    own manifest rather than written down here, because a list copied into a
+ *    second place is a list that goes stale.
+ *
+ *     const { getDefaultConfig } = require("expo/metro-config")
+ *     const { withMath } = require("react-native-marcus/metro")
+ *
+ *     module.exports = withMath(getDefaultConfig(__dirname))
+ *
+ * It does one thing more, which is to *remove* rather than add: the speech
+ * engine's rules for every language but the ones named in `{ locales }` are
+ * resolved away to nothing, so a display that reads its maths in English does
+ * not carry the twelve other languages it will never speak.
+ */
+import { dirname } from "path"
+
+/** As much of a Metro config as this touches. */
+interface MetroConfig {
+  resolver?: {
+    resolveRequest?: ResolveRequest
+    [key: string]: unknown
+  }
+  [key: string]: unknown
+}
+
+type Resolution = { type?: string; filePath?: string } | null | undefined
+
+type ResolveRequest = (
+  context: unknown,
+  moduleName: string,
+  platform: string | null,
+) => Resolution
+
+interface ResolutionContext {
+  resolveRequest: ResolveRequest
+}
+
+interface WithMathOptions {
+  /**
+   * The package whose `imports` field is read, in case a fork or a pinned copy
+   * is installed under another name.
+   *
+   * @default "@mathjax/src"
+   */
+  packageName?: string
+  /**
+   * This library, in case it is installed under another name. Its own directory
+   * is what bounds the backend swap below.
+   *
+   * @default "react-native-marcus"
+   */
+  libraryName?: string
+  /**
+   * The languages maths can be read out in, as the speech engine names them:
+   * `af`, `ca`, `da`, `de`, `en`, `es`, `fr`, `hi`, `it`, `ko`, `nb`, `nn` and
+   * `sv`. Everything else is left out of the bundle.
+   *
+   * This is what `MarkdownText`'s `mathAccessibilityLocale` can choose between.
+   * It is a build-time list rather than a runtime one for the same reason
+   * MathJax itself is: the rules for one language are around 300KB, and an
+   * application that reads its maths in English has no business carrying the
+   * four megabytes of rules for the twelve languages it does not. Naming the
+   * ones you support is the same thing you already do for your own
+   * translations.
+   *
+   * A locale asked for and not bundled is read in English instead, with a
+   * warning in the console saying so.
+   *
+   * @default ["en"]
+   */
+  locales?: string[]
+}
+
+/**
+ * MathJax's `#…` specifiers, as prefix pairs.
+ *
+ * Its manifest maps `"#js/*": "@mathjax/src/mjs/*"` and so on, and every entry
+ * is a single trailing wildcard, so a prefix swap is the whole of it. Anything
+ * shaped differently is skipped rather than guessed at.
+ */
+function importPrefixes(packageName: string): [string, string][] {
+  let manifest: { imports?: Record<string, unknown> }
+
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    manifest = require(`${packageName}/package.json`) as { imports?: Record<string, unknown> }
+  } catch {
+    throw new Error(
+      `[react-native-marcus] withMath() could not read "${packageName}/package.json". ` +
+        `Install it (npm install ${packageName}) or drop withMath() from your Metro config.`,
+    )
+  }
+
+  const prefixes: [string, string][] = []
+
+  for (const [specifier, target] of Object.entries(manifest.imports ?? {})) {
+    if (
+      typeof target === "string" &&
+      specifier.endsWith("/*") &&
+      target.endsWith("/*") &&
+      specifier.startsWith("#")
+    ) {
+      prefixes.push([specifier.slice(0, -1), target.slice(0, -1)])
+    }
+  }
+
+  return prefixes
+}
+
+/**
+ * The stub backend, whichever build of the library Metro picked: `lib/module`,
+ * `lib/commonjs`, or `src` itself when the library is being worked on.
+ */
+const STUB = /([\\/])math([\\/])index\.(js|jsx|ts|tsx|mjs|cjs)$/
+
+/**
+ * One of the speech engine's rule tables, by the locale it holds.
+ *
+ * They are all named in `math/mathjax.ts` -- a `require` has to be written down
+ * for a bundler to find it -- and all but the ones the application asked for
+ * are resolved away here.
+ */
+const MATHMAPS = /[\\/]mathmaps[\\/]([a-z]+)\.json$/
+
+/** The table every locale shares, which is never the one to leave out. */
+const BASE_LOCALE = "base"
+
+/** Where this library lives, so that nothing else's `math/index.js` is touched. */
+function libraryRoot(libraryName: string): string {
+  try {
+    return dirname(require.resolve(`${libraryName}/package.json`))
+  } catch {
+    throw new Error(
+      `[react-native-marcus] withMath() could not locate "${libraryName}". ` +
+        `Pass { libraryName } if it is installed under another name.`,
+    )
+  }
+}
+
+/**
+ * Where the speech engine lives, when it is installed at all.
+ *
+ * It bounds the locale trimming the way the library root bounds the backend
+ * swap: only files under this directory are considered rule tables. Missing is
+ * not an error -- an application can draw maths without ever reading it out --
+ * and then there is nothing to trim.
+ */
+function speechRoot(): string | null {
+  try {
+    return dirname(require.resolve("speech-rule-engine/package.json"))
+  } catch {
+    return null
+  }
+}
+
+/**
+ * Teaches Metro to resolve MathJax, and points the library's maths backend at
+ * it.
+ *
+ * Any `resolveRequest` already on the config is kept and called for everything
+ * else, so this composes with whatever else the application is doing.
+ */
+function withMath(config: MetroConfig, options: WithMathOptions = {}): MetroConfig {
+  const prefixes = importPrefixes(options.packageName ?? "@mathjax/src")
+  const root = libraryRoot(options.libraryName ?? "react-native-marcus")
+  const speech = speechRoot()
+  const locales = new Set(options.locales ?? ["en"])
+  const existing = config.resolver?.resolveRequest
+
+  const resolveRequest: ResolveRequest = (context, moduleName, platform) => {
+    let name = moduleName
+
+    if (name.charCodeAt(0) === 35 /* # */) {
+      for (const [from, to] of prefixes) {
+        if (name.startsWith(from)) {
+          name = to + name.slice(from.length)
+          break
+        }
+      }
+    }
+
+    const resolution = existing
+      ? existing(context, name, platform)
+      : (context as ResolutionContext).resolveRequest(context, name, platform)
+
+    // Matched on what it resolved to rather than on what was asked for: the
+    // import is written `"./math"`, relative to whichever of the three builds
+    // of this library the application ended up with, and a relative specifier
+    // says nothing about which package it is in.
+    const filePath = resolution?.filePath
+
+    if (filePath && STUB.test(filePath) && isInside(root, filePath)) {
+      return { ...resolution, filePath: filePath.replace(STUB, "$1math$2mathjax.$3") }
+    }
+
+    if (filePath && speech !== null && isInside(speech, filePath)) {
+      const locale = MATHMAPS.exec(filePath)?.[1]
+
+      // An empty module rather than a missing one: the backend requires every
+      // locale it knows of, unconditionally, and reads the empty object back as
+      // "this one was not bundled" -- which is what it warns about and falls
+      // back from. A resolution error here would be a bundle that does not
+      // build.
+      if (locale !== undefined && locale !== BASE_LOCALE && !locales.has(locale)) {
+        return { type: "empty" }
+      }
+    }
+
+    return resolution
+  }
+
+  return { ...config, resolver: { ...config.resolver, resolveRequest } }
+}
+
+function isInside(root: string, filePath: string) {
+  return filePath.length > root.length && filePath.startsWith(root)
+}
+
+export type { MetroConfig, WithMathOptions }
+
+export { withMath }

package/src/parser/index.ts

@@ -2,13 +2,16 @@ import emojiRegex from "emoji-regex"
 import { parse, postprocess, preprocess } from "micromark"
 import { gfmAutolinkLiteral } from "micromark-extension-gfm-autolink-literal"
 import { gfmStrikethrough } from "micromark-extension-gfm-strikethrough"
+import { math } from "micromark-extension-math"
 import type { Token, TokenType } from "micromark-util-types"
 
-import type { MarkdownRange, MarkdownType } from "../commonTypes"
+import type { MarkdownParser, MarkdownRange, MarkdownType, ParserOptions } from "../commonTypes"
 import { mentions } from "./micromark-extension-mentions"
 
 const EMOJI_TYPE = "emoji"
 const SYNTAX_TYPE = "syntax"
+const LABEL_TYPE = "label"
+const ALT_TYPE = "alt"
 
 const MAX_DEPTH = 6
 
@@ -72,6 +75,20 @@ const TOKENS: {
     "codeTextSequence",
     "codeFencedFenceSequence",
 
+    // Maths. The padding is the space `$ x $` is written with, which is part of
+    // the markup rather than of the TeX.
+    "mathTextSequence",
+    "mathTextPadding",
+    "mathFlowFenceSequence",
+
+    // Whatever follows the opening `$$`, which nothing reads.
+    "mathFlowFenceMeta",
+
+    // Whatever follows a fenced block's language. The language itself is
+    // `language` -- something will want to read it -- but the rest of
+    // that line is markup like any other fence.
+    "codeFencedFenceMeta",
+
     // Character references.
     "characterReferenceMarker",
     "characterReferenceMarkerNumeric",
@@ -89,22 +106,45 @@ const TOKENS: {
   special: {
     codeFenced: "codeblock",
     codeIndented: "codeblock",
-    image: "inline-image",
+    image: "image",
+
+    // One range over the whole span, delimiters included, the way an image is
+    // one range over `![alt](url)`. What is inside is TeX: a display strips it
+    // whole and hands it to a renderer, and an input leaves it alone. Nothing
+    // is emitted for the content itself -- there is nothing markdown-shaped in
+    // there to style.
+    mathText: "math",
+    mathFlow: "math-block",
   },
   scope: {
-    emphasis: "italic",
-    strong: "bold",
+    // Identities: these carry CommonMark's own names, so what micromark calls
+    // a token and what a range is typed as are the same word.
+    emphasis: "emphasis",
+    strong: "strong",
     strikethrough: "strikethrough",
 
     // Headings are absent: their level is only known once the marker token
     // arrives, so they are emitted from emitHeading() instead.
   },
   content: {
-    mention: "mention-user",
+    mention: "mention",
 
     codeTextData: "code",
     codeFlowValue: "pre",
 
+    // The language a fenced block declares. Its own type rather than `syntax`
+    // because it is metadata something will want to read -- highlighting, at
+    // least -- and no formatter styles it, so it stays plain text in an input
+    // the way the rest of the fence line does not.
+    codeFencedFenceInfo: "language",
+
+    // The title after a destination -- `[a](b "title")`, on a link as much as on
+    // an image. Metadata for the same reasons the language above is: a display
+    // renderer removes it with the rest of the resource, and something that
+    // renders an image wants to read it. Left unstyled in an input, where it is
+    // text between two quotes.
+    resourceTitleString: "title",
+
     // TODO:
     // htmlFlow: 'html',
     // htmlText: 'html',
@@ -155,22 +195,31 @@ const EMOJI: {
     "autolinkProtocol",
     "autolinkEmail",
   ]),
-  exclude: new Set(["bold", "italic", "strikethrough"]),
+  exclude: new Set(["strong", "emphasis", "strikethrough"]),
   pattern: emojiRegex(),
 }
 
 // Bundler shenanigans: when bundling esm export ends up in wrapper function body
 // which is a syntax error so don't export anything at all.
 // Instead declare export so typing still works.
+//
+// The factory is synthesized by `scripts/bundle-parser.mjs`: the whole bundle
+// below ends up inside the worklet it returns, with the options baked into that
+// worklet's closure. Nothing native ever passes options -- the C++ bridge calls
+// a registered worklet with a string and nothing else -- so the only place they
+// can be chosen is where the worklet is made.
 declare module "./index" {
-  export function parse(markdown: string): MarkdownRange[]
+  export function createParser(options?: ParserOptions): MarkdownParser
 }
 
 declare global {
-  function __parse__micromark(markdown: string): MarkdownRange[]
+  function __parse__micromark(markdown: string, options?: ParserOptions): MarkdownRange[]
 }
 
-globalThis.__parse__micromark = function (markdown: string): MarkdownRange[] {
+globalThis.__parse__micromark = function (
+  markdown: string,
+  options?: ParserOptions,
+): MarkdownRange[] {
   // micromark's preprocessor drops a leading BOM, so every offset it reports is
   // one short of the caller's string -- a document that opens with one had every
   // range shifted left by a character, leaving the last one uncovered. Parse the
@@ -182,7 +231,30 @@ globalThis.__parse__micromark = function (markdown: string): MarkdownRange[] {
     markdown = markdown.slice(bomLength)
   }
 
-  const extensions = [gfmAutolinkLiteral(), gfmStrikethrough(), mentions()]
+  // Every extension is off unless it is asked for: a parser made with no
+  // options at all is CommonMark and nothing else. Each of these changes what
+  // some perfectly ordinary character means -- a tilde, an `@`, a dollar sign --
+  // and that is not a thing to do to a document on the way past.
+  const extensions = []
+
+  if (options?.gfm === true) {
+    extensions.push(gfmAutolinkLiteral(), gfmStrikethrough())
+  }
+
+  if (options?.mention === true) {
+    extensions.push(mentions())
+  }
+
+  if (options?.math === true) {
+    // Two dollars, never one. A single `$` is a currency sign far more often
+    // than it is a delimiter, and with the single form on, `$5 and $6` is one
+    // maths span whose TeX is "5 and ": prose silently turned into a picture,
+    // in the one place -- a document written by someone else -- where that is
+    // least recoverable. `$$x^2$$` costs a keystroke and can never be an
+    // accident.
+    extensions.push(math({ singleDollarTextMath: false }))
+  }
+
   const ranges: MarkdownRange[] = []
   // Counted block containers, innermost last. Never exceeds MAX_DEPTH per type.
   const openBlocks: MarkdownType[] = []
@@ -190,6 +262,16 @@ globalThis.__parse__micromark = function (markdown: string): MarkdownRange[] {
   const countedBlocks: boolean[] = []
   // Heading awaiting the marker that states its level.
   let pendingHeading: Token | null = null
+  // Open images. A link's label is a link; an image's label is alt text, and
+  // the two are the same token type distinguished only by what encloses them.
+  let openImages = 0
+  // Open labels inside an image's, so that the outermost one is the alt text
+  // and everything under it is suppressed.
+  let altDepth = 0
+  // Open link labels. A label carries everything else it is written with --
+  // emphasis, code, mentions, an image -- but not a link: a link inside a link
+  // is text, and the one wrapped around it is the one that is pressed.
+  let labelDepth = 0
   // The container markers on the line being built, in the order they appear.
   // Positional rather than keyed by type: one type can open more than once on a
   // line, and the occurrences need not be adjacent (`> - > `). Cleared per
@@ -220,6 +302,9 @@ globalThis.__parse__micromark = function (markdown: string): MarkdownRange[] {
   return ranges
 
   function enter(token: Token) {
+    // Ahead of the alt-text gate below, and deliberately: an emoji is not
+    // markup but a property of a character that survives, and alt text shown as
+    // prose has to draw it in the emoji font like any other.
     if (EMOJI.scan.has(token.type)) {
       emitEmoji(token)
     }
@@ -289,6 +374,16 @@ globalThis.__parse__micromark = function (markdown: string): MarkdownRange[] {
     }
 
     if (TOKENS.special[token.type]) {
+      if (token.type === "image") {
+        openImages++
+
+        // An image written inside another one's label is not an image. It is
+        // some of the characters that label is spelled with.
+        if (altDepth > 0) {
+          return
+        }
+      }
+
       emit(token, TOKENS.special[token.type]!)
       return
     }
@@ -298,6 +393,47 @@ globalThis.__parse__micromark = function (markdown: string): MarkdownRange[] {
       return
     }
 
+    // The visible half of a link, emitted so that a renderer which removes the
+    // markup around it has something left to style. Deliberately not `link`:
+    // while it is being edited the label is text you are typing, and only a
+    // renderer that has taken the brackets away has any business making it look
+    // like one.
+    if (token.type === "labelText") {
+      // A label nested inside an image's is part of that alt text, not a second
+      // one: `![a [b](c) d](e)` has exactly one.
+      if (altDepth > 0) {
+        altDepth++
+        return
+      }
+
+      // An image's label is alt text, not a link label. The two are the same
+      // token, distinguished only by what encloses them -- and they part ways
+      // downstream: a label is the link once the brackets are gone, while alt
+      // text is what stands in for an image nobody rendered.
+      if (openImages > 0) {
+        emit(token, ALT_TYPE)
+        altDepth = 1
+        return
+      }
+
+      emit(token, LABEL_TYPE)
+      labelDepth++
+      return
+    }
+
+    // Alt text is a string. CommonMark says as much -- it flattens a label to
+    // plain text to render the `alt` attribute -- but micromark reports the
+    // structure it found, so `![**a** [b](c)](d)` arrives carrying emphasis, a
+    // whole link and that link's own destination. None of it means anything:
+    // nothing can draw a link inside an image. Only the syntax handled above
+    // survives, so that whatever removes markup can still remove these
+    // characters, and everything downstream sees one `alt` range over one
+    // flat string. Emoji are the exception, emitted above: they are not markup
+    // and the characters they cover are still there to be drawn.
+    if (altDepth > 0) {
+      return
+    }
+
     if (TOKENS.scope[token.type]) {
       emit(token, TOKENS.scope[token.type]!)
       return
@@ -314,12 +450,40 @@ globalThis.__parse__micromark = function (markdown: string): MarkdownRange[] {
     }
 
     if (TOKENS.link[token.type]) {
+      // A link written inside a label is not one: `[<https://x.com>](y)` reads
+      // as its own URL and points somewhere else. Its markers are still markup
+      // and still go, but what they leave behind is prose, so no range is
+      // emitted over it -- a second `link` here would draw it as a link of its
+      // own in an input and, once the display has taken the brackets away,
+      // would cover the same characters twice.
+      //
+      // An image's destination is not that: it is inside the label but it is
+      // the image's, and something has to render the image.
+      if (labelDepth > 0 && openImages === 0) {
+        return
+      }
+
       emit(token, TOKENS.link[token.type]!)
       return
     }
   }
 
   function exit(token: Token) {
+    if (token.type === "labelText") {
+      if (altDepth > 0) {
+        altDepth--
+      } else if (labelDepth > 0) {
+        labelDepth--
+      }
+
+      return
+    }
+
+    if (token.type === "image") {
+      openImages--
+      return
+    }
+
     if (!TOKENS.block[token.type]) {
       return
     }
@@ -554,3 +718,4 @@ globalThis.__parse__micromark = function (markdown: string): MarkdownRange[] {
     ranges.push(...result)
   }
 }
+

package/src/parser/micromark-extension-mentions.ts

@@ -1,9 +1,20 @@
-import type { Code, Effects, Extension, State } from "micromark-util-types"
+import type {
+  Code,
+  Construct,
+  Effects,
+  Extension,
+  State,
+  TokenizeContext,
+} from "micromark-util-types"
 
 const LETTER = /^\p{L}$/u
-const WORD = /^[\p{L}\p{M}\p{N}_]$/u
 const ALPHANUMERIC = /^[\p{L}\p{M}\p{N}]$/u
 
+const AT = 64
+const DOT = 46
+const HYPHEN = 45
+const UNDERSCORE = 95
+
 declare module "micromark-util-types" {
   interface TokenTypeMap {
     mention: "mention"
@@ -13,8 +24,20 @@ declare module "micromark-util-types" {
 const mention = {
   name: "mention",
   tokenize,
+  previous,
 } as const
 
+/**
+ * A `-`, `_` or `@` and the part of the name after it.
+ *
+ * Attempted rather than consumed outright, because micromark cannot give a
+ * character back: a name that ends in front of one -- `@user@ ` -- has to leave
+ * the separator outside the token, and an attempt that fails rewinds to it.
+ * That is also what lets `_@someone_` be a mention in italics: the closing `_`
+ * has no name after it, so it stays markup.
+ */
+const join: Construct = { tokenize: tokenizeJoin, partial: true }
+
 export function mentions(): Extension {
   return {
     text: {
@@ -23,6 +46,20 @@ export function mentions(): Extension {
   }
 }
 
+/**
+ * A mention starts where a name starts: not inside a word, and not straight
+ * after another `@`.
+ *
+ * That is what keeps `user@example.com` an address rather than an address with
+ * a mention buried in it, and `@@user` a stray `@` rather than a name beginning
+ * with one. Everything else may sit against it -- `**@someone**`, `[@user](/u)`
+ * and `(@user)` all hold one -- because the markup is gone by the time a
+ * display draws it, and punctuation was never part of a name to begin with.
+ */
+function previous(this: TokenizeContext, code: Code): boolean {
+  return code === null || (code !== AT && !test(ALPHANUMERIC, code))
+}
+
 function tokenize(effects: Effects, ok: State, nok: State): State {
   return start
 
@@ -44,32 +81,88 @@ function tokenize(effects: Effects, ok: State, nok: State): State {
 
     effects.consume(code)
 
-    return body
+    return part
   }
 
-  function body(code: Code): State {
-    if (code !== null && test(WORD, code)) {
+  /** Inside a part of the name: letters, marks and digits. */
+  function part(code: Code): State {
+    if (code !== null && test(ALPHANUMERIC, code)) {
+      effects.consume(code)
+      return part
+    }
+
+    if (code === DOT) {
       effects.consume(code)
-      return body
+      return afterDot
     }
 
-    if (code === 45) {
-      // `-` is allowed only when followed by an alphanumeric character.
-      return hyphen
+    if (code === HYPHEN || code === UNDERSCORE || code === AT) {
+      return effects.attempt(join, part, done)
     }
 
+    return done(code)
+  }
+
+  /**
+   * Just after a dot, which belongs to the name wherever it falls -- `@bullet.`
+   * is one mention rather than a mention and a full stop.
+   *
+   * Only the first of several does, though: a second dot in a row ends the name
+   * here, in front of it, so `@bullet..` is `@bullet.` and a full stop that
+   * still reads as one.
+   */
+  function afterDot(code: Code): State {
+    if (code === DOT) {
+      return done(code)
+    }
+
+    return part(code)
+  }
+
+  /**
+   * The name is finished.
+   *
+   * Nothing is checked here: `part` consumes every character a name is made of,
+   * so whatever stopped it is not one -- whitespace, punctuation, or the markup
+   * the mention was written inside.
+   */
+  function done(code: Code): State {
     effects.exit("mention")
+
     return ok(code)!
   }
+}
+
+function tokenizeJoin(effects: Effects, ok: State, nok: State): State {
+  return separator
+
+  function separator(code: Code): State {
+    // A `-`, `_` or `@`: `part` only attempts this construct on one of the
+    // three. An `@` opens a new name and so wants a letter; the other two join
+    // two parts of one and take whatever a part is made of.
+    effects.consume(code)
 
-  function hyphen(code: Code): State {
+    return code === AT ? afterAt : afterJoin
+  }
+
+  function afterJoin(code: Code): State {
     if (code === null || !test(ALPHANUMERIC, code)) {
       return nok(code)!
     }
 
     effects.consume(code)
 
-    return body
+    return ok
+  }
+
+  function afterAt(code: Code): State {
+    if (code === null || !test(LETTER, code)) {
+      return nok(code)!
+    }
+
+    effects.consume(code)
+
+    return ok
   }
 }