unicode-input-toolconverter 0.2.0 → 0.2.3

package/tools/parseUnicodeCharts.js

@@ -1,5 +1,5 @@
-import fs from 'fs/promises';
-import path from 'path';
+import fs from 'node:fs/promises';
+import path from 'node:path';
 
 // eslint-disable-next-line no-shadow -- Remove?
 import fetch from 'node-fetch';
@@ -18,8 +18,8 @@ const {JSDOM} = jsdom;
 function getChromeSafeLocaleKey (key) {
   return key.replaceAll(
     String.raw`\n`, ''
-  ).replaceAll(/\s+/gu, ' '). // Getting some extra WS
-    replaceAll(/\W/gu, '_');
+  ).replaceAll(/\s+/gv, ' '). // Getting some extra WS
+    replaceAll(/\W/gv, '_');
 }
 
 let text;
@@ -47,23 +47,38 @@ const uniqueTextPlaceholder = localize
   ? '___placeholder___'
   : ''; // We know it is not present inside Unicode script names!
 
+/**
+ * @param {string} txt
+ */
 const cleanupText = (txt) => {
   return txt.trim().
-    replace(/(?<initWS>\s)\s+/u, '$<initWS>').
-    replace(/\s*\((?:ASCII|Odia)\)$/u, '').
-    replace(/\s\(\d\.?\d*MB\)$/u, ''); // Remove MB size
+    replace(/(?<initWS>\s)\s+/v, '$<initWS>').
+    replace(/\s*\((?:ASCII|Odia)\)$/v, '').
+    replace(/\s\(\d\.?\d*MB\)$/v, ''); // Remove MB size
 };
 
+/**
+ * @type {{
+ *   script: string,
+ *   startRange: string
+ * }[]}
+ */
 const scriptsAndStartRanges = [];
 
 const jamilih = scriptMaps.map((scriptMap) => {
   const majorHeading = cleanupText(
-    scriptMap.previousElementSibling.textContent
+    /** @type {HTMLElement} */
+    (scriptMap.previousElementSibling).textContent
   );
   // const scriptGroups = [...scriptMap.querySelectorAll('table td p')];
 
-  const scriptGroups = [...scriptMap.querySelectorAll('table td p.sg')];
+  // Note: as of the Unicode 18.0 charts redesign, `p.sg` sits directly in
+  //   `table.map`'s own `td` (no more nested `table`), so this must not
+  //   require an extra `table` ancestor or it silently matches nothing.
+  const scriptGroups = [...scriptMap.querySelectorAll(':scope td p.sg')];
   // sg, mb, pb/sb
+
+  /** @type {import('jamilih/dist/jml.js').JamilihChildren|null} */
   let lastChildren;
   return ['li', [
     majorHeading,
@@ -87,11 +102,12 @@ const jamilih = scriptMaps.map((scriptMap) => {
               );
             } else if (scriptGroup.matches('.pb,.sb')) {
               cleanedText = cleanupText(scriptGroup.textContent);
-              const children = [
+              // eslint-disable-next-line @stylistic/max-len -- Long
+              const children = /** @type {import('jamilih/dist/jml.js').JamilihChildren} */ ([
                 ['i', [
                   uniqueTextPlaceholder + cleanedText
                 ]]
-              ];
+              ]);
               if (!lastChildren) { // A few rare cases to handle, e.g., "Other"
                 lists.push(
                   ['li', {title}, children]
@@ -100,18 +116,27 @@ const jamilih = scriptMaps.map((scriptMap) => {
                 if (!lastChildren[1]) {
                   lastChildren[1] = ['ul', []];
                 }
-                lastChildren[1][1].push(
+                /** @type {import('jamilih/dist/jml.js').JamilihChildren} */ (
+                  /** @type {import('jamilih/dist/jml.js').JamilihArray} */
+                  (lastChildren[1])[1]
+                ).push(
                   ['li', {title}, children]
                 );
               }
             }
             if (cleanedText && title) {
+              // Range separators are inconsistent across entries: some use
+              //   a plain hyphen, others an en dash (U+2013).
               scriptsAndStartRanges.push({
                 script: getChromeSafeLocaleKey(cleanedText),
-                startRange: title.replace(/-.*$/u, '')
+                startRange: title.replace(/[\-–].*$/v, '')
               });
             }
-            scriptGroup = scriptGroup.nextElementSibling;
+
+            // May actually be `null` too, but next condition will catch
+            scriptGroup = /** @type {HTMLElement} */ (
+              scriptGroup.nextElementSibling
+            );
           } while (scriptGroup && !scriptGroup.matches('p.sg'));
           lastChildren = null;
           if (!lists.length) { // Just be safe
@@ -126,14 +151,15 @@ const jamilih = scriptMaps.map((scriptMap) => {
 // console.log('m', majorHeading, scriptGroups);
 
 /**
-* @typedef {JSON} LocaleFileContents
-* @todo Indicate more precise format
-*/
+ * @typedef {{
+ *  body: Record<string, {message: string, description?: string}>
+ * }} LocaleFileContents
+ */
 
 /**
  * @param {string[]} localeFiles
- * @param {LocaleFileContents} localeFileContents
- * @returns {Promise<void>}
+ * @param {LocaleFileContents[]} localeFileContents
+ * @returns {Promise<void[]>}
  */
 async function saveLocalesWithoutDupes (localeFiles, localeFileContents) {
   return await Promise.all(
@@ -147,13 +173,40 @@ async function saveLocalesWithoutDupes (localeFiles, localeFileContents) {
 
 /**
  * @param {string[]} newScriptNames
- * @param {LocaleFileContents} localeFileContents
+ * @param {LocaleFileContents[]} localeFileContents
  * @returns {Promise<void>}
  */
 async function deleteUnusedScriptNames (newScriptNames, localeFileContents) {
-  const {lastScriptNames} = JSON.parse(
+  // eslint-disable-next-line prefer-destructuring -- TS
+  const lastScriptNames = /** @type {string[]} */ (JSON.parse(
     await fs.readFile(lastScriptNamesFile, 'utf8')
-  );
+  ).lastScriptNames);
+
+  // Safety guard: a parsing failure (e.g., unicode.org restructuring the
+  //   charts page, as happened for the Unicode 18.0 redesign, breaking the
+  //   `scriptGroups` selector above) can silently yield few or no
+  //   `newScriptNames`. Without this check, every "missing" script name
+  //   would look "removed from the standard" and get deleted from every
+  //   locale file. Script counts barely change between Unicode versions,
+  //   so a drop this large almost certainly means parsing broke, not that
+  //   the scripts really vanished.
+  const survivingCount = lastScriptNames.filter(
+    (lastScriptName) => newScriptNames.includes(lastScriptName)
+  ).length;
+  const survivalRatio = lastScriptNames.length > 0
+    ? survivingCount / lastScriptNames.length
+    : 1;
+  if (survivalRatio < 0.5) {
+    throw new Error(
+      `Refusing to update: only ${survivingCount} of ` +
+      `${lastScriptNames.length} previously known script names were found ` +
+      `in this run (parsed ${newScriptNames.length} total). This usually ` +
+      'means the unicode.org charts page structure changed and the ' +
+      'scraper needs fixing, not that the scripts were actually removed ' +
+      'from the standard. No files were written.'
+    );
+  }
+
   lastScriptNames.forEach((lastScriptName) => {
     if (!newScriptNames.includes(lastScriptName)) {
       localeFileContents.forEach((lfc) => {
@@ -171,6 +224,9 @@ await fs.writeFile(
 // Do not edit this file; this is an auto-generated file used to
 //   build a hierarchy of script names. It is built by
 //   \`parseUnicodeCharts.js\` using ${chartsURL}.
+/**
+ * @param {import('intl-dom').I18NCallback} _
+ */
 function unicodeScripts (_) {
   return ` + (localize
     ? await (async () => {
@@ -181,34 +237,38 @@ function unicodeScripts (_) {
       const localeFiles = dirs.map(
         (dir) => path.join(localesDir, dir, 'messages.json')
       );
-      const localeFileContents = (
+      const localeFileContents = /** @type {LocaleFileContents[]} */ ((
         await Promise.all(localeFiles.map((localeFile) => {
           return fs.readFile(localeFile, 'utf8');
         }))
       ).map((fileContents) => {
         return JSON.parse(fileContents);
-      });
+      }));
 
-      const dirResults = (
+      const dirResults = /** @type {{path: string, keys: string[]}[]} */ (
         await recurseDirectory({directory: 'browser_action'})
       ).filter(({keys}) => {
         return keys.length;
       });
       // console.log('dirResults', JSON.stringify(dirResults, null, 2));
+
+      /** @type {Record<string, {paths: string[], locales: string[]}>} */
       const keyMap = {};
       dirResults.forEach(({path: pth, keys}) => {
         keys.forEach((key) => {
           localeFileContents.forEach((lfc, i) => {
-            if (!(key in lfc.body)) {
-              if (!keyMap[key]) {
-                keyMap[key] = {paths: [], locales: []};
-              }
-              if (!keyMap[key].paths.includes(pth)) {
-                keyMap[key].paths.push(pth);
-              }
-              if (!keyMap[key].locales.includes(localeFiles[i])) {
-                keyMap[key].locales.push(localeFiles[i]);
-              }
+            if (Object.hasOwn(lfc.body, key)) {
+              return;
+            }
+
+            if (!Object.hasOwn(keyMap, key)) {
+              keyMap[key] = {paths: [], locales: []};
+            }
+            if (!keyMap[key].paths.includes(pth)) {
+              keyMap[key].paths.push(pth);
+            }
+            if (!keyMap[key].locales.includes(localeFiles[i])) {
+              keyMap[key].locales.push(localeFiles[i]);
             }
           });
         });
@@ -229,12 +289,12 @@ function unicodeScripts (_) {
           return;
         }
         Object.keys(lfc.body).forEach((localeKey) => {
-          if (!dirResults.some(({
+          if (dirResults.every(({
             keys
             // path: pth
           }) => {
             // console.log('keys', keys);
-            return keys.includes(localeKey);
+            return !keys.includes(localeKey);
           })) {
             // Working:
             // console.log(`Locale key "${localeKey}" not present in keys
@@ -243,11 +303,12 @@ function unicodeScripts (_) {
         });
       });
 
+      /** @type {string[]} */
       const newScriptNames = [];
       const ret = JSON.stringify(['ul', jamilih], null, 2).replaceAll(
         new RegExp(
           String.raw`^(\s*)"` + uniqueTextPlaceholder + '(.*)"(,)?$',
-          'gum'
+          'gvm'
         ),
         (_, initialWS, key, possibleComma = '') => {
           const chromeSafeLocaleKey = getChromeSafeLocaleKey(key);
@@ -273,7 +334,7 @@ function unicodeScripts (_) {
             if (lfc.body.langCode.message !== 'hu-HU') {
               // return;
             }
-            if (!(chromeSafeLocaleKey in lfc.body)) {
+            if (!Object.hasOwn(lfc.body, chromeSafeLocaleKey)) {
               lfc.body[chromeSafeLocaleKey] = {
                 message: key
               };
@@ -335,29 +396,29 @@ scriptsAndStartRanges.forEach(({script, startRange}) => {
   } else if (num < 0x${startRange}) {
     codePointStart = '${lastStartRange}';
     script = _('${lastScript}');${
-  // Not in chart:
-  // startRange === 'DB80' ? "surrogate = _('High_Private_Use_Surrogate')" :
-  lastStartRange === 'D800'
-    ? `
+      // Not in chart:
+      // startRange === 'DB80' ? "surrogate = _('High_Private_Use_Surrogate')" :
+      lastStartRange === 'D800'
+        ? `
 surrogate = _('High_Surrogate');`
-    : lastStartRange === 'DC00'
-      ? `
+        : lastStartRange === 'DC00'
+          ? `
 surrogate = _('Low_Surrogate');`
-      : ['E000', 'F0000', '100000'].includes(lastStartRange)
-        ? `
+          : ['E000', 'F0000', '100000'].includes(lastStartRange)
+            ? `
 privateuse = true;`
-        : ''
-}`;
+            : ''
+    }`;
   lastStartRange = startRange;
   lastScript = script;
 });
 
 /**
- * @param {PlainObject} cfg
+ * @param {object} cfg
  * @param {string} cfg.directory
  * @param {string} [cfg.basePath]
- * @param {{path, keys}} cfg.results
- * @returns {Promise<{path, keys}|void>}
+ * @param {{path: string, keys: string[]}[]} [cfg.results]
+ * @returns {Promise<{path: string, keys: string[]}[]|void>}
  */
 async function recurseDirectory ({directory, basePath = './', results}) {
   const dirPath = path.join(basePath, directory);
@@ -370,26 +431,31 @@ async function recurseDirectory ({directory, basePath = './', results}) {
   await Promise.all(
     filesAndDirs.filter((fileOrDir) => {
       // Is a directory or JavaScript file
-      return !fileOrDir.includes('.') || (/\.js$/u).test(fileOrDir);
+      return !fileOrDir.includes('.') || (/\.js$/v).test(fileOrDir);
     }).map(async (fileOrDir) => {
       if (!fileOrDir.includes('.')) { // Should check `isDirectory()` or such
-        return recurseDirectory({
-          directory: fileOrDir,
-          basePath: dirPath,
-          results
-        });
+        return /** @type {Promise<{path: string, keys: string[]}[]>} */ (
+          recurseDirectory({
+            directory: fileOrDir,
+            basePath: dirPath,
+            results
+          })
+        );
       }
       const pth = path.join(dirPath, fileOrDir);
       const contents = await fs.readFile(pth, 'utf8');
 
       const i18nRegex = // /_\((?:['"]([^'"]*)['"])\)/g;
       // Todo: Works to find variable forms:
-        /_\((?:['"](?<key1>[^'"]*)['"]|(?<key2>[^"')][^{)]*|[^{)]*[^"')]))\)/gu;
+        /_\((?:['"](?<key1>[^'"]*)['"]|(?<key2>[^"'\)][^\{\)]*|[^\{\)]*[^"'\)]))\)/gv;
 
+      /** @type {string[]} */
       const keys = [];
       let result;
       while ((result = (i18nRegex.exec(contents))) !== null) {
-        keys.push(result.groups.key1 || result.groups.key2);
+        keys.push(/** @type {{key1: string}} */ (
+          result.groups
+        ).key1 || /** @type {{key2: string}} */ (result.groups).key2);
       }
       results.push({path: pth, keys});
       return undefined;

package/tools/ucd-import.js

@@ -2,23 +2,14 @@
 //  this file for the browser, but we're keeping this file for demonstrating
 //  an approach for live-obtaining the live version.
 
-import fs from 'fs/promises';
-
 import download from 'download';
-import extract from 'extract-zip';
 
 const args = process.argv.slice(2);
 
 const targetDir = `${process.cwd()}/download/UCD`;
-const ucdZip = `${targetDir}/UCD.zip`;
 
 if (args.includes('download')) {
   await download(
-    'https://www.unicode.org/Public/UCD/latest/ucd/UCD.zip', targetDir
+    'https://www.unicode.org/Public/UCD/latest/ucd/UCD.zip', targetDir, {extract: true}
   );
 }
-
-if (args.includes('extract') || args.includes('download')) {
-  await extract(ucdZip, {dir: targetDir});
-  await fs.unlink(ucdZip);
-}