shelving 1.309.0 → 1.310.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "shelving",
3
- "version": "1.309.0",
3
+ "version": "1.310.0",
4
4
  "author": "Dave Houlbrooke <dave@shax.com>",
5
5
  "repository": {
6
6
  "type": "git",
package/util/string.d.ts CHANGED
@@ -157,14 +157,19 @@ export declare function sanitizeWord(str: string): string;
157
157
  export declare function sanitizeMultilineText(str: string): string;
158
158
  /**
159
159
  * Simplify a string by removing anything that isn't a number, letter, or space.
160
- * - Normalizes the string by
161
- * - Useful when you're running a query against a string entered by a user.
160
+ * - Useful when you're running a query against a string entered by a user, or making a slug from a title.
161
+ * - Uses `NFKD` normalisation to convert styled letters to plain letters, e.g. circled `Ⓒ`, italic `𝐶`, script `𝒞`, fraktur `ℭ`, double-struck `ℂ`, fullwidth `C`, and superscript `ᶜ` all become `c`.
162
+ * - `NFKD` also splits ligatures (e.g. `ff` becomes `ff`) and removes accents (e.g. `é` becomes `e`).
163
+ * - Converts a small set of letterlike characters that `NFKD` does not cover: small capitals (e.g. `ᴄ`), negative circled and squared letters (e.g. `🅒` and `🅲`), regional indicators (e.g. `🇨`), and the estimate symbol `℮`.
164
+ * - Keeps letters from other scripts (e.g. `ж`, `π`, `ℵ` alef becomes `א`), and does not convert them to Latin letters that look the same.
165
+ * - Removes symbols, punctuation, and emoji (e.g. `©`, `!`, `😂`).
166
+ * - Removes marks that annotate text (`™`, `℠`, `🅪`, `🅫`, `🅬`), so `Acme™` becomes `acme`, not `acmetm`.
167
+ * - Keeps the letters of units and abbreviations, because they are part of the meaning (e.g. `㎏` becomes `kg`, `№` becomes `no`, `℃` becomes `c`).
168
+ * - Converts runs of spaces, tabs, newlines, underscores, and dashes to a single space, then trims and lowercases the result.
162
169
  *
163
170
  * @param str The string to simplify.
164
171
  * @returns The simplified, lowercased string containing only numbers, letters, and single spaces.
165
172
  * @see https://shelving.cc/util/string/simplifyString
166
- *
167
- * @todo Convert confusables (e.g. `ℵ` alef symbol or `℮` estimate symbol) to their letterlike equivalent (e.g. `N` and `e`).
168
173
  */
169
174
  export declare function simplifyString(str: string): string;
170
175
  /**
package/util/string.js CHANGED
@@ -187,22 +187,47 @@ export function sanitizeMultilineText(str) {
187
187
  .replace(/[^\S\t\n]{4}/g, "\t") // Normalise leading ` ` four whitespace characters to a single `\t` tab.
188
188
  .replace(/(^|\t|\n)[^\S\t\n]+/g, "$1"); // Remove leading whitespace that isn't a tab.
189
189
  }
190
+ // Characters where `NFKD` normalisation does not give the plain letter we want.
191
+ const _LETTERS = "abcdefghijklmnopqrstuvwxyz";
192
+ const _LETTERLIKE = new Map([
193
+ ["℮", "e"], // Estimate symbol.
194
+ ...Array.from("™℠🅪🅫🅬", char => [char, ""]), // Marks that annotate text are removed (these are every multi-letter symbol with a `<super>` decomposition, e.g. `NFKD` converts `™` to `TM`).
195
+ ...Array.from("ᴀʙᴄᴅᴇꜰɢʜɪᴊᴋʟᴍɴᴏᴘꞯʀꜱᴛᴜᴠᴡxʏᴢ", (char, i) => [char, _LETTERS.charAt(i)]), // Small capitals `ᴀ` to `ᴢ` (no small capital `x` exists).
196
+ ..._getLetterRange(0x1f150), // Negative circled capitals `🅐` to `🅩`.
197
+ ..._getLetterRange(0x1f170), // Negative squared capitals `🅰` to `🆉`.
198
+ ..._getLetterRange(0x1f1e6), // Regional indicators `🇦` to `🇿` (pairs show as flags, e.g. `🇬🇧` becomes `gb`).
199
+ ]);
200
+ const _LETTERLIKE_REGEXP = new RegExp(`[${Array.from(_LETTERLIKE.keys()).join("")}]`, "gu");
201
+ /** Get the 26 `[char, letter]` pairs for a run of letterlike characters that starts at `start` and runs `A` to `Z`. */
202
+ function _getLetterRange(start) {
203
+ return Array.from(_LETTERS, (letter, i) => [String.fromCodePoint(start + i), letter]);
204
+ }
205
+ /** Replace a letterlike character with its plain letter. */
206
+ function _replaceLetterlike(char) {
207
+ return _LETTERLIKE.get(char) ?? char;
208
+ }
190
209
  /**
191
210
  * Simplify a string by removing anything that isn't a number, letter, or space.
192
- * - Normalizes the string by
193
- * - Useful when you're running a query against a string entered by a user.
211
+ * - Useful when you're running a query against a string entered by a user, or making a slug from a title.
212
+ * - Uses `NFKD` normalisation to convert styled letters to plain letters, e.g. circled `Ⓒ`, italic `𝐶`, script `𝒞`, fraktur `ℭ`, double-struck `ℂ`, fullwidth `C`, and superscript `ᶜ` all become `c`.
213
+ * - `NFKD` also splits ligatures (e.g. `ff` becomes `ff`) and removes accents (e.g. `é` becomes `e`).
214
+ * - Converts a small set of letterlike characters that `NFKD` does not cover: small capitals (e.g. `ᴄ`), negative circled and squared letters (e.g. `🅒` and `🅲`), regional indicators (e.g. `🇨`), and the estimate symbol `℮`.
215
+ * - Keeps letters from other scripts (e.g. `ж`, `π`, `ℵ` alef becomes `א`), and does not convert them to Latin letters that look the same.
216
+ * - Removes symbols, punctuation, and emoji (e.g. `©`, `!`, `😂`).
217
+ * - Removes marks that annotate text (`™`, `℠`, `🅪`, `🅫`, `🅬`), so `Acme™` becomes `acme`, not `acmetm`.
218
+ * - Keeps the letters of units and abbreviations, because they are part of the meaning (e.g. `㎏` becomes `kg`, `№` becomes `no`, `℃` becomes `c`).
219
+ * - Converts runs of spaces, tabs, newlines, underscores, and dashes to a single space, then trims and lowercases the result.
194
220
  *
195
221
  * @param str The string to simplify.
196
222
  * @returns The simplified, lowercased string containing only numbers, letters, and single spaces.
197
223
  * @see https://shelving.cc/util/string/simplifyString
198
- *
199
- * @todo Convert confusables (e.g. `ℵ` alef symbol or `℮` estimate symbol) to their letterlike equivalent (e.g. `N` and `e`).
200
224
  */
201
225
  export function simplifyString(str) {
202
226
  return str
227
+ .replace(_LETTERLIKE_REGEXP, _replaceLetterlike) // Convert letterlike characters that `NFKD` doesn't (e.g. `ᴄ` small capital, `🅒` negative circled, `℮` estimate symbol), and remove marks (e.g. `™`).
203
228
  .normalize("NFKD") // Normalize ligatures (e.g. `ff` to `ff`), combined characters (e.g. `Ⓜ` to `m`), accents (e.g. `å` to `a`).
204
- .replace(/[^\p{L}\p{N}\p{Z}\p{Pc}\p{Pd}]+/gu, "") // Strip characters that aren't `\p{L}` letters, `\p{N}` numbers, `\p{Z}` separators (e.g. ` ` space), `\p{Pc}` connector punctuation (e.g. `_` underscore_, `\p{Pd}` dash punctuation (e.g. `-` hyphen)
205
- .replace(/[\p{Z}\p{Pc}\p{Pd}]+/gu, " ") // Normalise runs of `\p{Z}` separators (e.g. ` ` space), `\p{Pc}` connector punctuation (e.g. `_` underscore_, `\p{Pd}` dash punctuation (e.g. `-` hyphen), to ` ` single space.
229
+ .replace(/[^\p{L}\p{N}\p{Z}\p{Pc}\p{Pd}\s]+/gu, "") // Strip characters that aren't `\p{L}` letters, `\p{N}` numbers, `\p{Z}` separators (e.g. ` ` space), `\p{Pc}` connector punctuation (e.g. `_` underscore), `\p{Pd}` dash punctuation (e.g. `-` hyphen), or `\s` whitespace (e.g. `\t` tab or `\n` newline).
230
+ .replace(/[\p{Z}\p{Pc}\p{Pd}\s]+/gu, " ") // Normalise runs of separators, connector punctuation, dash punctuation, and whitespace to ` ` single space.
206
231
  .trim()
207
232
  .toLowerCase();
208
233
  }