react-lib-tools 0.0.46 → 0.0.47

package/dist/react-lib-tools.cjs

@@ -1,2 +1,2 @@
-"use strict";Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const o=require("./index-BBqu13cy.cjs");exports.AppRoot=o.AppRoot;exports.Block=o.Block;exports.Box=o.Box;exports.Button=o.Button;exports.Callout=o.Callout;exports.Checkbox=o.Checkbox;exports.Code=o.Code;exports.ComponentProps=o.ComponentProps;exports.ComponentPropsSection=o.ComponentPropsSection;exports.DocsSection=o.DocsSection;exports.ErrorBoundary=o.ErrorBoundary;exports.ExternalLink=o.ExternalLink;exports.Header=o.Header;exports.ImperativeHandle=o.ImperativeHandle;exports.Input=o.Input;exports.Link=o.Link;exports.LoadingSpinner=o.LoadingSpinner;exports.NavLink=o.NavLink;exports.NavSection=o.NavSection;exports.Radio=o.Radio;exports.Select=o.Select;exports.Tooltip=o.Tooltip;exports.assert=o.assert;exports.cn=o.cn;exports.getIntentClassNames=o.getIntentClassNames;
+"use strict";Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const o=require("./index-DwdhsbJr.cjs");exports.AppRoot=o.AppRoot;exports.Block=o.Block;exports.Box=o.Box;exports.Button=o.Button;exports.Callout=o.Callout;exports.Checkbox=o.Checkbox;exports.Code=o.Code;exports.ComponentProps=o.ComponentProps;exports.ComponentPropsSection=o.ComponentPropsSection;exports.DocsSection=o.DocsSection;exports.ErrorBoundary=o.ErrorBoundary;exports.ExternalLink=o.ExternalLink;exports.Header=o.Header;exports.ImperativeHandle=o.ImperativeHandle;exports.Input=o.Input;exports.Link=o.Link;exports.LoadingSpinner=o.LoadingSpinner;exports.NavLink=o.NavLink;exports.NavSection=o.NavSection;exports.Radio=o.Radio;exports.Select=o.Select;exports.Tooltip=o.Tooltip;exports.assert=o.assert;exports.cn=o.cn;exports.getIntentClassNames=o.getIntentClassNames;
 //# sourceMappingURL=react-lib-tools.cjs.map

package/dist/react-lib-tools.js

@@ -1,4 +1,4 @@
-import { A as o, a as e, B as n, b as t, C as r, c as i, d as p, i as c, j as l, D as d, e as C, E as k, H as m, I as B, f as L, L as S, g as u, N as x, h as I, R as N, k as g, T as v, l as E, m as H, n as R } from "./index-Bq-3EQ1i.js";
+import { A as o, a as e, B as n, b as t, C as r, c as i, d as p, i as c, j as l, D as d, e as C, E as k, H as m, I as B, f as L, L as S, g as u, N as x, h as I, R as N, k as g, T as v, l as E, m as H, n as R } from "./index-DHqKf7uc.js";
 export {
   o as AppRoot,
   e as Block,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "react-lib-tools",
-  "version": "0.0.46",
+  "version": "0.0.47",
   "type": "module",
   "author": "Brian Vaughn <brian.david.vaughn@gmail.com> (https://github.com/bvaughn/)",
   "contributors": [

package/scripts/utils/search/crawlPage.ts

@@ -36,6 +36,8 @@ export async function crawlPage({
   await page.goto(`${host}${path}`);
   await page.waitForSelector("main");
 
+  page.on("console", (event) => console.log(event.text()));
+
   const result = await page.evaluate(
     async ([filterSelector, stopWords]) => {
       const paths: string[] = [];
@@ -53,38 +55,52 @@ export async function crawlPage({
         document.body.querySelector("header [data-title]")?.textContent ?? "";
 
       const words = new Set<string>();
-      const texts: string[] = [];
+      let text = "";
 
-      const main = document.body.querySelector("header")?.parentElement;
+      const main = document.body.querySelector("[data-main-scrollable]");
       if (main) {
-        for (const child of main.children) {
-          switch (child.tagName) {
-            //   case "CODE":
-            case "HEADER": {
-              continue;
-            }
-          }
+        text = main.innerHTML;
 
-          if (filterSelector && child.querySelector(filterSelector)) {
-            continue;
+        // Filter out special content (e.g. lorem ipsum type text)
+        if (filterSelector) {
+          const matches = document.querySelectorAll(filterSelector);
+          for (const match of matches) {
+            text = text.replace(match.outerHTML, "");
           }
+        }
 
-          if (child.textContent) {
-            texts.push(child.textContent);
-            child.textContent.split(/[\s.:]/).forEach((word) => {
-              word = word.trim().toLowerCase();
-              if (word && !stopWords.includes(word)) {
-                words.add(word);
-              }
-            });
-          }
+        // Ensure whitespace between HTML tags
+        text = text.replaceAll("><", "> <");
+
+        // Strip HTML tags
+        text = text.replaceAll(/<[^>]+>/g, "");
+
+        // Replace HTML entities
+        const HTML_ENTITIES: { [chars: string]: string } = {
+          "&amp;": "&",
+          "&lt;": "<",
+          "&gt;": ">",
+          "&quot;": '"',
+          "&#039;": "'",
+          "&ndash;": "-",
+          "&nbsp;": " "
+        };
+        for (const chars in HTML_ENTITIES) {
+          text = text.replaceAll(chars, HTML_ENTITIES[chars]);
         }
+
+        text.split(/[\s.:?]/).forEach((word) => {
+          word = word.trim().toLowerCase();
+          if (word && !stopWords.includes(word)) {
+            words.add(word);
+          }
+        });
       }
 
       return {
         paths,
         section,
-        text: texts.join(" "),
+        text,
         title,
         words: Array.from(words)
       };