react-lib-tools 0.0.43 → 0.0.45

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "react-lib-tools",
-  "version": "0.0.43",
+  "version": "0.0.45",
   "type": "module",
   "author": "Brian Vaughn <brian.david.vaughn@gmail.com> (https://github.com/bvaughn/)",
   "contributors": [

package/scripts/compile-search-index.ts

@@ -3,7 +3,9 @@ import { writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import type { SiteSearchRecord } from "react-lib-tools";
 import { crawlPage } from "./utils/search/crawlPage";
+import { scheduleWork } from "./utils/search/scheduleWork";
 import type { SiteSearchPage } from "./utils/search/types";
+import { waitForConnection } from "./utils/search/waitForConnection";
 
 export async function compileSearchIndex({
   chromeExecutablePath,
@@ -18,14 +20,23 @@ export async function compileSearchIndex({
 } = {}) {
   const recordsMap: Record<string, SiteSearchPage> = {};
 
-  await crawlPage({
-    chromeExecutablePath,
-    filterSelector,
-    host,
-    path: "/",
-    records: recordsMap
+  const url = new URL(host);
+
+  await waitForConnection({
+    host: url.hostname,
+    port: parseInt(url.port)
   });
 
+  await scheduleWork(async () =>
+    crawlPage({
+      chromeExecutablePath,
+      filterSelector,
+      host,
+      path: "/",
+      records: recordsMap
+    })
+  );
+
   const records = Object.values(recordsMap);
 
   const searchIndex = Fuse.createIndex<SiteSearchRecord>(

package/scripts/utils/search/crawlPage.ts

@@ -1,4 +1,5 @@
 import puppeteer from "puppeteer";
+import { scheduleWork } from "./scheduleWork";
 import { stopWords } from "./stopWords";
 import type { SiteSearchPage } from "./types";
 
@@ -64,10 +65,6 @@ export async function crawlPage({
             }
           }
 
-          if (child.hasAttribute("data-demo")) {
-            continue;
-          }
-
           if (filterSelector && child.querySelector(filterSelector)) {
             continue;
           }
@@ -102,24 +99,30 @@ export async function crawlPage({
     title: result.title
   };
 
-  await Promise.all(
-    result.paths
-      .filter((current) => !records[current])
-      .map((current) => {
-        records[current] = {
-          path: current,
-          text: "",
-          title: ""
-        };
+  const filteredPaths = result.paths.filter((current) => {
+    if (!records[current]) {
+      records[current] = {
+        path: current,
+        text: "",
+        title: ""
+      };
 
-        return crawlPage({
+      return path;
+    }
+  });
+
+  await browser.close();
+
+  scheduleWork(
+    ...filteredPaths.map(
+      (current) => () =>
+        crawlPage({
+          chromeExecutablePath,
           filterSelector,
           host,
           path: current,
           records
-        });
-      })
+        })
+    )
   );
-
-  await browser.close();
 }

package/scripts/utils/search/scheduleWork.ts

@@ -0,0 +1,51 @@
+type Work = () => Promise<unknown>;
+
+const CONCURRENCY_LIMIT = 5;
+
+const active = new Set<Work>();
+const pending: Work[] = [];
+
+let promise: Promise<unknown> | undefined = undefined;
+let promiseResolver: (() => void) | undefined = undefined;
+
+export async function scheduleWork(...work: Work[]) {
+  if (work.length === 0) {
+    return;
+  }
+
+  pending.push(...work);
+
+  if (!promise) {
+    promise = new Promise<void>((resolve) => {
+      promiseResolver = resolve;
+    });
+  }
+
+  for (let index = 0; index < CONCURRENCY_LIMIT; index++) {
+    doNextUnitOfWork();
+  }
+
+  await promise;
+}
+
+async function doNextUnitOfWork() {
+  if (active.size >= CONCURRENCY_LIMIT) {
+    return;
+  }
+
+  const next = pending.shift();
+  if (next) {
+    active.add(next);
+
+    await next();
+
+    active.delete(next);
+
+    doNextUnitOfWork();
+  } else if (active.size === 0) {
+    promiseResolver?.();
+
+    promise = undefined;
+    promiseResolver = undefined;
+  }
+}

package/scripts/utils/search/waitForConnection.ts

@@ -0,0 +1,46 @@
+import { createConnection } from "net";
+
+export async function waitForConnection({
+  host = "localhost",
+  port = 3000,
+  retryAttempts = 10,
+  retryDelay = 1_000
+}: {
+  host?: string;
+  port?: number;
+  retryAttempts?: number;
+  retryDelay?: number;
+} = {}) {
+  console.log(`Waiting for connection ${host}:${port} ...`);
+
+  return new Promise<void>((resolve, reject) => {
+    const socket = createConnection({ host, port });
+    socket.addListener("connect", () => {
+      console.log("Connected");
+
+      socket.destroy();
+
+      resolve();
+    });
+    socket.addListener("error", () => {
+      socket.destroy();
+
+      if (retryAttempts > 1) {
+        console.log(`Retrying after ${retryDelay} ms...`);
+
+        setTimeout(() => {
+          waitForConnection({
+            host,
+            port,
+            retryAttempts: retryAttempts - 1,
+            retryDelay
+          }).then(resolve, reject);
+        }, retryDelay);
+      } else {
+        console.error("Host not available");
+
+        process.exit(1);
+      }
+    });
+  });
+}