twl-generator 1.0.2 → 1.0.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "twl-generator",
-  "version": "1.0.2",
+  "version": "1.0.4",
   "description": "Generate term-to-article lists from unfoldingWord en_tw archive for Bible books. Works in both Node.js (CLI) and React.js (browser) environments.",
   "main": "src/index.js",
   "bin": {

package/src/utils/usfm-alignment-remover.js

@@ -1,8 +1,28 @@
 /* eslint-disable no-async-promise-executor, no-throw-literal */
 
-import fetch from 'node-fetch';
 import { BibleBookData } from '../common/books.js';
 
+// Environment detection
+const isNode = typeof window === 'undefined' && typeof process !== 'undefined' && process.versions?.node;
+
+// Get appropriate fetch implementation
+async function getFetch() {
+  if (isNode) {
+    const nodeFetch = await import('node-fetch');
+    return nodeFetch.default;
+  }
+  return globalThis.fetch;
+}
+
+// Get appropriate base64 decoder
+function decodeBase64(base64String) {
+  if (isNode) {
+    return Buffer.from(base64String, 'base64').toString('utf-8');
+  }
+  // Browser implementation
+  return atob(base64String);
+}
+
 // Note: This version doesn't use usfm-js to avoid external dependencies
 // It implements a simple USFM alignment remover for the specific case
 
@@ -53,11 +73,12 @@ export const removeAllTagsExceptChapterVerse = (usfmContent) => {
 export async function processUsfmForBook(book) {
   if (!BibleBookData[book]) throw new Error(`Unknown book: ${book}`);
 
+  const fetch = await getFetch();
   const usfmUrl = `https://git.door43.org/api/v1/repos/unfoldingWord/en_ult/contents/${BibleBookData[book].usfm}.usfm?ref=master`;
   const usfmRes = await fetch(usfmUrl);
   if (!usfmRes.ok) throw new Error(`Failed to download USFM file for ${book}`);
   const usfmData = await usfmRes.json();
-  const usfmContent = Buffer.from(usfmData.content, 'base64').toString('utf-8');
+  const usfmContent = decodeBase64(usfmData.content);
 
   // Remove alignments from USFM
   const cleanUsfm = removeAllTagsExceptChapterVerse(usfmContent);

package/src/utils/zipProcessor.js

@@ -9,7 +9,6 @@
  *   const terms = await generateTWTerms();
  */
 
-import AdmZip from 'adm-zip';
 import { BibleBookData } from '../common/books.js';
 
 // Environment detection
@@ -30,18 +29,20 @@ async function getNodeDeps() {
   if (!isNode) return null;
 
   try {
-    const [nodeModule, fsModule, pathModule, urlModule] = await Promise.all([
+    const [nodeModule, fsModule, pathModule, urlModule, admZipModule] = await Promise.all([
       import('node-fetch'),
       import('fs'),
       import('path'),
-      import('url')
+      import('url'),
+      import('adm-zip')
     ]);
 
     return {
       fetch: nodeModule.default,
       fs: fsModule.default,
       path: pathModule.default,
-      fileURLToPath: urlModule.fileURLToPath
+      fileURLToPath: urlModule.fileURLToPath,
+      AdmZip: admZipModule.default
     };
   } catch (error) {
     console.error('Failed to load Node.js dependencies:', error);
@@ -164,6 +165,23 @@ async function cacheTerms(termMap) {
   }
 }
 
+/**
+ * Process zip in Node.js environment using AdmZip
+ */
+function processNodeZip(AdmZip, Buffer, buffer) {
+  const zip = new AdmZip(Buffer.from(buffer));
+  return zip.getEntries().filter(e => e.entryName.match(/^en_tw\/bible\/.*\/.*\.md$/));
+}
+
+/**
+ * Process zip in browser environment using native APIs
+ */
+async function processBrowserZip(buffer) {
+  // For browser compatibility, we'll throw an error for now
+  // This would require a browser-compatible zip library or different approach
+  throw new Error('ZIP processing in browser requires cached terms. Please ensure article_terms.json is available or use Node.js environment for initial processing.');
+}
+
 export async function generateTWTerms() {
   // Try to get cached terms first
   const cachedTerms = await getCachedTerms();
@@ -174,25 +192,29 @@ export async function generateTWTerms() {
   console.log('Downloading TW archive...');
 
   try {
-    // Get appropriate fetch function
-    let fetchFn;
+    // Get appropriate fetch function and zip processing
+    let fetchFn, processZip;
+
     if (isBrowser) {
       fetchFn = window.fetch;
+      // Use browser-compatible zip processing
+      processZip = (buffer) => processBrowserZip(buffer);
     } else if (isNode) {
       const deps = await getNodeDeps();
-      fetchFn = deps?.fetch;
-      if (!fetchFn) throw new Error('Failed to load Node.js dependencies');
+      if (!deps) throw new Error('Failed to load Node.js dependencies');
+      fetchFn = deps.fetch;
+      const AdmZip = deps.AdmZip;
+      const Buffer = (await import('buffer')).Buffer;
+      processZip = (buffer) => processNodeZip(AdmZip, Buffer, buffer);
     }
 
     const res = await fetchFn(ZIP_URL);
     if (!res.ok) throw new Error(`Failed to download zip: ${res.status} ${res.statusText}`);
 
     const buffer = await res.arrayBuffer();
-    const zip = new AdmZip(Buffer.from(buffer));
+    const entries = await processZip(buffer);
 
     console.log('Processing TW articles...');
-
-    const entries = zip.getEntries().filter(e => e.entryName.match(/^en_tw\/bible\/.*\/.*\.md$/));
     entries.sort((a, b) => a.entryName.localeCompare(b.entryName));
 
     const termMap = {};