twl-generator 1.1.0 → 1.1.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "twl-generator",
-  "version": "1.1.0",
+  "version": "1.1.3",
   "description": "Generate term-to-article lists from unfoldingWord en_tw archive for Bible books. Works in both Node.js (CLI) and React.js (browser) environments.",
   "main": "src/index.js",
   "bin": {
@@ -46,7 +46,7 @@
     "node": ">=16.0.0"
   },
   "dependencies": {
-    "adm-zip": "^0.5.10",
+    "jszip": "^3.10.1",
     "node-fetch": "^3.3.2"
   },
   "peerDependencies": {

package/src/index.js

@@ -1,6 +1,6 @@
 // Main module for twl-generator
 import { generateTWTerms } from './utils/zipProcessor.js';
-import { processUsfmForBook, parseUsfmToVerses } from './utils/usfm-alignment-remover.js';
+import { processUsfmForBook, parseUsfmToVerses, removeAllTagsExceptChapterVerse } from './utils/usfm-alignment-remover.js';
 import { generateTWLMatches } from './utils/twl-matcher.js';
 
 export { generateTWTerms, processUsfmForBook };
@@ -17,8 +17,9 @@ export async function generateTWLWithUsfm(book, usfmContent = null) {
 
   let verses;
   if (usfmContent) {
-    // Parse provided USFM content
-    verses = parseUsfmToVerses(usfmContent);
+    // Parse provided USFM content (clean it first)
+    const cleanUsfm = removeAllTagsExceptChapterVerse(usfmContent);
+    verses = parseUsfmToVerses(cleanUsfm);
   } else {
     // Fetch USFM from git.door43.org
     if (!book) throw new Error('Book parameter required when no USFM content provided');

package/src/utils/zipProcessor.js

@@ -22,30 +22,28 @@ const CACHE_VERSION = '1.0';
 let processedTermsCache = null;
 
 /**
- * Get Node.js dependencies dynamically
+ * Get dependencies dynamically (JSZip works in both environments)
  */
-async function getNodeDeps() {
-  if (!isNode) return null;
-
+async function getDeps() {
   try {
-    const [nodeModule, admZipModule] = await Promise.all([
-      import('node-fetch'),
-      import('adm-zip')
-    ]);
-
-    return {
-      fetch: nodeModule.default,
-      AdmZip: admZipModule.default
+    const jsZipModule = await import('jszip');
+    const deps = {
+      JSZip: jsZipModule.default
     };
+
+    // Add Node.js-specific fetch if needed
+    if (isNode) {
+      const nodeModule = await import('node-fetch');
+      deps.fetch = nodeModule.default;
+    }
+
+    return deps;
   } catch (error) {
-    console.error('Failed to load Node.js dependencies:', error);
+    console.error('Failed to load dependencies:', error);
     return null;
   }
 }
 
-/**
- * Get cached ZIP data from appropriate storage
- */
 async function getCachedZip() {
   if (isBrowser) {
     // Browser: Use localStorage for ZIP cache
@@ -208,28 +206,30 @@ async function cacheTerms(termMap) {
  * Process ZIP buffer and extract term mappings
  */
 async function processZipBuffer(zipBuffer) {
-  let zip;
-
-  if (isNode) {
-    // Node.js: Use adm-zip
-    const deps = await getNodeDeps();
-    if (!deps) throw new Error('Failed to load Node.js dependencies');
-    const { AdmZip } = deps;
-    const Buffer = (await import('buffer')).Buffer;
-    zip = new AdmZip(Buffer.from(zipBuffer));
-  } else {
-    // Browser: Use browser-compatible zip processing
-    // For now, we'll use a simple approach that works with the TW archive structure
-    throw new Error('Browser ZIP processing not yet implemented. Use cached data or run initial processing in Node.js.');
-  }
+  // Use JSZip universally for both Node.js and Browser
+  const deps = await getDeps();
+  if (!deps) throw new Error('Failed to load dependencies');
+  const { JSZip } = deps;
+
+  const zip = new JSZip();
+  const zipData = await zip.loadAsync(zipBuffer);
+
+  const entries = [];
+  zipData.forEach((relativePath, file) => {
+    if (relativePath.match(/^en_tw\/bible\/.*\/.*\.md$/) && !file.dir) {
+      entries.push({
+        entryName: relativePath,
+        getData: () => file.async('string') // Return promise for string content
+      });
+    }
+  });
 
-  const entries = zip.getEntries().filter(e => e.entryName.match(/^en_tw\/bible\/.*\/.*\.md$/));
   entries.sort((a, b) => a.entryName.localeCompare(b.entryName));
 
   const termMap = {};
 
   for (const entry of entries) {
-    const content = entry.getData().toString('utf8');
+    const content = await entry.getData(); // Await the async string content
     const firstLine = content.split('\n')[0];
     const terms = firstLine.replace(/^#/, '').trim().split(',').map(t => t.trim()).filter(Boolean);
     const truncated = entry.entryName.replace('en_tw/bible/', '');
@@ -268,7 +268,14 @@ export async function generateTWTerms() {
     // Download fresh ZIP
     console.log('Downloading TW archive...');
 
-    const fetchFn = isBrowser ? window.fetch : (await getNodeDeps())?.fetch;
+    let fetchFn;
+    if (isBrowser) {
+      fetchFn = window.fetch;
+    } else {
+      const deps = await getDeps();
+      fetchFn = deps?.fetch;
+    }
+
     if (!fetchFn) throw new Error('Fetch not available');
 
     const res = await fetchFn(ZIP_URL);