twl-generator 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 TWL Generator Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,155 @@
+# twl-generator
+
+Generate term-to-article lists from unfoldingWord en_tw archive for Bible books. Works in both Node.js (CLI) and React.js (browser) environments with intelligent caching.
+
+## Features
+
+- ✅ **Universal**: Works in Node.js and browser environments
+- ✅ **Smart Caching**: File system (Node.js) or localStorage/sessionStorage (browser)
+- ✅ **Performance**: Optimized matching with PrefixTrie algorithm
+- ✅ **Case Sensitivity**: Proper God/god distinction (God→kt/god, god→kt/falsegod)
+- ✅ **Morphological Variants**: Handles plurals, possessives, verb forms
+- ✅ **Parentheses Normalization**: "Joseph (OT)" → "Joseph" for better coverage
+
+## Usage
+
+### CLI (Node.js)
+
+```bash
+# Install globally
+npm install -g twl-generator
+
+# Generate TSV for a Bible book
+twl-generator --book JHN --output john.tsv
+
+# Process local USFM file
+twl-generator --usfm my-file.usfm --output results.tsv
+```
+
+### React.js / Browser
+
+```jsx
+import { generateTWTerms, getCacheInfo, clearCache } from 'twl-generator/src/utils/zipProcessor.js';
+
+function MyTWLComponent() {
+  const [terms, setTerms] = useState(null);
+  const [loading, setLoading] = useState(false);
+
+  const loadTerms = async (book) => {
+    setLoading(true);
+    try {
+      // First load: Downloads and processes (~3-4 seconds)
+      // Subsequent loads: Uses browser cache (~instant)
+      const termData = await generateTWTerms(book);
+      setTerms(termData);
+
+      // Debug cache info
+      console.log('Cache info:', getCacheInfo());
+    } catch (error) {
+      console.error('Failed to load terms:', error);
+    }
+    setLoading(false);
+  };
+
+  const handleClearCache = async () => {
+    await clearCache();
+    console.log('Cache cleared - next load will download fresh data');
+  };
+
+  return (
+    <div>
+      {loading && <p>Loading translation words...</p>}
+      <button onClick={() => loadTerms('JHN')}>Load John</button>
+      <button onClick={() => loadTerms('RUT')}>Load Ruth</button>
+      <button onClick={handleClearCache}>Clear Cache</button>
+      {terms && <p>Loaded {Object.keys(terms).length} terms</p>}
+    </div>
+  );
+}
+```
+
+### Node.js Module
+
+```js
+import { generateTWL } from 'twl-generator';
+
+const result = await generateTWL('RUT');
+console.log(result);
+```
+
+## Browser Caching Strategy
+
+The package uses a multi-tier caching approach for optimal performance in React.js:
+
+1. **Memory Cache**: Fastest access during current session
+2. **localStorage**: Persistent across browser sessions
+3. **sessionStorage**: Fallback for private browsing mode
+4. **Auto-regeneration**: Downloads fresh data when cache is invalid
+
+### Cache Performance
+
+- **Cold start** (no cache): ~3-4 seconds
+- **Warm start** (browser cache): ~50-100ms
+- **Hot start** (memory cache): ~1-5ms
+
+## API Reference
+
+### `generateTWTerms(book)`
+
+Generate terms for a Bible book with caching.
+
+- **book**: Bible book code (e.g., 'JHN', 'RUT', 'GEN')
+- **Returns**: Promise<Object> - Term mapping object
+
+### `clearCache()`
+
+Clear all caches and force fresh download on next call.
+
+- **Returns**: Promise<boolean> - Success status
+
+### `getCacheInfo()`
+
+Get cache status for debugging.
+
+- **Returns**: Object with cache details
+
+## Installation
+
+```bash
+# Global installation (for CLI usage)
+npm install -g twl-generator
+
+# Local installation (for React.js projects)
+npm install twl-generator
+```
+
+## Supported Bible Books
+
+All standard Bible book abbreviations are supported:
+
+**Old Testament**: GEN, EXO, LEV, NUM, DEU, JOS, JDG, RUT, 1SA, 2SA, 1KI, 2KI, 1CH, 2CH, EZR, NEH, EST, JOB, PSA, PRO, ECC, SNG, ISA, JER, LAM, EZK, DAN, HOS, JOL, AMO, OBA, JON, MIC, NAH, HAB, ZEP, HAG, ZEC, MAL
+
+**New Testament**: MAT, MRK, LUK, JHN, ACT, ROM, 1CO, 2CO, GAL, EPH, PHP, COL, 1TH, 2TH, 1TI, 2TI, TIT, PHM, HEB, JAS, 1PE, 2PE, 1JN, 2JN, 3JN, JUD, REV
+
+## Output Format
+
+The generated TSV contains these columns:
+
+- **Reference**: Chapter:verse (e.g., "1:1")
+- **ID**: Unique 4-character hex identifier
+- **Tags**: Article category ("keyterm", "name", or empty)
+- **OrigWords**: The matched text from the source
+- **Occurrence**: Occurrence number for this term in this verse
+- **TWLink**: Link to the translation words article
+- **Disambiguation**: Multiple article options (if applicable)
+- **Context**: Verse text with [matched term] in brackets
+
+## Requirements
+
+- **Node.js**: >=16.0.0
+- **Browser**: Modern browser with ES6 modules support
+- **React.js**: >=16.8.0 (for React usage)
+
+## License
+
+MIT License - see LICENSE file for details.

package/package.json

@@ -0,0 +1,60 @@
+{
+  "name": "twl-generator",
+  "version": "1.0.0",
+  "description": "Generate term-to-article lists from unfoldingWord en_tw archive for Bible books. Works in both Node.js (CLI) and React.js (browser) environments.",
+  "main": "src/index.js",
+  "bin": {
+    "twl-generator": "src/cli.js"
+  },
+  "type": "module",
+  "scripts": {
+    "start": "node src/index.js",
+    "cli": "node src/cli.js",
+    "test": "node src/cli.js --book RUT --output test-output.tsv"
+  },
+  "keywords": [
+    "bible",
+    "twl",
+    "generator",
+    "cli",
+    "react",
+    "browser",
+    "unfoldingword",
+    "translation-words"
+  ],
+  "author": "Rich Mahn <richmahn@example.com>",
+  "license": "MIT",
+  "homepage": "https://github.com/yourusername/twl-generator#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/yourusername/twl-generator.git"
+  },
+  "bugs": {
+    "url": "https://github.com/yourusername/twl-generator/issues"
+  },
+  "files": [
+    "src/cli.js",
+    "src/index.js",
+    "src/common/",
+    "src/utils/twl-matcher.js",
+    "src/utils/zipProcessor.js",
+    "src/utils/usfm-alignment-remover.js",
+    "README.md",
+    "LICENSE"
+  ],
+  "engines": {
+    "node": ">=16.0.0"
+  },
+  "dependencies": {
+    "adm-zip": "^0.5.10",
+    "node-fetch": "^3.3.2"
+  },
+  "peerDependencies": {
+    "react": ">=16.8.0"
+  },
+  "peerDependenciesMeta": {
+    "react": {
+      "optional": true
+    }
+  }
+}

package/src/cli.js

@@ -0,0 +1,86 @@
+#!/usr/bin/env node
+import { generateTWLWithUsfm } from './index.js';
+import fs from 'fs';
+import path from 'path';
+
+const args = process.argv.slice(2);
+
+function printHelp() {
+  console.log(`Usage: generate-twls [options]
+
+Options:
+  --book <book>           Specify the Bible book (e.g., rut)
+  --usfm <path>          Path to USFM file to process
+  --output <path>        Path to output TSV file
+  --help                 Show this help message
+
+Examples:
+  generate-twls --book rut
+  generate-twls --usfm ./41-MAT.usfm --output ./mat_twl.tsv
+  generate-twls --usfm ./file.usfm --book rut`);
+}
+
+let book = null;
+let usfmPath = null;
+let outputPath = null;
+
+for (let i = 0; i < args.length; i++) {
+  if (args[i] === '--book' && args[i + 1]) {
+    book = args[i + 1].toLowerCase();
+    i++;
+  } else if (args[i] === '--usfm' && args[i + 1]) {
+    usfmPath = args[i + 1];
+    i++;
+  } else if (args[i] === '--output' && args[i + 1]) {
+    outputPath = args[i + 1];
+    i++;
+  } else if (args[i] === '--help') {
+    printHelp();
+    process.exit(0);
+  }
+}
+
+// Validate arguments
+if (!book && !usfmPath) {
+  console.error('Error: Either --book or --usfm parameter is required');
+  printHelp();
+  process.exit(1);
+}
+
+if (usfmPath && !fs.existsSync(usfmPath)) {
+  console.error(`Error: USFM file not found: ${usfmPath}`);
+  process.exit(1);
+}
+
+(async () => {
+  try {
+    let usfmContent = null;
+    if (usfmPath) {
+      usfmContent = fs.readFileSync(usfmPath, 'utf8');
+      console.log(`Reading USFM from: ${usfmPath}`);
+    }
+
+    const tsv = await generateTWLWithUsfm(book, usfmContent);
+
+    // Determine output filename
+    let filename;
+    if (outputPath) {
+      filename = outputPath;
+    } else if (book) {
+      filename = `twl_${book.toUpperCase()}.tsv`;
+    } else if (usfmPath) {
+      const baseName = path.basename(usfmPath, path.extname(usfmPath));
+      filename = `${baseName}.tsv`;
+    } else {
+      filename = 'output.tsv';
+    }
+
+    // Save TSV to file
+    fs.writeFileSync(filename, tsv, 'utf8');
+    console.log(`TSV file saved as ${filename}`);
+    console.log(`Found ${tsv.split('\n').length - 1} matches`);
+  } catch (error) {
+    console.error('Error:', error.message);
+    process.exit(1);
+  }
+})();

package/src/common/books.js

@@ -0,0 +1,180 @@
+export const BibleBookData = {
+  "gen": { "id": "gen", "title": "Genesis", "usfm": "01-GEN", "testament": "old", "verseCount": 1533, "chapters": [31, 25, 24, 26, 32, 22, 24, 22, 29, 32, 32, 20, 18, 24, 21, 16, 27, 33, 38, 18, 34, 24, 20, 67, 34, 35, 46, 22, 35, 43, 55, 32, 20, 31, 29, 43, 36, 30, 23, 23, 57, 38, 34, 34, 28, 34, 31, 22, 33, 26] },
+  "exo": { "id": "exo", "title": "Exodus", "usfm": "02-EXO", "testament": "old", "verseCount": 1213, "chapters": [22, 25, 22, 31, 23, 30, 25, 32, 35, 29, 10, 51, 22, 31, 27, 36, 16, 27, 25, 26, 36, 31, 33, 18, 40, 37, 21, 43, 46, 38, 18, 35, 23, 35, 35, 38, 29, 31, 43, 38] },
+  "lev": { "id": "lev", "title": "Leviticus", "usfm": "03-LEV", "testament": "old", "verseCount": 859, "chapters": [17, 16, 17, 35, 19, 30, 38, 36, 24, 20, 47, 8, 59, 57, 33, 34, 16, 30, 37, 27, 24, 33, 44, 23, 55, 46, 34] },
+  "num": { "id": "num", "title": "Numbers", "usfm": "04-NUM", "testament": "old", "verseCount": 1288, "chapters": [54, 34, 51, 49, 31, 27, 89, 26, 23, 36, 35, 16, 33, 45, 41, 50, 13, 32, 22, 29, 35, 41, 30, 25, 18, 65, 23, 31, 40, 16, 54, 42, 56, 29, 34, 13] },
+  "deu": { "id": "deu", "title": "Deuteronomy", "usfm": "05-DEU", "testament": "old", "verseCount": 959, "chapters": [46, 37, 29, 49, 33, 25, 26, 20, 29, 22, 32, 32, 18, 29, 23, 22, 20, 22, 21, 20, 23, 30, 25, 22, 19, 19, 26, 68, 29, 20, 30, 52, 29, 12] },
+  "jos": { "id": "jos", "title": "Joshua", "usfm": "06-JOS", "testament": "old", "verseCount": 658, "chapters": [18, 24, 17, 24, 15, 27, 26, 35, 27, 43, 23, 24, 33, 15, 63, 10, 18, 28, 51, 9, 45, 34, 16, 33] },
+  "jdg": { "id": "jdg", "title": "Judges", "usfm": "07-JDG", "testament": "old", "verseCount": 618, "chapters": [36, 23, 31, 24, 31, 40, 25, 35, 57, 18, 40, 15, 25, 20, 20, 31, 13, 31, 30, 48, 25] },
+  "rut": { "id": "rut", "title": "Ruth", "usfm": "08-RUT", "testament": "old", "verseCount": 85, "chapters": [22, 23, 18, 22] },
+  "1sa": { "id": "1sa", "title": "1 Samuel", "usfm": "09-1SA", "testament": "old", "verseCount": 810, "chapters": [28, 36, 21, 22, 12, 21, 17, 22, 27, 27, 15, 25, 23, 52, 35, 23, 58, 30, 24, 42, 15, 23, 29, 22, 44, 25, 12, 25, 11, 31, 13] },
+  "2sa": { "id": "2sa", "title": "2 Samuel", "usfm": "10-2SA", "testament": "old", "verseCount": 695, "chapters": [27, 32, 39, 12, 25, 23, 29, 18, 13, 19, 27, 31, 39, 33, 37, 23, 29, 33, 43, 26, 22, 51, 39, 25] },
+  "1ki": { "id": "1ki", "title": "1 Kings", "usfm": "11-1KI", "testament": "old", "verseCount": 816, "chapters": [53, 46, 28, 34, 18, 38, 51, 66, 28, 29, 43, 33, 34, 31, 34, 34, 24, 46, 21, 43, 29, 53] },
+  "2ki": { "id": "2ki", "title": "2 Kings", "usfm": "12-2KI", "testament": "old", "verseCount": 719, "chapters": [18, 25, 27, 44, 27, 33, 20, 29, 37, 36, 21, 21, 25, 29, 38, 20, 41, 37, 37, 21, 26, 20, 37, 20, 30] },
+  "1ch": { "id": "1ch", "title": "1 Chronicles", "usfm": "13-1CH", "testament": "old", "verseCount": 942, "chapters": [54, 55, 24, 43, 26, 81, 40, 40, 44, 14, 47, 40, 14, 17, 29, 43, 27, 17, 19, 8, 30, 19, 32, 31, 31, 32, 34, 21, 30] },
+  "2ch": { "id": "2ch", "title": "2 Chronicles", "usfm": "14-2CH", "testament": "old", "verseCount": 822, "chapters": [17, 18, 17, 22, 14, 42, 22, 18, 31, 19, 23, 16, 22, 15, 19, 14, 19, 34, 11, 37, 20, 12, 21, 27, 28, 23, 9, 27, 36, 27, 21, 33, 25, 33, 27, 23] },
+  "ezr": { "id": "ezr", "title": "Ezra", "usfm": "15-EZR", "testament": "old", "verseCount": 280, "chapters": [11, 70, 13, 24, 17, 22, 28, 36, 15, 44] },
+  "neh": { "id": "neh", "title": "Nehemiah", "usfm": "16-NEH", "testament": "old", "verseCount": 406, "chapters": [11, 20, 32, 23, 19, 19, 73, 18, 38, 39, 36, 47, 31] },
+  "est": { "id": "est", "title": "Esther", "usfm": "17-EST", "testament": "old", "verseCount": 167, "chapters": [22, 23, 15, 17, 14, 14, 10, 17, 32, 3] },
+  "job": { "id": "job", "title": "Job", "usfm": "18-JOB", "testament": "old", "verseCount": 1070, "chapters": [22, 13, 26, 21, 27, 30, 21, 22, 35, 22, 20, 25, 28, 22, 35, 22, 16, 21, 29, 29, 34, 30, 17, 25, 6, 14, 23, 28, 25, 31, 40, 22, 33, 37, 16, 33, 24, 41, 30, 24, 34, 17] },
+  "psa": { "id": "psa", "title": "Psalms", "usfm": "19-PSA", "testament": "old", "verseCount": 2461, "chapters": [6, 12, 8, 8, 12, 10, 17, 9, 20, 18, 7, 8, 6, 7, 5, 11, 15, 50, 14, 9, 13, 31, 6, 10, 22, 12, 14, 9, 11, 12, 24, 11, 22, 22, 28, 12, 40, 22, 13, 17, 13, 11, 5, 26, 17, 11, 9, 14, 20, 23, 19, 9, 6, 7, 23, 13, 11, 11, 17, 12, 8, 12, 11, 10, 13, 20, 7, 35, 36, 5, 24, 20, 28, 23, 10, 12, 20, 72, 13, 19, 16, 8, 18, 12, 13, 17, 7, 18, 52, 17, 16, 15, 5, 23, 11, 13, 12, 9, 9, 5, 8, 28, 22, 35, 45, 48, 43, 13, 31, 7, 10, 10, 9, 8, 18, 19, 2, 29, 176, 7, 8, 9, 4, 8, 5, 6, 5, 6, 8, 8, 3, 18, 3, 3, 21, 26, 9, 8, 24, 13, 10, 7, 12, 15, 21, 10, 20, 14, 9, 6] },
+  "pro": { "id": "pro", "title": "Proverbs", "usfm": "20-PRO", "testament": "old", "verseCount": 915, "chapters": [33, 22, 35, 27, 23, 35, 27, 36, 18, 32, 31, 28, 25, 35, 33, 33, 28, 24, 29, 30, 31, 29, 35, 34, 28, 28, 27, 28, 27, 33, 31] },
+  "ecc": { "id": "ecc", "title": "Ecclesiastes", "usfm": "21-ECC", "testament": "old", "verseCount": 222, "chapters": [18, 26, 22, 16, 20, 12, 29, 17, 18, 20, 10, 14] },
+  "sng": { "id": "sng", "title": "Song of Songs", "usfm": "22-SNG", "testament": "old", "verseCount": 117, "chapters": [17, 17, 11, 16, 16, 13, 13, 14] },
+  "isa": { "id": "isa", "title": "Isaiah", "usfm": "23-ISA", "testament": "old", "verseCount": 1292, "chapters": [31, 22, 26, 6, 30, 13, 25, 22, 21, 34, 16, 6, 22, 32, 9, 14, 14, 7, 25, 6, 17, 25, 18, 23, 12, 21, 13, 29, 24, 33, 9, 20, 24, 17, 10, 22, 38, 22, 8, 31, 29, 25, 28, 28, 25, 13, 15, 22, 26, 11, 23, 15, 12, 17, 13, 12, 21, 14, 21, 22, 11, 12, 19, 12, 25, 24] },
+  "jer": { "id": "jer", "title": "Jeremiah", "usfm": "24-JER", "testament": "old", "verseCount": 1364, "chapters": [19, 37, 25, 31, 31, 30, 34, 22, 26, 25, 23, 17, 27, 22, 21, 21, 27, 23, 15, 18, 14, 30, 40, 10, 38, 24, 22, 17, 32, 24, 40, 44, 26, 22, 19, 32, 21, 28, 18, 16, 18, 22, 13, 30, 5, 28, 7, 47, 39, 46, 64, 34] },
+  "lam": { "id": "lam", "title": "Lamentations", "usfm": "25-LAM", "testament": "old", "verseCount": 154, "chapters": [22, 22, 66, 22, 22] },
+  "ezk": { "id": "ezk", "title": "Ezekiel", "usfm": "26-EZK", "testament": "old", "verseCount": 1273, "chapters": [28, 10, 27, 17, 17, 14, 27, 18, 11, 22, 25, 28, 23, 23, 8, 63, 24, 32, 14, 49, 32, 31, 49, 27, 17, 21, 36, 26, 21, 26, 18, 32, 33, 31, 15, 38, 28, 23, 29, 49, 26, 20, 27, 31, 25, 24, 23, 35] },
+  "dan": { "id": "dan", "title": "Daniel", "usfm": "27-DAN", "testament": "old", "verseCount": 357, "chapters": [21, 49, 30, 37, 31, 28, 28, 27, 27, 21, 45, 13] },
+  "hos": { "id": "hos", "title": "Hosea", "usfm": "28-HOS", "testament": "old", "verseCount": 197, "chapters": [11, 23, 5, 19, 15, 11, 16, 14, 17, 15, 12, 14, 16, 9] },
+  "jol": { "id": "jol", "title": "Joel", "usfm": "29-JOL", "testament": "old", "verseCount": 73, "chapters": [20, 32, 21] },
+  "amo": { "id": "amo", "title": "Amos", "usfm": "30-AMO", "testament": "old", "verseCount": 146, "chapters": [15, 16, 15, 13, 27, 14, 17, 14, 15] },
+  "oba": { "id": "oba", "title": "Obadiah", "usfm": "31-OBA", "testament": "old", "verseCount": 21, "chapters": [21] },
+  "jon": { "id": "jon", "title": "Jonah", "usfm": "32-JON", "testament": "old", "verseCount": 48, "chapters": [17, 10, 10, 11] },
+  "mic": { "id": "mic", "title": "Micah", "usfm": "33-MIC", "testament": "old", "verseCount": 105, "chapters": [16, 13, 12, 13, 15, 16, 20] },
+  "nam": { "id": "nam", "title": "Nahum", "usfm": "34-NAM", "testament": "old", "verseCount": 47, "chapters": [15, 13, 19] },
+  "hab": { "id": "hab", "title": "Habakkuk", "usfm": "35-HAB", "testament": "old", "verseCount": 56, "chapters": [17, 20, 19] },
+  "zep": { "id": "zep", "title": "Zephaniah", "usfm": "36-ZEP", "testament": "old", "verseCount": 53, "chapters": [18, 15, 20] },
+  "hag": { "id": "hag", "title": "Haggai", "usfm": "37-HAG", "testament": "old", "verseCount": 38, "chapters": [15, 23] },
+  "zec": { "id": "zec", "title": "Zechariah", "usfm": "38-ZEC", "testament": "old", "verseCount": 211, "chapters": [21, 13, 10, 14, 11, 15, 14, 23, 17, 12, 17, 14, 9, 21] },
+  "mal": { "id": "mal", "title": "Malachi", "usfm": "39-MAL", "testament": "old", "verseCount": 55, "chapters": [14, 17, 18, 6] },
+  "mat": { "id": "mat", "title": "Matthew", "usfm": "41-MAT", "testament": "new", "verseCount": 1071, "chapters": [25, 23, 17, 25, 48, 34, 29, 34, 38, 42, 30, 50, 58, 36, 39, 28, 27, 35, 30, 34, 46, 46, 39, 51, 46, 75, 66, 20] },
+  "mrk": { "id": "mrk", "title": "Mark", "usfm": "42-MRK", "testament": "new", "verseCount": 678, "chapters": [45, 28, 35, 41, 43, 56, 37, 38, 50, 52, 33, 44, 37, 72, 47, 20] },
+  "luk": { "id": "luk", "title": "Luke", "usfm": "43-LUK", "testament": "new", "verseCount": 1151, "chapters": [80, 52, 38, 44, 39, 49, 50, 56, 62, 42, 54, 59, 35, 35, 32, 31, 37, 43, 48, 47, 38, 71, 56, 53] },
+  "jhn": { "id": "jhn", "title": "John", "usfm": "44-JHN", "testament": "new", "verseCount": 879, "chapters": [51, 25, 36, 54, 47, 71, 53, 59, 41, 42, 57, 50, 38, 31, 27, 33, 26, 40, 42, 31, 25] },
+  "act": { "id": "act", "title": "Acts", "usfm": "45-ACT", "testament": "new", "verseCount": 1007, "chapters": [26, 47, 26, 37, 42, 15, 60, 40, 43, 48, 30, 25, 52, 28, 41, 40, 34, 28, 41, 38, 40, 30, 35, 27, 27, 32, 44, 31] },
+  "rom": { "id": "rom", "title": "Romans", "usfm": "46-ROM", "testament": "new", "verseCount": 433, "chapters": [32, 29, 31, 25, 21, 23, 25, 39, 33, 21, 36, 21, 14, 23, 33, 27] },
+  "1co": { "id": "1co", "title": "1 Corinthians", "usfm": "47-1CO", "testament": "new", "verseCount": 437, "chapters": [31, 16, 23, 21, 13, 20, 40, 13, 27, 33, 34, 31, 13, 40, 58, 24] },
+  "2co": { "id": "2co", "title": "2 Corinthians", "usfm": "48-2CO", "testament": "new", "verseCount": 257, "chapters": [24, 17, 18, 18, 21, 18, 16, 24, 15, 18, 33, 21, 14] },
+  "gal": { "id": "gal", "title": "Galatians", "usfm": "49-GAL", "testament": "new", "verseCount": 149, "chapters": [24, 21, 29, 31, 26, 18] },
+  "eph": { "id": "eph", "title": "Ephesians", "usfm": "50-EPH", "testament": "new", "verseCount": 155, "chapters": [23, 22, 21, 32, 33, 24] },
+  "php": { "id": "php", "title": "Philippians", "usfm": "51-PHP", "testament": "new", "verseCount": 104, "chapters": [30, 30, 21, 23] },
+  "col": { "id": "col", "title": "Colossians", "usfm": "52-COL", "testament": "new", "verseCount": 95, "chapters": [29, 23, 25, 18] },
+  "1th": { "id": "1th", "title": "1 Thessalonians", "usfm": "53-1TH", "testament": "new", "verseCount": 89, "chapters": [10, 20, 13, 18, 28] },
+  "2th": { "id": "2th", "title": "2 Thessalonians", "usfm": "54-2TH", "testament": "new", "verseCount": 47, "chapters": [12, 17, 18] },
+  "1ti": { "id": "1ti", "title": "1 Timothy", "usfm": "55-1TI", "testament": "new", "verseCount": 113, "chapters": [20, 15, 16, 16, 25, 21] },
+  "2ti": { "id": "2ti", "title": "2 Timothy", "usfm": "56-2TI", "testament": "new", "verseCount": 83, "chapters": [18, 26, 17, 22] },
+  "tit": { "id": "tit", "title": "Titus", "usfm": "57-TIT", "testament": "new", "verseCount": 46, "chapters": [16, 15, 15] },
+  "phm": { "id": "phm", "title": "Philemon", "usfm": "58-PHM", "testament": "new", "verseCount": 25, "chapters": [25] },
+  "heb": { "id": "heb", "title": "Hebrews", "usfm": "59-HEB", "testament": "new", "verseCount": 303, "chapters": [14, 18, 19, 16, 14, 20, 28, 13, 28, 39, 40, 29, 25] },
+  "jas": { "id": "jas", "title": "James", "usfm": "60-JAS", "testament": "new", "verseCount": 108, "chapters": [27, 26, 18, 17, 20] },
+  "1pe": { "id": "1pe", "title": "1 Peter", "usfm": "61-1PE", "testament": "new", "verseCount": 105, "chapters": [25, 25, 22, 19, 14] },
+  "2pe": { "id": "2pe", "title": "2 Peter", "usfm": "62-2PE", "testament": "new", "verseCount": 61, "chapters": [21, 22, 18] },
+  "1jn": { "id": "1jn", "title": "1 John", "usfm": "63-1JN", "testament": "new", "verseCount": 105, "chapters": [10, 29, 24, 21, 21] },
+  "2jn": { "id": "2jn", "title": "2 John", "usfm": "64-2JN", "testament": "new", "verseCount": 13, "chapters": [13] },
+  "3jn": { "id": "3jn", "title": "3 John", "usfm": "65-3JN", "testament": "new", "verseCount": 15, "chapters": [15] },
+  "jud": { "id": "jud", "title": "Jude", "usfm": "66-JUD", "testament": "new", "verseCount": 25, "chapters": [25] },
+  "rev": { "id": "rev", "title": "Revelation", "usfm": "67-REV", "testament": "new", "verseCount": 404, "chapters": [20, 29, 22, 11, 14, 17, 17, 13, 21, 11, 19, 17, 18, 20, 8, 21, 18, 24, 21, 15, 27, 21] }
+}
+
+const oftenMissingBCVList = [
+  // See https://en.wikipedia.org/wiki/List_of_New_Testament_verses_not_included_in_modern_English_translations
+  ['NEH', 7, 68], // ?
+  ['MAT', 16, 3],
+  ['MAT', 17, 21],
+  ['MAT', 18, 11],
+  // ['MAT', 20, 16b],
+  ['MAT', 23, 14],
+  // ['MRK', 6, 11b],
+  ['MRK', 7, 16],
+  ['MRK', 9, 44], ['MRK', 9, 46],
+  ['MRK', 11, 26],
+  ['MRK', 15, 28],
+  ['MRK', 16, 9], ['MRK', 16, 10], ['MRK', 16, 11], ['MRK', 16, 12], ['MRK', 16, 13], ['MRK', 16, 14],
+  ['MRK', 16, 15], ['MRK', 16, 16], ['MRK', 16, 17], ['MRK', 16, 18], ['MRK', 16, 19], ['MRK', 16, 20],
+  // ['LUK', 4, 8b],
+  // ['LUK', 9, 55], ['LUK', 9, 56],
+  ['LUK', 17, 36],
+  ['LUK', 22, 43], // ?
+  ['LUK', 22, 44], // ?
+  ['LUK', 23, 17],
+  ['JHN', 5, 3], ['JHN', 5, 4],
+  ['JHN', 7, 53], // ?
+  ['JHN', 8, 1], // ?
+  ['ACT', 8, 37],
+  // ['ACT', 9, 5], ['ACT', 9, 6],
+  // ['ACT', 13, 42],
+  ['ACT', 15, 34],
+  // ['ACT', 23, 9b],
+  ['ACT', 24, 6], ['ACT', 24, 7], ['ACT', 24, 8],
+  ['ACT', 28, 29],
+  ['ROM', 16, 24],
+  ['2CO', 13, 14], // ?
+  ['1JN', 5, 7], ['1JN', 5, 8],
+];
+
+
+const extraBookList = ['FRT', 'BAK'];
+export const isValidBookID = (bookId) => {
+  return bookId.toLowerCase() in BibleBookData || extraBookList.includes(bookId);
+}
+export const isOptionalValidBookID = (bookId) => {
+  return !bookId || bookId.toLowerCase() in BibleBookData || extraBookList.includes(bookId);
+}
+export const isExtraBookID = (bookId) => {
+  return extraBookList.includes(bookId);
+}
+
+
+export const usfmNumberName = (bookId) => {
+  try { return BibleBookData[bookId.toLowerCase()].usfm; }
+  catch (err) { throw new Error(`usfmNumberName() given invalid bookId: '${bookId}'`); }
+}
+
+export const chaptersInBook = (bookId) => {
+  // parameterAssert(bookId.toLowerCase() !== 'obs', `chaptersInBook shouldn’t be passed '${bookId}'`);
+  let chapters;
+  try {
+    chapters = BibleBookData[bookId.toLowerCase()].chapters;
+  } catch (err) {
+    throw new Error(`chaptersInBook() given invalid bookId: '${bookId}'`);
+  }
+  if (chapters === undefined) {
+    throw new Error(`chaptersInBook(): Invalid bookId: '${bookId}'`);
+  }
+  return chapters;
+};
+
+export const isOneChapterBook = (bookId) => {
+  return chaptersInBook(bookId).length === 1;
+};
+
+export const versesInChapter = (bookId, chapter) => {
+  // parameterAssert(bookId.toLowerCase() !== 'obs', `versesInChapter shouldn’t be passed '${bookId}'`);
+  const verses = chaptersInBook(bookId)[chapter - 1];
+  if (verses === undefined) {
+    throw new Error(`versesInChapter(${bookId}) given invalid chapter: ${chapter}`);
+  }
+  return verses;
+};
+
+export const testament = (bookId) => {
+  const _testament = BibleBookData[bookId.toLowerCase()].testament;
+  return _testament;
+};
+
+export function getEnglishBookName(bookId) {
+  return BibleBookData[bookId.toLowerCase()].title;
+}
+
+export function isGoodEnglishBookName(givenBookName) {
+  // debugLog(`isGoodEnglishBookName(${givenBookName})…`);
+  const partialMatches = [];
+  const givenBookNameLower = givenBookName.toLowerCase();
+  for (const bk in BibleBookData) {
+    const thisBookName = BibleBookData[bk].title;
+    // debugLog("thisBookName", thisBookName);
+    if (thisBookName === givenBookName) return true;
+    const thisBookNameLower = thisBookName.toLowerCase();
+    if (thisBookNameLower === givenBookNameLower) return 1;
+    if (thisBookNameLower.startsWith(givenBookNameLower)) partialMatches.push(thisBookName);
+  }
+  if (partialMatches.length === 1) return 2; // We got an unambiguous partial match, e.g., Gen for Genesis
+  return false;
+}
+
+export function isOftenMissing(bookID, C, V) {
+  function matchBCV(entry) { return entry[0] === bookID && entry[1] === C && entry[2] === V; }
+
+  return oftenMissingBCVList.find(matchBCV) !== undefined;
+}

package/src/index.js

@@ -0,0 +1,31 @@
+// Main module for twl-generator
+import { generateTWTerms } from './utils/zipProcessor.js';
+import { processUsfmForBook, parseUsfmToVerses } from './utils/usfm-alignment-remover.js';
+import { generateTWLMatches } from './utils/twl-matcher.js';
+
+export { generateTWTerms, processUsfmForBook };
+
+/**
+ * Main function that processes both TW articles and USFM file
+ * @param {string} book - The book identifier (optional if usfmContent is provided)
+ * @param {string} usfmContent - Optional USFM content to process instead of fetching
+ * @return {Promise<string>} - TSV string
+ */
+export async function generateTWLWithUsfm(book, usfmContent = null) {
+  // Generate TW terms (with caching)
+  const terms = await generateTWTerms(book || 'gen'); // Use 'gen' as fallback for terms generation
+
+  let verses;
+  if (usfmContent) {
+    // Parse provided USFM content
+    verses = parseUsfmToVerses(usfmContent);
+  } else {
+    // Fetch USFM from git.door43.org
+    if (!book) throw new Error('Book parameter required when no USFM content provided');
+    verses = await processUsfmForBook(book);
+  }
+
+  // Generate TWL matches and return TSV
+  const tsv = generateTWLMatches(terms, verses);
+  return tsv;
+}