synset 0.9.0 → 0.9.2

package/dist/index.cjs

@@ -46,6 +46,7 @@ __export(src_exports, {
   PartsOfSpeech: () => PartsOfSpeech,
   PartsOfSpeechLabels: () => PartsOfSpeech2,
   Pronunciation: () => Pronunciation,
+  SCHEMA: () => SCHEMA,
   Sense: () => Sense,
   SenseId: () => SenseId,
   SenseRelation: () => SenseRelation,
@@ -65,6 +66,7 @@ __export(src_exports, {
   createParser: () => createParser,
   decodeXmlEntities: () => decodeXmlEntities,
   ensureWordNetCached: () => ensureWordNetCached,
+  exportToSQLite: () => exportToSQLite,
   fetchWordNet: () => fetchWordNet,
   findLatestVersion: () => findLatestVersion,
   findSenses: () => findSenses,
@@ -88,6 +90,9 @@ __export(src_exports, {
 });
 module.exports = __toCommonJS(src_exports);
 
+// src/export-sqlite.ts
+var import_bun_sqlite = require("bun:sqlite");
+
 // src/types.ts
 var import_zod = require("zod");
 var LexiconId = import_zod.z.string();
@@ -253,7 +258,290 @@ var Lexicon = import_zod.z.object({
   synsets: import_zod.z.array(Synset).min(0),
   syntacticBehaviors: import_zod.z.array(SyntacticBehavior).min(0)
 });
-var partsOfSpeechList = PartsOfSpeech.options.map((v) => v.value);
+var partsOfSpeechList = PartsOfSpeech.options.map(
+  (v) => v.value
+);
+
+// src/helpers.ts
+function PronunciationNode(node) {
+  const obj = {
+    variety: optAttr(node, "variety"),
+    inner: node.innerText
+  };
+  return Pronunciation.parse(extendWithRestAttr(node, obj, (s) => s));
+}
+function LemmaNode(node) {
+  const obj = {
+    writtenForm: attr(node, "writtenForm"),
+    partOfSpeech: PartsOfSpeech.parse(attr(node, "partOfSpeech")),
+    pronunciations: (
+      //
+      children(node, "Pronunciation", (v) => PronunciationNode(v))
+    )
+  };
+  return Lemma.parse(extendWithRestAttr(node, obj, (s) => s));
+}
+function SenseRelationNode(node) {
+  const obj = {
+    relType: SenseRelationRelType.parse(attr(node, "relType")),
+    target: attr(node, "target"),
+    dcType: optAttr(node, "dc:type")
+  };
+  return SenseRelation.parse(
+    extendWithRestAttr(node, obj, (s) => s === "dc:type" ? "dcType" : s)
+  );
+}
+function SenseNode(node) {
+  const adjPos = optAttr(node, "adjposition");
+  const obj = {
+    id: attr(node, "id"),
+    synset: SynsetId.parse(attr(node, "synset")),
+    senseRelations: children(node, "SenseRelation", SenseRelationNode),
+    subCat: optAttr(node, "subcat"),
+    adjPosition: adjPos ? AdjPosition.parse(adjPos) : void 0
+  };
+  return Sense.parse(
+    extendWithRestAttr(
+      node,
+      obj,
+      (s) => s === "subcat" ? "subCat" : s === "adjposition" ? "adjPosition" : s
+    )
+  );
+}
+function FormNode(node) {
+  const obj = {
+    writtenForm: attr(node, "writtenForm")
+  };
+  return Form.parse(extendWithRestAttr(node, obj, (s) => s));
+}
+function LexicalEntryNode(node) {
+  const obj = {
+    id: attr(node, "id"),
+    lemmas: children(node, "Lemma", LemmaNode),
+    senses: children(node, "Sense", SenseNode),
+    forms: children(node, "Form", FormNode)
+  };
+  return LexicalEntry.parse(extendWithRestAttr(node, obj, (s) => s));
+}
+function DefinitionNode(node) {
+  const obj = {
+    inner: node.innerText
+  };
+  return Definition.parse(extendWithRestAttr(node, obj, (s) => s));
+}
+function ExampleNode(node) {
+  const obj = {
+    inner: node.innerText,
+    dcSource: optAttr(node, "dc:source")
+  };
+  return Example.parse(
+    extendWithRestAttr(node, obj, (s) => s === "dc:source" ? "dcSource" : s)
+  );
+}
+function ILIDefinitionNode(node) {
+  const obj = {
+    inner: node.innerText
+  };
+  return ILIDefinition.parse(extendWithRestAttr(node, obj, (s) => s));
+}
+function SynsetRelationNode(node) {
+  const obj = {
+    relType: SynsetRelationRelType.parse(attr(node, "relType")),
+    target: attr(node, "target")
+  };
+  return SynsetRelation.parse(extendWithRestAttr(node, obj, (s) => s));
+}
+function SyntacticBehaviorNode(node) {
+  const obj = {
+    id: attr(node, "id"),
+    subcategorizationFrame: attr(node, "subcategorizationFrame")
+  };
+  return SyntacticBehavior.parse(extendWithRestAttr(node, obj, (s) => s));
+}
+function SynsetNode(node) {
+  const obj = {
+    id: attr(node, "id"),
+    ili: attr(node, "ili"),
+    lexfile: attr(node, "lexfile"),
+    members: attr(node, "members").split(" "),
+    dcSource: optAttr(node, "dc:source"),
+    partOfSpeech: PartsOfSpeech.parse(attr(node, "partOfSpeech")),
+    definitions: children(node, "Definition", (v) => DefinitionNode(v)),
+    examples: children(node, "Example", (v) => ExampleNode(v)),
+    iliDefinitions: children(node, "ILIDefinition", ILIDefinitionNode),
+    synsetRelations: children(node, "SynsetRelation", SynsetRelationNode)
+  };
+  return Synset.parse(
+    extendWithRestAttr(node, obj, (s) => s === "dc:source" ? "dcSource" : s)
+  );
+}
+function LexiconNode(node) {
+  const obj = {
+    id: attr(node, "id"),
+    label: attr(node, "label"),
+    language: attr(node, "language"),
+    email: attr(node, "email"),
+    license: attr(node, "license"),
+    version: attr(node, "version"),
+    citation: optAttr(node, "citation"),
+    url: attr(node, "url"),
+    lexicalEntries: children(node, "LexicalEntry", LexicalEntryNode),
+    synsets: children(node, "Synset", SynsetNode),
+    syntacticBehaviors: (
+      //
+      children(node, "SyntacticBehaviour", SyntacticBehaviorNode)
+    )
+  };
+  return Lexicon.parse(extendWithRestAttr(node, obj, (s) => s));
+}
+var decodeXmlEntities = (s) => {
+  if (s === void 0) return void 0;
+  return s.replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&apos;/g, "'").replace(/&quot;/g, '"');
+};
+var attr = (node, attrName) => {
+  const value = decodeXmlEntities(node.attributes[attrName]);
+  if (value === void 0) {
+    throw new Error(
+      `Missing required attribute "${attrName}" on node "${node.type}"`
+    );
+  }
+  return value;
+};
+var optAttr = (node, attrName) => {
+  return decodeXmlEntities(node.attributes[attrName]);
+};
+var restAttrs = (node, obj, proxy) => {
+  const result = {};
+  Object.keys(node.attributes).filter((a) => !(proxy(a) in obj)).forEach((k) => {
+    result[k] = decodeXmlEntities(node.attributes[k]) ?? node.attributes[k];
+  });
+  return result;
+};
+var extendWithRestAttr = (node, obj, proxy) => {
+  return Object.assign(obj, restAttrs(node, obj, proxy));
+};
+var children = (node, type, fn) => {
+  return node.children.filter((v) => v.type === type).map((v) => fn(v));
+};
+
+// src/export-sqlite.ts
+var SCHEMA = `
+CREATE TABLE IF NOT EXISTS words (
+  id INTEGER PRIMARY KEY,
+  word TEXT NOT NULL,
+  word_display TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_words_word ON words(word);
+
+CREATE TABLE IF NOT EXISTS synsets (
+  id TEXT PRIMARY KEY,
+  pos TEXT NOT NULL,
+  definition TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS word_synsets (
+  word_id INTEGER NOT NULL,
+  synset_id TEXT NOT NULL,
+  PRIMARY KEY (word_id, synset_id)
+);
+CREATE INDEX IF NOT EXISTS idx_ws_word ON word_synsets(word_id);
+`;
+function exportToSQLite(lexicon, outputPath, options = {}) {
+  const { onProgress } = options;
+  const db = new import_bun_sqlite.Database(outputPath, { create: true });
+  db.exec("PRAGMA journal_mode = OFF");
+  db.exec("PRAGMA synchronous = OFF");
+  db.exec(SCHEMA);
+  const wordToEntries = /* @__PURE__ */ new Map();
+  for (const entry of lexicon.lexicalEntries) {
+    const word = entry.lemmas[0]?.writtenForm;
+    if (word) {
+      const lower = word.toLowerCase();
+      const existing = wordToEntries.get(lower) || [];
+      existing.push(entry);
+      wordToEntries.set(lower, existing);
+    }
+  }
+  const synsetMap = /* @__PURE__ */ new Map();
+  for (const synset of lexicon.synsets) {
+    synsetMap.set(synset.id, synset);
+  }
+  const insertWord = db.prepare(
+    "INSERT INTO words (word, word_display) VALUES (?, ?)"
+  );
+  const wordIds = /* @__PURE__ */ new Map();
+  const words = Array.from(wordToEntries.keys()).sort();
+  const totalWords = words.length;
+  db.exec("BEGIN TRANSACTION");
+  let wordId = 0;
+  for (let i = 0; i < words.length; i++) {
+    const word = words[i];
+    const entries = wordToEntries.get(word);
+    if (!entries) continue;
+    const display = entries[0].lemmas[0]?.writtenForm || word;
+    insertWord.run(word, display);
+    wordId++;
+    wordIds.set(word, wordId);
+    if (onProgress && i % 1e4 === 0) {
+      onProgress({ phase: "words", current: i, total: totalWords });
+    }
+  }
+  db.exec("COMMIT");
+  const usedSynsetIds = /* @__PURE__ */ new Set();
+  for (const entries of wordToEntries.values()) {
+    for (const entry of entries) {
+      for (const sense of entry.senses) {
+        usedSynsetIds.add(sense.synset);
+      }
+    }
+  }
+  const insertSynset = db.prepare(
+    "INSERT OR IGNORE INTO synsets (id, pos, definition) VALUES (?, ?, ?)"
+  );
+  const synsetList = Array.from(usedSynsetIds);
+  const totalSynsets = synsetList.length;
+  db.exec("BEGIN TRANSACTION");
+  for (let i = 0; i < synsetList.length; i++) {
+    const synsetId = synsetList[i];
+    const synset = synsetMap.get(synsetId);
+    if (synset) {
+      const def = decodeXmlEntities(synset.definitions[0]?.inner) || "";
+      insertSynset.run(synsetId, synset.partOfSpeech, def);
+    }
+    if (onProgress && i % 1e4 === 0) {
+      onProgress({ phase: "synsets", current: i, total: totalSynsets });
+    }
+  }
+  db.exec("COMMIT");
+  const insertRelation = db.prepare(
+    "INSERT OR IGNORE INTO word_synsets (word_id, synset_id) VALUES (?, ?)"
+  );
+  let relationCount = 0;
+  const totalRelations = Array.from(wordToEntries.values()).reduce(
+    (sum, entries) => sum + entries.reduce((s, e) => s + e.senses.length, 0),
+    0
+  );
+  db.exec("BEGIN TRANSACTION");
+  for (const [word, entries] of wordToEntries) {
+    const wordId2 = wordIds.get(word);
+    if (!wordId2) continue;
+    for (const entry of entries) {
+      for (const sense of entry.senses) {
+        insertRelation.run(wordId2, sense.synset);
+        relationCount++;
+        if (onProgress && relationCount % 1e4 === 0) {
+          onProgress({
+            phase: "relations",
+            current: relationCount,
+            total: totalRelations
+          });
+        }
+      }
+    }
+  }
+  db.exec("COMMIT");
+  db.close();
+}
 
 // src/literals.ts
 var PartsOfSpeech2 = {
@@ -408,6 +696,7 @@ var AdjPosition2 = {
 // src/loader.ts
 var import_node_fs = require("fs");
 var import_node_path = __toESM(require("path"), 1);
+var import_node_stream = require("stream");
 
 // node_modules/@dbushell/xml-streamify/src/node.ts
 var Node = class {
@@ -651,166 +940,6 @@ async function* parse(input, options) {
   return document;
 }
 
-// src/helpers.ts
-function PronunciationNode(node) {
-  const obj = {
-    variety: optAttr(node, "variety"),
-    inner: node.innerText
-  };
-  return Pronunciation.parse(extendWithRestAttr(node, obj, (s) => s));
-}
-function LemmaNode(node) {
-  const obj = {
-    writtenForm: attr(node, "writtenForm"),
-    partOfSpeech: PartsOfSpeech.parse(attr(node, "partOfSpeech")),
-    pronunciations: (
-      //
-      children(node, "Pronunciation", (v) => PronunciationNode(v))
-    )
-  };
-  return Lemma.parse(extendWithRestAttr(node, obj, (s) => s));
-}
-function SenseRelationNode(node) {
-  const obj = {
-    relType: SenseRelationRelType.parse(attr(node, "relType")),
-    target: attr(node, "target"),
-    dcType: optAttr(node, "dc:type")
-  };
-  return SenseRelation.parse(
-    extendWithRestAttr(node, obj, (s) => s == "dc:type" ? "dcType" : s)
-  );
-}
-function SenseNode(node) {
-  const adjPos = optAttr(node, "adjposition");
-  const obj = {
-    id: attr(node, "id"),
-    synset: SynsetId.parse(attr(node, "synset")),
-    senseRelations: children(node, "SenseRelation", SenseRelationNode),
-    subCat: optAttr(node, "subcat"),
-    adjPosition: adjPos ? AdjPosition.parse(adjPos) : void 0
-  };
-  return Sense.parse(
-    extendWithRestAttr(
-      node,
-      obj,
-      (s) => s == "subcat" ? "subCat" : s == "adjposition" ? "adjPosition" : s
-    )
-  );
-}
-function FormNode(node) {
-  const obj = {
-    writtenForm: attr(node, "writtenForm")
-  };
-  return Form.parse(extendWithRestAttr(node, obj, (s) => s));
-}
-function LexicalEntryNode(node) {
-  const obj = {
-    id: attr(node, "id"),
-    lemmas: children(node, "Lemma", LemmaNode),
-    senses: children(node, "Sense", SenseNode),
-    forms: children(node, "Form", FormNode)
-  };
-  return LexicalEntry.parse(extendWithRestAttr(node, obj, (s) => s));
-}
-function DefinitionNode(node) {
-  const obj = {
-    inner: node.innerText
-  };
-  return Definition.parse(extendWithRestAttr(node, obj, (s) => s));
-}
-function ExampleNode(node) {
-  const obj = {
-    inner: node.innerText,
-    dcSource: optAttr(node, "dc:source")
-  };
-  return Example.parse(
-    extendWithRestAttr(node, obj, (s) => s == "dc:source" ? "dcSource" : s)
-  );
-}
-function ILIDefinitionNode(node) {
-  const obj = {
-    inner: node.innerText
-  };
-  return ILIDefinition.parse(extendWithRestAttr(node, obj, (s) => s));
-}
-function SynsetRelationNode(node) {
-  const obj = {
-    relType: SynsetRelationRelType.parse(attr(node, "relType")),
-    target: attr(node, "target")
-  };
-  return SynsetRelation.parse(extendWithRestAttr(node, obj, (s) => s));
-}
-function SyntacticBehaviorNode(node) {
-  const obj = {
-    id: attr(node, "id"),
-    subcategorizationFrame: attr(node, "subcategorizationFrame")
-  };
-  return SyntacticBehavior.parse(extendWithRestAttr(node, obj, (s) => s));
-}
-function SynsetNode(node) {
-  const obj = {
-    id: attr(node, "id"),
-    ili: attr(node, "ili"),
-    lexfile: attr(node, "lexfile"),
-    members: attr(node, "members").split(" "),
-    dcSource: optAttr(node, "dc:source"),
-    partOfSpeech: PartsOfSpeech.parse(attr(node, "partOfSpeech")),
-    definitions: children(node, "Definition", (v) => DefinitionNode(v)),
-    examples: children(node, "Example", (v) => ExampleNode(v)),
-    iliDefinitions: children(node, "ILIDefinition", ILIDefinitionNode),
-    synsetRelations: children(node, "SynsetRelation", SynsetRelationNode)
-  };
-  return Synset.parse(
-    extendWithRestAttr(node, obj, (s) => s == "dc:source" ? "dcSource" : s)
-  );
-}
-function LexiconNode(node) {
-  const obj = {
-    id: attr(node, "id"),
-    label: attr(node, "label"),
-    language: attr(node, "language"),
-    email: attr(node, "email"),
-    license: attr(node, "license"),
-    version: attr(node, "version"),
-    citation: optAttr(node, "citation"),
-    url: attr(node, "url"),
-    lexicalEntries: children(node, "LexicalEntry", LexicalEntryNode),
-    synsets: children(node, "Synset", SynsetNode),
-    syntacticBehaviors: (
-      //
-      children(node, "SyntacticBehaviour", SyntacticBehaviorNode)
-    )
-  };
-  return Lexicon.parse(extendWithRestAttr(node, obj, (s) => s));
-}
-var decodeXmlEntities = (s) => {
-  if (s === void 0) return void 0;
-  return s.replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&apos;/g, "'").replace(/&quot;/g, '"');
-};
-var attr = (node, attrName) => {
-  const value = decodeXmlEntities(node.attributes[attrName]);
-  if (value === void 0) {
-    throw new Error(`Missing required attribute "${attrName}" on node "${node.type}"`);
-  }
-  return value;
-};
-var optAttr = (node, attrName) => {
-  return decodeXmlEntities(node.attributes[attrName]);
-};
-var restAttrs = (node, obj, proxy) => {
-  const result = {};
-  Object.keys(node.attributes).filter((a) => !(proxy(a) in obj)).forEach((k) => {
-    result[k] = decodeXmlEntities(node.attributes[k]) ?? node.attributes[k];
-  });
-  return result;
-};
-var extendWithRestAttr = (node, obj, proxy) => {
-  return Object.assign(obj, restAttrs(node, obj, proxy));
-};
-var children = (node, type, fn) => {
-  return node.children.filter((v) => v.type == type).map((v) => fn(v));
-};
-
 // src/loader.ts
 var BASE_VERSION = "2024";
 function getFilename(version) {
@@ -876,7 +1005,6 @@ async function findLatestVersion(onProgress, cacheDir) {
     for (let year = baseYear + 1; year <= lastReleasableYear; year++) {
       const version = year.toString();
       if (await urlExists(getDownloadUrl(version))) {
-        continue;
       } else {
         return (year - 1).toString();
       }
@@ -897,9 +1025,13 @@ async function downloadWordNet(version, destPath) {
   const url = getDownloadUrl(version);
   const response = await fetch(url);
   if (!response.ok || !response.body) {
-    throw new Error(`Failed to download WordNet ${version}: ${response.statusText}`);
+    throw new Error(
+      `Failed to download WordNet ${version}: ${response.statusText}`
+    );
   }
-  const decompressed = response.body.pipeThrough(new DecompressionStream("gzip"));
+  const decompressed = response.body.pipeThrough(
+    new DecompressionStream("gzip")
+  );
   const arrayBuffer = await new Response(decompressed).arrayBuffer();
   const dir = import_node_path.default.dirname(destPath);
   if (!(0, import_node_fs.existsSync)(dir)) {
@@ -909,8 +1041,9 @@ async function downloadWordNet(version, destPath) {
 }
 function createParser(filePath) {
   const resolvedPath = import_node_path.default.resolve(filePath);
-  const fileUrl = resolvedPath.startsWith("/") ? `file://${resolvedPath}` : `file:///${resolvedPath.replace(/\\/g, "/")}`;
-  return parse(fileUrl, {
+  const nodeStream = (0, import_node_fs.createReadStream)(resolvedPath);
+  const webStream = import_node_stream.Readable.toWeb(nodeStream);
+  return parse(webStream, {
     ignoreDeclaration: false,
     silent: false
   });
@@ -1096,6 +1229,7 @@ function getSynsetWords(index, synset) {
   PartsOfSpeech,
   PartsOfSpeechLabels,
   Pronunciation,
+  SCHEMA,
   Sense,
   SenseId,
   SenseRelation,
@@ -1115,6 +1249,7 @@ function getSynsetWords(index, synset) {
   createParser,
   decodeXmlEntities,
   ensureWordNetCached,
+  exportToSQLite,
   fetchWordNet,
   findLatestVersion,
   findSenses,