synset 0.9.6 → 0.9.8

package/dist/cli.cjs

@@ -24,6 +24,7 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 ));
 
 // src/export-sqlite.ts
+var import_node_fs = require("fs");
 var import_libsql = __toESM(require("libsql"), 1);
 
 // src/helpers.ts
@@ -375,9 +376,11 @@ CREATE TABLE IF NOT EXISTS synsets (
 CREATE TABLE IF NOT EXISTS word_synsets (
   word_id INTEGER NOT NULL,
   synset_id TEXT NOT NULL,
+  sense_order INTEGER NOT NULL DEFAULT 0,
   PRIMARY KEY (word_id, synset_id)
 );
 CREATE INDEX IF NOT EXISTS idx_ws_word ON word_synsets(word_id);
+CREATE INDEX IF NOT EXISTS idx_ws_order ON word_synsets(word_id, sense_order);
 
 CREATE TABLE IF NOT EXISTS synset_relations (
   source_id TEXT NOT NULL,
@@ -397,9 +400,26 @@ CREATE TABLE IF NOT EXISTS sense_relations (
   PRIMARY KEY (source_word_id, source_synset_id, target_word_id, target_synset_id, rel_type)
 );
 CREATE INDEX IF NOT EXISTS idx_sense_rel_source ON sense_relations(source_word_id, source_synset_id);
+
+CREATE TABLE IF NOT EXISTS synset_examples (
+  synset_id TEXT NOT NULL,
+  example TEXT NOT NULL,
+  example_order INTEGER NOT NULL DEFAULT 0,
+  PRIMARY KEY (synset_id, example_order)
+);
+CREATE INDEX IF NOT EXISTS idx_examples_synset ON synset_examples(synset_id);
 `;
 function exportToSQLite(lexicon, outputPath, options = {}) {
-  const { onProgress } = options;
+  const { onProgress, overwrite } = options;
+  if ((0, import_node_fs.existsSync)(outputPath)) {
+    if (overwrite) {
+      (0, import_node_fs.unlinkSync)(outputPath);
+    } else {
+      throw new Error(
+        `File already exists: ${outputPath}. Use --overwrite to replace it.`
+      );
+    }
+  }
   const db = new import_libsql.default(outputPath);
   db.exec("PRAGMA journal_mode = OFF");
   db.exec("PRAGMA synchronous = OFF");
@@ -466,7 +486,7 @@ function exportToSQLite(lexicon, outputPath, options = {}) {
   }
   db.exec("COMMIT");
   const insertWordSynset = db.prepare(
-    "INSERT OR IGNORE INTO word_synsets (word_id, synset_id) VALUES (?, ?)"
+    "INSERT OR IGNORE INTO word_synsets (word_id, synset_id, sense_order) VALUES (?, ?, ?)"
   );
   let wsCount = 0;
   const totalWordSynsets = Array.from(wordToEntries.values()).reduce(
@@ -477,9 +497,11 @@ function exportToSQLite(lexicon, outputPath, options = {}) {
   for (const [word, entries] of wordToEntries) {
     const wId = wordIds.get(word);
     if (!wId) continue;
+    let senseOrder = 0;
     for (const entry of entries) {
       for (const sense of entry.senses) {
-        insertWordSynset.run(wId, sense.synset);
+        insertWordSynset.run(wId, sense.synset, senseOrder);
+        senseOrder++;
         wsCount++;
         if (onProgress && wsCount % 1e4 === 0) {
           onProgress({
@@ -492,6 +514,37 @@ function exportToSQLite(lexicon, outputPath, options = {}) {
     }
   }
   db.exec("COMMIT");
+  const insertExample = db.prepare(
+    "INSERT OR IGNORE INTO synset_examples (synset_id, example, example_order) VALUES (?, ?, ?)"
+  );
+  let totalExamples = 0;
+  for (const synsetId of usedSynsetIds) {
+    const synset = synsetMap.get(synsetId);
+    if (synset?.examples) {
+      totalExamples += synset.examples.length;
+    }
+  }
+  db.exec("BEGIN TRANSACTION");
+  let exCount = 0;
+  for (const synsetId of usedSynsetIds) {
+    const synset = synsetMap.get(synsetId);
+    if (!synset?.examples) continue;
+    for (let i = 0; i < synset.examples.length; i++) {
+      const example = decodeXmlEntities(synset.examples[i].inner);
+      if (example) {
+        insertExample.run(synsetId, example, i);
+        exCount++;
+        if (onProgress && exCount % 1e4 === 0) {
+          onProgress({
+            phase: "synset_examples",
+            current: exCount,
+            total: totalExamples
+          });
+        }
+      }
+    }
+  }
+  db.exec("COMMIT");
   const insertSynsetRelation = db.prepare(
     "INSERT OR IGNORE INTO synset_relations (source_id, target_id, rel_type) VALUES (?, ?, ?)"
   );
@@ -701,7 +754,7 @@ var SynsetRelationRelType2 = {
 };
 
 // src/loader.ts
-var import_node_fs = require("fs");
+var import_node_fs2 = require("fs");
 var import_node_path = __toESM(require("path"), 1);
 var import_node_stream = require("stream");
 
@@ -960,8 +1013,8 @@ function getDefaultCacheDir() {
   return import_node_path.default.join(homeDir, ".cache", "synset");
 }
 function fileExists(filePath) {
-  if ((0, import_node_fs.existsSync)(filePath)) {
-    const stat = (0, import_node_fs.statSync)(filePath);
+  if ((0, import_node_fs2.existsSync)(filePath)) {
+    const stat = (0, import_node_fs2.statSync)(filePath);
     return stat.isFile();
   }
   return false;
@@ -979,8 +1032,8 @@ function extractVersionFromFilename(filename) {
   return match ? parseInt(match[1], 10) : null;
 }
 function findCachedVersion(cacheDir) {
-  if (!(0, import_node_fs.existsSync)(cacheDir)) return null;
-  const files = (0, import_node_fs.readdirSync)(cacheDir);
+  if (!(0, import_node_fs2.existsSync)(cacheDir)) return null;
+  const files = (0, import_node_fs2.readdirSync)(cacheDir);
   const wordnetFiles = files.map((f) => ({ file: f, year: extractVersionFromFilename(f) })).filter((x) => x.year !== null).sort((a, b) => b.year - a.year);
   return wordnetFiles.length > 0 ? wordnetFiles[0].year.toString() : null;
 }
@@ -1041,14 +1094,14 @@ async function downloadWordNet(version, destPath) {
   );
   const arrayBuffer = await new Response(decompressed).arrayBuffer();
   const dir = import_node_path.default.dirname(destPath);
-  if (!(0, import_node_fs.existsSync)(dir)) {
-    (0, import_node_fs.mkdirSync)(dir, { recursive: true });
+  if (!(0, import_node_fs2.existsSync)(dir)) {
+    (0, import_node_fs2.mkdirSync)(dir, { recursive: true });
   }
-  (0, import_node_fs.writeFileSync)(destPath, Buffer.from(arrayBuffer));
+  (0, import_node_fs2.writeFileSync)(destPath, Buffer.from(arrayBuffer));
 }
 function createParser(filePath) {
   const resolvedPath = import_node_path.default.resolve(filePath);
-  const nodeStream = (0, import_node_fs.createReadStream)(resolvedPath);
+  const nodeStream = (0, import_node_fs2.createReadStream)(resolvedPath);
   const webStream = import_node_stream.Readable.toWeb(nodeStream);
   return parse(webStream, {
     ignoreDeclaration: false,
@@ -1223,6 +1276,7 @@ Commands:
 
 Options:
   --file <path>       Use a local WordNet XML file instead of cache
+  --overwrite         Overwrite existing file (for export-sqlite)
   --help, -h          Show this help message
 
 Examples:
@@ -1258,10 +1312,12 @@ async function main() {
       console.error("Error: Missing output path for export-sqlite");
       process.exit(1);
     }
+    const overwrite = args.includes("--overwrite");
     console.log("Loading WordNet data...");
     const lexicon2 = filePath ? await loadWordNet(filePath) : (await fetchWordNet({ onProgress: console.log })).lexicon;
     console.log(`Exporting to ${outputPath}...`);
     exportToSQLite(lexicon2, outputPath, {
+      overwrite,
       onProgress: ({ phase, current, total }) => {
         process.stdout.write(`\r${phase}: ${current}/${total}`);
       }