zotero-bridge 1.0.3 → 1.1.1

package/src/database.ts

@@ -10,9 +10,10 @@
  */
 
 import initSqlJs, { Database as SqlJsDatabase } from 'sql.js';
-import { existsSync, readFileSync, writeFileSync } from 'fs';
+import { existsSync, readFileSync, writeFileSync, copyFileSync, unlinkSync, statSync } from 'fs';
 import { homedir } from 'os';
-import { join } from 'path';
+import { join, dirname } from 'path';
+import { execSync } from 'child_process';
 
 export interface ZoteroItem {
   itemID: number;
@@ -48,6 +49,8 @@ export class ZoteroDatabase {
   private dbPath: string;
   private readonly: boolean;
   private SQL: any = null;
+  private backupPath: string | null = null;
+  private hasUnsavedChanges: boolean = false;
 
   constructor(dbPath?: string, readonly: boolean = false) {
     this.dbPath = dbPath || this.findDefaultZoteroDB();
@@ -94,6 +97,58 @@ export class ZoteroDatabase {
     return join(home, 'Zotero', 'zotero.sqlite');
   }
 
+  /**
+   * Check if Zotero is currently running
+   */
+  private isZoteroRunning(): boolean {
+    try {
+      if (process.platform === 'win32') {
+        const result = execSync('tasklist /FI "IMAGENAME eq zotero.exe" /NH', { encoding: 'utf8' });
+        return result.toLowerCase().includes('zotero.exe');
+      } else if (process.platform === 'darwin') {
+        const result = execSync('pgrep -x Zotero', { encoding: 'utf8' });
+        return result.trim().length > 0;
+      } else {
+        const result = execSync('pgrep -x zotero', { encoding: 'utf8' });
+        return result.trim().length > 0;
+      }
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Check if WAL files exist (indicates active Zotero session)
+   */
+  private hasWALFiles(): boolean {
+    const walPath = this.dbPath + '-wal';
+    const shmPath = this.dbPath + '-shm';
+    return existsSync(walPath) || existsSync(shmPath);
+  }
+
+  /**
+   * Create a backup of the database before modification
+   */
+  private createBackup(): string {
+    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+    const backupPath = this.dbPath.replace('.sqlite', `.backup-${timestamp}.sqlite`);
+    copyFileSync(this.dbPath, backupPath);
+    return backupPath;
+  }
+
+  /**
+   * Verify database integrity
+   */
+  private verifyIntegrity(): boolean {
+    if (!this.db) return false;
+    try {
+      const result = this.queryOne('PRAGMA integrity_check');
+      return result && result.integrity_check === 'ok';
+    } catch {
+      return false;
+    }
+  }
+
   /**
    * Connect to the Zotero database
    */
@@ -106,6 +161,18 @@ export class ZoteroDatabase {
       throw new Error(`Zotero database not found at: ${this.dbPath}`);
     }
 
+    // WARNING: Check if Zotero is running (write operations are dangerous)
+    if (!this.readonly && this.isZoteroRunning()) {
+      console.warn('⚠️  WARNING: Zotero is currently running. Write operations may corrupt the database!');
+      console.warn('   Please close Zotero before making changes, or use readonly mode.');
+    }
+
+    // WARNING: Check for WAL files
+    if (!this.readonly && this.hasWALFiles()) {
+      console.warn('⚠️  WARNING: WAL files detected. Zotero may be running or was not closed properly.');
+      console.warn('   Writing to the database may cause corruption!');
+    }
+
     // Initialize sql.js
     this.SQL = await initSqlJs();
     
@@ -116,19 +183,57 @@ export class ZoteroDatabase {
 
     // Enable foreign keys
     db.run('PRAGMA foreign_keys = ON');
+
+    // Verify database integrity on connect
+    if (!this.verifyIntegrity()) {
+      console.error('❌ ERROR: Database integrity check failed! The database may already be corrupted.');
+    }
   }
 
   /**
    * Save changes to the database file
+   * WARNING: This can corrupt the database if Zotero is running!
    */
   save(): void {
-    if (!this.db || this.readonly) {
+    if (!this.db || this.readonly || !this.hasUnsavedChanges) {
       return;
     }
+
+    // Safety check before save
+    if (this.isZoteroRunning()) {
+      throw new Error('Cannot save: Zotero is currently running. Please close Zotero first to avoid database corruption.');
+    }
+
+    if (this.hasWALFiles()) {
+      throw new Error('Cannot save: WAL files detected. Please close Zotero and wait for WAL files to be cleaned up.');
+    }
+
+    // Create backup before saving
+    if (!this.backupPath) {
+      this.backupPath = this.createBackup();
+      console.log(`📁 Backup created at: ${this.backupPath}`);
+    }
+
+    // Verify integrity before save (only check a few important tables to speed up)
+    try {
+      this.queryOne("SELECT 1 FROM items LIMIT 1");
+      this.queryOne("SELECT 1 FROM itemData LIMIT 1");
+    } catch (e) {
+      throw new Error('Cannot save: Basic database check failed. Database may be corrupted.');
+    }
     
     const data = this.db.export();
     const buffer = Buffer.from(data);
     writeFileSync(this.dbPath, buffer);
+    this.hasUnsavedChanges = false;
+    console.log('✅ Database saved successfully.');
+  }
+
+  /**
+   * Mark that changes have been made (for tracking unsaved changes)
+   */
+  private markDirty(): void {
+    this.hasUnsavedChanges = true;
   }
 
   /**
@@ -136,8 +241,12 @@ export class ZoteroDatabase {
    */
   disconnect(): void {
     if (this.db) {
-      if (!this.readonly) {
-        this.save();
+      if (!this.readonly && this.hasUnsavedChanges) {
+        try {
+          this.save();
+        } catch (e) {
+          console.error('Failed to save on disconnect:', e);
+        }
       }
       this.db.close();
       this.db = null;
@@ -161,6 +270,34 @@ export class ZoteroDatabase {
     return this.dbPath;
   }
 
+  /**
+   * Get current timestamp in Zotero SQL format
+   */
+  private getCurrentTimestamp(): string {
+    return new Date().toISOString().replace('T', ' ').replace('Z', '').slice(0, -4);
+  }
+
+  /**
+   * Update item metadata after modification (dateModified, version, synced)
+   * This is CRITICAL for Zotero compatibility!
+   * 
+   * According to Zotero's official code:
+   * - dateModified must be updated on every change
+   * - version must be incremented
+   * - synced must be set to 0 (false) to indicate local change
+   */
+  private updateItemMetadata(itemID: number): void {
+    const timestamp = this.getCurrentTimestamp();
+    this.db!.run(`
+      UPDATE items 
+      SET dateModified = ?, 
+          clientDateModified = ?,
+          version = version + 1,
+          synced = 0
+      WHERE itemID = ?
+    `, [timestamp, timestamp, itemID]);
+  }
+
   /**
    * Execute a query and return all results
    */
@@ -198,7 +335,13 @@ export class ZoteroDatabase {
       throw new Error('Database not connected');
     }
     
+    // Safety check before modifying database
+    if (this.isZoteroRunning()) {
+      throw new Error('Cannot modify database: Zotero is currently running. Please close Zotero first.');
+    }
+    
     this.db.run(sql, params);
+    this.markDirty(); // Mark that we have unsaved changes
     
     const changes = this.db.getRowsModified();
     const lastId = this.queryOne('SELECT last_insert_rowid() as id');
@@ -373,6 +516,8 @@ export class ZoteroDatabase {
 
   /**
    * Add tag to item
+   * 
+   * Following Zotero's pattern: modifying item tags should update item metadata
    */
   addTagToItem(itemID: number, tagName: string, type: number = 0): boolean {
     // Get or create tag
@@ -389,6 +534,9 @@ export class ZoteroDatabase {
     
     this.execute('INSERT INTO itemTags (itemID, tagID) VALUES (?, ?)', [itemID, tagID]);
     
+    // CRITICAL: Update item metadata for Zotero compatibility
+    this.updateItemMetadata(itemID);
+    
     if (!this.readonly) {
       this.save();
     }
@@ -407,6 +555,11 @@ export class ZoteroDatabase {
     
     const result = this.execute('DELETE FROM itemTags WHERE itemID = ? AND tagID = ?', [itemID, tag.tagID]);
     
+    if (result.changes > 0) {
+      // CRITICAL: Update item metadata for Zotero compatibility
+      this.updateItemMetadata(itemID);
+    }
+    
     if (!this.readonly && result.changes > 0) {
       this.save();
     }
@@ -446,6 +599,8 @@ export class ZoteroDatabase {
 
   /**
    * Add item to collection
+   * 
+   * Following Zotero's pattern: collection membership changes update item metadata
    */
   addItemToCollection(itemID: number, collectionID: number): boolean {
     // Check if already in collection
@@ -459,6 +614,9 @@ export class ZoteroDatabase {
     
     this.execute('INSERT INTO collectionItems (itemID, collectionID) VALUES (?, ?)', [itemID, collectionID]);
     
+    // CRITICAL: Update item metadata for Zotero compatibility
+    this.updateItemMetadata(itemID);
+    
     if (!this.readonly) {
       this.save();
     }
@@ -468,10 +626,17 @@ export class ZoteroDatabase {
 
   /**
    * Remove item from collection
+   * 
+   * Following Zotero's pattern: collection membership changes update item metadata
    */
   removeItemFromCollection(itemID: number, collectionID: number): boolean {
     const result = this.execute('DELETE FROM collectionItems WHERE itemID = ? AND collectionID = ?', [itemID, collectionID]);
     
+    if (result.changes > 0) {
+      // CRITICAL: Update item metadata for Zotero compatibility
+      this.updateItemMetadata(itemID);
+    }
+    
     if (!this.readonly && result.changes > 0) {
       this.save();
     }
@@ -587,6 +752,11 @@ export class ZoteroDatabase {
 
   /**
    * Set item abstract
+   * 
+   * Following Zotero's official pattern for modifying item data:
+   * 1. Get or create value in itemDataValues
+   * 2. Insert or update itemData
+   * 3. Update item metadata (dateModified, version, synced)
    */
   setItemAbstract(itemID: number, abstract: string): boolean {
     // Get abstractNote field ID
@@ -595,7 +765,7 @@ export class ZoteroDatabase {
       return false;
     }
     
-    // Get or create value
+    // Get or create value (Zotero stores unique values in itemDataValues)
     let valueRow = this.queryOne('SELECT valueID FROM itemDataValues WHERE value = ?', [abstract]);
     
     if (!valueRow) {
@@ -618,13 +788,15 @@ export class ZoteroDatabase {
         [itemID, field.fieldID, valueRow.valueID]);
     }
     
+    // CRITICAL: Update item metadata for Zotero compatibility
+    this.updateItemMetadata(itemID);
+    
     if (!this.readonly) {
       this.save();
     }
     
     return true;
   }
-
   /**
    * Get item notes
    */
@@ -638,6 +810,11 @@ export class ZoteroDatabase {
 
   /**
    * Add note to item
+   * 
+   * Following Zotero's official pattern for creating notes:
+   * 1. Create item with note type
+   * 2. Create itemNotes entry
+   * 3. Update parent item's metadata (as adding a child note is a modification)
    */
   addItemNote(parentItemID: number, noteContent: string, title: string = ''): number {
     // Get parent item's library ID
@@ -649,22 +826,32 @@ export class ZoteroDatabase {
     // Get note item type ID
     const noteType = this.queryOne("SELECT itemTypeID FROM itemTypes WHERE typeName = 'note'");
     
-    // Create item entry
+    // Create item entry with proper timestamp format
     const key = this.generateKey();
-    const now = new Date().toISOString().replace('T', ' ').replace('Z', '');
+    const now = this.getCurrentTimestamp();
     
+    // Note: synced=0 for new local items, clientDateModified is also set
     const itemResult = this.execute(`
-      INSERT INTO items (itemTypeID, dateAdded, dateModified, key, libraryID, version)
-      VALUES (?, ?, ?, ?, ?, 0)
-    `, [noteType.itemTypeID, now, now, key, parent.libraryID]);
+      INSERT INTO items (itemTypeID, dateAdded, dateModified, clientDateModified, key, libraryID, version, synced)
+      VALUES (?, ?, ?, ?, ?, ?, 0, 0)
+    `, [noteType.itemTypeID, now, now, now, key, parent.libraryID]);
     
     const itemID = itemResult.lastInsertRowid;
     
+    // Wrap note content in Zotero's expected format if not already
+    let formattedNote = noteContent;
+    if (!noteContent.startsWith('<div class="zotero-note')) {
+      formattedNote = `<div class="zotero-note znv1">${noteContent}</div>`;
+    }
+    
     // Create note entry
     this.execute(`
       INSERT INTO itemNotes (itemID, parentItemID, note, title)
       VALUES (?, ?, ?, ?)
-    `, [itemID, parentItemID, noteContent, title]);
+    `, [itemID, parentItemID, formattedNote, title]);
+    
+    // CRITICAL: Update parent item metadata (adding a note is considered a modification)
+    this.updateItemMetadata(parentItemID);
     
     if (!this.readonly) {
       this.save();
@@ -1222,4 +1409,339 @@ export class ZoteroDatabase {
   run(sql: string, params: any[] = []): { changes: number; lastInsertRowid: number } {
     return this.execute(sql, params);
   }
+
+  // ============================================
+  // Duplicate Detection & Attachment Validation
+  // ============================================
+
+  /**
+   * Find duplicate items based on title, DOI, or ISBN
+   */
+  findDuplicates(field: 'title' | 'doi' | 'isbn' = 'title', libraryID: number = 1): any[] {
+    if (field === 'title') {
+      // Find items with the same title
+      return this.queryAll(`
+        SELECT 
+          iv.value as title,
+          GROUP_CONCAT(i.itemID) as itemIDs,
+          COUNT(*) as count
+        FROM items i
+        JOIN itemData id ON i.itemID = id.itemID
+        JOIN itemDataValues iv ON id.valueID = iv.valueID
+        JOIN fields f ON id.fieldID = f.fieldID
+        WHERE f.fieldName = 'title'
+          AND i.libraryID = ?
+          AND i.itemTypeID NOT IN (1, 14)  -- Exclude notes and attachments
+        GROUP BY iv.value
+        HAVING COUNT(*) > 1
+        ORDER BY count DESC
+      `, [libraryID]);
+    } else if (field === 'doi') {
+      return this.queryAll(`
+        SELECT 
+          iv.value as doi,
+          GROUP_CONCAT(i.itemID) as itemIDs,
+          COUNT(*) as count
+        FROM items i
+        JOIN itemData id ON i.itemID = id.itemID
+        JOIN itemDataValues iv ON id.valueID = iv.valueID
+        JOIN fields f ON id.fieldID = f.fieldID
+        WHERE f.fieldName = 'DOI'
+          AND i.libraryID = ?
+          AND iv.value != ''
+        GROUP BY LOWER(iv.value)
+        HAVING COUNT(*) > 1
+        ORDER BY count DESC
+      `, [libraryID]);
+    } else {
+      return this.queryAll(`
+        SELECT 
+          iv.value as isbn,
+          GROUP_CONCAT(i.itemID) as itemIDs,
+          COUNT(*) as count
+        FROM items i
+        JOIN itemData id ON i.itemID = id.itemID
+        JOIN itemDataValues iv ON id.valueID = iv.valueID
+        JOIN fields f ON id.fieldID = f.fieldID
+        WHERE f.fieldName = 'ISBN'
+          AND i.libraryID = ?
+          AND iv.value != ''
+        GROUP BY REPLACE(REPLACE(iv.value, '-', ''), ' ', '')
+        HAVING COUNT(*) > 1
+        ORDER BY count DESC
+      `, [libraryID]);
+    }
+  }
+
+  /**
+   * Validate attachment files exist on disk
+   */
+  validateAttachments(itemID?: number, checkAll: boolean = false): {
+    valid: any[];
+    missing: any[];
+    total: number;
+  } {
+    let attachments: any[];
+    
+    if (itemID) {
+      attachments = this.queryAll(`
+        SELECT ia.itemID, ia.parentItemID, ia.path, ia.contentType, i.key
+        FROM itemAttachments ia
+        JOIN items i ON ia.itemID = i.itemID
+        WHERE ia.parentItemID = ? AND ia.path IS NOT NULL
+      `, [itemID]);
+    } else if (checkAll) {
+      attachments = this.queryAll(`
+        SELECT ia.itemID, ia.parentItemID, ia.path, ia.contentType, i.key
+        FROM itemAttachments ia
+        JOIN items i ON ia.itemID = i.itemID
+        WHERE ia.path IS NOT NULL
+        LIMIT 1000
+      `);
+    } else {
+      return { valid: [], missing: [], total: 0 };
+    }
+
+    const valid: any[] = [];
+    const missing: any[] = [];
+
+    for (const att of attachments) {
+      const fullPath = this.getAttachmentPath(att.itemID);
+      if (fullPath && existsSync(fullPath)) {
+        valid.push({
+          ...att,
+          fullPath,
+          exists: true
+        });
+      } else {
+        missing.push({
+          ...att,
+          fullPath,
+          exists: false
+        });
+      }
+    }
+
+    return {
+      valid,
+      missing,
+      total: attachments.length
+    };
+  }
+
+  /**
+   * Get a valid (existing) attachment for an item
+   * Useful when multiple attachment records exist but only one file is present
+   */
+  getValidAttachment(parentItemID: number, contentType: string = 'application/pdf'): any | null {
+    const attachments = this.queryAll(`
+      SELECT ia.itemID, ia.path, ia.contentType, i.key
+      FROM itemAttachments ia
+      JOIN items i ON ia.itemID = i.itemID
+      WHERE ia.parentItemID = ? 
+        AND ia.contentType = ?
+        AND ia.path IS NOT NULL
+    `, [parentItemID, contentType]);
+
+    // Return the first attachment that actually exists
+    for (const att of attachments) {
+      const fullPath = this.getAttachmentPath(att.itemID);
+      if (fullPath && existsSync(fullPath)) {
+        return {
+          ...att,
+          fullPath,
+          exists: true
+        };
+      }
+    }
+
+    return null;
+  }
+
+  /**
+   * Find items with valid (existing) PDF files
+   */
+  findItemsWithValidPDF(options: {
+    title?: string;
+    doi?: string;
+    requireValidPDF?: boolean;
+  }): any[] {
+    let items: any[];
+    
+    if (options.doi) {
+      // Search by DOI
+      const normalizedDOI = options.doi.replace(/^https?:\/\/doi\.org\//i, '').replace(/^doi:/i, '').trim();
+      items = this.queryAll(`
+        SELECT DISTINCT i.itemID, i.key, i.dateAdded, iv.value as doi
+        FROM items i
+        JOIN itemData id ON i.itemID = id.itemID
+        JOIN itemDataValues iv ON id.valueID = iv.valueID
+        JOIN fields f ON id.fieldID = f.fieldID
+        WHERE f.fieldName = 'DOI' AND LOWER(iv.value) = LOWER(?)
+      `, [normalizedDOI]);
+    } else if (options.title) {
+      // Search by title
+      items = this.queryAll(`
+        SELECT DISTINCT i.itemID, i.key, i.dateAdded, iv.value as title
+        FROM items i
+        JOIN itemData id ON i.itemID = id.itemID
+        JOIN itemDataValues iv ON id.valueID = iv.valueID
+        JOIN fields f ON id.fieldID = f.fieldID
+        WHERE f.fieldName = 'title' AND iv.value LIKE ?
+        LIMIT 50
+      `, [`%${options.title}%`]);
+    } else {
+      return [];
+    }
+
+    if (!options.requireValidPDF) {
+      return items.map(item => ({
+        ...this.getItemDetails(item.itemID),
+        hasValidPDF: this.getValidAttachment(item.itemID) !== null
+      }));
+    }
+
+    // Filter to only items with valid PDF
+    const results: any[] = [];
+    for (const item of items) {
+      const validAttachment = this.getValidAttachment(item.itemID);
+      if (validAttachment) {
+        results.push({
+          ...this.getItemDetails(item.itemID),
+          validAttachment
+        });
+      }
+    }
+    return results;
+  }
+
+  /**
+   * Find orphan attachments (records without files)
+   */
+  findOrphanAttachments(limit: number = 100): any[] {
+    const attachments = this.queryAll(`
+      SELECT ia.itemID, ia.parentItemID, ia.path, ia.contentType, i.key,
+             parent.itemID as parentExists
+      FROM itemAttachments ia
+      JOIN items i ON ia.itemID = i.itemID
+      LEFT JOIN items parent ON ia.parentItemID = parent.itemID
+      WHERE ia.path LIKE 'storage:%'
+      LIMIT ?
+    `, [limit]);
+
+    const orphans: any[] = [];
+
+    for (const att of attachments) {
+      const fullPath = this.getAttachmentPath(att.itemID);
+      if (!fullPath || !existsSync(fullPath)) {
+        orphans.push({
+          itemID: att.itemID,
+          parentItemID: att.parentItemID,
+          key: att.key,
+          path: att.path,
+          expectedPath: fullPath,
+          reason: !fullPath ? 'invalid_path' : 'file_not_found'
+        });
+      }
+    }
+
+    return orphans;
+  }
+
+  /**
+   * Delete orphan attachment records (use with caution!)
+   */
+  deleteOrphanAttachments(dryRun: boolean = true): {
+    orphans: any[];
+    deleted: number;
+    dryRun: boolean;
+  } {
+    const orphans = this.findOrphanAttachments(500);
+
+    if (dryRun || this.readonly) {
+      return {
+        orphans,
+        deleted: 0,
+        dryRun: true
+      };
+    }
+
+    let deleted = 0;
+    for (const orphan of orphans) {
+      try {
+        this.execute('DELETE FROM itemAttachments WHERE itemID = ?', [orphan.itemID]);
+        this.execute('DELETE FROM items WHERE itemID = ?', [orphan.itemID]);
+        deleted++;
+      } catch (error) {
+        console.error(`Failed to delete orphan ${orphan.itemID}:`, error);
+      }
+    }
+
+    return {
+      orphans,
+      deleted,
+      dryRun: false
+    };
+  }
+
+  /**
+   * Merge items by transferring notes and tags from source items to target
+   */
+  mergeItems(targetItemID: number, sourceItemIDs: number[]): {
+    success: boolean;
+    transferred: {
+      notes: number;
+      tags: number;
+    };
+    errors: string[];
+  } {
+    const errors: string[] = [];
+    let notesTransferred = 0;
+    let tagsTransferred = 0;
+
+    // Verify target exists
+    const target = this.getItemDetails(targetItemID);
+    if (!target) {
+      return {
+        success: false,
+        transferred: { notes: 0, tags: 0 },
+        errors: ['Target item not found']
+      };
+    }
+
+    for (const sourceID of sourceItemIDs) {
+      if (sourceID === targetItemID) continue;
+
+      // Transfer notes
+      try {
+        const notes = this.getItemNotes(sourceID);
+        for (const note of notes) {
+          this.addItemNote(targetItemID, note.note, `[Merged] ${note.title || ''}`);
+          notesTransferred++;
+        }
+      } catch (error) {
+        errors.push(`Failed to transfer notes from ${sourceID}: ${error}`);
+      }
+
+      // Transfer tags
+      try {
+        const tags = this.getItemTags(sourceID);
+        for (const tag of tags) {
+          this.addTagToItem(targetItemID, tag.name, tag.type);
+          tagsTransferred++;
+        }
+      } catch (error) {
+        errors.push(`Failed to transfer tags from ${sourceID}: ${error}`);
+      }
+    }
+
+    return {
+      success: errors.length === 0,
+      transferred: {
+        notes: notesTransferred,
+        tags: tagsTransferred
+      },
+      errors
+    };
+  }
 }