w-orm-lmdb 1.0.11 → 1.0.12

package/src/WOrmLmdb.mjs

@@ -82,13 +82,8 @@ function WOrmLmdb(opt = {}) {
             return cloneDeep(_cache) //與外部使用數據脫勾
         }
 
-        //waitFun
-        await waitFun(() => {
-            if (client.status === 'closed') {
-                console.log(`client.status[${client.status}], level is closed`)
-            }
-            return client.status === 'open'
-        })
+        //waitOpen
+        await waitOpen()
         // console.log('client.status',client.status)
 
         //ltdt
@@ -105,6 +100,16 @@ function WOrmLmdb(opt = {}) {
         return ltdt
     }
 
+    //waitOpen, 等待client開啟, 因getRange與get於client關閉時會拋錯
+    let waitOpen = async () => {
+        await waitFun(() => {
+            if (client.status === 'closed') {
+                console.log(`client.status[${client.status}], level is closed`)
+            }
+            return client.status === 'open'
+        })
+    }
+
     //getValue
     let getValue = async (key) => {
         let value = null
@@ -176,6 +181,54 @@ function WOrmLmdb(opt = {}) {
         return res
     }
 
+    /**
+     * 由主鍵id查詢單筆數據，因直接由LMDB取值，不需如select提取全表數據再過濾，故數據量大時效能較佳
+     *
+     * @memberOf WOrmLmdb
+     * @param {String} id 輸入主鍵字串
+     * @returns {Promise} 回傳Promise，resolve回傳數據物件，若無此id則回傳null，reject回傳錯誤訊息
+     */
+    async function selectById(id) {
+        let isErr = false
+        let res = null
+
+        try {
+
+            //check
+            if (!isestr(id)) {
+                //未給有效id視為查無數據
+                return null
+            }
+
+            //waitOpen
+            await waitOpen()
+
+            //查找資料表內id
+            let v = await getValue(id) //不會有catch
+
+            //check, 判定基準與insert、save、del內對既有數據之認定一致
+            if (iseobj(v)) {
+                res = v
+            }
+            else {
+                //不存在id, 回傳null
+                res = null
+            }
+
+        }
+        catch (err) {
+            isErr = true
+            res = err
+        }
+
+        //check
+        if (isErr) {
+            return Promise.reject(res)
+        }
+
+        return res
+    }
+
     /**
      * 插入數據
      *
@@ -216,29 +269,29 @@ function WOrmLmdb(opt = {}) {
             })
 
             //each
+            //ifNoExists, 由LMDB於同一寫交易內原子完成[檢查v.id未存在]與[寫入], 併發時同一v.id僅有一次成功,
+            //同批含重複v.id時亦僅首筆成功, 故不逐筆await而一次送出, 由LMDB寫執行緒依序處理並合併為批次寫交易
+            //注意: 回調函數須為同步函數, 且client.put須於回調函數內直接呼叫,
+            //若將client.put置於await之後, 該寫入會逸出條件批次而變成無條件寫入, 並使回傳值與實際狀態不一致
             let nAll = size(data)
-            let nPush = 0
-            for (let v of data) {
+            let ltdone = await Promise.all(map(data, function(v) {
                 // console.log(v)
+                return client.ifNoExists(v.id, () => {
+                    client.put(v.id, v)
+                })
+            }))
 
-                //查找資料表內v.id
-                let vv = await getValue(v.id) //不會有catch
-                // console.log('getValue',vv)
-
-                //check
-                if (!iseobj(vv)) {
-                    //未存在v.id
-
-                    //put
-                    await client.put(v.id, v)
-
+            //nPush
+            let nPush = 0
+            each(ltdone, function(done) {
+                if (done) {
+                    //未存在v.id, 已寫入
                     nPush++
                 }
                 else {
                     //已存在v.id則不push
                 }
-
-            }
+            })
 
             //res
             res = {
@@ -323,33 +376,74 @@ function WOrmLmdb(opt = {}) {
                 //rest
                 let rest = null
 
-                //查找資料表內v.id
-                let vv = await getValue(v.id) //不會有catch
+                //快速路徑, 先不開寫交易直接預讀, 內容相同者本就不須寫入, 可省去開啟寫交易之成本
+                //預讀值縱使已被其他寫入者更動而過期亦不影響正確性, 因內容相同時本次save等價於無操作,
+                //無操作可視為於預讀當下即已完成, 其後他人之寫入結果與[本次save先執行再輪到他人]相同
+                let vp = await getValue(v.id) //不會有catch
+                if (iseobj(vp) && isEqual(v, vp)) {
+                    //內容相同不更新
+                    return {
+                        n: 0,
+                        nModified: 0,
+                        ok: 1,
+                    }
+                }
 
-                //existed
-                let existed = iseobj(vv)
+                //existed, updated, inserted
+                let existed = false
+                let updated = false
+                let inserted = false
 
-                //check
-                if (existed) {
-                    //已存在v.id
-                    if (isEqual(v, vv)) {
-                        //內容相同不更新
+                //transaction, 由LMDB於同一寫交易內原子完成[查找v.id]與[合併寫入],
+                //併發時不會有兩方各自讀到同一舊值再各自寫入而導致遺失更新
+                //注意: 回調函數須為同步函數, client.get於寫交易內為同步取值, 不可使用await,
+                //若將client.get或client.put置於await之後, 該操作會逸出本交易而失去原子性
+                await client.transaction(() => {
+
+                    //查找資料表內v.id
+                    let vv = client.get(v.id)
+
+                    //existed
+                    existed = iseobj(vv)
+
+                    //check
+                    if (existed) {
+                        //已存在v.id
+                        if (isEqual(v, vv)) {
+                            //內容相同不更新
+                        }
+                        else {
+                            //內容不同須更新
+
+                            //merge and put
+                            client.put(v.id, merge(vv, v))
+
+                            updated = true
+                        }
                     }
                     else {
-                        //內容不同須更新
+                        //內容不存在, 若autoInsert則須插入
+                        if (autoInsert) {
 
-                        //merge and put
-                        await client.put(v.id, merge(vv, v))
+                            //put
+                            client.put(v.id, v)
 
-                        rest = { update: true }
+                            inserted = true
+                        }
                     }
-                }
-                else {
-                    //內容不存在
-                }
+
+                })
 
                 //rest
-                if (iseobj(rest)) {
+                if (inserted) {
+                    //內容不存在而自動插入, 回傳形狀同insert
+                    rest = {
+                        n: 1,
+                        nInserted: 1,
+                        ok: 1,
+                    }
+                }
+                else if (updated) {
                     rest = {
                         n: 1,
                         nModified: 1,
@@ -357,20 +451,33 @@ function WOrmLmdb(opt = {}) {
                     }
                 }
                 else {
+                    //rest.n === 0:
+                    // 內容相同不更新, 不須update
+                    // 內容不存在且未開啟autoInsert
                     rest = {
                         n: 0,
                         nModified: 0,
                         ok: 1,
                     }
                 }
-                //rest.n === 0:
-                // 內容相同不更新, 不須update
-                // 內容不存在, 若autoInsert則須insert
-
-                //autoInsert
-                if (autoInsert && rest.n === 0 && !existed) {
-                    // console.log('insert(v)', v)
-                    rest = await insert(v)
+
+                //emit, 原autoInsert係轉呼叫insert故會發出insert事件, 此處維持既有行為,
+                //須於交易外發出, 避免外部訂閱函數於寫交易內執行
+                if (inserted) {
+
+                    //update, 不能保證插入多少, 一律重設快取
+                    _cache = null
+
+                    try {
+
+                        //emit
+                        ee.emit('change', 'insert', [v], rest)
+
+                    }
+                    catch (err) {
+                        console.log(err)
+                    }
+
                 }
 
                 return rest
@@ -644,6 +751,7 @@ function WOrmLmdb(opt = {}) {
 
     //save
     ee.select = select
+    ee.selectById = selectById
     ee.insert = insert
     ee.save = save
     ee.del = del

package/test/{basic.test.mjs → unit-basic.test.mjs}

@@ -184,6 +184,36 @@ describe('basic', function() {
             })
         vget[5] = rt
 
+        //selectById
+        rt = null
+        // vans[12] = { id: 'id-rosemary', name: 'rosemary(modify)', value: 123.456 }
+        await wo.selectById('id-rosemary')
+            .then(function(msg) {
+                // console.log('selectById then', msg)
+                // selectById { id: 'id-rosemary', name: 'rosemary(modify)', value: 123.456 }
+                rt = msg
+            })
+            .catch(function(msg) {
+                // console.log('selectById catch', msg)
+                rt = msg.toString()
+            })
+        vget[12] = rt
+
+        //selectById by id not existed
+        rt = null
+        // vans[13] = null
+        await wo.selectById('id-not-existed')
+            .then(function(msg) {
+                // console.log('selectById by id not existed then', msg)
+                // selectById by id not existed null
+                rt = msg
+            })
+            .catch(function(msg) {
+                // console.log('selectById by id not existed catch', msg)
+                rt = msg.toString()
+            })
+        vget[13] = rt
+
         //select by $and, $gt, $lt
         rt = null
         // vans[6] = [{ id: 'id-rosemary', name: 'rosemary(modify)', value: 123.456 }]
@@ -370,6 +400,17 @@ describe('basic', function() {
         assert.strict.deepStrictEqual(vget[5], vans[5])
     })
 
+    //vans[12]、vans[13]為後續新增之selectById, 為不更動既有編號故接續於末號之後
+    vans[12] = { id: 'id-rosemary', name: 'rosemary(modify)', value: 123.456 }
+    it(`should get ${JSON.stringify(vans[12])} for selectById`, async function() {
+        assert.strict.deepStrictEqual(vget[12], vans[12])
+    })
+
+    vans[13] = null
+    it(`should get ${JSON.stringify(vans[13])} for selectById by id not existed`, async function() {
+        assert.strict.deepStrictEqual(vget[13], vans[13])
+    })
+
     vans[6] = [{ id: 'id-rosemary', name: 'rosemary(modify)', value: 123.456 }]
     it(`should get ${JSON.stringify(vans[6])} for select by $and, $gt, $lt`, async function() {
         assert.strict.deepStrictEqual(vget[6], vans[6])

package/test/unit-insert.test.mjs

@@ -0,0 +1,170 @@
+import assert from 'assert'
+import { spawn } from 'child_process'
+import path from 'path'
+import _ from 'lodash-es'
+import w from 'wsemi'
+import WOrm from '../src/WOrmLmdb.mjs'
+
+
+//runProc, 另起行程對同一資料表插入相同id, 用於驗證跨行程之插入原子性
+function runProc(tag, url, ids) {
+    return new Promise(function(resolve) {
+        let code = `
+import { pathToFileURL } from 'url'
+let { default: WOrm } = await import(pathToFileURL(process.env.SRC).href)
+let wo = WOrm({ url: process.env.URL, db: 'worm', cl: 'race' })
+let ids = JSON.parse(process.env.IDS)
+let n = 0
+for (let id of ids) {
+    let r = await wo.insert({ id, from: process.env.TAG })
+    n += r.nInserted
+}
+console.log(JSON.stringify({ tag: process.env.TAG, nInserted: n }))
+await wo.close()
+`
+        let out = ''
+        let p = spawn(process.execPath, ['--input-type=module', '-e', code], {
+            shell: false,
+            env: {
+                ...process.env,
+                SRC: path.resolve('./src/WOrmLmdb.mjs'),
+                URL: url,
+                IDS: JSON.stringify(ids),
+                TAG: tag,
+            },
+        })
+        p.stdout.on('data', function(d) {
+            out += d.toString()
+        })
+        p.stderr.on('data', function(d) {
+            out += d.toString()
+        })
+        p.on('close', function() {
+            let r = null
+            try {
+                r = JSON.parse(_.trim(out))
+            }
+            catch (err) {
+                r = { tag, error: _.trim(out) }
+            }
+            resolve(r)
+        })
+    })
+}
+
+
+describe('insert', function() {
+    let rt = null
+    let vans = {}
+    let vget = {}
+
+    before(async function() {
+
+        w.fsDeleteFolder('./_db_insert')
+
+        let url = './_db_insert'
+
+        //insert同批含重複id, 僅首筆應被插入
+        rt = null
+        // vans[1] = { n: 3, nInserted: 2, ok: 1 }
+        let woDup = WOrm({ url, db: 'worm', cl: 'dup' })
+        await woDup.insert([
+            { id: 'id-dup', name: 'first' },
+            { id: 'id-dup', name: 'second' },
+            { id: 'id-other', name: 'other' },
+        ])
+            .then(function(msg) {
+                // console.log('insert(含重複id) then', msg)
+                // insert(含重複id) then { n: 3, nInserted: 2, ok: 1 }
+                rt = msg
+            })
+            .catch(function(msg) {
+                // console.log('insert(含重複id) catch', msg)
+                rt = msg.toString()
+            })
+        vget[1] = rt
+
+        //同批重複id時應保留首筆內容
+        vget[2] = await woDup.selectById('id-dup')
+
+        //對已存在id再insert, 應不插入且不覆寫
+        rt = null
+        // vans[3] = { n: 1, nInserted: 0, ok: 1 }
+        await woDup.insert({ id: 'id-dup', name: 'third' })
+            .then(function(msg) {
+                rt = msg
+            })
+            .catch(function(msg) {
+                rt = msg.toString()
+            })
+        vget[3] = rt
+        vget[4] = await woDup.selectById('id-dup')
+        await woDup.close()
+
+        //單行程併發: 對同一id併發insert 100次, nInserted總和須為1
+        let woPar = WOrm({ url, db: 'worm', cl: 'par' })
+        let rs = await Promise.all(_.times(100, function(k) {
+            return woPar.insert({ id: 'id-race', k })
+        }))
+        vget[5] = _.sum(_.map(rs, 'nInserted'))
+        vget[6] = _.size(await woPar.select())
+        await woPar.close()
+
+        //跨行程併發: 兩行程各自對同50個id插入, nInserted總和須為50且資料表僅50筆
+        let ids = _.times(50, function(k) {
+            return `id-x${k}`
+        })
+        let rps = await Promise.all([
+            runProc('P1', url, ids),
+            runProc('P2', url, ids),
+        ])
+        // console.log('跨行程結果', rps)
+        // 跨行程結果 [ { tag: 'P1', nInserted: 34 }, { tag: 'P2', nInserted: 16 } ]
+        vget[7] = _.sum(_.map(rps, 'nInserted'))
+        let woX = WOrm({ url, db: 'worm', cl: 'race' })
+        vget[8] = _.size(await woX.select())
+        await woX.close()
+
+    })
+
+    vans[1] = { n: 3, nInserted: 2, ok: 1 }
+    it(`should get ${JSON.stringify(vans[1])} for insert with duplicated id in same batch`, async function() {
+        assert.strict.deepStrictEqual(vget[1], vans[1])
+    })
+
+    vans[2] = { id: 'id-dup', name: 'first' }
+    it(`should get ${JSON.stringify(vans[2])} for keeping first one of duplicated id`, async function() {
+        assert.strict.deepStrictEqual(vget[2], vans[2])
+    })
+
+    vans[3] = { n: 1, nInserted: 0, ok: 1 }
+    it(`should get ${JSON.stringify(vans[3])} for insert existed id`, async function() {
+        assert.strict.deepStrictEqual(vget[3], vans[3])
+    })
+
+    vans[4] = { id: 'id-dup', name: 'first' }
+    it(`should get ${JSON.stringify(vans[4])} for not overwriting existed id`, async function() {
+        assert.strict.deepStrictEqual(vget[4], vans[4])
+    })
+
+    vans[5] = 1
+    it(`should get ${JSON.stringify(vans[5])} for sum of nInserted by 100 concurrent insert with same id`, async function() {
+        assert.strict.deepStrictEqual(vget[5], vans[5])
+    })
+
+    vans[6] = 1
+    it(`should get ${JSON.stringify(vans[6])} for records after 100 concurrent insert with same id`, async function() {
+        assert.strict.deepStrictEqual(vget[6], vans[6])
+    })
+
+    vans[7] = 50
+    it(`should get ${JSON.stringify(vans[7])} for sum of nInserted by 2 processes inserting same 50 ids`, async function() {
+        assert.strict.deepStrictEqual(vget[7], vans[7])
+    })
+
+    vans[8] = 50
+    it(`should get ${JSON.stringify(vans[8])} for records after 2 processes inserting same 50 ids`, async function() {
+        assert.strict.deepStrictEqual(vget[8], vans[8])
+    })
+
+})