w-orm-mongodb 1.1.41 → 1.1.43

package/src/WOrmMongodb.mjs

@@ -2,13 +2,14 @@ import mongodb from 'mongodb'
 import stream from 'stream'
 import cloneDeep from 'lodash-es/cloneDeep.js'
 import every from 'lodash-es/every.js'
+import filter from 'lodash-es/filter.js'
 import get from 'lodash-es/get.js'
 import map from 'lodash-es/map.js'
 import omit from 'lodash-es/omit.js'
 import size from 'lodash-es/size.js'
 import evem from 'wsemi/src/evem.mjs'
 import genPm from 'wsemi/src/genPm.mjs'
-import genID from 'wsemi/src/genID.mjs'
+import genIDSeq from 'wsemi/src/genIDSeq.mjs'
 import isbol from 'wsemi/src/isbol.mjs'
 import isestr from 'wsemi/src/isestr.mjs'
 import isarr from 'wsemi/src/isarr.mjs'
@@ -63,6 +64,10 @@ function WOrmMongodb(opt = {}) {
     let _indexGfsReady = false
 
 
+    //_canUseTx, 拓樸判定只須一次, 以旗標記錄避免每次操作皆多一次round-trip
+    let _canUseTx = null
+
+
     //ee, 採wsemi之evem(即eventemitter3), 其於'error'無監聽者時僅回傳false而不拋出,
     //故本套件之操作行為不因呼叫端有無註冊監聽而改變; Node內建之EventEmitter具該拋出語義, 不可用
     let ee = evem()
@@ -154,8 +159,8 @@ function WOrmMongodb(opt = {}) {
                 throw new Error(`invalid data[${k}].id, autoGenPk is false`)
             }
 
-            //genID
-            v.id = genID()
+            //genIDSeq
+            v.id = genIDSeq()
 
         }
 
@@ -514,6 +519,208 @@ function WOrmMongodb(opt = {}) {
     }
 
 
+    /**
+     * 判定本次部署是否支援交易
+     * 註: MongoDB之交易須replica set或分片叢集，standalone不支援
+     * 判定依據為hello回應：replica set帶setName，連至mongos者msg為isdbgrid，standalone兩者皆無
+     *
+     * @ignore
+     * @param {Object} client 輸入MongoClient物件
+     * @returns {Promise} 回傳Promise，resolve回傳是否支援交易布林值，reject回傳錯誤訊息
+     */
+    async function canUseTx(client) {
+
+        //check
+        if (isbol(_canUseTx)) {
+            return _canUseTx
+        }
+
+        //hello
+        let hello = await client.db(opt.db).admin().command({ hello: 1 })
+
+        //_canUseTx
+        _canUseTx = isestr(get(hello, 'setName')) || get(hello, 'msg') === 'isdbgrid'
+
+        return _canUseTx
+    }
+
+
+    /**
+     * 以交易達成insertBulk之全有全無
+     * 交易內任一筆失敗即整批回滾，故不會寫入任何一筆，無須補償動作
+     *
+     * @ignore
+     * @param {Object} client 輸入MongoClient物件
+     * @param {Object} collection 輸入資料表物件
+     * @param {Array} data 輸入數據陣列
+     * @returns {Promise} 回傳Promise，resolve回傳undefined，reject回傳錯誤訊息
+     */
+    async function _insertBulkByTx(client, collection, data) {
+
+        //session
+        let session = client.startSession()
+
+        try {
+
+            //withTransaction, ordered:true令首筆衝突即停止, 由交易回滾保證未寫入任何一筆
+            await session.withTransaction(async () => {
+                await collection.insertMany(data, { session, ordered: true })
+            })
+
+        }
+        finally {
+            await session.endSession()
+        }
+
+    }
+
+
+    /**
+     * 以補償動作達成insertBulk之全有全無，供無交易可用之standalone使用
+     * 註: 補償係刪除本次呼叫已寫入者，其_id由驅動於送出前在用戶端產生，
+     * 與呼叫前既有數據之_id必不相同，故縱使該筆之id已存在亦不會誤刪既有數據；
+     * 刪除未寫入者為無操作，故不須精確得知哪幾筆已寫入
+     * 註: 行程若於補償途中中止(斷電、強制終止)，已寫入之部份可能殘留，此為無交易可用之固有限制
+     *
+     * @ignore
+     * @param {Object} collection 輸入資料表物件
+     * @param {Array} data 輸入數據陣列
+     * @returns {Promise} 回傳Promise，resolve回傳undefined，reject回傳錯誤訊息
+     */
+    async function _insertBulkByCompensate(collection, data) {
+
+        //errIns
+        let errIns = null
+
+        try {
+            await collection.insertMany(data, { ordered: false })
+        }
+        catch (err) {
+            errIns = err
+        }
+
+        //check, 全數插入成功即無須補償
+        if (errIns === null) {
+            return
+        }
+
+        //idDels, 取本次由驅動所產生之全部_id
+        let idDels = map(data, function(v) {
+            return v._id
+        })
+        idDels = filter(idDels, function(v) {
+            return v !== undefined && v !== null
+        })
+
+        //補償, 刪除本次已寫入者
+        if (size(idDels) > 0) {
+            await collection.deleteMany({ _id: { $in: idDels } })
+        }
+
+        throw errIns
+    }
+
+
+    /**
+     * 批次插入數據，全批視為一個單位：全部插入成功，或一筆都不寫入
+     * 任一筆之id已存在(含同批含重複id)即以reject拋出整批性錯誤，且不寫入任何一筆
+     * 註: 本函數非insert之加速版，兩者衝突政策不同——insert跳過已存在者而整批ok為1，
+     * 本函數則整批reject；需要逐筆處置者改用insert
+     * 註: n為輸入筆數，nInserted於成功時恆等於n；不提供逐筆結果，故不出現ok為0與err
+     * 註: opt.autoGenPk為true(預設)時未帶有效id者自動產生，為false時未帶有效id即reject且同批皆不寫入
+     * 註: 全有全無之達成方式依部署而異——具replica set或分片叢集者以交易包覆，
+     * standalone無交易可用而以補償動作達成，其限制詳見_insertBulkByCompensate
+     *
+     * @memberOf WOrmMongodb
+     * @param {Object|Array} data 輸入數據物件或陣列
+     * @returns {Promise} 回傳Promise，resolve回傳插入結果，reject回傳錯誤訊息
+     */
+    async function insertBulk(data) {
+        let isErr = false
+
+        //check
+        if (!iseobj(data) && !isearr(data)) {
+            return {
+                n: 0,
+                nInserted: 0,
+                ok: 1,
+            }
+        }
+
+        //cloneDeep
+        data = cloneDeep(data)
+
+        //client
+        let client = new MongoClient(opt.url)
+
+        //res
+        let res = null
+        try {
+
+            //connect, 令連線失敗於寫入前即拋出, 屬影響全部數據之整批性錯誤
+            await client.connect()
+
+            //database, collection
+            let database = client.db(opt.db)
+            let collection = database.collection(opt.cl)
+
+            //check
+            if (!isarr(data)) {
+                data = [data]
+            }
+
+            //check id, 須於任何寫入之前一次完成
+            data = map(data, procPk)
+
+            //ensureIndex
+            await ensureIndex(collection)
+
+            //nAll, n之基準為輸入筆數
+            let nAll = size(data)
+
+            //check, 有交易可用者一律優先採用交易, 其回滾即為全有全無之保證
+            let bTx = await canUseTx(client)
+            if (bTx) {
+                await _insertBulkByTx(client, collection, data)
+            }
+            else {
+                await _insertBulkByCompensate(collection, data)
+            }
+
+            //res, 執行至此表全數插入成功, 故nInserted恆等於n
+            res = {
+                n: nAll,
+                nInserted: nAll,
+                ok: 1,
+            }
+
+        }
+        catch (err) {
+            isErr = true
+            res = err
+        }
+        finally {
+            await client.close()
+            client = null
+        }
+
+        //emit
+        if (!isErr) {
+            emitChange('insertBulk', data, res)
+        }
+
+        //check
+        if (isErr) {
+
+            //emit, 整批性錯誤須於reject之前發出
+            emitError('insertBulk', data, res)
+
+            return Promise.reject(res)
+        }
+        return res
+    }
+
+
     /**
      * 儲存單筆數據，由MongoDB於單一updateOne內原子完成[查找id]與[更新或插入]
      * 註: MongoDB單一文件操作本即為原子，故不須開啟transaction(transaction另須replica set，standalone不支援)
@@ -1380,6 +1587,7 @@ function WOrmMongodb(opt = {}) {
     ee.select = select
     ee.selectByPk = selectByPk
     ee.insert = insert
+    ee.insertBulk = insertBulk
     ee.save = save
     ee.del = del
     ee.delAll = delAll

package/test/api-basic.test.mjs

@@ -599,6 +599,206 @@ describe('insert', function() {
 })
 
 
+//insertBulk之語義與insert不同: insert跳過已存在者而整批ok為1, insertBulk則整批reject且不寫入任何一筆
+//本檔之容器為standalone, 無交易可用, 故走補償動作路徑; 交易路徑另見api-insertbulk-rs.test.mjs
+
+
+describe('insertBulk', function() {
+    let rt = null
+    let vans = {}
+    let vget = {}
+
+    before(async function() {
+        this.timeout(120000)
+
+        let cl = 'bulkcnt'
+        let wo = WOrm(genOpt(cl))
+        await wo.delAll()
+
+        //全新3筆, nInserted須等於n
+        vget[1] = await wo.insertBulk([
+            { id: 'b1', name: 'peter' },
+            { id: 'b2', name: 'rosemary' },
+            { id: 'b3', name: 'kettle' },
+        ])
+        vget[2] = (await wo.select()).length
+
+        //撞既有主鍵須整批reject
+        rt = null
+        let nBefore = (await wo.select()).length
+        await wo.insertBulk([
+            { id: 'b4', name: 'new1' },
+            { id: 'b2', name: 'dup' },
+            { id: 'b5', name: 'new2' },
+        ])
+            .then(function(msg) {
+                rt = `不應resolve: ${JSON.stringify(msg)}`
+            })
+            .catch(function(msg) {
+                rt = msg.toString()
+            })
+        vget[3] = rt.indexOf('E11000') >= 0 || rt.indexOf('duplicate key') >= 0
+
+        //失敗後資料表不得有任何新增
+        vget[4] = (await wo.select()).length - nBefore
+        vget[5] = [
+            await wo.selectByPk('b4'),
+            await wo.selectByPk('b5'),
+        ]
+
+        //撞既有主鍵者本身不得被改動
+        vget[6] = (await wo.selectByPk('b2')).name
+
+        //同批含重複主鍵須整批reject
+        rt = null
+        nBefore = (await wo.select()).length
+        await wo.insertBulk([
+            { id: 'b6', name: 'a' },
+            { id: 'b6', name: 'b' },
+            { id: 'b7', name: 'c' },
+        ])
+            .then(function(msg) {
+                rt = `不應resolve: ${JSON.stringify(msg)}`
+            })
+            .catch(function(msg) {
+                rt = msg.toString()
+            })
+        vget[7] = rt.indexOf('E11000') >= 0 || rt.indexOf('duplicate key') >= 0
+        vget[8] = (await wo.select()).length - nBefore
+        vget[9] = [
+            await wo.selectByPk('b6'),
+            await wo.selectByPk('b7'),
+        ]
+
+        //單一物件輸入亦須可用
+        vget[10] = await wo.insertBulk({ id: 'b8', name: 'single' })
+
+        //輸入無效視為空結果
+        vget[11] = await wo.insertBulk(null)
+
+        //鍵集合須與insert完全相同
+        let ri = await wo.insert({ id: 'b9', name: 'x' })
+        vget[12] = [genKeys(vget[1]), genKeys(vget[10]), genKeys(vget[11]), genKeys(ri)]
+
+        //autoGenPk預設為true, 未帶id者自動產生
+        vget[13] = await wo.insertBulk([{ name: 'nopk1' }, { name: 'nopk2' }])
+
+        //autoGenPk為false且未帶id者須reject, 且同批之有效筆數不得被寫入
+        let woOff = WOrm(genOptPk('bulkpkoff', false))
+        await woOff.delAll()
+        rt = null
+        await woOff.insertBulk([
+            { id: 'p-ok1', name: 'ok1' },
+            { name: 'no-id' },
+        ])
+            .then(function(msg) {
+                rt = `不應resolve: ${JSON.stringify(msg)}`
+            })
+            .catch(function(msg) {
+                rt = msg.toString()
+            })
+        vget[14] = rt
+        vget[15] = await woOff.selectByPk('p-ok1')
+
+        //事件, 成功時發change, 失敗時發error且mode為insertBulk
+        let woEv = WOrm(genOpt('bulkevent'))
+        await woEv.delAll()
+        let evs = []
+        woEv.on('change', function(mode) {
+            evs.push(`change:${mode}`)
+        })
+        woEv.on('error', function(mode) {
+            evs.push(`error:${mode}`)
+        })
+        await woEv.insertBulk({ id: 'e1', name: 'a' })
+        await woEv.insertBulk({ id: 'e1', name: 'b' }).catch(() => {})
+        vget[16] = evs
+
+    })
+
+    vans[1] = { n: 3, nInserted: 3, ok: 1 }
+    it(`should get ${JSON.stringify(vans[1])} for insertBulk 3 new records`, async function() {
+        assert.strict.deepStrictEqual(vget[1], vans[1])
+    })
+
+    vans[2] = 3
+    it(`should get ${JSON.stringify(vans[2])} for records after insertBulk`, async function() {
+        assert.strict.deepStrictEqual(vget[2], vans[2])
+    })
+
+    vans[3] = true
+    it(`should get ${JSON.stringify(vans[3])} for reject of insertBulk with existed id`, async function() {
+        assert.strict.deepStrictEqual(vget[3], vans[3])
+    })
+
+    vans[4] = 0
+    it(`should get ${JSON.stringify(vans[4])} for no record added after rejected insertBulk`, async function() {
+        assert.strict.deepStrictEqual(vget[4], vans[4])
+    })
+
+    vans[5] = [null, null]
+    it(`should get ${JSON.stringify(vans[5])} for not writing any record in rejected insertBulk`, async function() {
+        assert.strict.deepStrictEqual(vget[5], vans[5])
+    })
+
+    vans[6] = 'rosemary'
+    it(`should get ${JSON.stringify(vans[6])} for not modifying existed record by rejected insertBulk`, async function() {
+        assert.strict.deepStrictEqual(vget[6], vans[6])
+    })
+
+    vans[7] = true
+    it(`should get ${JSON.stringify(vans[7])} for reject of insertBulk with duplicated id in same batch`, async function() {
+        assert.strict.deepStrictEqual(vget[7], vans[7])
+    })
+
+    vans[8] = 0
+    it(`should get ${JSON.stringify(vans[8])} for no record added after insertBulk with duplicated id`, async function() {
+        assert.strict.deepStrictEqual(vget[8], vans[8])
+    })
+
+    vans[9] = [null, null]
+    it(`should get ${JSON.stringify(vans[9])} for not writing any record by duplicated id in same batch`, async function() {
+        assert.strict.deepStrictEqual(vget[9], vans[9])
+    })
+
+    vans[10] = { n: 1, nInserted: 1, ok: 1 }
+    it(`should get ${JSON.stringify(vans[10])} for insertBulk with single object`, async function() {
+        assert.strict.deepStrictEqual(vget[10], vans[10])
+    })
+
+    vans[11] = { n: 0, nInserted: 0, ok: 1 }
+    it(`should get ${JSON.stringify(vans[11])} for insertBulk with invalid data`, async function() {
+        assert.strict.deepStrictEqual(vget[11], vans[11])
+    })
+
+    vans[12] = ['n,nInserted,ok', 'n,nInserted,ok', 'n,nInserted,ok', 'n,nInserted,ok']
+    it(`should get ${JSON.stringify(vans[12])} for same key set between insertBulk and insert`, async function() {
+        assert.strict.deepStrictEqual(vget[12], vans[12])
+    })
+
+    vans[13] = { n: 2, nInserted: 2, ok: 1 }
+    it(`should get ${JSON.stringify(vans[13])} for insertBulk without id by default autoGenPk`, async function() {
+        assert.strict.deepStrictEqual(vget[13], vans[13])
+    })
+
+    vans[14] = 'Error: invalid data[1].id, autoGenPk is false'
+    it(`should get ${JSON.stringify(vans[14])} for insertBulk without id by autoGenPk=false`, async function() {
+        assert.strict.deepStrictEqual(vget[14], vans[14])
+    })
+
+    vans[15] = null
+    it(`should get ${JSON.stringify(vans[15])} for not writing valid records in rejected insertBulk by autoGenPk`, async function() {
+        assert.strict.deepStrictEqual(vget[15], vans[15])
+    })
+
+    vans[16] = ['change:insertBulk', 'error:insertBulk']
+    it(`should get ${JSON.stringify(vans[16])} for events of insertBulk`, async function() {
+        assert.strict.deepStrictEqual(vget[16], vans[16])
+    })
+
+})
+
+
 describe('save', function() {
     let vans = {}
     let vget = {}