w-orm-postgresql 1.0.10 → 1.0.12

package/g-basic.mjs

@@ -110,6 +110,19 @@ let rsa = [
     },
 ]
 
+let rsb = [
+    {
+        time: '2025-01-01T00:20:00Z',
+        name: 'bulk-1',
+        value: 201,
+    },
+    {
+        time: '2025-01-01T00:21:00Z',
+        name: 'bulk-2',
+        value: 202,
+    },
+]
+
 async function test() {
 
     //wo
@@ -132,6 +145,9 @@ async function test() {
     wo.on('change', function(mode, data, res) {
         console.log('change', mode)
     })
+    wo.on('error', function(mode, data, err) {
+        console.log('error', mode, err)
+    })
 
     //delAll
     await wo.delAll()
@@ -151,6 +167,24 @@ async function test() {
             console.log('insert catch', msg)
         })
 
+    //insertBulk, 全批視為一個單位, 全部插入成功或一筆都不寫入
+    await wo.insertBulk(rsb)
+        .then(function(msg) {
+            console.log('insertBulk then', msg)
+        })
+        .catch(function(msg) {
+            console.log('insertBulk catch', msg)
+        })
+
+    //insertBulk by pk existed, 任一筆之主鍵已存在即整批reject且不寫入任何一筆
+    await wo.insertBulk(rsb)
+        .then(function(msg) {
+            console.log('insertBulk by pk existed then', msg)
+        })
+        .catch(function(msg) {
+            console.log('insertBulk by pk existed catch', `${msg}`) //reject回傳Error物件, 以樣板字串取其訊息以免印出堆疊
+        })
+
     //save
     await wo.save(rsm, { autoInsert: false })
         .then(function(msg) {
@@ -213,6 +247,15 @@ async function test() {
             console.log('del catch', msg)
         })
 
+    //del by data without pk, 該筆無法處理故ok為0並附err, 整批仍resolve且另發出error事件
+    await wo.del({ name: 'no-pk' })
+        .then(function(msg) {
+            console.log('del by data without pk then', msg)
+        })
+        .catch(function(msg) {
+            console.log('del by data without pk catch', msg)
+        })
+
     //del
     let ssDel = await wo.select()
     let ds = ssDel.filter(function(v) {
@@ -298,6 +341,10 @@ test()
 // delAll then { n: 0, nDeleted: 0, ok: 1 }
 // change insert
 // insert then { n: 13, nInserted: 13, ok: 1 }
+// change insertBulk
+// insertBulk then { n: 2, nInserted: 2, ok: 1 }
+// error insertBulk duplicate key value violates unique constraint "users_pkey"
+// insertBulk by pk existed catch error: duplicate key value violates unique constraint "users_pkey"
 // change save
 // save then [
 //   { n: 1, nInserted: 0, nModified: 0, ok: 1 },
@@ -317,6 +364,8 @@ test()
 //   { time: 2025-01-01T00:07:00.000Z, name: 'rosemary', value: 124.76 },
 //   { time: 2025-01-01T00:08:00.000Z, name: 'kettle', value: 524 },
 //   { time: 2025-01-01T00:09:00.000Z, name: 'peter', value: 127 },
+//   { time: 2025-01-01T00:20:00.000Z, name: 'bulk-1', value: 201 },
+//   { time: 2025-01-01T00:21:00.000Z, name: 'bulk-2', value: 202 },
 //   {
 //     time: 2025-01-01T00:10:00.000Z,
 //     name: 'rosemary(modify)',
@@ -352,6 +401,8 @@ test()
 //   { time: 2025-01-01T00:03:00.000Z, name: 'peter', value: 200 },
 //   { time: 2025-01-01T00:05:00.000Z, name: 'kettle', value: 488 },
 //   { time: 2025-01-01T00:08:00.000Z, name: 'kettle', value: 524 },
+//   { time: 2025-01-01T00:20:00.000Z, name: 'bulk-1', value: 201 },
+//   { time: 2025-01-01T00:21:00.000Z, name: 'bulk-2', value: 202 },
 //   {
 //     time: 2025-01-01T00:11:00.000Z,
 //     name: 'kettle(modify)',
@@ -369,10 +420,14 @@ test()
 //     value: 447
 //   }
 // ]
+// change insert
 // change save
 // save then [ { n: 1, nInserted: 1, nModified: 0, ok: 1 } ]
 // change del
 // del then [ { n: 0, nDeleted: 0, ok: 1 } ]
+// error del invalid time[undefined]
+// change del
+// del by data without pk then [ { n: 0, nDeleted: 0, ok: 0, err: 'invalid time[undefined]' } ]
 // change del
 // del then [
 //   { n: 1, nDeleted: 1, ok: 1 },
@@ -390,6 +445,8 @@ test()
 //   { time: 2025-01-01T00:01:00.000Z, name: 'rosemary', value: 123.456 },
 //   { time: 2025-01-01T00:04:00.000Z, name: 'rosemary', value: 123.1236 },
 //   { time: 2025-01-01T00:07:00.000Z, name: 'rosemary', value: 124.76 },
+//   { time: 2025-01-01T00:20:00.000Z, name: 'bulk-1', value: 201 },
+//   { time: 2025-01-01T00:21:00.000Z, name: 'bulk-2', value: 202 },
 //   {
 //     time: 2025-01-01T00:10:00.000Z,
 //     name: 'rosemary(modify)',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "w-orm-postgresql",
-  "version": "1.0.10",
+  "version": "1.0.12",
   "main": "dist/w-orm-postgresql.umd.js",
   "dependencies": {
     "mongo-sql": "^6.2.0",

package/src/WOrmPostgresql.mjs

@@ -1,4 +1,3 @@
-import events from 'events'
 import pg from 'pg'
 import mongoSql from 'mongo-sql'
 import size from 'lodash-es/size.js'
@@ -20,6 +19,7 @@ import isint from 'wsemi/src/isint.mjs'
 import isbol from 'wsemi/src/isbol.mjs'
 import isDate from 'wsemi/src/isDate.mjs'
 import haskey from 'wsemi/src/haskey.mjs'
+import evem from 'wsemi/src/evem.mjs'
 import pmSeries from 'wsemi/src/pmSeries.mjs'
 
 
@@ -173,8 +173,40 @@ function WOrmPostgresql(opt = {}) {
     let connectionString = `${url}/${db}`
     // console.log('connectionString', connectionString)
 
-    //ee
-    let ee = new events.EventEmitter()
+    //ee, 採eventemitter3而不採Node內建之events.EventEmitter,
+    //因後者於'error'無監聽者時會將錯誤直接拋出, 將使操作行為隨呼叫端是否註冊監聽而異
+    let ee = evem()
+
+    //getErrMsg, 取錯誤訊息字串, 供逐筆結果之err欄位與error事件使用
+    let getErrMsg = (err) => {
+        let m = get(err, 'message')
+        if (isestr(m)) {
+            return m
+        }
+        return String(err)
+    }
+
+    //emitChange, 資料實際異動成功後發出, 事件僅為附加通知不承擔回傳義務
+    //訂閱函數拋錯不得影響本次操作, 故一律以try/catch包覆, 收斂於此以免逐處遺漏
+    let emitChange = (mode, data, res) => {
+        try {
+            ee.emit('change', mode, data, res)
+        }
+        catch (errEmit) {
+            console.log(errEmit)
+        }
+    }
+
+    //emitError, 操作發生錯誤時發出, 錯誤訊息一律轉為字串
+    //正常結果(如查無數據、主鍵未命中、內容相同而未寫入)不得發出, 以免監聽者將常態當異常
+    let emitError = (mode, data, err) => {
+        try {
+            ee.emit('error', mode, data, getErrMsg(err))
+        }
+        catch (errEmit) {
+            console.log(errEmit)
+        }
+    }
 
     //PgClient
     let PgClient = pg.Client
@@ -213,6 +245,77 @@ function WOrmPostgresql(opt = {}) {
         _cache[key] = cloneDeep(data) //與外部使用數據脫勾
     }
 
+    //insertBatches, 分批送出插入語句並以交易包覆, 回傳實際插入筆數, 供insert與insertBulk共用
+    //PostgreSQL之協定以int16記綁定參數個數故上限為65535, 單一語句可送之筆數受欄位數所限,
+    //超出者須分批送出; 分批後單語句之原子性已不足以維持[失敗即不留下部份寫入], 故一律以交易包覆,
+    //令單批與分批之保證來源一致而不因批數而異
+    //兩函數之衝突政策由useConflict區分, 語句各自獨立:
+    //  true : 添加ON CONFLICT DO NOTHING, 已存在主鍵者跳過而不中斷整批, 為insert之語義
+    //  false: 不添加, 任一筆撞主鍵之唯一約束即整句失敗且不寫入任何一筆, 為insertBulk之全有全無語義
+    let insertBatches = async (client, data, useConflict) => {
+
+        //nCols, 取各筆欄位數之最大值
+        let nCols = 1
+        data.forEach(function(v) {
+            nCols = Math.max(nCols, size(keys(v)))
+        })
+
+        //nPerBatch
+        let nPerBatch = Math.max(1, Math.floor(65535 / nCols))
+
+        //ltdata, 分批
+        let ltdata = []
+        for (let i = 0; i < size(data); i += nPerBatch) {
+            ltdata.push(data.slice(i, i + nPerBatch))
+        }
+
+        //transaction
+        await client.query('BEGIN')
+
+        try {
+
+            //nIns
+            let nIns = 0
+
+            //each batch
+            for (let dt of ltdata) {
+
+                //mr
+                let mr = mongoSql.sql({
+                    type: 'insert',
+                    table: cl,
+                    values: dt,
+                })
+
+                //sql
+                let sql = mr.query
+                if (useConflict) {
+                    sql = `${sql} ON CONFLICT (${pkName}) DO NOTHING`
+                }
+
+                //insert
+                let r = await client.query(sql, mr.values)
+
+                //nIns
+                nIns += r.rowCount
+
+            }
+
+            //commit
+            await client.query('COMMIT')
+
+            return nIns
+        }
+        catch (err) {
+
+            //rollback, 令已送出之批次一併回滾
+            await client.query('ROLLBACK').catch(() => {})
+
+            throw err
+        }
+
+    }
+
     /**
      * 創建資料表
      * 註: pk未給時採建構時之opt.pk。因insert與save倚賴主鍵之唯一約束達成原子性，
@@ -252,6 +355,10 @@ function WOrmPostgresql(opt = {}) {
 
         //check
         if (isErr) {
+
+            //emit, 整批性錯誤於reject之前發出
+            emitError('createTable', arr, res)
+
             return Promise.reject(res)
         }
 
@@ -302,6 +409,10 @@ function WOrmPostgresql(opt = {}) {
 
         //check
         if (isErr) {
+
+            //emit, 整批性錯誤於reject之前發出
+            emitError('createTable', arr, res)
+
             return Promise.reject(res)
         }
 
@@ -346,6 +457,10 @@ function WOrmPostgresql(opt = {}) {
 
         //check
         if (isErr) {
+
+            //emit, 整批性錯誤於reject之前發出, 讀取函數無輸入數據故data為null
+            emitError('select', null, res)
+
             return Promise.reject(res)
         }
 
@@ -394,6 +509,10 @@ function WOrmPostgresql(opt = {}) {
 
         //check
         if (isErr) {
+
+            //emit, 整批性錯誤於reject之前發出, 讀取函數無輸入數據故data為null
+            emitError('select', null, res)
+
             return Promise.reject(res)
         }
 
@@ -452,6 +571,10 @@ function WOrmPostgresql(opt = {}) {
 
         //check
         if (isErr) {
+
+            //emit, 整批性錯誤於reject之前發出, 讀取函數無輸入數據故data為null
+            emitError('selectByPk', null, res)
+
             return Promise.reject(res)
         }
 
@@ -518,6 +641,11 @@ function WOrmPostgresql(opt = {}) {
 
         //check
         if (isErr) {
+
+            //emit, 整批性錯誤於reject之前發出, 讀取函數無輸入數據故data為null
+            //主鍵未命中與主鍵值無效皆為正常結果而不設isErr, 故不會誤發error
+            emitError('selectByPk', null, res)
+
             return Promise.reject(res)
         }
 
@@ -569,6 +697,10 @@ function WOrmPostgresql(opt = {}) {
 
         //check
         if (isErr) {
+
+            //emit, 整批性錯誤於reject之前發出
+            emitError('insert', data, res)
+
             return Promise.reject(res)
         }
 
@@ -587,40 +719,137 @@ function WOrmPostgresql(opt = {}) {
                 return v
             })
 
-            //mr
-            let mr = mongoSql.sql({
-                type: 'insert',
-                table: cl,
-                values: data,
-            })
-            // console.log('mr', mr)
-            // console.log('mr.query', mr.query)
-            // console.log('mr.values', mr.values)
+            //nAll
+            let nAll = size(data)
 
-            //添加conflict, 令已存在主鍵者跳過而不中斷整批插入, 主鍵認定與genConflictSQL一致
+            //insert, 以ON CONFLICT DO NOTHING令已存在主鍵者跳過而不中斷整批插入,
             //由PostgreSQL於單一語句內原子完成[檢查主鍵未存在]與[寫入], 併發時同一主鍵僅有一次成功,
             //同批含重複主鍵時亦僅首筆成功, 故不須逐筆插入即可取得實際插入筆數
-            let sql = `${mr.query} ON CONFLICT (${pkName}) DO NOTHING`
+            //筆數過多而超出綁定參數上限者由insertBatches分批送出, 主鍵認定與genConflictSQL一致
+            let nIns = await insertBatches(client, data, true)
+
+            //res
+            res = {
+                n: nAll,
+                nInserted: nIns,
+                ok: 1,
+            }
+
+        }
+        catch (err) {
+            isErr = true
+            res = err
+        }
+        finally {
+            await client.end()
+            client = null
+        }
+
+        //update
+        if (useCache) {
+            clearCache()
+        }
+
+        //emit
+        if (!isErr) {
+            emitChange('insert', data, res)
+        }
+
+        //check
+        if (isErr) {
+
+            //emit, 整批性錯誤於reject之前發出
+            emitError('insert', data, res)
+
+            return Promise.reject(res)
+        }
+
+        return res
+    }
+
+    /**
+     * 批次插入數據，全批視為一個單位，全部插入成功或一筆都不寫入
+     *
+     * 本函數非insert之加速版，兩者衝突政策不同：insert於主鍵已存在時跳過該筆且整批ok為1，
+     * 本函數則於任一筆之主鍵已存在時整批reject且不寫入任何一筆，同批含重複主鍵者亦視為衝突。
+     * n為輸入筆數，nInserted為實際插入筆數且成功時恆等於n。
+     * 不提供逐筆結果，故不出現ok:0與err，需要逐筆處置者改用insert。
+     * 因主鍵得承載業務語義，未帶有效主鍵值時不自動補值，一律以reject回報，
+     * 且主鍵檢查於任何寫入之前一次完成，故整批reject時同批之有效筆數亦不會被寫入。
+     *
+     * @memberOf WOrmPostgresql
+     * @param {Object|Array} data 輸入數據物件或陣列
+     * @returns {Promise} 回傳Promise，resolve回傳插入結果物件{n,nInserted,ok}，reject回傳錯誤訊息
+     */
+    async function insertBulk(data) {
+        let isErr = false
+        let res = null
+
+        //check
+        if (!iseobj(data) && !isearr(data)) {
+            return {
+                n: 0,
+                nInserted: 0,
+                ok: 1,
+            }
+        }
+
+        //cloneDeep
+        data = cloneDeep(data)
+
+        //client
+        let client = new PgClient({ connectionString })
+
+        //connect
+        try {
+
+            //connect
+            await client.connect()
+
+        }
+        catch (err) {
+            isErr = true
+            res = err
+            client = null
+        }
+
+        //check
+        if (isErr) {
+
+            //emit, 整批性錯誤於reject之前發出
+            emitError('insertBulk', data, res)
+
+            return Promise.reject(res)
+        }
+
+        try {
+
+            //check
+            if (!isarr(data)) {
+                data = [data]
+            }
+
+            //check pk, 因主鍵得承載業務語義故不自動補值, 未帶有效主鍵值者屬整批性錯誤
+            data = map(data, function(v, k) {
+                if (!checkPk(get(v, pkName))) {
+                    throw new Error(`invalid data[${k}].${pkName}[${get(v, pkName)}]`)
+                }
+                return v
+            })
 
             //nAll
             let nAll = size(data)
 
-            //insert, ON CONFLICT DO NOTHING時rowCount即為實際插入筆數
-            await client.query(sql, mr.values)
-                .then((r) => {
-
-                    //res
-                    res = {
-                        n: nAll,
-                        nInserted: r.rowCount,
-                        ok: 1,
-                    }
+            //insertBulk, 不添加conflict, 令任一筆撞主鍵之唯一約束即整句失敗且不寫入任何一筆,
+            //同批含重複主鍵者亦於此被偵測為衝突; 筆數過多者由insertBatches分批送出並以交易維持全有全無
+            let nIns = await insertBatches(client, data, false)
 
-                })
-                .catch((err) => {
-                    isErr = true
-                    res = err.message
-                })
+            //res
+            res = {
+                n: nAll,
+                nInserted: nIns,
+                ok: 1,
+            }
 
         }
         catch (err) {
@@ -639,16 +868,15 @@ function WOrmPostgresql(opt = {}) {
 
         //emit
         if (!isErr) {
-            try {
-                ee.emit('change', 'insert', data, res)
-            }
-            catch (err) {
-                console.log(err)
-            }
+            emitChange('insertBulk', data, res)
         }
 
         //check
         if (isErr) {
+
+            //emit, 整批性錯誤於reject之前發出
+            emitError('insertBulk', data, res)
+
             return Promise.reject(res)
         }
 
@@ -656,7 +884,7 @@ function WOrmPostgresql(opt = {}) {
     }
 
     /**
-     * 儲存數據，以主鍵time為準更新既有數據，未給之欄位保留
+     * 儲存數據，以主鍵為準更新既有數據，未給之欄位保留
      *
      * 回傳陣列恆與輸入等長，輸入單一物件亦回傳長度1之陣列。
      * 各筆之n為主鍵命中筆數，值為0或1，命中(不論內容有無變更)或經插入而產生皆為1；
@@ -705,6 +933,10 @@ function WOrmPostgresql(opt = {}) {
 
         //check
         if (isErr) {
+
+            //emit, 整批性錯誤於reject之前發出
+            emitError('save', data, res)
+
             return Promise.reject(res)
         }
 
@@ -970,6 +1202,16 @@ function WOrmPostgresql(opt = {}) {
 
                 }
 
+                //emit, 該筆經插入而產生者另發mode為insert之change事件, 供呼叫端區辨新增與更新
+                if (rest.nInserted === 1) {
+                    emitChange('insert', [v], rest)
+                }
+
+                //emit, 逐筆失敗於該筆結果定案後發出, 每筆一次
+                if (rest.ok === 0) {
+                    emitError('save', [v], rest.err)
+                }
+
                 return rest
             })
 
@@ -990,16 +1232,15 @@ function WOrmPostgresql(opt = {}) {
 
         //emit
         if (!isErr) {
-            try {
-                ee.emit('change', 'save', data, res)
-            }
-            catch (err) {
-                console.log(err)
-            }
+            emitChange('save', data, res)
         }
 
         //check
         if (isErr) {
+
+            //emit, 整批性錯誤於reject之前發出
+            emitError('save', data, res)
+
             return Promise.reject(res)
         }
 
@@ -1048,6 +1289,10 @@ function WOrmPostgresql(opt = {}) {
 
         //check
         if (isErr) {
+
+            //emit, 整批性錯誤於reject之前發出
+            emitError('del', data, res)
+
             return Promise.reject(res)
         }
 
@@ -1083,6 +1328,9 @@ function WOrmPostgresql(opt = {}) {
                         err: `invalid ${pkName}[${pkv}]`,
                     }
 
+                    //emit, 逐筆失敗於該筆結果定案後發出, 每筆一次
+                    emitError('del', [v], rest.err)
+
                     return rest
                 }
 
@@ -1122,6 +1370,11 @@ function WOrmPostgresql(opt = {}) {
 
                     })
 
+                //emit, 逐筆失敗於該筆結果定案後發出, 每筆一次
+                if (rest.ok === 0) {
+                    emitError('del', [v], rest.err)
+                }
+
                 return rest
             })
 
@@ -1142,16 +1395,15 @@ function WOrmPostgresql(opt = {}) {
 
         //emit
         if (!isErr) {
-            try {
-                ee.emit('change', 'del', data, res)
-            }
-            catch (err) {
-                console.log(err)
-            }
+            emitChange('del', data, res)
         }
 
         //check
         if (isErr) {
+
+            //emit, 整批性錯誤於reject之前發出
+            emitError('del', data, res)
+
             return Promise.reject(res)
         }
 
@@ -1190,6 +1442,10 @@ function WOrmPostgresql(opt = {}) {
 
         //check
         if (isErr) {
+
+            //emit, 整批性錯誤於reject之前發出, 無輸入數據故data為null
+            emitError('delAll', null, res)
+
             return Promise.reject(res)
         }
 
@@ -1253,16 +1509,15 @@ function WOrmPostgresql(opt = {}) {
 
         //emit
         if (!isErr) {
-            try {
-                ee.emit('change', 'delAll', null, res)
-            }
-            catch (err) {
-                console.log(err)
-            }
+            emitChange('delAll', null, res)
         }
 
         //check
         if (isErr) {
+
+            //emit, 整批性錯誤於reject之前發出
+            emitError('delAll', null, res)
+
             return Promise.reject(res)
         }
 
@@ -1274,6 +1529,7 @@ function WOrmPostgresql(opt = {}) {
     ee.select = select
     ee.selectByPk = selectByPk
     ee.insert = insert
+    ee.insertBulk = insertBulk
     ee.save = save
     ee.del = del
     ee.delAll = delAll