w-converhp 2.0.36 → 2.0.38

package/sclc.mjs

@@ -27,7 +27,7 @@ wo.on('error', (err) => {
 function uploadLargeFile() {
     let core = async() => {
 
-        let fsz = '1000mb' //0mb 10mb 100mb 1000mb
+        let fsz = '10mb' //0mb 10mb 100mb 1000mb
 
         //u8a
         let u8a = fs.readFileSync(`../_data/${fsz}.7z`) //用fs讀有檔案大小上限
@@ -54,6 +54,9 @@ function uploadLargeFile() {
 
     }
     core()
+        .catch((err) => {
+            console.log(err)
+        })
 }
 
 uploadLargeFile()

package/scld.mjs

@@ -49,6 +49,9 @@ function downloadLargeFile() {
 
     }
     core()
+        .catch((err) => {
+            console.log(err)
+        })
 }
 
 downloadLargeFile()

package/src/WConverhpClient.mjs

@@ -164,12 +164,12 @@ function WConverhpClient(opt) {
         else if (type === 'slice') {
             urlUse = `${url}slc`
         }
-        else if (type === 'upload-check') {
-            urlUse = `${url}ulck`
-        }
-        else if (type === 'slice-merge') {
-            urlUse = `${url}slcm`
+        else if (type === 'upload-controller') {
+            urlUse = `${url}ulctr`
         }
+        // else if (type === 'slice-merge') {
+        //     urlUse = `${url}slcm`
+        // }
         else if (type === 'download-get-filename') {
             urlUse = `${url}dwgfn`
         }
@@ -682,11 +682,11 @@ function WConverhpClient(opt) {
         let fileTotalHash = await calcHash(bb)
         // console.log(`calc hash for fileTotalSize[${fileTotalSize}] done`, fileTotalHash)
 
-        //send upload-check
+        //send check-total-hash
         // console.log(`send check-total-hash...`)
-        let resCk = await send('upload-check', { mode: 'check-total-hash', fileHash: fileTotalHash, fileSize: fileTotalSize }, { dataType: 'json' })
-        // console.log('resCk', resCk)
-        // resCk {
+        let resUpCkt = await send('upload-controller', { mode: 'check-total-hash', fileHash: fileTotalHash, filename: fileTotalName, fileSize: fileTotalSize }, { dataType: 'json' })
+        // console.log('resUpCkt', resUpCkt)
+        // resUpCkt {
         //   path: 'uploadTemp\\2429b7ef08ce6ba9',
         //   bAllExist: false,
         //   bAllSize: false,
@@ -702,30 +702,30 @@ function WConverhpClient(opt) {
         // }
 
         //check
-        if (resCk.bAllHash) {
+        if (resUpCkt.bAllHash) {
             // console.log('已有上傳大檔')
             let resMg = {
                 filename: fileTotalName,
-                path: resCk.path,
+                path: resUpCkt.path,
             }
             return resMg
         }
 
         //針對伺服器上已有切片檔案計算hash與比對
-        if (resCk.slks.length > 0) {
-            // console.log('receive slks...', resCk.slks[0], size(resCk.slks))
+        if (resUpCkt.slks.length > 0) {
+            // console.log('receive slks...', resUpCkt.slks[0], size(resUpCkt.slks))
 
             //fileSliceHashs
             let fileSliceHashs = []
-            // let n = Math.max(resCk.slks.length, 1)
+            // let n = Math.max(resUpCkt.slks.length, 1)
             // let nr = Math.floor(n / 100)
-            for (let k = 0; k < resCk.slks.length; k++) {
+            for (let k = 0; k < resUpCkt.slks.length; k++) {
                 // if (k % nr === 0) {
-                //     console.log(`calc hash for slices`, round(k / resCk.slks.length * 100, 1), '%')
+                //     console.log(`calc hash for slices`, round(k / resUpCkt.slks.length * 100, 1), '%')
                 // }
 
                 //i
-                let i = resCk.slks[k]
+                let i = resUpCkt.slks[k]
 
                 //start
                 let start = i * sizeSlice
@@ -749,11 +749,11 @@ function WConverhpClient(opt) {
             }
             // console.log('fileSliceHashs', fileSliceHashs)
 
-            //send upload-check
+            //send check-slices-hash
             // console.log(`send check-slices-hash...`)
-            let resCks = await send('upload-check', { mode: 'check-slices-hash', fileHash: fileTotalHash, fileSliceHashs }, { dataType: 'json' })
-            // console.log('resCks', resCks)
-            // resCks {
+            let resUpCks = await send('upload-controller', { mode: 'check-slices-hash', fileHash: fileTotalHash, fileSliceHashs }, { dataType: 'json' })
+            // console.log('resUpCks', resUpCks)
+            // resUpCks {
             //   slks: [
             //      0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11,
             //     12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
@@ -763,8 +763,8 @@ function WConverhpClient(opt) {
             //   ]
             // }
 
-            //update
-            resCk.slks = resCks.slks
+            //update, 伺服器針對各切片計算hash與比對, 回傳resUpCks.slks代表hash一致的切片編號, 非一致hash的切片則須重傳, 尚未傳切片亦須繼續傳
+            resUpCkt.slks = resUpCks.slks
 
         }
 
@@ -804,7 +804,7 @@ function WConverhpClient(opt) {
         for (let i = 0; i < chunkTotal; i++) {
 
             //check
-            if (resCk.slks.indexOf(i) >= 0) {
+            if (resUpCkt.slks.indexOf(i) >= 0) {
                 // console.log('已有上傳切片檔')
                 cbProgressSlice({ prog: 100, m: 'upload' }) //直接觸發更新進度
                 continue
@@ -820,7 +820,7 @@ function WConverhpClient(opt) {
             let chunk = bb.slice(start, end)
 
             //send slice
-            let hd = {
+            let hd = { //用header傳key與value時, key不分大小寫, 故使用kebabCase
                 'chunk-index': i,
                 'chunk-total': chunkTotal,
                 'package-id': packageId,
@@ -830,16 +830,64 @@ function WConverhpClient(opt) {
 
         }
 
-        //send slice-merge
-        let msg = {
-            'filename': fileTotalName,
-            'chunk-total': chunkTotal,
-            'package-id': packageId,
+        //send merge-slices-push
+        // console.log(`send merge-slices-push...`)
+        let resUpMgp = await send('upload-controller', { mode: 'merge-slices-push', fileHash: fileTotalHash, chunkTotal }, { dataType: 'json' })
+        // console.log('resUpMgp', resUpMgp)
+
+        //checkMerging
+        let checkMerging = () => {
+            let pm = genPm()
+
+            //queueId
+            let queueId = resUpMgp.queueId
+            // console.log('queueId', queueId)
+
+            let t = setInterval(() => {
+
+                //send merge-slices-get
+                // console.log(`send merge-slices-get...`)
+                send('upload-controller', { mode: 'merge-slices-get', fileHash: fileTotalHash, filename: fileTotalName, queueId }, { dataType: 'json' })
+                    .then((res) => {
+                        // console.log('res', res)
+                        // res => {
+                        //   queueId,
+                        //   state,
+                        //   filename,
+                        //   path,
+                        //   msg: ...
+                        // }
+
+                        //check
+                        if (res.state === 'done') {
+
+                            //cbProgressMerge
+                            cbProgressMerge({ prog: 100, m: 'download' })
+
+                            //resolve, 前端偵測state為'done'時提取msg顯示
+                            pm.resolve(res.msg)
+
+                        }
+
+                    })
+                    .catch((err) => {
+                        console.log(err)
+                    })
+                    .finally(() => {
+                        clearInterval(t)
+                    })
+
+            }, 2000)
+
+            return pm
         }
-        let resMg = await send('slice-merge', msg, { dataType: 'json', cbProgress: cbProgressMerge })
+
+        //checkMerging
+        let resUpMgr = await checkMerging()
+        // console.log('resUpMgr', resUpMgr)
 
         // console.log(`upload slice done`)
-        return resMg
+        return resUpMgr
     }
 
     //execute

package/src/WConverhpServer.mjs

@@ -23,8 +23,7 @@ import obj2u8arr from 'wsemi/src/obj2u8arr.mjs'
 import u8arr2obj from 'wsemi/src/u8arr2obj.mjs'
 import fsIsFolder from 'wsemi/src/fsIsFolder.mjs'
 import fsCreateFolder from 'wsemi/src/fsCreateFolder.mjs'
-// import mergeFiles from './mergeFiles.mjs'
-import mergeFiles from './mergeFiles.wk.umd.js'
+import managerMergeSlices from './managerMergeSlices.mjs'
 // import checkTotalHash from './checkTotalHash.mjs'
 import checkTotalHash from './checkTotalHash.wk.umd.js'
 // import checkSlicesHash from './checkSlicesHash.mjs'
@@ -36,6 +35,10 @@ import checkSlicesHash from './checkSlicesHash.wk.umd.js'
 //回傳前端(browser)時, 針對超大檔, 可用POST並用stream回傳但還要處理進度條, 若要交由瀏覽器下載器處理, 只能用GET並用stream回傳, 且前端只能用window.location.href或a.href+a.click()下載
 
 
+//mmg
+let mmg = managerMergeSlices()
+
+
 /**
  * 建立Hapi伺服器
  *
@@ -605,7 +608,7 @@ function WConverhpServer(opt = {}) {
 
     //apiUploadCheck
     let apiUploadCheck = {
-        path: `/${apiName}ulck`,
+        path: `/${apiName}ulctr`,
         method: 'POST',
         options: {
             payload: {
@@ -642,7 +645,7 @@ function WConverhpServer(opt = {}) {
             if (true) {
 
                 //verifyConn
-                let m = verifyConn({ apiType: 'upload-check', authorization, headers, query })
+                let m = verifyConn({ apiType: 'upload-controller', authorization, headers, query })
                 if (ispm(m)) {
                     m = await m
                 }
@@ -665,13 +668,14 @@ function WConverhpServer(opt = {}) {
             let mode = get(req, 'payload.mode', '')
 
             //check
-            if (mode !== 'check-total-hash' && mode !== 'check-slices-hash') {
+            if (mode !== 'check-total-hash' && mode !== 'check-slices-hash' && mode !== 'merge-slices-push' && mode !== 'merge-slices-get') {
                 // console.log('invalid mode in payload')
                 return responseU8aStreamWithError(res, `invalid mode[${mode}] in payload`)
             }
 
             //fileHash, 從payload接收
             let fileHash = get(req, 'payload.fileHash', '')
+            // console.log(mode, 'fileHash', fileHash)
 
             //check
             if (!isestr(fileHash)) {
@@ -679,36 +683,143 @@ function WConverhpServer(opt = {}) {
                 return responseU8aStreamWithError(res, 'invalid fileHash in payload')
             }
 
-            //out
-            let out = null
-            if (mode === 'check-total-hash') {
+            //procCore
+            let procCore = async() => {
+                let out = null
+                if (mode === 'check-total-hash') {
 
-                //fileSize, 從payload接收
-                let fileSize = get(req, 'payload.fileSize', '')
+                    //filename, 從payload接收
+                    let filename = get(req, 'payload.filename', '')
+                    // console.log(mode, 'filename', filename)
 
-                //checkTotalHash
-                out = await checkTotalHash(fileSize, sizeSlice, fileHash, pathUploadTemp)
+                    //fileSize, 從payload接收
+                    let fileSize = get(req, 'payload.fileSize', '')
+                    // console.log(mode, 'fileSize', fileSize)
 
-            }
-            else if (mode === 'check-slices-hash') {
+                    //checkTotalHash
+                    out = await checkTotalHash(fileSize, sizeSlice, fileHash, pathUploadTemp)
+                    // console.log(mode, 'out', out)
+
+                    //check, 因合併大檔後可能非預期中斷而重傳, 每次偵測有合併完成大檔, 就得調用procUpload讓伺服器攔截函數處理
+                    if (out.bAllHash) {
+
+                        //ri
+                        let ri = {
+                            filename,
+                            path: out.path,
+                        }
+
+                        //procUpload
+                        // console.log('procUpload start')
+                        let ro = await procUpload(ri)
+                        // console.log('procUpload done', ro)
+
+                        //out merge, 直接添加ro就回傳, 此處不使用msg儲存
+                        out = {
+                            ...out,
+                            ...ro,
+                        }
+
+                    }
+
+                }
+                else if (mode === 'check-slices-hash') {
+
+                    //fileSliceHashs, 從payload接收
+                    let fileSliceHashs = get(req, 'payload.fileSliceHashs', [])
+                    // console.log(mode, 'fileSliceHashs', fileSliceHashs)
+
+                    //checkSlicesHash
+                    out = await checkSlicesHash(fileSliceHashs, fileHash, pathUploadTemp)
+                    // console.log(mode, 'out', out)
+
+                }
+                else if (mode === 'merge-slices-push') {
+
+                    //chunkTotal, 從payload接收
+                    let chunkTotal = get(req, 'payload.chunkTotal', '')
+                    // console.log(mode, 'chunkTotal', chunkTotal)
+
+                    //mmg.push
+                    let queueId = mmg.push(fileHash, chunkTotal, pathUploadTemp)
+
+                    //out
+                    out = {
+                        queueId,
+                    }
+
+                }
+                else if (mode === 'merge-slices-get') {
+
+                    //filename, 從payload接收
+                    let filename = get(req, 'payload.filename', '')
+                    // console.log(mode, 'filename', filename)
+
+                    //queueId, 從payload接收
+                    let queueId = get(req, 'payload.queueId', '')
+                    // console.log(mode, 'queueId', queueId)
+
+                    //mmg.get
+                    let r = mmg.get(queueId, pathUploadTemp)
+
+                    //out
+                    out = {
+                        queueId,
+                        state: r.state,
+                        filename,
+                        path: r.path,
+                    }
+
+                    //check
+                    if (r.state === 'done') {
 
-                //fileSliceHashs, 從payload接收
-                let fileSliceHashs = get(req, 'payload.fileSliceHashs', [])
-                // console.log('fileSliceHashs', fileSliceHashs)
+                        //ri
+                        let ri = {
+                            filename,
+                            path: r.path,
+                        }
 
-                //checkSlicesHash
-                out = await checkSlicesHash(fileSliceHashs, fileHash, pathUploadTemp)
+                        //procUpload, 偵測有合併完成大檔, 得調用procUpload讓伺服器攔截函數處理
+                        // console.log('procUpload start')
+                        let ro = await procUpload(ri)
+                        // console.log('procUpload done', ro)
 
+                        //out merge, ro須附加至msg, 才供前端偵測state為'done'時提取msg顯示
+                        out = {
+                            ...out,
+                            msg: ro,
+                        }
+
+                    }
+
+                }
+                // console.log('out', out)
+
+                return out
             }
+
+            //procCore
+            let out = {}
+            let returnType = ''
+            let returnMsg = ''
+            await procCore()
+                .then((res) => {
+                    out.success = res
+                    returnType = 'success'
+                    returnMsg = 'need to parse'
+                })
+                .catch((err) => {
+                    out.error = err
+                    returnType = 'error'
+                    returnMsg = 'need to parse'
+                })
             // console.log('out', out)
 
-            if (out.error) {
-                return responseU8aStreamWithError(res, out.error)
-            }
-            else {
-                let u8aOut = obj2u8arr(out)
-                return responseU8aStream(res, u8aOut, { returnType: 'success', returnMsg: 'need to parse' })
-            }
+            //u8aOut
+            let u8aOut = obj2u8arr(out)
+            // console.log('u8aOut', u8aOut)
+
+            return responseU8aStream(res, u8aOut, { returnType, returnMsg })
         },
     }
 
@@ -777,18 +888,18 @@ function WConverhpServer(opt = {}) {
 
             //check
             if (!isp0int(chunkIndex)) {
-                console.log('invalid chunkIndex in headers')
-                return res.response({ error: 'invalid chunkIndex in headers' }).code(400)
+                // console.log('invalid chunkIndex in headers')
+                return responseU8aStreamWithError(res, 'invalid chunkIndex in headers')
             }
             chunkIndex = cint(chunkIndex)
             if (!isp0int(chunkTotal)) {
-                console.log('invalid chunkTotal in headers')
-                return res.response({ error: 'invalid chunkTotal in headers' }).code(400)
+                // console.log('invalid chunkTotal in headers')
+                return responseU8aStreamWithError(res, 'invalid chunkTotal in headers')
             }
             chunkTotal = cint(chunkTotal)
             if (!isestr(packageId)) {
-                console.log('invalid packageId in headers')
-                return res.response({ error: 'invalid packageId in headers' }).code(400)
+                // console.log('invalid packageId in headers')
+                return responseU8aStreamWithError(res, 'invalid packageId in headers')
             }
 
             //pathFileChunk
@@ -855,126 +966,6 @@ function WConverhpServer(opt = {}) {
         },
     }
 
-    //apiUploadSliceMerge
-    let apiUploadSliceMerge = {
-        path: `/${apiName}slcm`,
-        method: 'POST',
-        options: {
-            payload: {
-                maxBytes: 1024 * 1024 * 1024 * 1024, //預設為1mb, 調整至1tb, 也就是給予3次方
-                maxParts: 1000 * 1000 * 1000, //預設為1000, 給予3次方
-                timeout: false, //避免請求未完成時中斷
-                // output: 'stream',
-                parse: true, //前端送obj過來須自動解析
-            },
-            timeout: {
-                server: false, //關閉伺服器超時
-                socket: false, //關閉socket超時
-            },
-        },
-        handler: async function (req, res) {
-            // console.log(req, res)
-            // console.log('payload', req.payload)
-
-            //headers
-            let headers = get(req, 'headers')
-            headers = iseobj(headers) ? headers : ''
-            // console.log('headers', headers)
-
-            //query
-            let query = get(req, 'query')
-            query = iseobj(query) ? query : ''
-            // console.log('query', query)
-
-            //authorization
-            let authorization = get(headers, 'authorization', '')
-            authorization = isestr(authorization) ? authorization : ''
-
-            //check
-            if (true) {
-
-                //verifyConn
-                let m = verifyConn({ apiType: 'upload-slice-merge', authorization, headers, query })
-                if (ispm(m)) {
-                    m = await m
-                }
-
-                //check
-                if (m !== true) {
-                    return responseU8aStreamWithError(res, 'permission denied')
-                }
-
-            }
-
-            //eeEmit
-            eeEmit('handler', {
-                api: 'apiUploadSliceMerge',
-                headers,
-                query,
-            })
-
-            //filename, chunkTotal, packageId, 從payload接收
-            let filename = get(req, 'payload.filename', '')
-            let chunkTotal = get(req, 'payload.chunk-total', '')
-            let packageId = get(req, 'payload.package-id', '')
-
-            //check
-            if (!isestr(filename)) {
-                console.log('invalid filename in payload')
-                return res.response({ error: 'invalid filename in payload' }).code(400)
-            }
-            if (!isp0int(chunkTotal)) {
-                console.log('invalid chunkTotal in payload')
-                return res.response({ error: 'invalid chunkTotal in payload' }).code(400)
-            }
-            chunkTotal = cint(chunkTotal)
-            if (!isestr(packageId)) {
-                console.log('invalid packageId in payload')
-                return res.response({ error: 'invalid packageId in payload' }).code(400)
-            }
-
-            //mergeAndPorc
-            let mergeAndPorc = async() => {
-
-                //merge
-                // console.log('merge start')
-                let rMerg = await mergeFiles(pathUploadTemp, packageId, chunkTotal, filename)
-                // console.log('merge done', rMerg)
-
-                //procUpload
-                // console.log('procUpload start')
-                let rProc = await procUpload(rMerg)
-                // console.log('procUpload done', rProc)
-
-                return rProc
-            }
-
-            //mergeAndPorc
-            let out = {}
-            let returnType = ''
-            let returnMsg = ''
-            await mergeAndPorc()
-                .then((res) => {
-                    out.success = res
-                    returnType = 'success'
-                    returnMsg = 'need to parse'
-                })
-                .catch((err) => {
-                    out.error = err
-                    returnType = 'error'
-                    returnMsg = 'need to parse'
-                    eeEmit('error', `${err.message}`)
-                })
-            // console.log('out', out)
-
-            //u8aOut
-            let u8aOut = obj2u8arr(out)
-            // console.log('u8aOut', u8aOut)
-
-            return responseU8aStream(res, u8aOut, { returnType, returnMsg })
-        },
-    }
-
     //apiDownloadGetFilename
     let apiDownloadGetFilename = {
         path: `/${apiName}dwgfn`,
@@ -1386,7 +1377,7 @@ function WConverhpServer(opt = {}) {
                 apiMain,
                 apiUploadCheck,
                 apiUploadSlice,
-                apiUploadSliceMerge,
+                // apiUploadSliceMerge,
                 apiDownloadGetFilename,
                 apiDownloadGetFile,
                 apiDownload,
@@ -1405,7 +1396,8 @@ function WConverhpServer(opt = {}) {
 
     //start
     if (get(opt, 'serverHapi')) {
-        server.route([apiMain, apiUploadCheck, apiUploadSlice, apiUploadSliceMerge, apiDownloadGetFilename, apiDownloadGetFile, apiDownload])
+        // server.route([apiMain, apiUploadCheck, apiUploadSlice, apiUploadSliceMerge, apiDownloadGetFilename, apiDownloadGetFile, apiDownload])
+        server.route([apiMain, apiUploadCheck, apiUploadSlice, apiDownloadGetFilename, apiDownloadGetFile, apiDownload])
     }
     else {
         startServer()

package/src/checkSlicesHash.mjs

@@ -58,9 +58,7 @@ let checkSlicesHash = async(fileSliceHashs, fileHash, pathUploadTemp) => {
 
     //r
     let r = {
-        success: {
-            slks: slksCfm,
-        },
+        slks: slksCfm,
     }
 
     return r