w-converhp 2.0.35 → 2.0.37

package/src/WConverhpServer.mjs

@@ -4,8 +4,6 @@ import stream from 'stream'
 import Hapi from '@hapi/hapi'
 import Inert from '@hapi/inert' //提供靜態檔案
 import get from 'lodash-es/get.js'
-import each from 'lodash-es/each.js'
-import size from 'lodash-es/size.js'
 import isNumber from 'lodash-es/isNumber.js'
 import genPm from 'wsemi/src/genPm.mjs'
 import evem from 'wsemi/src/evem.mjs'
@@ -15,7 +13,6 @@ import isp0int from 'wsemi/src/isp0int.mjs'
 import ispint from 'wsemi/src/ispint.mjs'
 import isearr from 'wsemi/src/isearr.mjs'
 import isbol from 'wsemi/src/isbol.mjs'
-import isnum from 'wsemi/src/isnum.mjs'
 import isfun from 'wsemi/src/isfun.mjs'
 import ispm from 'wsemi/src/ispm.mjs'
 import cint from 'wsemi/src/cint.mjs'
@@ -24,15 +21,13 @@ import str2b64 from 'wsemi/src/str2b64.mjs'
 import haskey from 'wsemi/src/haskey.mjs'
 import obj2u8arr from 'wsemi/src/obj2u8arr.mjs'
 import u8arr2obj from 'wsemi/src/u8arr2obj.mjs'
-import fsIsFile from 'wsemi/src/fsIsFile.mjs'
 import fsIsFolder from 'wsemi/src/fsIsFolder.mjs'
 import fsCreateFolder from 'wsemi/src/fsCreateFolder.mjs'
-import fsGetFilesInFolder from 'wsemi/src/fsGetFilesInFolder.mjs'
-import getFileXxHash from 'wsemi/src/getFileXxHash.mjs'
-// import calcFileHash from './calcFileHash.mjs'
-import calcFileHash from './calcFileHash.wk.umd.js'
-// import mergeFiles from './mergeFiles.mjs'
-import mergeFiles from './mergeFiles.wk.umd.js'
+import managerMergeSlices from './managerMergeSlices.mjs'
+// import checkTotalHash from './checkTotalHash.mjs'
+import checkTotalHash from './checkTotalHash.wk.umd.js'
+// import checkSlicesHash from './checkSlicesHash.mjs'
+import checkSlicesHash from './checkSlicesHash.wk.umd.js'
 
 
 //回傳前端stream時(POST或GET皆可), 前端會須等stream傳完才能判斷是否為大檔或錯誤訊息, 此會導致若回傳超大檔, 會需要對超大檔進行解析會有記憶體上限問題, 故需要通過header提供基本成功或失敗訊息, 讓前端能進行解析判斷
@@ -40,6 +35,10 @@ import mergeFiles from './mergeFiles.wk.umd.js'
 //回傳前端(browser)時, 針對超大檔, 可用POST並用stream回傳但還要處理進度條, 若要交由瀏覽器下載器處理, 只能用GET並用stream回傳, 且前端只能用window.location.href或a.href+a.click()下載
 
 
+//mmg
+let mmg = managerMergeSlices()
+
+
 /**
  * 建立Hapi伺服器
  *
@@ -609,7 +608,7 @@ function WConverhpServer(opt = {}) {
 
     //apiUploadCheck
     let apiUploadCheck = {
-        path: `/${apiName}ulck`,
+        path: `/${apiName}ulctr`,
         method: 'POST',
         options: {
             payload: {
@@ -646,7 +645,7 @@ function WConverhpServer(opt = {}) {
             if (true) {
 
                 //verifyConn
-                let m = verifyConn({ apiType: 'upload-check', authorization, headers, query })
+                let m = verifyConn({ apiType: 'upload-controller', authorization, headers, query })
                 if (ispm(m)) {
                     m = await m
                 }
@@ -669,13 +668,14 @@ function WConverhpServer(opt = {}) {
             let mode = get(req, 'payload.mode', '')
 
             //check
-            if (mode !== 'check-total-hash' && mode !== 'check-slices-hash') {
+            if (mode !== 'check-total-hash' && mode !== 'check-slices-hash' && mode !== 'merge-slices-push' && mode !== 'merge-slices-get') {
                 // console.log('invalid mode in payload')
                 return responseU8aStreamWithError(res, `invalid mode[${mode}] in payload`)
             }
 
             //fileHash, 從payload接收
             let fileHash = get(req, 'payload.fileHash', '')
+            // console.log(mode, 'fileHash', fileHash)
 
             //check
             if (!isestr(fileHash)) {
@@ -683,189 +683,143 @@ function WConverhpServer(opt = {}) {
                 return responseU8aStreamWithError(res, 'invalid fileHash in payload')
             }
 
-            //checkTotalHash
-            let checkTotalHash = async () => {
+            //procCore
+            let procCore = async() => {
+                let out = null
+                if (mode === 'check-total-hash') {
 
-                //pathFile
-                let pathFile = path.join(pathUploadTemp, fileHash)
-                // console.log('pathFile', pathFile)
+                    //filename, 從payload接收
+                    let filename = get(req, 'payload.filename', '')
+                    // console.log(mode, 'filename', filename)
 
-                //bAllExist
-                let bAllExist = false
-                if (true) {
-                    // console.log(`check exist for pathFile[${pathFile}]...`)
+                    //fileSize, 從payload接收
+                    let fileSize = get(req, 'payload.fileSize', '')
+                    // console.log(mode, 'fileSize', fileSize)
 
-                    bAllExist = fsIsFile(pathFile)
+                    //checkTotalHash
+                    out = await checkTotalHash(fileSize, sizeSlice, fileHash, pathUploadTemp)
+                    // console.log(mode, 'out', out)
 
-                    // console.log(`check exist for pathFile[${pathFile}] done`, bAllExist)
-                }
+                    //check, 因合併大檔後可能非預期中斷而重傳, 每次偵測有合併完成大檔, 就得調用procUpload讓伺服器攔截函數處理
+                    if (out.bAllHash) {
 
-                //fileSize, 從payload接收
-                let fileSize = get(req, 'payload.fileSize', '')
+                        //ri
+                        let ri = {
+                            filename,
+                            path: out.path,
+                        }
+
+                        //procUpload
+                        // console.log('procUpload start')
+                        let ro = await procUpload(ri)
+                        // console.log('procUpload done', ro)
+
+                        //out merge, 直接添加ro就回傳, 此處不使用msg儲存
+                        out = {
+                            ...out,
+                            ...ro,
+                        }
 
-                //check
-                if (!isnum(fileSize)) {
-                    // console.log('invalid fileSize in payload')
-                    let r = {
-                        error: 'invalid fileSize in payload',
                     }
-                    return r
-                }
 
-                //bAllSize
-                let bAllSize = false
-                if (bAllExist) {
-                    // console.log(`check size for pathFile[${pathFile}]...`)
+                }
+                else if (mode === 'check-slices-hash') {
 
-                    //_fileSize
-                    let stats = fs.statSync(pathFile)
-                    let _fileSize = stats.size
+                    //fileSliceHashs, 從payload接收
+                    let fileSliceHashs = get(req, 'payload.fileSliceHashs', [])
+                    // console.log(mode, 'fileSliceHashs', fileSliceHashs)
 
-                    //bAllSize
-                    bAllSize = fileSize === _fileSize
+                    //checkSlicesHash
+                    out = await checkSlicesHash(fileSliceHashs, fileHash, pathUploadTemp)
+                    // console.log(mode, 'out', out)
 
-                    // console.log(`check size for pathFile[${pathFile}] done`, bAllSize)
                 }
+                else if (mode === 'merge-slices-push') {
 
-                //bAllHash
-                let bAllHash = false
-                if (bAllExist && bAllSize) {
-                    // console.log(`check hash for pathFile[${pathFile}]...`)
-
-                    //bAllHash
-                    await calcFileHash(pathFile)
-                        .then((_fileHash) => {
-                            bAllHash = fileHash === _fileHash
-                        })
-                        .catch((err) => {
-                            console.log(err)
-                            console.log(`fsIsFile(pathFile)`, pathFile, fsIsFile(pathFile))
-                            bAllHash = false
-                        })
-
-                    // console.log(`check hash for pathFile[${pathFile}] done`, bAllHash)
-                }
+                    //chunkTotal, 從payload接收
+                    let chunkTotal = get(req, 'payload.chunkTotal', '')
+                    // console.log(mode, 'chunkTotal', chunkTotal)
+
+                    //mmg.push
+                    let queueId = mmg.push(fileHash, chunkTotal, pathUploadTemp)
+
+                    //out
+                    out = {
+                        queueId,
+                    }
 
-                //vfps
-                let vfps = fsGetFilesInFolder(pathUploadTemp)
-                // console.log('vfps', vfps)
-
-                //slks
-                let slks = []
-                if (true) {
-                    each(vfps, (v, k) => {
-                        let b1 = (v.name).indexOf(`${fileHash}_`) >= 0
-                        let stats = fs.statSync(v.path)
-                        let b2 = stats.size === sizeSlice
-                        let b = b1 && b2
-                        if (b) {
-                            slks.push(k)
-                        }
-                    })
-                }
-                // console.log('slks', slks)
-
-                //bSls
-                let bSls = size(slks) > 0
-                // console.log('bSls', bSls)
-
-                //r
-                let r = {
-                    success: {
-                        path: pathFile,
-                        bAllExist,
-                        bAllSize,
-                        bAllHash,
-                        bSls,
-                        slks,
-                    },
                 }
+                else if (mode === 'merge-slices-get') {
 
-                return r
-            }
+                    //filename, 從payload接收
+                    let filename = get(req, 'payload.filename', '')
+                    // console.log(mode, 'filename', filename)
 
-            //checkSlicesHash
-            let checkSlicesHash = async() => {
+                    //queueId, 從payload接收
+                    let queueId = get(req, 'payload.queueId', '')
+                    // console.log(mode, 'queueId', queueId)
 
-                //fileSliceHashs, 從payload接收
-                let fileSliceHashs = get(req, 'payload.fileSliceHashs', [])
-                // console.log('fileSliceHashs', fileSliceHashs)
+                    //mmg.get
+                    let r = mmg.get(queueId, pathUploadTemp)
 
-                //check, 前端須檢核, 若之前已回應無切片, 就不能再調用檢測切片hash的API
-                if (size(fileSliceHashs) === 0) {
-                    let r = {
-                        error: 'no fileSliceHashs',
+                    //out
+                    out = {
+                        queueId,
+                        state: r.state,
+                        filename,
+                        path: r.path,
                     }
-                    return r
-                }
-
-                //slksCfm, 已確定hash值一致的切片
-                // console.log(`check hash for slices fileHash[${fileHash}]...`, fileSliceHashs[0], size(fileSliceHashs))
-                let slksCfm = []
-                // let n = Math.max(fileSliceHashs.length, 1)
-                // let nr = Math.floor(n / 200)
-                for (let k = 0; k < fileSliceHashs.length; k++) {
-                    // if (k % nr === 0) {
-                    //     console.log(`calc hash for slices`, round(k / fileSliceHashs.length * 100, 1), '%')
-                    // }
-
-                    //v
-                    let v = fileSliceHashs[k]
-
-                    //_pathFile
-                    let _pathFile = path.join(pathUploadTemp, `${fileHash}_${v.i}`)
-                    // console.log('_pathFile', _pathFile)
-
-                    //_fileHash
-                    let _fileHash = ''
-                    // await calcFileHash(_pathFile)
-                    await getFileXxHash(new Blob([fs.readFileSync(_pathFile)])) //計算切片因檔案很小, 直接用getFileXxHash速度比較快
-                        .then((res) => {
-                            _fileHash = res
-                        })
-                        .catch((err) => {
-                            console.log(err)
-                            console.log(`fsIsFile(_pathFile)`, _pathFile, fsIsFile(_pathFile))
-                        })
-                    // console.log('_fileHash', _fileHash)
 
                     //check
-                    if (v.h === _fileHash) {
-                        slksCfm.push(v.i)
-                    }
-                    // else {
-                    //     console.log(`hash is not equal`, `hash(front)`, v.h, `hash(backend)`, _fileHash)
-                    // }
+                    if (r.state === 'done') {
 
-                }
-                // console.log(`check hash for slices fileHash[${fileHash}] done`, slksCfm[0], size(slksCfm))
+                        //ri
+                        let ri = {
+                            filename,
+                            path: r.path,
+                        }
+
+                        //procUpload, 偵測有合併完成大檔, 得調用procUpload讓伺服器攔截函數處理
+                        // console.log('procUpload start')
+                        let ro = await procUpload(ri)
+                        // console.log('procUpload done', ro)
+
+                        //out merge, ro須附加至msg, 才供前端偵測state為'done'時提取msg顯示
+                        out = {
+                            ...out,
+                            msg: ro,
+                        }
+
+                    }
 
-                //r
-                let r = {
-                    success: {
-                        slks: slksCfm,
-                    },
                 }
+                // console.log('out', out)
 
-                return r
+                return out
             }
 
-            //out
-            let out = null
-            if (mode === 'check-total-hash') {
-                out = await checkTotalHash()
-            }
-            else if (mode === 'check-slices-hash') {
-                out = await checkSlicesHash()
-            }
+            //procCore
+            let out = {}
+            let returnType = ''
+            let returnMsg = ''
+            await procCore()
+                .then((res) => {
+                    out.success = res
+                    returnType = 'success'
+                    returnMsg = 'need to parse'
+                })
+                .catch((err) => {
+                    out.error = err
+                    returnType = 'error'
+                    returnMsg = 'need to parse'
+                })
+            // console.log('out', out)
 
-            if (out.error) {
-                return responseU8aStreamWithError(res, out.error)
-            }
-            else {
-                let u8aOut = obj2u8arr(out)
-                return responseU8aStream(res, u8aOut, { returnType: 'success', returnMsg: 'need to parse' })
-            }
+            //u8aOut
+            let u8aOut = obj2u8arr(out)
+            // console.log('u8aOut', u8aOut)
+
+            return responseU8aStream(res, u8aOut, { returnType, returnMsg })
         },
     }
 
@@ -934,18 +888,18 @@ function WConverhpServer(opt = {}) {
 
             //check
             if (!isp0int(chunkIndex)) {
-                console.log('invalid chunkIndex in headers')
-                return res.response({ error: 'invalid chunkIndex in headers' }).code(400)
+                // console.log('invalid chunkIndex in headers')
+                return responseU8aStreamWithError(res, 'invalid chunkIndex in headers')
             }
             chunkIndex = cint(chunkIndex)
             if (!isp0int(chunkTotal)) {
-                console.log('invalid chunkTotal in headers')
-                return res.response({ error: 'invalid chunkTotal in headers' }).code(400)
+                // console.log('invalid chunkTotal in headers')
+                return responseU8aStreamWithError(res, 'invalid chunkTotal in headers')
             }
             chunkTotal = cint(chunkTotal)
             if (!isestr(packageId)) {
-                console.log('invalid packageId in headers')
-                return res.response({ error: 'invalid packageId in headers' }).code(400)
+                // console.log('invalid packageId in headers')
+                return responseU8aStreamWithError(res, 'invalid packageId in headers')
             }
 
             //pathFileChunk
@@ -1012,126 +966,6 @@ function WConverhpServer(opt = {}) {
         },
     }
 
-    //apiUploadSliceMerge
-    let apiUploadSliceMerge = {
-        path: `/${apiName}slcm`,
-        method: 'POST',
-        options: {
-            payload: {
-                maxBytes: 1024 * 1024 * 1024 * 1024, //預設為1mb, 調整至1tb, 也就是給予3次方
-                maxParts: 1000 * 1000 * 1000, //預設為1000, 給予3次方
-                timeout: false, //避免請求未完成時中斷
-                // output: 'stream',
-                parse: true, //前端送obj過來須自動解析
-            },
-            timeout: {
-                server: false, //關閉伺服器超時
-                socket: false, //關閉socket超時
-            },
-        },
-        handler: async function (req, res) {
-            // console.log(req, res)
-            // console.log('payload', req.payload)
-
-            //headers
-            let headers = get(req, 'headers')
-            headers = iseobj(headers) ? headers : ''
-            // console.log('headers', headers)
-
-            //query
-            let query = get(req, 'query')
-            query = iseobj(query) ? query : ''
-            // console.log('query', query)
-
-            //authorization
-            let authorization = get(headers, 'authorization', '')
-            authorization = isestr(authorization) ? authorization : ''
-
-            //check
-            if (true) {
-
-                //verifyConn
-                let m = verifyConn({ apiType: 'upload-slice-merge', authorization, headers, query })
-                if (ispm(m)) {
-                    m = await m
-                }
-
-                //check
-                if (m !== true) {
-                    return responseU8aStreamWithError(res, 'permission denied')
-                }
-
-            }
-
-            //eeEmit
-            eeEmit('handler', {
-                api: 'apiUploadSliceMerge',
-                headers,
-                query,
-            })
-
-            //filename, chunkTotal, packageId, 從payload接收
-            let filename = get(req, 'payload.filename', '')
-            let chunkTotal = get(req, 'payload.chunk-total', '')
-            let packageId = get(req, 'payload.package-id', '')
-
-            //check
-            if (!isestr(filename)) {
-                console.log('invalid filename in payload')
-                return res.response({ error: 'invalid filename in payload' }).code(400)
-            }
-            if (!isp0int(chunkTotal)) {
-                console.log('invalid chunkTotal in payload')
-                return res.response({ error: 'invalid chunkTotal in payload' }).code(400)
-            }
-            chunkTotal = cint(chunkTotal)
-            if (!isestr(packageId)) {
-                console.log('invalid packageId in payload')
-                return res.response({ error: 'invalid packageId in payload' }).code(400)
-            }
-
-            //mergeAndPorc
-            let mergeAndPorc = async() => {
-
-                //merge
-                // console.log('merge start')
-                let r = await mergeFiles(pathUploadTemp, packageId, chunkTotal, filename)
-                // console.log('merge done', r)
-
-                //procUpload
-                // console.log('procUpload start')
-                let rr = await procUpload(r)
-                // console.log('procUpload done', rr)
-
-                return rr
-            }
-
-            //mergeAndPorc
-            let out = {}
-            let returnType = ''
-            let returnMsg = ''
-            await mergeAndPorc()
-                .then((res) => {
-                    out.success = res
-                    returnType = 'success'
-                    returnMsg = 'need to parse'
-                })
-                .catch((err) => {
-                    out.error = err
-                    returnType = 'error'
-                    returnMsg = 'need to parse'
-                    eeEmit('error', `${err.message}`)
-                })
-            // console.log('out', out)
-
-            //u8aOut
-            let u8aOut = obj2u8arr(out)
-            // console.log('u8aOut', u8aOut)
-
-            return responseU8aStream(res, u8aOut, { returnType, returnMsg })
-        },
-    }
-
     //apiDownloadGetFilename
     let apiDownloadGetFilename = {
         path: `/${apiName}dwgfn`,
@@ -1349,14 +1183,6 @@ function WConverhpServer(opt = {}) {
             //streamRead
             let streamRead = get(r, 'streamRead')
 
-            // //filename
-            // let filename = get(r, 'filename')
-            // if (!isestr(filename)) {
-            //     return responseU8aStreamWithError(res, 'invalid filename')
-            // }
-            // filename = str2b64(filename) //headers內對中文支援度不佳須用base64傳
-            // console.log('filename', filename)
-
             //fileSize
             let fileSize = get(r, 'fileSize')
             if (!isNumber(fileSize)) {
@@ -1551,7 +1377,7 @@ function WConverhpServer(opt = {}) {
                 apiMain,
                 apiUploadCheck,
                 apiUploadSlice,
-                apiUploadSliceMerge,
+                // apiUploadSliceMerge,
                 apiDownloadGetFilename,
                 apiDownloadGetFile,
                 apiDownload,
@@ -1570,7 +1396,8 @@ function WConverhpServer(opt = {}) {
 
     //start
     if (get(opt, 'serverHapi')) {
-        server.route([apiMain, apiUploadCheck, apiUploadSlice, apiUploadSliceMerge, apiDownloadGetFilename, apiDownloadGetFile, apiDownload])
+        // server.route([apiMain, apiUploadCheck, apiUploadSlice, apiUploadSliceMerge, apiDownloadGetFilename, apiDownloadGetFile, apiDownload])
+        server.route([apiMain, apiUploadCheck, apiUploadSlice, apiDownloadGetFilename, apiDownloadGetFile, apiDownload])
     }
     else {
         startServer()

package/src/checkSlicesHash.mjs

@@ -0,0 +1,68 @@
+import path from 'path'
+import fs from 'fs'
+import size from 'lodash-es/size.js'
+import fsIsFile from 'wsemi/src/fsIsFile.mjs'
+import getFileXxHash from 'wsemi/src/getFileXxHash.mjs'
+
+
+let checkSlicesHash = async(fileSliceHashs, fileHash, pathUploadTemp) => {
+
+    //check, 前端須檢核, 若之前已回應無切片, 就不能再調用檢測切片hash的API
+    if (size(fileSliceHashs) === 0) {
+        let r = {
+            error: 'no fileSliceHashs',
+        }
+        return r
+    }
+
+    //slksCfm, 已確定hash值一致的切片
+    // console.log(`check hash for slices fileHash[${fileHash}]...`, fileSliceHashs[0], size(fileSliceHashs))
+    let slksCfm = []
+    // let n = Math.max(fileSliceHashs.length, 1)
+    // let nr = Math.floor(n / 100)
+    for (let k = 0; k < fileSliceHashs.length; k++) {
+        // if (k % nr === 0) {
+        //     console.log(`calc hash for slices`, (k / fileSliceHashs.length * 100).toFixed(1), '%')
+        // }
+
+        //v
+        let v = fileSliceHashs[k]
+
+        //_pathFile
+        let _pathFile = path.join(pathUploadTemp, `${fileHash}_${v.i}`)
+        // console.log('_pathFile', _pathFile)
+
+        //_fileHash
+        let _fileHash = ''
+        // await calcFileHash(_pathFile)
+        await getFileXxHash(new Blob([fs.readFileSync(_pathFile)])) //計算切片因檔案很小, 直接用getFileXxHash速度比較快
+            .then((res) => {
+                _fileHash = res
+            })
+            .catch((err) => {
+                console.log(err)
+                console.log(`fsIsFile(_pathFile)`, _pathFile, fsIsFile(_pathFile))
+            })
+        // console.log('_fileHash', _fileHash)
+
+        //check
+        if (v.h === _fileHash) {
+            slksCfm.push(v.i)
+        }
+        // else {
+        //     console.log(`hash is not equal`, `hash(front)`, v.h, `hash(backend)`, _fileHash)
+        // }
+
+    }
+    // console.log(`check hash for slices fileHash[${fileHash}] done`, slksCfm[0], size(slksCfm))
+
+    //r
+    let r = {
+        slks: slksCfm,
+    }
+
+    return r
+}
+
+
+export default checkSlicesHash