w-converhp 2.0.32 → 2.0.34

package/src/WConverhpClient.mjs

@@ -673,37 +673,100 @@ function WConverhpClient(opt) {
         //fileTotalSize
         let fileTotalSize = n
 
+        //chunkTotal
+        let chunkTotal = Math.ceil(fileTotalSize / sizeSlice)
+        // console.log('chunkTotal', chunkTotal)
+
         //fileTotalHash
+        // console.log(`calc hash for fileTotalSize[${fileTotalSize}]...`)
         let fileTotalHash = await calcHash(bb)
-        // console.log('fileTotalHash', fileTotalHash)
-
-        //ckh
-        let ckh = async({ filename, fileSize, fileHash }) => {
-
-            //send upload-check
-            let resCk = await send('upload-check', { filename, fileSize, fileHash }, { dataType: 'json' })
-            // console.log('resCk', resCk)
-
-            return resCk
-        }
-
-        //rAll
-        let rAll = await ckh({ filename: fileTotalHash, fileSize: fileTotalSize, fileHash: fileTotalHash })
-        // console.log('rAll', rAll)
+        // console.log(`calc hash for fileTotalSize[${fileTotalSize}] done`, fileTotalHash)
+
+        //send upload-check
+        // console.log(`send check-total-hash...`)
+        let resCk = await send('upload-check', { mode: 'check-total-hash', fileHash: fileTotalHash, fileSize: fileTotalSize }, { dataType: 'json' })
+        // console.log('resCk', resCk)
+        // resCk {
+        //   path: 'uploadTemp\\2429b7ef08ce6ba9',
+        //   bAllExist: false,
+        //   bAllSize: false,
+        //   bAllHash: false,
+        //   bSls: true,
+        //   slks: [
+        //      0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11,
+        //     12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
+        //     24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
+        //     36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
+        //     48, 49, 50, 51, 52, 53
+        //   ]
+        // }
 
         //check
-        if (rAll.existed) {
+        if (resCk.bAllHash) {
             // console.log('已有上傳大檔')
             let resMg = {
                 filename: fileTotalName,
-                path: rAll.path,
+                path: resCk.path,
             }
             return resMg
         }
 
-        //chunkTotal
-        let chunkTotal = Math.ceil(n / sizeSlice)
-        // console.log('chunkTotal', chunkTotal)
+        //針對伺服器上已有切片檔案計算hash與比對
+        if (resCk.slks.length > 0) {
+            // console.log('receive slks...', resCk.slks[0], size(resCk.slks))
+
+            //fileSliceHashs
+            let fileSliceHashs = []
+            // let n = Math.max(resCk.slks.length, 1)
+            // let nr = Math.floor(n / 200)
+            for (let k = 0; k < resCk.slks.length; k++) {
+                // if (k % nr === 0) {
+                //     console.log(`calc hash for slices`, round(k / resCk.slks.length * 100, 1), '%')
+                // }
+
+                //i
+                let i = resCk.slks[k]
+
+                //start
+                let start = i * sizeSlice
+
+                //end
+                let end = Math.min(start + sizeSlice, fileTotalSize)
+
+                //chunk
+                let chunk = bb.slice(start, end)
+
+                //fileSliceHash
+                let fileSliceHash = await calcHash(chunk)
+                // console.log('fileSliceHash', fileSliceHash)
+
+                //push
+                fileSliceHashs.push({
+                    i,
+                    h: fileSliceHash,
+                })
+
+            }
+            // console.log('fileSliceHashs', fileSliceHashs)
+
+            //send upload-check
+            // console.log(`send check-slices-hash...`)
+            let resCks = await send('upload-check', { mode: 'check-slices-hash', fileHash: fileTotalHash, fileSliceHashs }, { dataType: 'json' })
+            // console.log('resCks', resCks)
+            // resCks {
+            //   slks: [
+            //      0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11,
+            //     12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
+            //     24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
+            //     36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
+            //     48, 49, 50, 51, 52, 53
+            //   ]
+            // }
+
+            //update
+            resCk.slks = resCks.slks
+
+        }
 
         //packageId
         let packageId = fileTotalHash
@@ -722,7 +785,7 @@ function WConverhpClient(opt) {
                 progCount++
                 let r = progCount / chunkTotal
                 let prog = r * progWeightSlice * 100
-                let psiz = r * n
+                let psiz = r * fileTotalSize
                 cbProgress({ prog, p: psiz, m: 'upload' })
             }
         }
@@ -732,44 +795,30 @@ function WConverhpClient(opt) {
             let perc = msg.prog
             let dir = msg.m
             if (dir === 'download' && perc === 100) {
-                cbProgress({ prog: 100, p: n, m: 'upload' })
+                cbProgress({ prog: 100, p: fileTotalSize, m: 'upload' })
             }
         }
 
         //upload slice
+        // console.log(`upload slice...`)
         for (let i = 0; i < chunkTotal; i++) {
 
-            //fileSliceName
-            let fileSliceName = `${fileTotalHash}_${i}`
-            // console.log('fileSliceName', fileSliceName)
+            //check
+            if (resCk.slks.indexOf(i) >= 0) {
+                // console.log('已有上傳切片檔')
+                cbProgressSlice({ prog: 100, m: 'upload' }) //直接觸發更新進度
+                continue
+            }
 
             //start
             let start = i * sizeSlice
 
             //end
-            let end = Math.min(start + sizeSlice, n)
+            let end = Math.min(start + sizeSlice, fileTotalSize)
 
             //chunk
             let chunk = bb.slice(start, end)
 
-            //fileSliceHash
-            let fileSliceHash = await calcHash(chunk)
-            // console.log('fileSliceHash', fileSliceHash)
-
-            //rChunk
-            let rChunk = await ckh({ filename: fileSliceName, fileSize: sizeSlice, fileHash: fileSliceHash })
-            // console.log('fileSliceName', fileSliceName)
-            // console.log('fileSliceHash', fileSliceHash)
-            // console.log('rChunk', rChunk)
-            // console.log('')
-
-            //check
-            if (rChunk.existed) {
-                // console.log('已有上傳切片檔')
-                cbProgressSlice({ prog: 100, m: 'upload' }) //直接觸發更新進度
-                continue
-            }
-
             //send slice
             let hd = {
                 'chunk-index': i,
@@ -789,6 +838,7 @@ function WConverhpClient(opt) {
         }
         let resMg = await send('slice-merge', msg, { dataType: 'json', cbProgress: cbProgressMerge })
 
+        // console.log(`upload slice done`)
         return resMg
     }
 

package/src/WConverhpServer.mjs

@@ -4,6 +4,8 @@ import stream from 'stream'
 import Hapi from '@hapi/hapi'
 import Inert from '@hapi/inert' //提供靜態檔案
 import get from 'lodash-es/get.js'
+import each from 'lodash-es/each.js'
+import size from 'lodash-es/size.js'
 import isNumber from 'lodash-es/isNumber.js'
 import genPm from 'wsemi/src/genPm.mjs'
 import evem from 'wsemi/src/evem.mjs'
@@ -25,6 +27,8 @@ import u8arr2obj from 'wsemi/src/u8arr2obj.mjs'
 import fsIsFile from 'wsemi/src/fsIsFile.mjs'
 import fsIsFolder from 'wsemi/src/fsIsFolder.mjs'
 import fsCreateFolder from 'wsemi/src/fsCreateFolder.mjs'
+import fsGetFilesInFolder from 'wsemi/src/fsGetFilesInFolder.mjs'
+import getFileXxHash from 'wsemi/src/getFileXxHash.mjs'
 // import calcFileHash from './calcFileHash.mjs'
 import calcFileHash from './calcFileHash.wk.umd.js'
 // import mergeFiles from './mergeFiles.mjs'
@@ -47,6 +51,7 @@ import mergeFiles from './mergeFiles.wk.umd.js'
  * @param {String} [opt.apiName='api'] 輸入API名稱字串，預設'api'
  * @param {String} [opt.pathUploadTemp='./uploadTemp'] 輸入暫時存放切片上傳檔案資料夾字串，預設'./uploadTemp'
  * @param {String} [opt.tokenType='Bearer'] 輸入token類型字串，預設'Bearer'
+ * @param {Integer} [opt.sizeSlice=1024*1024] 輸入切片上傳檔案之切片檔案大小整數，單位為Byte，預設為1024*1024
  * @param {Function} [opt.verifyConn=()=>{return true}] 輸入呼叫API時檢測函數，預設()=>{return true}
  * @param {Array} [opt.corsOrigins=['*']] 輸入允許跨域網域陣列，若給予['*']代表允許全部，預設['*']
  * @param {Integer} [opt.delayForBasic=0] 輸入基本API用延遲響應時間，單位ms，預設0
@@ -231,6 +236,12 @@ function WConverhpServer(opt = {}) {
         tokenType = 'Bearer'
     }
 
+    //sizeSlice
+    let sizeSlice = get(opt, 'sizeSlice')
+    if (!ispint(sizeSlice)) {
+        sizeSlice = 1024 * 1024 //1m
+    }
+
     //verifyConn
     let verifyConn = get(opt, 'verifyConn')
     if (!isfun(verifyConn)) {
@@ -424,12 +435,6 @@ function WConverhpServer(opt = {}) {
         }
 
         return r
-        // return res.response(smr)
-        //     .header('Return-Type', returnType)
-        //     .header('Return-Msg', returnMsg)
-        //     .header('Cache-Control', 'no-cache, no-store, must-revalidate')
-        //     .header('Content-Type', 'application/octet-stream')
-        //     .header('Content-Length', smr.readableLength)
     }
 
     //responseU8aStreamWithError
@@ -660,25 +665,17 @@ function WConverhpServer(opt = {}) {
                 query,
             })
 
-            //filename, fileSize, fileHash, 從payload接收
-            let filename = get(req, 'payload.filename', '')
-            let fileSize = get(req, 'payload.fileSize', '')
-            let fileHash = get(req, 'payload.fileHash', '')
-            // console.log('filename', filename)
-            // console.log('fileSize', fileSize)
-            // console.log('fileHash', fileHash)
+            //mode, 從payload接收
+            let mode = get(req, 'payload.mode', '')
 
             //check
-            if (!isestr(filename)) {
-                // console.log('invalid filename in payload')
-                return responseU8aStreamWithError(res, 'invalid filename in payload')
+            if (mode !== 'check-total-hash' && mode !== 'check-slices-hash') {
+                // console.log('invalid mode in payload')
+                return responseU8aStreamWithError(res, `invalid mode[${mode}] in payload`)
             }
 
-            //check
-            if (!isnum(fileSize)) {
-                // console.log('invalid fileSize in payload')
-                return responseU8aStreamWithError(res, 'invalid fileSize in payload')
-            }
+            //fileHash, 從payload接收
+            let fileHash = get(req, 'payload.fileHash', '')
 
             //check
             if (!isestr(fileHash)) {
@@ -686,69 +683,189 @@ function WConverhpServer(opt = {}) {
                 return responseU8aStreamWithError(res, 'invalid fileHash in payload')
             }
 
-            //pathFile
-            let pathFile = path.join(pathUploadTemp, filename)
-            // console.log('pathFile', pathFile)
+            //checkTotalHash
+            let checkTotalHash = async () => {
 
-            //check
-            if (!fsIsFile(pathFile)) {
-                let out = {
-                    success: {
-                        existed: false,
-                        msg: 'not exist',
-                    },
+                //pathFile
+                let pathFile = path.join(pathUploadTemp, fileHash)
+                // console.log('pathFile', pathFile)
+
+                //bAllExist
+                let bAllExist = false
+                if (true) {
+                    // console.log(`check exist for pathFile[${pathFile}]...`)
+
+                    bAllExist = fsIsFile(pathFile)
+
+                    // console.log(`check exist for pathFile[${pathFile}] done`, bAllExist)
                 }
-                let u8aOut = obj2u8arr(out)
-                return responseU8aStream(res, u8aOut, { returnType: 'success', returnMsg: 'need to parse' })
-            }
 
-            //_fileSize
-            let stats = fs.statSync(pathFile)
-            let _fileSize = stats.size
+                //fileSize, 從payload接收
+                let fileSize = get(req, 'payload.fileSize', '')
 
-            //check
-            if (fileSize !== _fileSize) {
-                let out = {
-                    success: {
-                        existed: false,
-                        msg: 'not equal size',
-                    },
+                //check
+                if (!isnum(fileSize)) {
+                    // console.log('invalid fileSize in payload')
+                    let r = {
+                        error: 'invalid fileSize in payload',
+                    }
+                    return r
                 }
-                let u8aOut = obj2u8arr(out)
-                return responseU8aStream(res, u8aOut, { returnType: 'success', returnMsg: 'need to parse' })
-            }
 
-            //_fileHash
-            let _fileHash = await calcFileHash(pathFile)
-            // console.log('path', pathFile)
-            // console.log('hash(front)', fileHash)
-            // console.log('hash(backend)', _fileHash)
-            // console.log('')
+                //bAllSize
+                let bAllSize = false
+                if (bAllExist) {
+                    // console.log(`check size for pathFile[${pathFile}]...`)
 
-            //check
-            if (fileHash !== _fileHash) {
-                let out = {
+                    //_fileSize
+                    let stats = fs.statSync(pathFile)
+                    let _fileSize = stats.size
+
+                    //bAllSize
+                    bAllSize = fileSize === _fileSize
+
+                    // console.log(`check size for pathFile[${pathFile}] done`, bAllSize)
+                }
+
+                //bAllHash
+                let bAllHash = false
+                if (bAllExist && bAllSize) {
+                    // console.log(`check hash for pathFile[${pathFile}]...`)
+
+                    //bAllHash
+                    await calcFileHash(pathFile)
+                        .then((_fileHash) => {
+                            bAllHash = fileHash === _fileHash
+                        })
+                        .catch((err) => {
+                            console.log(err)
+                            console.log(`fsIsFile(pathFile)`, pathFile, fsIsFile(pathFile))
+                            bAllHash = false
+                        })
+
+                    // console.log(`check hash for pathFile[${pathFile}] done`, bAllHash)
+                }
+
+                //vfps
+                let vfps = fsGetFilesInFolder(pathUploadTemp)
+                // console.log('vfps', vfps)
+
+                //slks
+                let slks = []
+                if (true) {
+                    each(vfps, (v, k) => {
+                        let b1 = (v.name).indexOf(`${fileHash}_`) >= 0
+                        let stats = fs.statSync(v.path)
+                        let b2 = stats.size === sizeSlice
+                        let b = b1 && b2
+                        if (b) {
+                            slks.push(k)
+                        }
+                    })
+                }
+                // console.log('slks', slks)
+
+                //bSls
+                let bSls = size(slks) > 0
+                // console.log('bSls', bSls)
+
+                //r
+                let r = {
                     success: {
-                        existed: false,
-                        msg: 'not equal hash',
+                        path: pathFile,
+                        bAllExist,
+                        bAllSize,
+                        bAllHash,
+                        bSls,
+                        slks,
                     },
                 }
-                let u8aOut = obj2u8arr(out)
-                return responseU8aStream(res, u8aOut, { returnType: 'success', returnMsg: 'need to parse' })
+
+                return r
             }
 
-            if (true) {
-                let out = {
+            //checkSlicesHash
+            let checkSlicesHash = async() => {
+
+                //fileSliceHashs, 從payload接收
+                let fileSliceHashs = get(req, 'payload.fileSliceHashs', [])
+                // console.log('fileSliceHashs', fileSliceHashs)
+
+                //check, 前端須檢核, 若之前已回應無切片, 就不能再調用檢測切片hash的API
+                if (size(fileSliceHashs) === 0) {
+                    let r = {
+                        error: 'no fileSliceHashs',
+                    }
+                    return r
+                }
+
+                //slksCfm, 已確定hash值一致的切片
+                // console.log(`check hash for slices fileHash[${fileHash}]...`, fileSliceHashs[0], size(fileSliceHashs))
+                let slksCfm = []
+                // let n = Math.max(fileSliceHashs.length, 1)
+                // let nr = Math.floor(n / 200)
+                for (let k = 0; k < fileSliceHashs.length; k++) {
+                    // if (k % nr === 0) {
+                    //     console.log(`calc hash for slices`, round(k / fileSliceHashs.length * 100, 1), '%')
+                    // }
+
+                    //v
+                    let v = fileSliceHashs[k]
+
+                    //_pathFile
+                    let _pathFile = path.join(pathUploadTemp, `${fileHash}_${v.i}`)
+                    // console.log('_pathFile', _pathFile)
+
+                    //_fileHash
+                    let _fileHash = ''
+                    // await calcFileHash(_pathFile)
+                    await getFileXxHash(new Blob([fs.readFileSync(_pathFile)])) //計算切片因檔案很小, 直接用getFileXxHash速度比較快
+                        .then((res) => {
+                            _fileHash = res
+                        })
+                        .catch((err) => {
+                            console.log(err)
+                            console.log(`fsIsFile(_pathFile)`, _pathFile, fsIsFile(_pathFile))
+                        })
+                    // console.log('_fileHash', _fileHash)
+
+                    //check
+                    if (v.h === _fileHash) {
+                        slksCfm.push(v.i)
+                    }
+                    // else {
+                    //     console.log(`hash is not equal`, `hash(front)`, v.h, `hash(backend)`, _fileHash)
+                    // }
+
+                }
+                // console.log(`check hash for slices fileHash[${fileHash}] done`, slksCfm[0], size(slksCfm))
+
+                //r
+                let r = {
                     success: {
-                        existed: true,
-                        msg: 'equal',
-                        path: pathFile,
+                        slks: slksCfm,
                     },
                 }
+
+                return r
+            }
+
+            //out
+            let out = null
+            if (mode === 'check-total-hash') {
+                out = await checkTotalHash()
+            }
+            else if (mode === 'check-slices-hash') {
+                out = await checkSlicesHash()
+            }
+
+            if (out.error) {
+                return responseU8aStreamWithError(res, out.error)
+            }
+            else {
                 let u8aOut = obj2u8arr(out)
                 return responseU8aStream(res, u8aOut, { returnType: 'success', returnMsg: 'need to parse' })
             }
-
         },
     }
 

package/srv.mjs

@@ -137,5 +137,4 @@ setTimeout(() => {
     wo.stop()
 }, 3000)
 
-
-//node --experimental-modules srv.mjs
+//node --max-old-space-size=120000 srv.mjs