w-converhp 2.0.26 → 2.0.28

package/src/WConverhpClient.mjs

@@ -3,7 +3,8 @@ import get from 'lodash-es/get.js'
 import isWindow from 'wsemi/src/isWindow.mjs'
 import evem from 'wsemi/src/evem.mjs'
 import genPm from 'wsemi/src/genPm.mjs'
-import genID from 'wsemi/src/genID.mjs'
+// import genID from 'wsemi/src/genID.mjs'
+// import now2strp from 'wsemi/src/now2strp.mjs'
 import haskey from 'wsemi/src/haskey.mjs'
 import isfun from 'wsemi/src/isfun.mjs'
 import ispint from 'wsemi/src/ispint.mjs'
@@ -19,7 +20,7 @@ import blob2u8arr from 'wsemi/src/blob2u8arr.mjs'
 import obj2u8arr from 'wsemi/src/obj2u8arr.mjs'
 import u8arr2obj from 'wsemi/src/u8arr2obj.mjs'
 import pmConvertResolve from 'wsemi/src/pmConvertResolve.mjs'
-import now2strp from 'wsemi/src/now2strp.mjs'
+// import getFileHash from 'wsemi/src/getFileHash.mjs'
 
 
 /**
@@ -72,7 +73,7 @@ import now2strp from 'wsemi/src/now2strp.mjs'
  *                 }
  *             },
  *             {
- *                 fdDownload: './', //於後端nodejs環境才能提供
+ *                 fdDownload: './', //於nodejs環境才能提供
  *             })
  *             .then(function(res) {
  *                 console.log('client web: download: then', res)
@@ -165,6 +166,9 @@ function WConverhpClient(opt) {
         else if (type === 'slice') {
             urlUse = `${url}slc`
         }
+        else if (type === 'upload-check-slice-hash') {
+            urlUse = `${url}ulckh`
+        }
         else if (type === 'slice-merge') {
             urlUse = `${url}slcm`
         }
@@ -436,11 +440,11 @@ function WConverhpClient(opt) {
                 //nodejs通過fs與stream接收檔案, stream出錯只會觸發error事件, 此處try catch為攔截其他非stream程式碼錯誤
                 try {
 
-                    //path, fs
+                    //path, fs, 使用動態import供nodejs使用, 否則會被webpack偵測報錯
                     let path = await import('path')
                     let fs = await import('fs')
 
-                    //fdDownload, 只有後端下載才使用fdDownload
+                    //fdDownload, 只有nodejs下載才使用fdDownload
                     let fdDownload = get(opt, 'fdDownload', '')
                     fs.mkdirSync(fdDownload, { recursive: true }) //須使用mkdirSync, 不要用fsIsFolder與fsCreateFolder避免編譯
                     // console.log('fdDownload', fdDownload)
@@ -623,8 +627,36 @@ function WConverhpClient(opt) {
         }
     }
 
+    //calcHash
+    async function calcHash(inp) {
+        let pm = genPm()
+        if (env === 'browser') {
+            //於browser時, 因是給予blob或file上傳, 且使用非同步async版計算hash, 可支援大檔
+
+            let ab = await inp.arrayBuffer()
+            let hashBuffer = await crypto.subtle.digest('SHA-256', ab)
+            let hashArray = Array.from(new Uint8Array(hashBuffer))
+            let hashHex = hashArray.map(b => b.toString(16).padStart(2, '0')).join('')
+
+            pm.resolve(hashHex)
+        }
+        else {
+            //於nodejs時, 因尚無法提供檔名上傳, 故會是readFileSync讀入的buffer, 同步sync版無法支援大檔
+
+            //crypto, 使用動態import供nodejs使用, 否則會被webpack偵測報錯
+            let crypto = await import('crypto')
+
+            let hash = crypto.createHash('sha256')
+            hash.update(inp, 'utf8')
+            let hashHex = hash.digest('hex')
+
+            pm.resolve(hashHex)
+        }
+        return pm
+    }
+
     //sendDataSlice
-    async function sendDataSlice(tempId, bb, cbProgress) {
+    async function sendDataSlice(fileTotalName, bb, cbProgress) {
 
         //n
         let n = 0
@@ -637,7 +669,7 @@ function WConverhpClient(opt) {
         }
         if (n === 0) {
             try {
-                n = bb.length //for Uint8Array //後端用fs讀有檔案大小上限, 除非改傳入檔名用stream讀, 否則無法支援超大檔
+                n = bb.length //for Uint8Array //nodejs用fs讀有檔案大小上限, 除非改傳入檔名用stream讀, 否則無法支援超大檔
                 n = cint(n)
             }
             catch (err) {}
@@ -649,12 +681,40 @@ function WConverhpClient(opt) {
         }
         // console.log('n', n)
 
+        //getFileHash
+        let fileTotalHash = await calcHash(bb)
+        // console.log('fileTotalHash', fileTotalHash)
+
+        //ckh
+        let ckh = async({ filename, fileHash }) => {
+
+            //send upload-check-slice-hash
+            let resCk = await send('upload-check-slice-hash', { filename, fileHash }, { dataType: 'json' })
+            // console.log('resCk', resCk)
+
+            return resCk
+        }
+
+        //rAll
+        let rAll = await ckh({ filename: fileTotalHash, fileHash: fileTotalHash })
+        // console.log('rAll', rAll)
+
+        //check
+        if (rAll.existed) {
+            //已有上傳大檔
+            let resMg = {
+                filename: fileTotalName,
+                path: rAll.path,
+            }
+            return resMg
+        }
+
         //chunkTotal
         let chunkTotal = Math.ceil(n / sizeSlice)
         // console.log('chunkTotal', chunkTotal)
 
         //packageId
-        let packageId = `${now2strp()}-${genID()}`
+        let packageId = fileTotalHash
         // console.log('packageId', packageId)
 
         //progCount, progWeightSlice
@@ -687,6 +747,10 @@ function WConverhpClient(opt) {
         //upload slice
         for (let i = 0; i < chunkTotal; i++) {
 
+            //fileSliceName
+            let fileSliceName = `${fileTotalHash}_${i}`
+            // console.log('fileSliceName', fileSliceName)
+
             //start
             let start = i * sizeSlice
 
@@ -696,6 +760,23 @@ function WConverhpClient(opt) {
             //chunk
             let chunk = bb.slice(start, end)
 
+            //fileSliceHash
+            let fileSliceHash = await calcHash(chunk)
+            // console.log('fileSliceHash', fileSliceHash)
+
+            //rChunk
+            let rChunk = await ckh({ filename: fileSliceName, fileHash: fileSliceHash })
+            // console.log('fileSliceName', fileSliceName)
+            // console.log('fileSliceHash', fileSliceHash)
+            // console.log('rChunk', rChunk)
+            // console.log('')
+
+            //check
+            if (rChunk.existed) {
+                //已有上傳切片檔
+                continue
+            }
+
             //send slice
             let hd = {
                 'chunk-index': i,
@@ -703,12 +784,13 @@ function WConverhpClient(opt) {
                 'package-id': packageId,
             }
             await send('slice', chunk, { headers: hd, dataType: 'blob', cbProgress: cbProgressSlice })
+            // console.log('resSl', resSl)
 
         }
 
         //send slice-merge
         let msg = {
-            'filename': tempId,
+            'filename': fileTotalName,
             'chunk-total': chunkTotal,
             'package-id': packageId,
         }
@@ -784,12 +866,12 @@ function WConverhpClient(opt) {
     }
 
     //upload
-    async function upload(tempId, input, cbProgress) {
+    async function upload(filename, input, cbProgress) {
 
         //bb
         let bb = input
 
-        return sendDataSlice(tempId, bb, cbProgress)
+        return sendDataSlice(filename, bb, cbProgress)
     }
 
     //downloadNodejs

package/src/WConverhpServer.mjs

@@ -1,3 +1,4 @@
+import path from 'path'
 import fs from 'fs'
 import stream from 'stream'
 import Hapi from '@hapi/hapi'
@@ -20,8 +21,10 @@ import str2b64 from 'wsemi/src/str2b64.mjs'
 import haskey from 'wsemi/src/haskey.mjs'
 import obj2u8arr from 'wsemi/src/obj2u8arr.mjs'
 import u8arr2obj from 'wsemi/src/u8arr2obj.mjs'
+import fsIsFile from 'wsemi/src/fsIsFile.mjs'
 import fsIsFolder from 'wsemi/src/fsIsFolder.mjs'
 import fsCreateFolder from 'wsemi/src/fsCreateFolder.mjs'
+import fsGetFileHash from 'wsemi/src/fsGetFileHash.mjs'
 import mergeFiles from './mergeFiles.wk.umd.js'
 // import mergeFiles from './mergeFiles.mjs'
 
@@ -406,12 +409,25 @@ function WConverhpServer(opt = {}) {
         //returnMsg
         let returnMsg = get(opt, 'returnMsg', '')
 
-        return res.response(smr)
-            .header('Return-Type', returnType)
-            .header('Return-Msg', returnMsg)
+        //r
+        let r = res.response(smr)
             .header('Cache-Control', 'no-cache, no-store, must-revalidate')
             .header('Content-Type', 'application/octet-stream')
             .header('Content-Length', smr.readableLength)
+        if (isestr(returnType)) {
+            r.header('Return-Type', returnType)
+        }
+        if (isestr(returnMsg)) {
+            r.header('Return-Msg', returnMsg)
+        }
+
+        return r
+        // return res.response(smr)
+        //     .header('Return-Type', returnType)
+        //     .header('Return-Msg', returnMsg)
+        //     .header('Cache-Control', 'no-cache, no-store, must-revalidate')
+        //     .header('Content-Type', 'application/octet-stream')
+        //     .header('Content-Length', smr.readableLength)
     }
 
     //responseU8aStreamWithError
@@ -584,6 +600,132 @@ function WConverhpServer(opt = {}) {
         },
     }
 
+    //apiUploadCheckHash
+    let apiUploadCheckHash = {
+        path: `/${apiName}ulckh`,
+        method: 'POST',
+        options: {
+            payload: {
+                maxBytes: 1024 * 1024 * 1024 * 1024, //預設為1mb, 調整至1tb, 也就是給予3次方
+                maxParts: 1000 * 1000 * 1000, //預設為1000, 給予3次方
+                timeout: false, //避免請求未完成時中斷
+                // output: 'stream',
+                parse: true, //前端送obj過來須自動解析
+            },
+            timeout: {
+                server: false, //關閉伺服器超時
+                socket: false, //關閉socket超時
+            },
+        },
+        handler: async function (req, res) {
+            // console.log(req, res)
+            // console.log('payload', req.payload)
+
+            //headers
+            let headers = get(req, 'headers')
+            headers = iseobj(headers) ? headers : ''
+            // console.log('headers', headers)
+
+            //query
+            let query = get(req, 'query')
+            query = iseobj(query) ? query : ''
+            // console.log('query', query)
+
+            //authorization
+            let authorization = get(headers, 'authorization', '')
+            authorization = isestr(authorization) ? authorization : ''
+
+            //check
+            if (true) {
+
+                //verifyConn
+                let m = verifyConn({ apiType: 'upload-check-slice-hash', authorization, headers, query })
+                if (ispm(m)) {
+                    m = await m
+                }
+
+                //check
+                if (m !== true) {
+                    return responseU8aStreamWithError(res, 'permission denied')
+                }
+
+            }
+
+            //eeEmit
+            eeEmit('handler', {
+                api: 'apiUploadCheckHash',
+                headers,
+                query,
+            })
+
+            //filename, fileHash, 從payload接收
+            let filename = get(req, 'payload.filename', '')
+            let fileHash = get(req, 'payload.fileHash', '')
+            // console.log('filename', filename)
+            // console.log('fileHash', fileHash)
+
+            //check
+            if (!isestr(filename)) {
+                // console.log('invalid filename in payload')
+                return responseU8aStreamWithError(res, 'invalid filename in payload')
+            }
+
+            //check
+            if (!isestr(fileHash)) {
+                // console.log('invalid fileHash in payload')
+                return responseU8aStreamWithError(res, 'invalid fileHash in payload')
+            }
+
+            //pathFile
+            let pathFile = path.join(pathUploadTemp, filename)
+            // console.log('pathFile', pathFile)
+
+            //check
+            if (!fsIsFile(pathFile)) {
+                let out = {
+                    success: {
+                        existed: false,
+                        msg: 'not exist',
+                    },
+                }
+                let u8aOut = obj2u8arr(out)
+                return responseU8aStream(res, u8aOut, { returnType: 'success', returnMsg: 'need to parse' })
+            }
+
+            //hash, fsGetFileHash要用非同步stream模式才能支援大檔
+            let hash = await fsGetFileHash(pathFile, { useSync: false, type: 'sha256' })
+            // console.log('path', pathFile)
+            // console.log('hash(front)', fileHash)
+            // console.log('hash(backend)', hash)
+            // console.log('')
+
+            //check
+            if (fileHash !== hash) {
+                let out = {
+                    success: {
+                        existed: false,
+                        msg: 'not equal',
+                    },
+                }
+                let u8aOut = obj2u8arr(out)
+                return responseU8aStream(res, u8aOut, { returnType: 'success', returnMsg: 'need to parse' })
+            }
+
+            if (true) {
+                let out = {
+                    success: {
+                        existed: true,
+                        msg: 'equal',
+                        path: pathFile,
+                    },
+                }
+                let u8aOut = obj2u8arr(out)
+                return responseU8aStream(res, u8aOut, { returnType: 'success', returnMsg: 'need to parse' })
+            }
+
+        },
+    }
+
     //apiUploadSlice
     let apiUploadSlice = {
         path: `/${apiName}slc`,
@@ -664,7 +806,7 @@ function WConverhpServer(opt = {}) {
             }
 
             //pathFileChunk
-            let pathFileChunk = `${pathUploadTemp}/${packageId}_${chunkIndex}` //path.join(pathUploadTemp, `${packageId}_${chunkIndex}`)
+            let pathFileChunk = path.join(pathUploadTemp, `${packageId}_${chunkIndex}`)
             // console.log('pathFileChunk', pathFileChunk)
 
             //streamWrite
@@ -1264,6 +1406,7 @@ function WConverhpServer(opt = {}) {
             apiRoutes = [
                 ...apiRoutes,
                 apiMain,
+                apiUploadCheckHash,
                 apiUploadSlice,
                 apiUploadSliceMerge,
                 apiDownloadGetFilename,
@@ -1284,7 +1427,7 @@ function WConverhpServer(opt = {}) {
 
     //start
     if (get(opt, 'serverHapi')) {
-        server.route([apiMain, apiUploadSlice, apiUploadSliceMerge, apiDownloadGetFilename, apiDownloadGetFile, apiDownload])
+        server.route([apiMain, apiUploadCheckHash, apiUploadSlice, apiUploadSliceMerge, apiDownloadGetFilename, apiDownloadGetFile, apiDownload])
     }
     else {
         startServer()

package/srv.mjs

@@ -1,3 +1,4 @@
+import path from 'path'
 import fs from 'fs'
 import _ from 'lodash-es'
 import w from 'wsemi'
@@ -134,7 +135,7 @@ setTimeout(() => {
     console.log('ms', ms)
     // console.log('ms', JSON.stringify(ms))
     wo.stop()
-}, 3000)
+}, 3000) //bbb
 
 
 //node --experimental-modules srv.mjs

package/toolg/gDistRollup.mjs

@@ -21,10 +21,12 @@ async function core() {
         globals: {
             'path': 'path',
             'fs': 'fs',
+            'crypto': 'crypto',
         },
         external: [
             'path',
             'fs',
+            'crypto',
         ],
     })
         .catch((err) => {