w-converhp 2.0.26 → 2.0.27

package/docs/WConverhpServer.mjs.html

@@ -45,7 +45,8 @@
     
     <section>
         <article>
-            <pre class="prettyprint source linenums"><code>import fs from 'fs'
+            <pre class="prettyprint source linenums"><code>import path from 'path'
+import fs from 'fs'
 import stream from 'stream'
 import Hapi from '@hapi/hapi'
 import Inert from '@hapi/inert' //提供靜態檔案
@@ -67,8 +68,10 @@ import str2b64 from 'wsemi/src/str2b64.mjs'
 import haskey from 'wsemi/src/haskey.mjs'
 import obj2u8arr from 'wsemi/src/obj2u8arr.mjs'
 import u8arr2obj from 'wsemi/src/u8arr2obj.mjs'
+import fsIsFile from 'wsemi/src/fsIsFile.mjs'
 import fsIsFolder from 'wsemi/src/fsIsFolder.mjs'
 import fsCreateFolder from 'wsemi/src/fsCreateFolder.mjs'
+import fsGetFileHash from 'wsemi/src/fsGetFileHash.mjs'
 import mergeFiles from './mergeFiles.wk.umd.js'
 // import mergeFiles from './mergeFiles.mjs'
 
@@ -453,12 +456,25 @@ function WConverhpServer(opt = {}) {
         //returnMsg
         let returnMsg = get(opt, 'returnMsg', '')
 
-        return res.response(smr)
-            .header('Return-Type', returnType)
-            .header('Return-Msg', returnMsg)
+        //r
+        let r = res.response(smr)
             .header('Cache-Control', 'no-cache, no-store, must-revalidate')
             .header('Content-Type', 'application/octet-stream')
             .header('Content-Length', smr.readableLength)
+        if (isestr(returnType)) {
+            r.header('Return-Type', returnType)
+        }
+        if (isestr(returnMsg)) {
+            r.header('Return-Msg', returnMsg)
+        }
+
+        return r
+        // return res.response(smr)
+        //     .header('Return-Type', returnType)
+        //     .header('Return-Msg', returnMsg)
+        //     .header('Cache-Control', 'no-cache, no-store, must-revalidate')
+        //     .header('Content-Type', 'application/octet-stream')
+        //     .header('Content-Length', smr.readableLength)
     }
 
     //responseU8aStreamWithError
@@ -631,6 +647,132 @@ function WConverhpServer(opt = {}) {
         },
     }
 
+    //apiUploadCheckHash
+    let apiUploadCheckHash = {
+        path: `/${apiName}ulckh`,
+        method: 'POST',
+        options: {
+            payload: {
+                maxBytes: 1024 * 1024 * 1024 * 1024, //預設為1mb, 調整至1tb, 也就是給予3次方
+                maxParts: 1000 * 1000 * 1000, //預設為1000, 給予3次方
+                timeout: false, //避免請求未完成時中斷
+                // output: 'stream',
+                parse: true, //前端送obj過來須自動解析
+            },
+            timeout: {
+                server: false, //關閉伺服器超時
+                socket: false, //關閉socket超時
+            },
+        },
+        handler: async function (req, res) {
+            // console.log(req, res)
+            // console.log('payload', req.payload)
+
+            //headers
+            let headers = get(req, 'headers')
+            headers = iseobj(headers) ? headers : ''
+            // console.log('headers', headers)
+
+            //query
+            let query = get(req, 'query')
+            query = iseobj(query) ? query : ''
+            // console.log('query', query)
+
+            //authorization
+            let authorization = get(headers, 'authorization', '')
+            authorization = isestr(authorization) ? authorization : ''
+
+            //check
+            if (true) {
+
+                //verifyConn
+                let m = verifyConn({ apiType: 'upload-check-slice-hash', authorization, headers, query })
+                if (ispm(m)) {
+                    m = await m
+                }
+
+                //check
+                if (m !== true) {
+                    return responseU8aStreamWithError(res, 'permission denied')
+                }
+
+            }
+
+            //eeEmit
+            eeEmit('handler', {
+                api: 'apiUploadCheckHash',
+                headers,
+                query,
+            })
+
+            //filename, fileHash, 從payload接收
+            let filename = get(req, 'payload.filename', '')
+            let fileHash = get(req, 'payload.fileHash', '')
+            // console.log('filename', filename)
+            // console.log('fileHash', fileHash)
+
+            //check
+            if (!isestr(filename)) {
+                // console.log('invalid filename in payload')
+                return responseU8aStreamWithError(res, 'invalid filename in payload')
+            }
+
+            //check
+            if (!isestr(fileHash)) {
+                // console.log('invalid fileHash in payload')
+                return responseU8aStreamWithError(res, 'invalid fileHash in payload')
+            }
+
+            //pathFile
+            let pathFile = path.join(pathUploadTemp, filename)
+            // console.log('pathFile', pathFile)
+
+            //check
+            if (!fsIsFile(pathFile)) {
+                let out = {
+                    success: {
+                        existed: false,
+                        msg: 'not exist',
+                    },
+                }
+                let u8aOut = obj2u8arr(out)
+                return responseU8aStream(res, u8aOut, { returnType: 'success', returnMsg: 'need to parse' })
+            }
+
+            //hash, fsGetFileHash要用非同步stream模式才能支援大檔
+            let hash = await fsGetFileHash(pathFile, { useSync: false, type: 'sha256' })
+            // console.log('path', pathFile)
+            // console.log('hash(front)', fileHash)
+            // console.log('hash(backend)', hash)
+            // console.log('')
+
+            //check
+            if (fileHash !== hash) {
+                let out = {
+                    success: {
+                        existed: false,
+                        msg: 'not equal',
+                    },
+                }
+                let u8aOut = obj2u8arr(out)
+                return responseU8aStream(res, u8aOut, { returnType: 'success', returnMsg: 'need to parse' })
+            }
+
+            if (true) {
+                let out = {
+                    success: {
+                        existed: true,
+                        msg: 'equal',
+                        path: pathFile,
+                    },
+                }
+                let u8aOut = obj2u8arr(out)
+                return responseU8aStream(res, u8aOut, { returnType: 'success', returnMsg: 'need to parse' })
+            }
+
+        },
+    }
+
     //apiUploadSlice
     let apiUploadSlice = {
         path: `/${apiName}slc`,
@@ -711,7 +853,7 @@ function WConverhpServer(opt = {}) {
             }
 
             //pathFileChunk
-            let pathFileChunk = `${pathUploadTemp}/${packageId}_${chunkIndex}` //path.join(pathUploadTemp, `${packageId}_${chunkIndex}`)
+            let pathFileChunk = path.join(pathUploadTemp, `${packageId}_${chunkIndex}`)
             // console.log('pathFileChunk', pathFileChunk)
 
             //streamWrite
@@ -1311,6 +1453,7 @@ function WConverhpServer(opt = {}) {
             apiRoutes = [
                 ...apiRoutes,
                 apiMain,
+                apiUploadCheckHash,
                 apiUploadSlice,
                 apiUploadSliceMerge,
                 apiDownloadGetFilename,
@@ -1331,7 +1474,7 @@ function WConverhpServer(opt = {}) {
 
     //start
     if (get(opt, 'serverHapi')) {
-        server.route([apiMain, apiUploadSlice, apiUploadSliceMerge, apiDownloadGetFilename, apiDownloadGetFile, apiDownload])
+        server.route([apiMain, apiUploadCheckHash, apiUploadSlice, apiUploadSliceMerge, apiDownloadGetFilename, apiDownloadGetFile, apiDownload])
     }
     else {
         startServer()
@@ -1364,7 +1507,7 @@ export default WConverhpServer
 <br class="clear">
 
 <footer>
-    Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.2</a> on Thu Apr 10 2025 15:40:07 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
+    Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.2</a> on Thu Apr 10 2025 22:27:22 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
 </footer>
 
 <script>prettyPrint();</script>

package/docs/index.html

@@ -71,7 +71,7 @@
 <br class="clear">
 
 <footer>
-    Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.2</a> on Thu Apr 10 2025 15:40:07 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
+    Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.2</a> on Thu Apr 10 2025 22:27:22 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
 </footer>
 
 <script>prettyPrint();</script>

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "w-converhp",
-  "version": "2.0.26",
+  "version": "2.0.27",
   "main": "dist/w-converhp-server.umd.js",
   "dependencies": {
     "@hapi/hapi": "^21.4.0",
     "@hapi/inert": "^7.1.0",
     "axios": "^1.8.4",
     "lodash-es": "^4.17.21",
-    "wsemi": "^1.7.79"
+    "wsemi": "^1.7.80"
   },
   "devDependencies": {
     "form-data": "^4.0.2",

package/sclc.mjs

@@ -27,7 +27,7 @@ wo.on('error', (err) => {
 function uploadLargeFile() {
     let core = async() => {
 
-        let fsz = '1000mb' //0mb 10mb 100mb 1000mb
+        let fsz = '10mb' //0mb 10mb 100mb 1000mb
 
         //u8a
         let u8a = new Uint8Array(fs.readFileSync(`../_data/${fsz}.7z`)) //後端並無blob格式, 用fs讀有檔案大小上限

package/script.txt

@@ -16,5 +16,3 @@ npm run deploy
 
 #npm publish
 
-#npm test bbb
-

package/src/WConverhpClient.mjs

@@ -1,9 +1,11 @@
+import crypto from 'crypto'
 import axios from 'axios'
 import get from 'lodash-es/get.js'
 import isWindow from 'wsemi/src/isWindow.mjs'
 import evem from 'wsemi/src/evem.mjs'
 import genPm from 'wsemi/src/genPm.mjs'
-import genID from 'wsemi/src/genID.mjs'
+// import genID from 'wsemi/src/genID.mjs'
+// import now2strp from 'wsemi/src/now2strp.mjs'
 import haskey from 'wsemi/src/haskey.mjs'
 import isfun from 'wsemi/src/isfun.mjs'
 import ispint from 'wsemi/src/ispint.mjs'
@@ -19,7 +21,7 @@ import blob2u8arr from 'wsemi/src/blob2u8arr.mjs'
 import obj2u8arr from 'wsemi/src/obj2u8arr.mjs'
 import u8arr2obj from 'wsemi/src/u8arr2obj.mjs'
 import pmConvertResolve from 'wsemi/src/pmConvertResolve.mjs'
-import now2strp from 'wsemi/src/now2strp.mjs'
+// import getFileHash from 'wsemi/src/getFileHash.mjs'
 
 
 /**
@@ -165,6 +167,9 @@ function WConverhpClient(opt) {
         else if (type === 'slice') {
             urlUse = `${url}slc`
         }
+        else if (type === 'upload-check-slice-hash') {
+            urlUse = `${url}ulckh`
+        }
         else if (type === 'slice-merge') {
             urlUse = `${url}slcm`
         }
@@ -623,8 +628,29 @@ function WConverhpClient(opt) {
         }
     }
 
+    //calcHash
+    async function calcHash(inp) {
+        let pm = genPm()
+        if (env === 'browser') {
+            //前端browser時, 因是給予blob或file上傳, 且使用非同步async版計算hash, 可支援大檔
+            let ab = await inp.arrayBuffer()
+            let hashBuffer = await crypto.subtle.digest('SHA-256', ab)
+            let hashArray = Array.from(new Uint8Array(hashBuffer))
+            let hashHex = hashArray.map(b => b.toString(16).padStart(2, '0')).join('')
+            pm.resolve(hashHex)
+        }
+        else {
+            //前端nodejs時, 因尚無法提供檔名上傳, 故會是readFileSync讀入的buffer, 同步sync版無法支援大檔
+            let hash = crypto.createHash('sha256')
+            hash.update(inp, 'utf8')
+            let hashHex = hash.digest('hex')
+            pm.resolve(hashHex)
+        }
+        return pm
+    }
+
     //sendDataSlice
-    async function sendDataSlice(tempId, bb, cbProgress) {
+    async function sendDataSlice(fileTotalName, bb, cbProgress) {
 
         //n
         let n = 0
@@ -649,12 +675,40 @@ function WConverhpClient(opt) {
         }
         // console.log('n', n)
 
+        //getFileHash
+        let fileTotalHash = await calcHash(bb)
+        // console.log('fileTotalHash', fileTotalHash)
+
+        //ckh
+        let ckh = async({ filename, fileHash }) => {
+
+            //send upload-check-slice-hash
+            let resCk = await send('upload-check-slice-hash', { filename, fileHash }, { dataType: 'json' })
+            // console.log('resCk', resCk)
+
+            return resCk
+        }
+
+        //rAll
+        let rAll = await ckh({ filename: fileTotalHash, fileHash: fileTotalHash })
+        // console.log('rAll', rAll)
+
+        //check
+        if (rAll.existed) {
+            //已有上傳大檔
+            let resMg = {
+                filename: fileTotalName,
+                path: rAll.path,
+            }
+            return resMg
+        }
+
         //chunkTotal
         let chunkTotal = Math.ceil(n / sizeSlice)
         // console.log('chunkTotal', chunkTotal)
 
         //packageId
-        let packageId = `${now2strp()}-${genID()}`
+        let packageId = fileTotalHash
         // console.log('packageId', packageId)
 
         //progCount, progWeightSlice
@@ -687,6 +741,10 @@ function WConverhpClient(opt) {
         //upload slice
         for (let i = 0; i < chunkTotal; i++) {
 
+            //fileSliceName
+            let fileSliceName = `${fileTotalHash}_${i}`
+            // console.log('fileSliceName', fileSliceName)
+
             //start
             let start = i * sizeSlice
 
@@ -696,6 +754,23 @@ function WConverhpClient(opt) {
             //chunk
             let chunk = bb.slice(start, end)
 
+            //fileSliceHash
+            let fileSliceHash = await calcHash(chunk)
+            // console.log('fileSliceHash', fileSliceHash)
+
+            //rChunk
+            let rChunk = await ckh({ filename: fileSliceName, fileHash: fileSliceHash })
+            // console.log('fileSliceName', fileSliceName)
+            // console.log('fileSliceHash', fileSliceHash)
+            // console.log('rChunk', rChunk)
+            // console.log('')
+
+            //check
+            if (rChunk.existed) {
+                //已有上傳切片檔
+                continue
+            }
+
             //send slice
             let hd = {
                 'chunk-index': i,
@@ -703,12 +778,13 @@ function WConverhpClient(opt) {
                 'package-id': packageId,
             }
             await send('slice', chunk, { headers: hd, dataType: 'blob', cbProgress: cbProgressSlice })
+            // console.log('resSl', resSl)
 
         }
 
         //send slice-merge
         let msg = {
-            'filename': tempId,
+            'filename': fileTotalName,
             'chunk-total': chunkTotal,
             'package-id': packageId,
         }
@@ -784,12 +860,12 @@ function WConverhpClient(opt) {
     }
 
     //upload
-    async function upload(tempId, input, cbProgress) {
+    async function upload(filename, input, cbProgress) {
 
         //bb
         let bb = input
 
-        return sendDataSlice(tempId, bb, cbProgress)
+        return sendDataSlice(filename, bb, cbProgress)
     }
 
     //downloadNodejs

package/src/WConverhpServer.mjs

@@ -1,3 +1,4 @@
+import path from 'path'
 import fs from 'fs'
 import stream from 'stream'
 import Hapi from '@hapi/hapi'
@@ -20,8 +21,10 @@ import str2b64 from 'wsemi/src/str2b64.mjs'
 import haskey from 'wsemi/src/haskey.mjs'
 import obj2u8arr from 'wsemi/src/obj2u8arr.mjs'
 import u8arr2obj from 'wsemi/src/u8arr2obj.mjs'
+import fsIsFile from 'wsemi/src/fsIsFile.mjs'
 import fsIsFolder from 'wsemi/src/fsIsFolder.mjs'
 import fsCreateFolder from 'wsemi/src/fsCreateFolder.mjs'
+import fsGetFileHash from 'wsemi/src/fsGetFileHash.mjs'
 import mergeFiles from './mergeFiles.wk.umd.js'
 // import mergeFiles from './mergeFiles.mjs'
 
@@ -406,12 +409,25 @@ function WConverhpServer(opt = {}) {
         //returnMsg
         let returnMsg = get(opt, 'returnMsg', '')
 
-        return res.response(smr)
-            .header('Return-Type', returnType)
-            .header('Return-Msg', returnMsg)
+        //r
+        let r = res.response(smr)
             .header('Cache-Control', 'no-cache, no-store, must-revalidate')
             .header('Content-Type', 'application/octet-stream')
             .header('Content-Length', smr.readableLength)
+        if (isestr(returnType)) {
+            r.header('Return-Type', returnType)
+        }
+        if (isestr(returnMsg)) {
+            r.header('Return-Msg', returnMsg)
+        }
+
+        return r
+        // return res.response(smr)
+        //     .header('Return-Type', returnType)
+        //     .header('Return-Msg', returnMsg)
+        //     .header('Cache-Control', 'no-cache, no-store, must-revalidate')
+        //     .header('Content-Type', 'application/octet-stream')
+        //     .header('Content-Length', smr.readableLength)
     }
 
     //responseU8aStreamWithError
@@ -584,6 +600,132 @@ function WConverhpServer(opt = {}) {
         },
     }
 
+    //apiUploadCheckHash
+    let apiUploadCheckHash = {
+        path: `/${apiName}ulckh`,
+        method: 'POST',
+        options: {
+            payload: {
+                maxBytes: 1024 * 1024 * 1024 * 1024, //預設為1mb, 調整至1tb, 也就是給予3次方
+                maxParts: 1000 * 1000 * 1000, //預設為1000, 給予3次方
+                timeout: false, //避免請求未完成時中斷
+                // output: 'stream',
+                parse: true, //前端送obj過來須自動解析
+            },
+            timeout: {
+                server: false, //關閉伺服器超時
+                socket: false, //關閉socket超時
+            },
+        },
+        handler: async function (req, res) {
+            // console.log(req, res)
+            // console.log('payload', req.payload)
+
+            //headers
+            let headers = get(req, 'headers')
+            headers = iseobj(headers) ? headers : ''
+            // console.log('headers', headers)
+
+            //query
+            let query = get(req, 'query')
+            query = iseobj(query) ? query : ''
+            // console.log('query', query)
+
+            //authorization
+            let authorization = get(headers, 'authorization', '')
+            authorization = isestr(authorization) ? authorization : ''
+
+            //check
+            if (true) {
+
+                //verifyConn
+                let m = verifyConn({ apiType: 'upload-check-slice-hash', authorization, headers, query })
+                if (ispm(m)) {
+                    m = await m
+                }
+
+                //check
+                if (m !== true) {
+                    return responseU8aStreamWithError(res, 'permission denied')
+                }
+
+            }
+
+            //eeEmit
+            eeEmit('handler', {
+                api: 'apiUploadCheckHash',
+                headers,
+                query,
+            })
+
+            //filename, fileHash, 從payload接收
+            let filename = get(req, 'payload.filename', '')
+            let fileHash = get(req, 'payload.fileHash', '')
+            // console.log('filename', filename)
+            // console.log('fileHash', fileHash)
+
+            //check
+            if (!isestr(filename)) {
+                // console.log('invalid filename in payload')
+                return responseU8aStreamWithError(res, 'invalid filename in payload')
+            }
+
+            //check
+            if (!isestr(fileHash)) {
+                // console.log('invalid fileHash in payload')
+                return responseU8aStreamWithError(res, 'invalid fileHash in payload')
+            }
+
+            //pathFile
+            let pathFile = path.join(pathUploadTemp, filename)
+            // console.log('pathFile', pathFile)
+
+            //check
+            if (!fsIsFile(pathFile)) {
+                let out = {
+                    success: {
+                        existed: false,
+                        msg: 'not exist',
+                    },
+                }
+                let u8aOut = obj2u8arr(out)
+                return responseU8aStream(res, u8aOut, { returnType: 'success', returnMsg: 'need to parse' })
+            }
+
+            //hash, fsGetFileHash要用非同步stream模式才能支援大檔
+            let hash = await fsGetFileHash(pathFile, { useSync: false, type: 'sha256' })
+            // console.log('path', pathFile)
+            // console.log('hash(front)', fileHash)
+            // console.log('hash(backend)', hash)
+            // console.log('')
+
+            //check
+            if (fileHash !== hash) {
+                let out = {
+                    success: {
+                        existed: false,
+                        msg: 'not equal',
+                    },
+                }
+                let u8aOut = obj2u8arr(out)
+                return responseU8aStream(res, u8aOut, { returnType: 'success', returnMsg: 'need to parse' })
+            }
+
+            if (true) {
+                let out = {
+                    success: {
+                        existed: true,
+                        msg: 'equal',
+                        path: pathFile,
+                    },
+                }
+                let u8aOut = obj2u8arr(out)
+                return responseU8aStream(res, u8aOut, { returnType: 'success', returnMsg: 'need to parse' })
+            }
+
+        },
+    }
+
     //apiUploadSlice
     let apiUploadSlice = {
         path: `/${apiName}slc`,
@@ -664,7 +806,7 @@ function WConverhpServer(opt = {}) {
             }
 
             //pathFileChunk
-            let pathFileChunk = `${pathUploadTemp}/${packageId}_${chunkIndex}` //path.join(pathUploadTemp, `${packageId}_${chunkIndex}`)
+            let pathFileChunk = path.join(pathUploadTemp, `${packageId}_${chunkIndex}`)
             // console.log('pathFileChunk', pathFileChunk)
 
             //streamWrite
@@ -1264,6 +1406,7 @@ function WConverhpServer(opt = {}) {
             apiRoutes = [
                 ...apiRoutes,
                 apiMain,
+                apiUploadCheckHash,
                 apiUploadSlice,
                 apiUploadSliceMerge,
                 apiDownloadGetFilename,
@@ -1284,7 +1427,7 @@ function WConverhpServer(opt = {}) {
 
     //start
     if (get(opt, 'serverHapi')) {
-        server.route([apiMain, apiUploadSlice, apiUploadSliceMerge, apiDownloadGetFilename, apiDownloadGetFile, apiDownload])
+        server.route([apiMain, apiUploadCheckHash, apiUploadSlice, apiUploadSliceMerge, apiDownloadGetFilename, apiDownloadGetFile, apiDownload])
     }
     else {
         startServer()

package/srv.mjs

@@ -1,3 +1,4 @@
+import path from 'path'
 import fs from 'fs'
 import _ from 'lodash-es'
 import w from 'wsemi'
@@ -134,7 +135,7 @@ setTimeout(() => {
     console.log('ms', ms)
     // console.log('ms', JSON.stringify(ms))
     wo.stop()
-}, 3000)
+}, 1113000) //bbb
 
 
 //node --experimental-modules srv.mjs

package/toolg/gDistRollup.mjs

@@ -21,10 +21,12 @@ async function core() {
         globals: {
             'path': 'path',
             'fs': 'fs',
+            'crypto': 'crypto',
         },
         external: [
             'path',
             'fs',
+            'crypto',
         ],
     })
         .catch((err) => {