w-dwdata-tweqod 1.0.0

package/docs/styles/prettify.css

@@ -0,0 +1,80 @@
+.pln {
+  color: #ddd;
+}
+
+/* string content */
+.str {
+  color: #61ce3c;
+}
+
+/* a keyword */
+.kwd {
+  color: #fbde2d;
+}
+
+/* a comment */
+.com {
+  color: #aeaeae;
+}
+
+/* a type name */
+.typ {
+  color: #8da6ce;
+}
+
+/* a literal value */
+.lit {
+  color: #fbde2d;
+}
+
+/* punctuation */
+.pun {
+  color: #ddd;
+}
+
+/* lisp open bracket */
+.opn {
+  color: #000000;
+}
+
+/* lisp close bracket */
+.clo {
+  color: #000000;
+}
+
+/* a markup tag name */
+.tag {
+  color: #8da6ce;
+}
+
+/* a markup attribute name */
+.atn {
+  color: #fbde2d;
+}
+
+/* a markup attribute value */
+.atv {
+  color: #ddd;
+}
+
+/* a declaration */
+.dec {
+  color: #EF5050;
+}
+
+/* a variable name */
+.var {
+  color: #c82829;
+}
+
+/* a function name */
+.fun {
+  color: #4271ae;
+}
+
+/* Specify class=linenums on a pre to get line numbering */
+ol.linenums {
+  margin-top: 0;
+  margin-bottom: 0;
+  padding-bottom: 2px;
+}

package/g.downloadEqs.mjs

@@ -0,0 +1,21 @@
+import path from 'path'
+import fs from 'fs'
+import _ from 'lodash-es'
+import w from 'wsemi'
+import downloadEqs from './src/downloadEqs.mjs'
+
+
+let j = fs.readFileSync('../_data/settings.json', 'utf8')
+let st = JSON.parse(j)
+let token = _.get(st, 'token')
+
+let fd = `./_dwAttime`
+if (!w.fsIsFolder(fd)) {
+    w.fsCreateFolder(fd)
+}
+
+let eqs = await downloadEqs(token, fd)
+console.log('eqs', eqs)
+
+
+//node g.downloadEqs.mjs

package/g.mjs

@@ -0,0 +1,60 @@
+// import path from 'path'
+import fs from 'fs'
+import _ from 'lodash-es'
+import w from 'wsemi'
+import WDwdataTweqod from './src/WDwdataTweqod.mjs'
+
+
+let j = fs.readFileSync('../_data/settings.json', 'utf8')
+let st = JSON.parse(j)
+let token = _.get(st, 'token')
+
+//fdDwStorage
+let fdDwStorage = `./_dwStorage`
+w.fsCleanFolder(fdDwStorage)
+
+//fdDwAttime
+let fdDwAttime = `./_dwAttime`
+w.fsCleanFolder(fdDwAttime)
+
+//fdDwCurrent
+let fdDwCurrent = `./_dwCurrent`
+w.fsCleanFolder(fdDwCurrent)
+
+//fdResult
+let fdResult = './_result'
+w.fsCleanFolder(fdResult)
+
+let opt = {
+    fdDwStorage,
+    fdDwAttime,
+    fdDwCurrent,
+    fdResult,
+    // funDownload,
+    // funGetCurrent,
+    // funRemove,
+    // funAdd,
+    // funModify,
+}
+let ev = await WDwdataTweqod(token, opt)
+    .catch((err) => {
+        console.log(err)
+    })
+ev.on('change', (msg) => {
+    delete msg.type
+    console.log('change', msg)
+})
+// change { event: 'start', msg: 'running...' }
+// change { event: 'proc-callfun-download', msg: 'start...' }
+// change { event: 'proc-callfun-download', msg: 'done' }
+// change { event: 'proc-callfun-getCurrent', msg: 'start...' }
+// change { event: 'proc-callfun-getCurrent', msg: 'done' }
+// change { event: 'compare', msg: 'start...' }
+// change { event: 'compare', msg: 'done' }
+// change { event: 'proc-add-callfun-add', id: '114101', msg: 'start...' }
+// change { event: 'proc-add-callfun-add', id: '114101', msg: 'done' }
+// change { event: 'proc-add-callfun-add', id: '114102', msg: 'start...' }
+// change { event: 'proc-add-callfun-add', id: '114102', msg: 'done' }
+// ...
+
+//node g.mjs

package/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "w-dwdata-tweqod",
+  "version": "1.0.0",
+  "main": "dist/w-dwdata-tweqod.umd.js",
+  "dependencies": {
+    "axios": "^1.11.0",
+    "lodash-es": "^4.17.21",
+    "w-dwdata-builder": "^1.0.1",
+    "wsemi": "^1.8.18"
+  },
+  "devDependencies": {
+    "w-package-tools": "^1.1.2"
+  },
+  "scripts": {
+    "test": "mocha --parallel --timeout 60000",
+    "deploy": "gh-pages -d docs"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/yuda-lyu/w-dwdata-tweqod.git"
+  },
+  "keywords": [
+    "package",
+    "tool",
+    "download",
+    "data",
+    "tw",
+    "eq",
+    "nodejs"
+  ],
+  "author": "yuda-lyu(semisphere)",
+  "license": "MIT"
+}

package/script.txt

@@ -0,0 +1,17 @@
+#node toolg/addVersion.mjs
+node toolg/modifyReadme.mjs
+
+node toolg/cleanFolder.mjs
+./node_modules/.bin/jsdoc -c .jsdoc
+
+node toolg/gDistRollup.mjs
+
+git add .  -A
+git commit -m 'modify: '
+git push origin master:master
+
+npm run deploy
+
+#npm test
+
+#npm publish

package/src/WDwdataTweqod.mjs

@@ -0,0 +1,373 @@
+import fs from 'fs'
+import get from 'lodash-es/get.js'
+import each from 'lodash-es/each.js'
+import reverse from 'lodash-es/reverse.js'
+import values from 'lodash-es/values.js'
+import isestr from 'wsemi/src/isestr.mjs'
+import isfun from 'wsemi/src/isfun.mjs'
+import ispm from 'wsemi/src/ispm.mjs'
+import haskey from 'wsemi/src/haskey.mjs'
+import pmSeries from 'wsemi/src/pmSeries.mjs'
+import fsIsFolder from 'wsemi/src/fsIsFolder.mjs'
+import fsCopyFile from 'wsemi/src/fsCopyFile.mjs'
+import fsCleanFolder from 'wsemi/src/fsCleanFolder.mjs'
+import fsCreateFolder from 'wsemi/src/fsCreateFolder.mjs'
+import fsDeleteFolder from 'wsemi/src/fsDeleteFolder.mjs'
+import fsTreeFolder from 'wsemi/src/fsTreeFolder.mjs'
+import fsGetFileXxHash from 'wsemi/src/fsGetFileXxHash.mjs'
+import WDwdataBuilder from 'w-dwdata-builder/src/WDwdataBuilder.mjs'
+import downloadEqs from './downloadEqs.mjs'
+import fsCopyFolder from 'wsemi/src/fsCopyFolder.mjs'
+
+
+/**
+ * 於OpenData下載台灣氣象署地震資料
+ *
+ * @param {String} token 輸入氣象署OpenData之API用token字串
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
+ * @param {String} [opt.fdDwStorage] 輸入完整下載數據資料夾字串，預設'./_dwStorage'
+ * @param {String} [opt.fdDwAttime] 輸入當前下載供比對hash用之數據資料夾字串，預設'./_dwAttime'
+ * @param {String} [opt.fdDwCurrent] 輸入已下載供比對hash用之數據資料夾字串，預設'./_dwCurrent'
+ * @param {String} [opt.fdResult] 輸入已下載數據所連動生成數據資料夾字串，預設'./_result'
+ * @param {String} [opt.fdTaskCpSrc] 輸入任務狀態之來源端資料夾字串，預設'./_taskCpSrc'
+ * @param {Function} [opt.funDownloadEqs] 輸入自定義下載地震數據函數，回傳資料陣列
+ * @param {Function} [opt.funDownload] 輸入自定義當前下載之hash數據處理函數，回傳資料陣列
+ * @param {Function} [opt.funGetCurrent] 輸入自定義已下載之hash數據處理函數，回傳資料陣列
+ * @param {Function} [opt.funAdd] 輸入當有新資料時，需要連動處理之函數
+ * @param {Function} [opt.funModify] 輸入當有資料需更新時，需要連動處理之函數
+ * @param {Function} [opt.funRemove] 輸入當有資料需刪除時，需要連動處理之函數
+ * @returns {Object} 回傳事件物件，可呼叫函數on監聽change事件
+ * @example
+ *
+ * import fs from 'fs'
+ * import _ from 'lodash-es'
+ * import w from 'wsemi'
+ * import WDwdataTweqod from './src/WDwdataTweqod.mjs'
+ *
+ * let j = fs.readFileSync('../_data/settings.json', 'utf8')
+ * let st = JSON.parse(j)
+ * let token = _.get(st, 'token')
+ *
+ * //fdDwStorage
+ * let fdDwStorage = `./_dwStorage`
+ * w.fsCleanFolder(fdDwStorage)
+ *
+ * //fdDwAttime
+ * let fdDwAttime = `./_dwAttime`
+ * w.fsCleanFolder(fdDwAttime)
+ *
+ * //fdDwCurrent
+ * let fdDwCurrent = `./_dwCurrent`
+ * w.fsCleanFolder(fdDwCurrent)
+ *
+ * //fdResult
+ * let fdResult = './_result'
+ * w.fsCleanFolder(fdResult)
+ *
+ * let opt = {
+ *     fdDwStorage,
+ *     fdDwAttime,
+ *     fdDwCurrent,
+ *     fdResult,
+ *     // funDownload,
+ *     // funGetCurrent,
+ *     // funRemove,
+ *     // funAdd,
+ *     // funModify,
+ * }
+ * let ev = await WDwdataTweqod(token, opt)
+ *     .catch((err) => {
+ *         console.log(err)
+ *     })
+ * ev.on('change', (msg) => {
+ *     delete msg.type
+ *     console.log('change', msg)
+ * })
+ * // change { event: 'start', msg: 'running...' }
+ * // change { event: 'proc-callfun-download', msg: 'start...' }
+ * // change { event: 'proc-callfun-download', msg: 'done' }
+ * // change { event: 'proc-callfun-getCurrent', msg: 'start...' }
+ * // change { event: 'proc-callfun-getCurrent', msg: 'done' }
+ * // change { event: 'compare', msg: 'start...' }
+ * // change { event: 'compare', msg: 'done' }
+ * // change { event: 'proc-add-callfun-add', id: '114101', msg: 'start...' }
+ * // change { event: 'proc-add-callfun-add', id: '114101', msg: 'done' }
+ * // change { event: 'proc-add-callfun-add', id: '114102', msg: 'start...' }
+ * // change { event: 'proc-add-callfun-add', id: '114102', msg: 'done' }
+ * // ...
+ *
+ */
+let WDwdataTweqod = async(token, opt = {}) => {
+
+    //fdDwStorage
+    let fdDwStorage = get(opt, 'fdDwStorage')
+    if (!isestr(fdDwStorage)) {
+        fdDwStorage = `./_dwStorage`
+    }
+    if (!fsIsFolder(fdDwStorage)) {
+        fsCreateFolder(fdDwStorage)
+    }
+
+    //fdDwAttime
+    let fdDwAttime = get(opt, 'fdDwAttime')
+    if (!isestr(fdDwAttime)) {
+        fdDwAttime = `./_dwAttime`
+    }
+    if (!fsIsFolder(fdDwAttime)) {
+        fsCreateFolder(fdDwAttime)
+    }
+
+    //fdDwCurrent
+    let fdDwCurrent = get(opt, 'fdDwCurrent')
+    if (!isestr(fdDwCurrent)) {
+        fdDwCurrent = `./_dwCurrent`
+    }
+    if (!fsIsFolder(fdDwCurrent)) {
+        fsCreateFolder(fdDwCurrent)
+    }
+
+    //fdResult
+    let fdResult = get(opt, 'fdResult')
+    if (!isestr(fdResult)) {
+        fdResult = './_result'
+    }
+    if (!fsIsFolder(fdResult)) {
+        fsCreateFolder(fdResult)
+    }
+
+    //fdTaskCpSrc
+    let fdTaskCpSrc = get(opt, 'fdTaskCpSrc')
+    if (!isestr(fdTaskCpSrc)) {
+        fdTaskCpSrc = './_taskCpSrc'
+    }
+    if (!fsIsFolder(fdTaskCpSrc)) {
+        fsCreateFolder(fdTaskCpSrc)
+    }
+
+    //funDownloadEqs
+    let funDownloadEqs = get(opt, 'funDownloadEqs')
+
+    //funDownload
+    let funDownload = get(opt, 'funDownload')
+
+    //funGetCurrent
+    let funGetCurrent = get(opt, 'funGetCurrent')
+
+    //funAdd
+    let funAdd = get(opt, 'funAdd')
+
+    //funModify
+    let funModify = get(opt, 'funModify')
+
+    //funRemove
+    let funRemove = get(opt, 'funRemove')
+
+    //treeFilesAndGetHashs
+    let treeFilesAndGetHashs = (fd) => {
+
+        //vfps
+        let vfps = fsTreeFolder(fd, 1)
+        // console.log('vfps', vfps)
+
+        //ltdtHash
+        let ltdtHash = []
+        each(vfps, (v) => {
+
+            let j = fs.readFileSync(v.path, 'utf8')
+            let o = JSON.parse(j)
+
+            ltdtHash.push(o)
+
+        })
+
+        return ltdtHash
+    }
+
+    //funDownloadDef
+    let funDownloadDef = async() => {
+
+        //eqs
+        let eqs = []
+        if (isfun(funDownloadEqs)) {
+            eqs = funDownloadEqs()
+            if (ispm(eqs)) {
+                eqs = await eqs
+            }
+        }
+        else {
+            eqs = await downloadEqs(token)
+            // console.log('eqs', eqs, size(eqs))
+            // eqs = [eqs[0], eqs[1]]
+            // fs.writeFileSync('./temp.json', JSON.stringify(eqs, null, 2), 'utf8')
+        }
+
+        //reverse
+        eqs = reverse(eqs)
+
+        //清空fdDwAttime
+        fsCleanFolder(fdDwAttime)
+
+        //複製已下載供比對hash用數據fdDwCurrent至fdDwAttime
+        fsCopyFolder(fdDwCurrent, fdDwAttime)
+
+        //ltdtHashOld, 數據來源為fdDwCurrent, 故為舊hash數據清單
+        let ltdtHashOld = treeFilesAndGetHashs(fdDwAttime)
+
+        //kpHash, 基於舊hash數據建構字典物件
+        let kpHash = {}
+        each(ltdtHashOld, (v) => {
+            kpHash[v.id] = v
+        })
+
+        //逐筆偵測與更新
+        await pmSeries(eqs, async(v) => {
+
+            //先儲存解析後數據至存放完整數據資料夾fdDwStorage
+            let fp = `${fdDwStorage}/${v.id}.json`
+            fs.writeFileSync(fp, JSON.stringify(v), 'utf8')
+
+            //計算檔案hash值, 為新hash
+            let hashNew = await fsGetFileXxHash(fp)
+
+            //check
+            if (haskey(kpHash, v.id)) {
+                //為已下載過之地震數據
+
+                //舊hash
+                let hashOld = kpHash[v.id].hash
+
+                //check
+                if (hashNew !== hashOld) {
+
+                    //update
+                    kpHash[v.id].hash = hashNew
+
+                    //儲存更新後之新地震hash數據檔案
+                    let fp = `${fdDwAttime}/${v.id}.json`
+                    fs.writeFileSync(fp, JSON.stringify(kpHash[v.id]), 'utf8')
+
+                }
+                else {
+                    //不用更新hash檔案
+                }
+
+            }
+            else {
+                //為新增之地震數據
+
+                //add
+                kpHash[v.id] = {
+                    id: v.id,
+                    hash: hashNew,
+                }
+
+                //儲存新地震hash數據檔案
+                let fp = `${fdDwAttime}/${v.id}.json`
+                fs.writeFileSync(fp, JSON.stringify(kpHash[v.id]), 'utf8')
+
+            }
+
+        })
+
+        //ltdtHashNew
+        let ltdtHashNew = values(kpHash)
+
+        return ltdtHashNew
+    }
+    if (!isfun(funDownload)) {
+        funDownload = funDownloadDef
+    }
+
+    //funGetCurrentDef
+    let funGetCurrentDef = async() => {
+
+        //vfps
+        let vfps = fsTreeFolder(fdDwCurrent, 1)
+        // console.log('vfps', vfps)
+
+        //ltdtHashOld
+        let ltdtHashOld = []
+        each(vfps, (v) => {
+
+            let j = fs.readFileSync(v.path, 'utf8')
+            let eq = JSON.parse(j)
+
+            ltdtHashOld.push(eq)
+
+        })
+
+        return ltdtHashOld
+    }
+    if (!isfun(funGetCurrent)) {
+        funGetCurrent = funGetCurrentDef
+    }
+
+    //funRemoveDef
+    let funRemoveDef = async(v) => {
+
+        let fd = `${fdResult}/${v.id}`
+
+        if (fsIsFolder(fd)) {
+            fsDeleteFolder(fd)
+        }
+
+    }
+    if (!isfun(funRemove)) {
+        funRemove = funRemoveDef
+    }
+
+    //funAddDef
+    let funAddDef = async(v) => {
+
+        let fd = `${fdResult}/${v.id}`
+
+        if (fsIsFolder(fd)) {
+            fsCleanFolder(fd)
+        }
+
+        let fpStorage = `${fdDwStorage}/${v.id}.json`
+        let fpResult = `${fd}/${v.id}.json`
+        fsCopyFile(fpStorage, fpResult)
+
+    }
+    if (!isfun(funAdd)) {
+        funAdd = funAddDef
+    }
+
+    //funModifyDef
+    let funModifyDef = async(v) => {
+
+        let fd = `${fdResult}/${v.id}`
+
+        if (fsIsFolder(fd)) {
+            fsCleanFolder(fd)
+        }
+
+        let fpStorage = `${fdDwStorage}/${v.id}.json`
+        let fpResult = `${fd}/${v.id}.json`
+        fsCopyFile(fpStorage, fpResult)
+
+    }
+    if (!isfun(funModify)) {
+        funModify = funModifyDef
+    }
+
+    //WDwdataBuilder
+    let optBdr = {
+        fdDwAttime,
+        fdDwCurrent,
+        fdResult,
+        fdTaskCpSrc,
+        funDownload,
+        funGetCurrent,
+        funRemove,
+        funAdd,
+        funModify,
+    }
+    let ev = await WDwdataBuilder(optBdr)
+
+    return ev
+}
+
+
+export default WDwdataTweqod