w-ipproxy 1.0.1 → 1.0.3

package/docs/styles/prettify.css

@@ -0,0 +1,80 @@
+.pln {
+  color: #ddd;
+}
+
+/* string content */
+.str {
+  color: #61ce3c;
+}
+
+/* a keyword */
+.kwd {
+  color: #fbde2d;
+}
+
+/* a comment */
+.com {
+  color: #aeaeae;
+}
+
+/* a type name */
+.typ {
+  color: #8da6ce;
+}
+
+/* a literal value */
+.lit {
+  color: #fbde2d;
+}
+
+/* punctuation */
+.pun {
+  color: #ddd;
+}
+
+/* lisp open bracket */
+.opn {
+  color: #000000;
+}
+
+/* lisp close bracket */
+.clo {
+  color: #000000;
+}
+
+/* a markup tag name */
+.tag {
+  color: #8da6ce;
+}
+
+/* a markup attribute name */
+.atn {
+  color: #fbde2d;
+}
+
+/* a markup attribute value */
+.atv {
+  color: #ddd;
+}
+
+/* a declaration */
+.dec {
+  color: #EF5050;
+}
+
+/* a variable name */
+.var {
+  color: #c82829;
+}
+
+/* a function name */
+.fun {
+  color: #4271ae;
+}
+
+/* Specify class=linenums on a pre to get line numbering */
+ol.linenums {
+  margin-top: 0;
+  margin-bottom: 0;
+  padding-bottom: 2px;
+}

package/g-getByTimer.mjs

@@ -0,0 +1,24 @@
+import _ from 'lodash-es'
+import wip from './src/WIpProxy.mjs'
+
+
+let wo = wip({
+    // tar: `https://www.google.com`,
+})
+wo.init()
+wo.on('getRawProxies', (prxsRaw) => {
+    console.log(`已抓取公開代理共 ${_.size(prxsRaw)} 個...`)
+})
+wo.on('add', (p, proxies) => {
+    // console.log('新加入代理', { host: p.host, port: p.port }, _.map(proxies, 'proxy'))
+})
+wo.on('delete', (p, proxies) => {
+    // console.log('刪除代理', { host: p.host, port: p.port }, _.map(proxies, 'proxy'))
+})
+wo.on('change', (proxies) => {
+    // console.log(`有效代理`, _.map(proxies, 'proxy'))
+    console.log(`已檢測有效代理共 ${_.size(proxies)} 個...`)
+})
+
+
+//node g-getByTimer.mjs

package/g-getOnce.mjs

@@ -0,0 +1,11 @@
+import wip from './src/WIpProxy.mjs'
+
+
+let wo = wip({
+    // tar: `https://www.google.com`,
+})
+let ps = await wo.getProxiesOnce()
+console.log('ps', ps)
+
+
+//node g-getOnce.mjs

package/g-srv.mjs

@@ -0,0 +1,28 @@
+import _ from 'lodash-es'
+import wip from './src/WIpProxy.mjs'
+
+
+let wo = wip({
+    // tar: `https://www.google.com`,
+    withServer: true,
+    serverPort: 9000,
+    serverCorsOrigins: ['*'],
+})
+wo.on('getRawProxies', (prxsRaw) => {
+    console.log(`已抓取公開代理共 ${_.size(prxsRaw)} 個...`)
+})
+wo.on('add', (p, proxies) => {
+    // console.log('新加入代理', { host: p.host, port: p.port }, _.map(proxies, 'proxy'))
+})
+wo.on('delete', (p, proxies) => {
+    // console.log('刪除代理', { host: p.host, port: p.port }, _.map(proxies, 'proxy'))
+})
+wo.on('change', (proxies) => {
+    // console.log(`有效代理`, _.map(proxies, 'proxy'))
+    console.log(`已檢測有效代理共 ${_.size(proxies)} 個...`)
+})
+
+//browser view: http://localhost:9000/getProxies
+
+
+//node g-srv.mjs

package/package.json

@@ -1,19 +1,19 @@
 {
   "name": "w-ipproxy",
-  "version": "1.0.1",
+  "version": "1.0.3",
   "main": "dist/w-ipproxy.umd.js",
   "dependencies": {
-    "axios": "^1.2.3",
-    "cheerio": "^1.0.0-rc.12",
-    "lodash": "^4.17.21",
-    "puppeteer": "^19.5.2",
-    "wsemi": "^1.6.87"
+    "@hapi/hapi": "^21.4.8",
+    "axios": "^1.13.6",
+    "cheerio": "^1.2.0",
+    "lodash-es": "^4.18.1",
+    "wsemi": "^1.8.47"
   },
   "devDependencies": {
-    "w-package-tools": "^1.0.66"
+    "w-package-tools": "^1.1.5"
   },
   "scripts": {
-    "test": "mocha --parallel --timeout 60000 --experimental-modules --es-module-specifier-resolution=node",
+    "test": "mocha --parallel --timeout 60000",
     "deploy": "gh-pages -d docs"
   },
   "repository": {
@@ -24,7 +24,9 @@
     "package",
     "tool",
     "ip",
-    "proxy"
+    "proxy",
+    "hapi",
+    "nodejs"
   ],
   "author": "yuda-lyu(semisphere)",
   "license": "MIT"

package/script.txt

@@ -0,0 +1,17 @@
+#node toolg/addVersion.mjs
+node toolg/modifyReadme.mjs
+
+node toolg/cleanFolder.mjs
+./node_modules/.bin/jsdoc -c .jsdoc
+
+node toolg/gDistRollup.mjs
+
+git add .  -A
+git commit -m 'modify: '
+git push origin master:master
+
+npm run deploy
+
+#npm test
+
+#npm publish

package/src/WIpProxy.mjs

@@ -1,23 +1,482 @@
-import getProxyListOrg from './getProxyListOrg.mjs'
+import axios from 'axios'
+import https from 'https'
+import get from 'lodash-es/get.js'
+import size from 'lodash-es/size.js'
+import each from 'lodash-es/each.js'
+import map from 'lodash-es/map.js'
+import trim from 'lodash-es/trim.js'
+import values from 'lodash-es/values.js'
+import evem from 'wsemi/src/evem.mjs'
+import sep from 'wsemi/src/sep.mjs'
+import isestr from 'wsemi/src/isestr.mjs'
+import ispint from 'wsemi/src/ispint.mjs'
+import isbol from 'wsemi/src/isbol.mjs'
+import isearr from 'wsemi/src/isearr.mjs'
+import iseobj from 'wsemi/src/iseobj.mjs'
+import cint from 'wsemi/src/cint.mjs'
+import waitFun from 'wsemi/src/waitFun.mjs'
+import provideServer from './provideServer.mjs'
 
-//freeproxylists
-//http://www.freeproxylists.net/zh/?c=&pt=&pr=&a%5B%5D=0&a%5B%5D=1&a%5B%5D=2&u=0
 
-//aliveproxy
-//http://www.aliveproxy.com/proxy-list/proxies.aspx/Taiwan-tw
+/**
+ * 抓取代理伺服器
+ *
+ * @class
+ * @param {Object} [opt={}] 輸入設定物件，預設{}
+ * @param {String} [opt.src='https://api.proxyscrape.com/v2/?request=displayproxies&protocol=http&timeout=5000&anonymity=all'] 輸入取得代理伺服器API字串，預設'https://api.proxyscrape.com/v2/?request=displayproxies&protocol=http&timeout=5000&anonymity=all'
+ * @param {String} [opt.tar='https://httpbin.org/ip'] 輸入檢測目標網址字串，預設'https://httpbin.org/ip'
+ * @param {Integer} [opt.timeGetProxies=30*1000] 輸入輪循抓取代理伺服器時間間隔整數，單位ms，預設30*1000
+ * @param {Integer} [opt.timeTestProxies=60*1000] 輸入輪循測試代理伺服器時間間隔整數，單位ms，預設60*1000
+ * @param {Boolean} [opt.anonymityCheck=true] 輸入是否對代理做匿名性檢測布林值，開啟時會先打https://httpbin.org/ip驗證代理不會將本機真實IP透過X-Forwarded-For等方式洩漏給目標端，預設true
+ * @param {Integer} [opt.anonymityCheckRepeats=2] 輸入匿名性檢測重複次數整數，用以偵測負載平衡型代理的機率性洩漏，任一次洩漏即判定失敗，預設2
+ * @param {Boolean} [opt.withServer=false] 輸入是否創建供數據供給伺服器布林值，預設false
+ * @param {Integer} [opt.serverPort=8080] 輸入創建伺服器所用port整數，預設8080
+ * @param {String} [opt.serverApiName='getProxies'] 輸入供給數據API名稱字串，預設'getProxies'
+ * @param {Array} [opt.serverCorsOrigins=['*']] 輸入允許跨域網域陣列，若給予['*']代表允許全部，預設['*']
+ * @returns {Object} 回傳事件物件，提供函數getProxies，可監聽事件getRawProxies、add、delete、change
+ * @example
+ *
+ * import _ from 'lodash-es'
+ * import wip from './src/WIpProxy.mjs'
+ *
+ * let wo = wip({
+ *     // tar: `https://www.google.com`,
+ *     withServer: true,
+ *     serverPort: 9000,
+ *     serverCorsOrigins: ['*'],
+ * })
+ * wo.on('getRawProxies', (prxsRaw) => {
+ *     console.log(`已抓取公開代理共 ${_.size(prxsRaw)} 個...`)
+ * })
+ * wo.on('add', (p, proxies) => {
+ *     // console.log('新加入代理', { host: p.host, port: p.port }, _.map(proxies, 'proxy'))
+ * })
+ * wo.on('delete', (p, proxies) => {
+ *     // console.log('刪除代理', { host: p.host, port: p.port }, _.map(proxies, 'proxy'))
+ * })
+ * wo.on('change', (proxies) => {
+ *     // console.log(`有效代理`, _.map(proxies, 'proxy'))
+ *     console.log(`已檢測有效代理共 ${_.size(proxies)} 個...`)
+ * })
+ *
+ * //browser view: http://localhost:9000/getProxies
+ *
+ */
+function WIpProxy(opt = {}) {
 
-//cnproxy
-//http://www.cnproxy.com/proxy1.html
+    //src
+    let src = get(opt, 'src', '')
+    if (!isestr(src)) {
+        src = 'https://api.proxyscrape.com/v2/?request=displayproxies' +
+        `&protocol=http` +
+        // `&protocol=https` +
+        `&timeout=5000` +
+        // `&country=${join(['tw', 'cn', 'hk', 'jp', 'kr', 'sg', 'my', 'in', 'id', 'th', 'vn'])}` +
+        // `&ssl=yes` +
+        `&anonymity=elite` //elite匿名代理, all會使用X-Forwarded-For儲存原本ip導致被識別
+    }
 
-async function WIpProxy() {
-    let rs = []
-    let r
+    //tar
+    let tar = get(opt, 'tar', '')
+    if (!isestr(tar)) {
+        tar = 'https://httpbin.org/ip' //'https://www.google.com'
+    }
 
-    //getProxyListOrg
-    r = await getProxyListOrg()
-    rs = [...rs, ...r]
+    //timeGetProxies
+    let timeGetProxies = get(opt, 'timeGetProxies')
+    if (!ispint(timeGetProxies)) {
+        timeGetProxies = 30 * 1000 //30s
+    }
+    timeGetProxies = cint(timeGetProxies)
 
-    return rs
+    //timeTestProxies
+    let timeTestProxies = get(opt, 'timeTestProxies')
+    if (!ispint(timeTestProxies)) {
+        timeTestProxies = 60 * 1000 //1min
+    }
+    timeTestProxies = cint(timeTestProxies)
+
+    //anonymityCheck
+    let anonymityCheck = get(opt, 'anonymityCheck')
+    if (!isbol(anonymityCheck)) {
+        anonymityCheck = true
+    }
+
+    //anonymityCheckRepeats
+    let anonymityCheckRepeats = get(opt, 'anonymityCheckRepeats')
+    if (!ispint(anonymityCheckRepeats)) {
+        anonymityCheckRepeats = 2
+    }
+    anonymityCheckRepeats = cint(anonymityCheckRepeats)
+
+    //withServer
+    let withServer = get(opt, 'withServer')
+    if (!isbol(withServer)) {
+        withServer = false
+    }
+
+    //serverPort
+    let serverPort = get(opt, 'serverPort')
+    if (!ispint(serverPort)) {
+        serverPort = 8080
+    }
+    serverPort = cint(serverPort)
+
+    //serverApiName
+    let serverApiName = get(opt, 'serverApiName')
+    if (!isestr(serverApiName)) {
+        serverApiName = 'getProxies'
+    }
+
+    //serverCorsOrigins
+    let serverCorsOrigins = get(opt, 'serverCorsOrigins', [])
+    if (!isearr(serverCorsOrigins)) {
+        serverCorsOrigins = ['*']
+    }
+
+    //prxsRaw, kpPrx
+    let prxsRaw = []
+    let kpPrx = {}
+
+    //myRealIp: 本機真實公網IP, 用於匿名性檢測時比對代理egress是否洩漏
+    let myRealIp = ''
+    let myRealIpReady = false
+    let myRealIpPm = null
+    async function _fetchMyRealIp() {
+        if (myRealIpReady) {
+            return
+        }
+        if (!myRealIpPm) {
+            myRealIpPm = (async () => {
+                try {
+                    let res = await axios.get('https://httpbin.org/ip', { timeout: 10000 })
+                    let origin = get(res, 'data.origin', '')
+                    if (isestr(origin)) {
+                        //origin可能為"1.2.3.4"或"1.2.3.4:port", 取首個逗號前且去port
+                        myRealIp = origin.split(',')[0].trim().split(':')[0]
+                    }
+                }
+                catch (err) {
+                    //取不到就算了, 匿名性檢查會自動跳過
+                }
+                myRealIpReady = true
+            })()
+        }
+        await myRealIpPm
+    }
+
+    async function _anonymityCheckCore(host, port) {
+        if (!anonymityCheck) {
+            return { ok: true }
+        }
+        if (!isestr(myRealIp)) {
+            return { ok: true } //沒抓到自己IP就無從比對, 放行
+        }
+        let agent = new https.Agent({ rejectUnauthorized: false })
+        for (let i = 0; i < anonymityCheckRepeats; i++) {
+            try {
+                let res = await axios.get('https://httpbin.org/ip', {
+                    proxy: { host, port, protocol: 'http' },
+                    httpsAgent: agent,
+                    timeout: 5000,
+                    headers: {
+                        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/122.0.0.0 Safari/537.36',
+                        'Accept': 'application/json',
+                    },
+                    validateStatus: () => true,
+                })
+                if (res.status !== 200) {
+                    return { ok: false, reason: `anonymity status[${res.status}]` }
+                }
+                let origin = get(res, 'data.origin', '')
+                if (!isestr(origin)) {
+                    return { ok: false, reason: 'anonymity non-json' }
+                }
+                if (origin.indexOf(myRealIp) >= 0) {
+                    return { ok: false, reason: `leaks real IP` }
+                }
+            }
+            catch (err) {
+                return { ok: false, reason: trim(err.message || '') }
+            }
+        }
+        return { ok: true }
+    }
+
+    async function _getProxiesCore(src) {
+        let ps = []
+
+        try {
+
+            //res
+            let res = await axios.get(src)
+
+            //c
+            let c = res.data
+            // console.log('c', c)
+
+            //sep
+            let ss = sep(c, '\n')
+            // console.log('ss', ss)
+
+            //ps
+            ps = map(ss, (v) => {
+                let [host, port] = v.split(':')
+                port = cint(port)
+                return {
+                    proxy: v,
+                    host,
+                    port,
+                }
+            })
+            // console.log('ps', ps)
+
+        }
+        catch (err) {
+            console.log('_getProxiesCore catch', err.message)
+        }
+        // console.log('ps', ps)
+
+        return ps
+    }
+
+    let b_getProxies = false
+    async function _getProxies() {
+        // console.log('call getProxies...')
+
+        let core = async() => {
+
+            //get
+            prxsRaw = await _getProxiesCore(src)
+
+            //emit
+            ev.emit('getRawProxies', prxsRaw)
+
+        }
+
+        //core
+        if (b_getProxies) {
+            return
+        }
+        b_getProxies = true
+        await core()
+            .catch(() => {}) //吃掉錯誤
+            .finally(() => {
+                b_getProxies = false
+            })
+
+    }
+
+    async function _testProxyCore(host, port, url) {
+
+        //ret
+        let ret = (state, msg) => {
+            return {
+                state,
+                msg,
+                p: {
+                    proxy: `${host}:${port}`,
+                    host,
+                    port,
+                },
+            }
+        }
+
+        //匿名性預檢: 先確認代理不會透過X-Forwarded-For等洩漏本機真實IP
+        let ac = await _anonymityCheckCore(host, port)
+        if (!ac.ok) {
+            return ret('error', ac.reason)
+        }
+
+        //r
+        let r = {}
+        try {
+
+            //agent
+            let agent = new https.Agent({ rejectUnauthorized: false }) //避免證書錯誤中止
+
+            //get
+            let res = await axios.get(url, {
+                proxy: {
+                    host,
+                    port,
+                    protocol: 'http',
+                },
+                httpsAgent: agent,
+                timeout: 5000,
+                headers: {
+                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/122.0.0.0 Safari/537.36',
+                    'Accept': 'application/json',
+                },
+            })
+
+            if (res.status === 200) {
+                r = ret('success', res.data)
+            }
+            else {
+                r = ret('error', `status[${res.status}] error`)
+            }
+
+        }
+        catch (err) {
+            r = ret('error', trim(err.message))
+            // if (r.msg.indexOf(`ssl3_get_record`) >= 0) {
+            //     r.msg = `ssl3_get_record error`
+            // }
+        }
+        // console.log(r)
+
+        return r
+    }
+
+    async function _testProxiesCore(prxs) {
+
+        //pms
+        let pms = map(prxs, (p) => {
+            let pm = _testProxyCore(p.host, p.port, tar)
+            return pm
+        })
+
+        //rrs, allSettled
+        let rrs = await Promise.allSettled(pms)
+        // console.log('rrs', rrs)
+
+        //psValid, psInvalid
+        let psValid = []
+        let psInvalid = []
+        each(rrs, (v) => {
+
+            //be
+            let p = get(v, 'value.p', null)
+            let be = iseobj(p) && isestr(get(p, 'proxy'))
+
+            //check
+            if (!be) {
+                return true //跳出換下一個
+            }
+
+            //b
+            let status = get(v, 'status', '')
+            let b1 = status === 'fulfilled'
+            let state = get(v, 'value.state', '')
+            let b2 = state === 'success'
+            let b = b1 && b2
+
+            //push
+            if (b) {
+                psValid.push(p)
+            }
+            else {
+                psInvalid.push(p)
+            }
+
+        })
+
+        return {
+            psValid,
+            psInvalid,
+        }
+    }
+
+    let b_testProxies = false
+    async function _testProxies() {
+        // console.log('call testProxies...')
+
+        let core = async() => {
+
+            //確保本機真實IP已取得, 供匿名性檢查比對
+            await _fetchMyRealIp()
+
+            //get
+            let r = await _testProxiesCore(prxsRaw)
+
+            //emit add
+            each(r.psValid, (p) => {
+                kpPrx[p.proxy] = p
+                ev.emit('add', p, values(kpPrx))
+            })
+
+            //emit delete
+            each(r.psInvalid, (p) => {
+                delete kpPrx[p.proxy]
+                ev.emit('delete', p, values(kpPrx))
+            })
+
+            //emit change
+            ev.emit('change', values(kpPrx))
+
+        }
+
+        //core
+        if (b_testProxies) {
+            return
+        }
+        b_testProxies = true
+        await core()
+            .catch(() => {}) //吃掉錯誤
+            .finally(() => {
+                b_testProxies = false
+            })
+
+    }
+
+    function init() {
+
+        //取得代理清單
+        if (true) {
+            _getProxies()
+            setInterval(() => {
+                _getProxies()
+            }, timeGetProxies)
+        }
+
+        //過濾出有效代理清單, 延遲3s觸發, 給時間抓取
+        if (true) {
+            setTimeout(() => {
+                _testProxies()
+                setInterval(() => {
+                    _testProxies()
+                }, timeTestProxies)
+            }, 3000)
+        }
+
+    }
+
+    async function getProxies() {
+        let vs
+        await waitFun(() => {
+            vs = values(kpPrx)
+            return size(vs) > 0
+        })
+        return vs
+    }
+
+    //ev
+    let ev = evem()
+
+    //withServer
+    if (withServer) {
+        provideServer(getProxies, {
+            port: serverPort,
+            apiName: serverApiName,
+            corsOrigins: serverCorsOrigins,
+        })
+    }
+
+    async function getProxiesOnce() {
+        await _getProxies()
+        await _testProxies()
+        let ps = await getProxies()
+        return ps
+    }
+
+    //save
+    ev.init = init
+    ev.getProxies = getProxies
+    ev.getProxiesOnce = getProxiesOnce
+
+    return ev
 }
 
+
 export default WIpProxy