w-ipproxy 1.0.1 → 1.0.3

package/src/provideServer.mjs

@@ -0,0 +1,65 @@
+import Hapi from '@hapi/hapi'
+import get from 'lodash-es/get.js'
+import ispint from 'wsemi/src/ispint.mjs'
+import isearr from 'wsemi/src/isearr.mjs'
+import isestr from 'wsemi/src/isestr.mjs'
+import cint from 'wsemi/src/cint.mjs'
+
+
+function provideServer(fun, opt = {}) {
+
+    //port
+    let port = get(opt, 'port')
+    if (!ispint(port)) {
+        port = 8080
+    }
+    port = cint(port)
+
+    //corsOrigins
+    let corsOrigins = get(opt, 'corsOrigins', [])
+    if (!isearr(corsOrigins)) {
+        corsOrigins = ['*']
+    }
+
+    //apiName
+    let apiName = get(opt, 'apiName')
+    if (!isestr(apiName)) {
+        apiName = 'getProxies'
+    }
+
+    //startServer
+    let startServer = async () => {
+        let server = Hapi.server({
+            // host: 'localhost',
+            port,
+            routes: {
+                cors: {
+                    origin: corsOrigins, //Access-Control-Allow-Origin
+                    credentials: false, //Access-Control-Allow-Credentials
+                },
+            },
+        })
+
+        //route
+        server.route({
+            method: 'GET',
+            path: `/${apiName}`,
+            handler: async () => {
+                let proxies = await fun()
+                return proxies
+            },
+        })
+
+        //start
+        await server.start()
+
+        console.log(`Server running at: ${server.info.uri}`)
+    }
+
+    //startServer
+    startServer()
+
+}
+
+
+export default provideServer

package/test/anonymity.test.mjs

@@ -0,0 +1,140 @@
+import assert from 'assert'
+import axios from 'axios'
+import https from 'https'
+import wip from '../src/WIpProxy.mjs'
+
+
+describe('anonymity (elite + anonymityCheck)', function() {
+    this.timeout(240 * 1000) //4min, 因為getProxiesOnce加上獨立leak驗證本身就耗時
+
+    let myIp = ''
+    let ps = []
+
+    before(async function() {
+
+        //取本機真實公網IP
+        try {
+            let r = await axios.get('https://httpbin.org/ip', { timeout: 10000 })
+            myIp = r.data?.origin || ''
+        }
+        catch (err) {
+            this.skip()
+        }
+        if (!myIp) {
+            this.skip()
+        }
+
+        //預設設定已是anonymity=elite且啟用anonymityCheck, 走完整過濾路徑
+        let wo = wip({})
+        ps = await wo.getProxiesOnce()
+
+    })
+
+    it('no returned proxy should leak real IP via httpbin origin', async function() {
+        if (ps.length === 0) {
+            this.skip()
+        }
+
+        let agent = new https.Agent({ rejectUnauthorized: false })
+        let sample = ps.slice(0, 30)
+        let pms = sample.map(async (p) => {
+            try {
+                let res = await axios.get('https://httpbin.org/ip', {
+                    proxy: { host: p.host, port: p.port, protocol: 'http' },
+                    httpsAgent: agent,
+                    timeout: 8000,
+                    headers: {
+                        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/122.0.0.0 Safari/537.36',
+                        'Accept': 'application/json',
+                    },
+                    validateStatus: () => true,
+                })
+                if (res.status !== 200) {
+                    return { proxy: p.proxy, ok: true, reason: 'non-200, ignored' }
+                }
+                let origin = res.data?.origin
+                if (typeof origin !== 'string') {
+                    return { proxy: p.proxy, ok: true, reason: 'non-json, ignored' }
+                }
+                //回傳origin含本機真實IP即代表該代理洩漏
+                let leaks = origin.indexOf(myIp) >= 0
+                return { proxy: p.proxy, ok: !leaks, origin }
+            }
+            catch (err) {
+                //網路錯誤不算洩漏, 直接放行
+                return { proxy: p.proxy, ok: true, reason: err.message }
+            }
+        })
+        let rrs = await Promise.all(pms)
+        let leaked = rrs.filter((r) => !r.ok)
+
+        assert.strictEqual(
+            leaked.length, 0,
+            `expected 0 proxies to leak real IP, got ${leaked.length}: ` +
+            leaked.slice(0, 5).map((r) => `${r.proxy} (origin=${r.origin})`).join('; ')
+        )
+    })
+
+    it('no returned proxy should carry X-Forwarded-For / Via when viewed from target', async function() {
+        if (ps.length === 0) {
+            this.skip()
+        }
+
+        let LEAK_IP_HEADERS = [
+            'x-forwarded-for', 'x-real-ip', 'x-client-ip', 'client-ip',
+            'true-client-ip', 'cf-connecting-ip', 'forwarded',
+            'x-originating-ip', 'x-cluster-client-ip',
+        ]
+        let LEAK_PROXY_HEADERS = [
+            'via', 'proxy-connection', 'x-proxy-id',
+        ]
+
+        let agent = new https.Agent({ rejectUnauthorized: false })
+        let sample = ps.slice(0, 30)
+        let pms = sample.map(async (p) => {
+            try {
+                let res = await axios.get('https://httpbin.org/headers', {
+                    proxy: { host: p.host, port: p.port, protocol: 'http' },
+                    httpsAgent: agent,
+                    timeout: 8000,
+                    headers: {
+                        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/122.0.0.0 Safari/537.36',
+                        'Accept': 'application/json',
+                    },
+                    validateStatus: () => true,
+                })
+                if (res.status !== 200 || typeof res.data?.headers !== 'object') {
+                    return { proxy: p.proxy, ok: true, reason: 'ignored' }
+                }
+                let hs = {}
+                for (let k in res.data.headers) {
+                    hs[k.toLowerCase()] = String(res.data.headers[k])
+                }
+                let bad = []
+                for (let k of LEAK_IP_HEADERS) {
+                    if (k in hs && hs[k].indexOf(myIp) >= 0) {
+                        bad.push(`${k}=${hs[k]}`)
+                    }
+                }
+                for (let k of LEAK_PROXY_HEADERS) {
+                    if (k in hs) {
+                        bad.push(`${k}=${hs[k]}`)
+                    }
+                }
+                return { proxy: p.proxy, ok: bad.length === 0, bad }
+            }
+            catch (err) {
+                return { proxy: p.proxy, ok: true, reason: err.message }
+            }
+        })
+        let rrs = await Promise.all(pms)
+        let leaked = rrs.filter((r) => !r.ok)
+
+        assert.strictEqual(
+            leaked.length, 0,
+            `expected 0 proxies with leaky headers, got ${leaked.length}: ` +
+            leaked.slice(0, 5).map((r) => `${r.proxy} (${r.bad.join(',')})`).join('; ')
+        )
+    })
+
+})

package/test/getProxiesOnce.test.mjs

@@ -0,0 +1,89 @@
+import assert from 'assert'
+import axios from 'axios'
+import https from 'https'
+import wip from '../src/WIpProxy.mjs'
+
+
+describe('getProxiesOnce', function() {
+    this.timeout(180 * 1000) //3min, 因為需要抓清單+檢測+獨立驗證
+
+    let myIp = ''
+    let ps = []
+
+    before(async function() {
+
+        //取本機真實公網IP (不走代理), 作為後續洩漏比對基準
+        try {
+            let r = await axios.get('https://httpbin.org/ip', { timeout: 10000 })
+            myIp = r.data?.origin || ''
+        }
+        catch (err) {
+            //網路異常則整組test跳過
+            this.skip()
+        }
+        if (!myIp) {
+            this.skip()
+        }
+
+        //取代理清單 (預設anonymity=elite + 匿名性檢查)
+        let wo = wip({})
+        ps = await wo.getProxiesOnce()
+
+    })
+
+    it('should return a non-empty array of proxy objects', function() {
+        //若上游proxyscrape當下無代理供應, 放行以免CI誤報 (非本專案bug)
+        if (ps.length === 0) {
+            this.skip()
+        }
+        assert.ok(Array.isArray(ps))
+        for (let p of ps) {
+            assert.strictEqual(typeof p.host, 'string')
+            assert.ok(p.host.length > 0)
+            assert.strictEqual(typeof p.port, 'number')
+            assert.ok(p.port > 0)
+            assert.strictEqual(p.proxy, `${p.host}:${p.port}`)
+        }
+    })
+
+    it('majority of returned proxies should be independently usable against httpbin', async function() {
+        if (ps.length === 0) {
+            this.skip()
+        }
+
+        //獨立再打一次httpbin.org/ip, 不倚賴內建的檢測結果
+        let agent = new https.Agent({ rejectUnauthorized: false })
+        let sample = ps.slice(0, 30) //取前30個避免長時間阻塞CI
+        let pms = sample.map(async (p) => {
+            try {
+                let res = await axios.get('https://httpbin.org/ip', {
+                    proxy: { host: p.host, port: p.port, protocol: 'http' },
+                    httpsAgent: agent,
+                    timeout: 8000,
+                    headers: {
+                        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/122.0.0.0 Safari/537.36',
+                        'Accept': 'application/json',
+                    },
+                    validateStatus: () => true,
+                })
+                return res.status === 200
+                    && typeof res.data === 'object'
+                    && res.data !== null
+                    && typeof res.data.origin === 'string'
+            }
+            catch (err) {
+                return false
+            }
+        })
+        let rrs = await Promise.all(pms)
+        let okCount = rrs.filter((v) => v).length
+        let ratio = okCount / rrs.length
+
+        //要求至少50%仍可用, 免疫於個別代理剛好在測試間掛掉的flakiness
+        assert.ok(
+            ratio >= 0.5,
+            `independent reachability ratio too low: ${okCount}/${rrs.length} (${(ratio * 100).toFixed(1)}%)`
+        )
+    })
+
+})

package/toolg/cleanFolder.mjs

@@ -0,0 +1,4 @@
+import cleanFolder from 'w-package-tools/src/cleanFolder.mjs'
+
+
+cleanFolder('docs')

package/toolg/gDistRollup.mjs

@@ -0,0 +1,29 @@
+import rollupFiles from 'w-package-tools/src/rollupFiles.mjs'
+import getFiles from 'w-package-tools/src/getFiles.mjs'
+import w from 'wsemi'
+
+
+let fdSrc = './src'
+let fdTar = './dist'
+let fns = getFiles(fdSrc)
+fns = fns.filter((v) => {
+    return w.strleft(v, 1) === 'W'
+})
+
+rollupFiles({
+    fns,
+    fdSrc,
+    fdTar,
+    nameDistType: 'kebabCase',
+    globals: {
+        '@hapi/hapi': '@hapi/hapi',
+        '@hapi/inert': '@hapi/inert',
+        'https': 'https',
+    },
+    external: [
+        '@hapi/hapi',
+        '@hapi/inert',
+        'https',
+    ],
+})
+

package/toolg/modifyReadme.mjs

@@ -0,0 +1,4 @@
+import modifyReadme from 'w-package-tools/src/modifyReadme.mjs'
+
+
+modifyReadme()

package/g.mjs

@@ -1,12 +0,0 @@
-import wip from './src/WIpProxy.mjs'
-
-wip()
-    .then((res) => {
-        console.log(res)
-        console.log(res.length)
-    })
-    .catch((err) => {
-        console.error(err)
-    })
-
-//node --experimental-modules --es-module-specifier-resolution=node g.mjs

package/src/getHtml.mjs

@@ -1,84 +0,0 @@
-import puppeteer from 'puppeteer'
-
-
-async function getHtml(url, selector, muti = false) {
-
-    async function view(url) {
-
-        //new page
-        let olaunch = {
-            //timeout: 0,
-            headless: false,
-            slowMo: 20,
-            // arg: [
-            //     //`--window-size=${viewport.width},${viewport.height}`, //無效
-            //     //`--disable-resize-lock`, //無效
-            // ],
-            // defaultViewport: { //無效
-            //     x: 0,
-            //     y: 0,
-            //     width: 800,
-            //     height: 600,
-            // }
-        }
-        let browser = await puppeteer.launch(olaunch)
-        let page = await browser.newPage()
-
-        //show page
-        await page.goto(url)
-        let viewport = {
-            x: 0,
-            y: 0,
-            width: 800,
-            height: 600,
-        }
-        await page.setViewport(viewport)
-
-        //wait
-        let waitsec = 5
-        await page.waitFor(waitsec * 1000)
-
-        return {
-            browser,
-            page,
-        }
-    }
-
-    async function close(p) {
-
-        //close
-        await p.page.close()
-        await p.browser.close()
-
-    }
-
-    async function g1(p, selector) {
-        return await p.page.$eval(selector, (node) => {
-            return node.innerHTML
-        })
-    }
-
-    async function gs(p, selector) {
-        return await p.page.$$eval(selector, (nodes) => {
-            return nodes.map((node) => {
-                return node.innerHTML
-            })
-        })
-    }
-
-    let p = await view(url)
-
-    let r
-    if (muti) {
-        r = await gs(p, selector)
-    }
-    else {
-        r = await g1(p, selector)
-    }
-
-    close(p)
-
-    return r
-}
-
-export default getHtml

package/src/getProxyListOrg.mjs

@@ -1,66 +0,0 @@
-// import fs from 'fs'
-import get from 'lodash/get'
-import sep from 'wsemi/src/sep.mjs'
-import pmMap from 'wsemi/src/pmMap.mjs'
-import parseHtml from './parseHtml.mjs'
-import getHtml from './getHtml.mjs'
-
-
-async function core(url) {
-    let ps = []
-
-    let selector = 'div.table'
-    let h = await getHtml(url, selector)
-    // fs.writeFileSync('temp.html', h)
-    // let h = fs.readFileSync('tempProxyListOrg.html', 'utf8')
-
-    //parseHtml
-    let $ = parseHtml(h)
-    $('ul').map((k, v) => {
-        let q = $(v)
-
-        let proxy = q.children('li.proxy').html()
-        proxy = sep(proxy, 'script>')
-        proxy = get(proxy, 1)
-        // console.log(k,'proxy',proxy)
-
-        let httpType = q.children('li.https').text()
-        // console.log(k,'httpType',httpType)
-
-        let country = q.find('span.country').text()
-        // console.log(k,'country',country)
-
-        let countryName = q.find('span.name').attr('name')
-        // console.log(k,'countryName',countryName)
-
-        ps.push({
-            proxy,
-            httpType,
-            country,
-            countryName,
-        })
-    })
-
-    return ps
-}
-
-async function getProxyListOrg() {
-    let psAll = []
-
-    await pmMap([1, 2, 3], async(v) => {
-        // console.log(v)
-
-        //url
-        //let url = 'https://proxy-list.org/chinese/index.php?setlang=chinese'
-        let url = `https://proxy-list.org/chinese/index.php?setlang=chinese&p=${v}`
-
-        //core
-        let ps = await core(url)
-        psAll = [...psAll, ...ps]
-
-    })
-
-    return psAll
-}
-
-export default getProxyListOrg

package/src/parseHtml.mjs

@@ -1,19 +0,0 @@
-
-import cheerio from 'cheerio'
-
-
-function parseHtml(h) {
-
-    let $setting = {
-        //ignoreWhitespace: true,
-        xmlMode: true,
-        decodeEntities: false,
-    }
-
-    //$
-    let $ = cheerio.load(h, $setting)
-
-    return $
-}
-
-export default parseHtml

package/test/all.test.mjs

@@ -1,10 +0,0 @@
-import assert from 'assert'
-
-
-describe('all', function() {
-
-    it('test in localhost', function() {
-        assert.strict.deepEqual(1, 1)
-    })
-
-})