swpp-backends 0.0.1-alpha.0 → 0.0.2-alpha

package/src/FileAnalyzer.ts

@@ -1,417 +0,0 @@
-import fs from 'fs'
-import nodePath from 'path'
-import {Request} from 'node-fetch'
-import {readRules} from './SwppRules'
-import * as crypto from 'crypto'
-import {Buffer} from 'buffer'
-import HTMLParser from 'fast-html-parser'
-import CSSParser from 'css'
-import {fetchFile, readEjectData} from './Utils'
-
-/**
- * 版本信息（可以用 JSON 序列化）
- * @see VersionMap
- */
-export interface VersionJson {
-    version: number,
-    list: VersionMap
-}
-
-/**
- * 版本列表
- *
- * + key 为文件的 URL
- * + value {string} 为 URL 对应文件的 md5 值
- * + value {string[]} 为 stable 文件其中包含的 URL
- */
-export interface VersionMap {
-    [propName: string]: any
-}
-
-/**
- * 遍历指定目录及其子目录中包含的所有文件（不遍历文件夹）
- * @param root 根目录
- * @param cb 回调函数（接收的参数是文件的相对路径）
- */
-function eachAllFile(root: string, cb: (path: string) => void) {
-    const stats = fs.statSync(root)
-    if (stats.isFile()) cb(root)
-    else {
-        const files = fs.readdirSync(root)
-        files.forEach(it => eachAllFile(nodePath.join(root, it), cb))
-    }
-}
-
-/**
- * 判断指定 URL 是否排除
- *
- * + **执行该函数前必须调用过 [loadRules]**
- *
- * @param webRoot 网站域名
- * @param url 要判断的 URL
- */
-export function isExclude(webRoot: string, url: string): boolean {
-    const exclude = readRules().config?.json?.exclude
-    if (!exclude) throw 'exclude 为空'
-    const list = isExternalLink(webRoot, url) ? exclude.other : exclude.localhost
-    for (let reg of list) {
-        if (url.match(reg)) return true
-    }
-    return false
-}
-
-/**
- * 判断指定 URL 是否是 stable 的
- *
- * + **执行该函数前必须调用过 [loadRules]**
- */
-export function isStable(url: string): boolean {
-    const stable = readRules().config?.external?.stable
-    if (!stable) throw 'stable 为空'
-    for (let reg of stable) {
-        if (url.match(reg)) return true
-    }
-    return false
-}
-
-/**
- * 从指定 URL 加载 cache json
- *
- * + **执行该函数前必须调用过 [loadRules]**
- */
-export async function loadVersionJson(url: string): Promise<VersionJson> {
-    const response = await fetchFile(url)
-    return _oldVersionJson = (await response.json()) as VersionJson
-}
-
-let _oldVersionJson: VersionJson
-let _newVersionJson: VersionJson
-let _mergeVersionMap: VersionMap
-
-/**
- * 读取最后一次加载的 version json
- *
- * + **执行该函数前必须调用过 [loadRules]**
- * + **调用该函数前必须调用过 [loadCacheJson]**
- */
-export function readOldVersionJson(): VersionJson {
-    if (!_oldVersionJson) throw 'cache json 尚未初始化'
-    return _oldVersionJson
-}
-
-/**
- * 读取最后一次构建的 VersionJson
- *
- * + **执行该函数前必须调用过 [loadRules]**
- * + **调用该函数前必须调用过 [loadCacheJson]**
- * + **执行该函数前必须调用过 [buildVersionJson]**
- * + **执行该函数前必须调用过 [calcEjectValues]**
- */
-export function readNewVersionJson(): VersionJson {
-    if (!_newVersionJson) throw 'cache json 尚未初始化'
-    return _newVersionJson
-}
-
-/**
- * 读取新旧版本文件合并后的版本地图
- *
- * + **执行该函数前必须调用过 [loadRules]**
- * + **调用该函数前必须调用过 [loadCacheJson]**
- * + **执行该函数前必须调用过 [buildVersionJson]**
- * + **执行该函数前必须调用过 [calcEjectValues]**
- */
-export function readMergeVersionMap(): VersionMap {
-    if (_mergeVersionMap) return _mergeVersionMap
-    const map: VersionMap = {}
-    Object.assign(map, readOldVersionJson().list)
-    Object.assign(map, readNewVersionJson().list)
-    return _mergeVersionMap = map
-}
-
-/**
- * 构建一个 cache json
- *
- * + **执行该函数前必须调用过 [loadRules]**
- * + **调用该函数前必须调用过 [loadCacheJson]**
- * + **执行该函数前必须调用过 [calcEjectValues]**
- *
- * @param protocol 网站的网络协议
- * @param webRoot 网站域名（包括二级域名）
- * @param root 网页根目录（首页 index.html 所在目录）
- */
-export async function buildVersionJson(
-    protocol: ('https://' | 'http://'), webRoot: string, root: string
-): Promise<VersionJson> {
-    const list: VersionMap = {}
-    eachAllFile(root, async path => {
-        const endIndex = path.length - (path.endsWith('/index.html') ? 10 : 0)
-        const url = new URL(protocol + nodePath.join(webRoot, path.substring(root.length, endIndex)))
-        const pathname = url.pathname
-        if (isExclude(webRoot, pathname)) return
-        let content = null
-        if (findCache(url)) {
-            content = fs.readFileSync(path, 'utf-8')
-            const key = decodeURIComponent(url.pathname)
-            list[key] = crypto.createHash('md5').update(content).digest('hex')
-        }
-        if (pathname.endsWith('/') || pathname.endsWith('.html')) {
-            if (!content) content = fs.readFileSync(path, 'utf-8')
-            await eachAllLinkInHtml(webRoot, content, list)
-        } else if (pathname.endsWith('.css')) {
-            if (!content) content = fs.readFileSync(path, 'utf-8')
-            await eachAllLinkInCss(webRoot, content, list)
-        } else if (pathname.endsWith('.js')) {
-            if (!content) content = fs.readFileSync(path, 'utf-8')
-            await eachAllLinkInJavaScript(webRoot, content, list)
-        }
-    })
-    return _newVersionJson = {
-        version: 3, list
-    }
-}
-
-/**
- * 检索一个 URL 指向的文件中所有地外部链接
- *
- * 该函数会处理该 URL 指向的文件和文件中直接或间接包含的所有 URL
- *
- * + **执行该函数前必须调用过 [loadRules]**
- * + **调用该函数前必须调用过 [loadCacheJson]**
- * + **执行该函数前必须调用过 [calcEjectValues]**
- *
- * @param webRoot 网站域名
- * @param url 要检索的 URL
- * @param result 存放结果的对象
- * @param event 检索到一个 URL 时触发的事件
- */
-export async function eachAllLinkInUrl(
-    webRoot: string, url: string, result: VersionMap, event?: (url: string) => void
-) {
-    if (url.startsWith('//')) url = 'http' + url
-    if (url in result) return event?.(url)
-    if (!url.startsWith('http') || isExclude(webRoot, url)) return
-    if (!(isExternalLink(webRoot, url) && findCache(new URL(url)))) return
-    event?.(url)
-    const stable = isStable(url)
-    if (stable) {
-        const old = readOldVersionJson().list
-        if (url in old) {
-            const copyTree = (key: string) => {
-                const value = old[key]
-                if (!value) return
-                result[key] = value
-                if (Array.isArray(value)) {
-                    result[key] = value
-                    for (let url of value) {
-                        copyTree(url)
-                    }
-                } else {
-                    event?.(value)
-                }
-            }
-            copyTree(url)
-            return
-        }
-    }
-    const response = await fetchFile(url)
-    if (![200, 301, 302, 307, 308].includes(response.status))
-        throw response
-    const pathname = new URL(url).pathname
-    let content: string | undefined
-    const relay: string[] = []
-    const nextEvent = (it: string) => {
-        relay.push(it)
-        event?.(it)
-    }
-    switch (true) {
-        case pathname.endsWith('.html'): case pathname.endsWith('/'):
-            content = await response.text()
-            await eachAllLinkInHtml(webRoot, content, result, nextEvent)
-            break
-        case pathname.endsWith('.css'):
-            content = await response.text()
-            await eachAllLinkInCss(webRoot, content, result, nextEvent)
-            break
-        case pathname.endsWith('.js'):
-            content = await response.text()
-            await eachAllLinkInJavaScript(webRoot, content, result, nextEvent)
-            break
-        default:
-            if (stable) {
-                result[url] = []
-            } else {
-                const buffer = Buffer.from(await response.arrayBuffer())
-                result[url] = crypto.createHash('md5').update(buffer).digest('hex')
-            }
-            break
-    }
-    if (content) {
-        if (stable) {
-            result[url] = relay
-        } else {
-            result[url] = crypto.createHash('md5').update(content).digest('hex')
-        }
-    }
-}
-
-/**
- * 检索 HTML 文件中的所有外部链接
- *
- * 该函数仅处理 HTML 当中直接或间接包含的 URL，不处理文件本身
- *
- * + **执行该函数前必须调用过 [loadRules]**
- * + **调用该函数前必须调用过 [loadCacheJson]**
- *
- * @param webRoot 网站域名
- * @param content HTML 文件内容
- * @param result 存放结果的对象
- * @param event 检索到 URL 时触发的事件
- */
-export async function eachAllLinkInHtml(
-    webRoot: string, content: string, result: VersionMap, event?: (url: string) => void
-) {
-    const each = async (node: HTMLParser.HTMLElement) => {
-        let url: string | undefined = undefined
-        switch (node.tagName) {
-            case 'link':
-                url = node.attributes.href
-                break
-            case 'script': case 'img': case 'source': case 'iframe': case 'embed':
-                url = node.attributes.src
-                break
-            case 'object':
-                url = node.attributes.data
-                break
-        }
-        if (url) {
-            await eachAllLinkInUrl(webRoot, url, result, event)
-        } else if (node.tagName === 'script') {
-            await eachAllLinkInJavaScript(webRoot, node.rawText, result, event)
-        } else if (node.tagName === 'style') {
-            await eachAllLinkInCss(webRoot, node.rawText, result, event)
-        }
-        for (let childNode of node.childNodes) {
-            await each(childNode)
-        }
-    }
-    await each(HTMLParser.parse(content, { style: true, script: true }))
-}
-
-/**
- * 检索 CSS 文件中的所有外部链
- *
- * 该函数仅处理 CSS 当中直接或间接包含的 URL，不处理文件本身
- *
- * + **执行该函数前必须调用过 [loadRules]**
- * + **调用该函数前必须调用过 [loadCacheJson]**
- *
- * @param webRoot 网站域名
- * @param content CSS 文件内容
- * @param result 存放结果的对象
- * @param event 当检索到一个 URL 后触发的事件
- */
-export async function eachAllLinkInCss(
-    webRoot: string, content: string, result: VersionMap, event?: (url: string) => void
-) {
-    const each = async (any: Array<any> | undefined) => {
-        if (!any) return
-        for (let rule of any) {
-            switch (rule.type) {
-                case 'rule':
-                    await each(rule.declarations)
-                    break
-                case 'declaration':
-                    const value: string = rule.value
-                    const list = value.match(/url\(['"]?([^'")]+)['"]?\)/g)
-                        ?.map(it => it.replace(/(^url\(['"])|(['"]\)$)/g, ''))
-                    if (list) {
-                        for (let url of list) {
-                            await eachAllLinkInUrl(webRoot, url, result, event)
-                        }
-                    }
-                    break
-                case 'import':
-                    const url = rule.import.trim().replace(/^["']|["']$/g, '')
-                    await eachAllLinkInUrl(webRoot, url, result, event)
-                    break
-            }
-        }
-    }
-    await each(CSSParser.parse(content).stylesheet?.rules)
-}
-
-/**
- * 遍历 JS 文件中地所有外部链接
- *
- * 该函数仅处理 JS 当中直接或间接包含的 URL，不处理文件本身
- *
- * + **执行该函数前必须调用过 [loadRules]**
- * + **调用该函数前必须调用过 [loadCacheJson]**
- *
- * @param webRoot 网站域名
- * @param content JS 文件内容
- * @param result 存放结果的对象
- * @param event 当检索到一个 URL 后触发的事件
- */
-export async function eachAllLinkInJavaScript(
-    webRoot: string, content: string, result: VersionMap, event?: (url: string) => void
-) {
-    const ruleList = readRules().config?.external?.js
-    if (!ruleList) throw 'ruleList 为空'
-    for (let value of ruleList) {
-        if (typeof value === 'function') {
-            const urls: string[] = value(content)
-            for (let url of urls) {
-                await eachAllLinkInUrl(webRoot, url, result, event)
-            }
-        } else {
-            const {head, tail} = value
-            const reg = new RegExp(`${head}(['"\`])(.*?)(['"\`])${tail}`, 'mg')
-            const list = content.match(reg)
-                ?.map(it => it.substring(head.length, it.length - tail.length).trim())
-                ?.map(it => it.replace(/^['"`]|['"`]$/g, ''))
-            if (list) {
-                for (let url of list) {
-                    await eachAllLinkInUrl(webRoot, url, result, event)
-                }
-            }
-        }
-    }
-}
-
-/** 判断一个 URL 是否是外部链接 */
-function isExternalLink(webRoot: string, url: string): boolean {
-    return new RegExp(`^(https?:)?\\/\\/${webRoot}`).test(url)
-}
-
-/**
- * 查询指定 URL 对应的缓存规则
- *
- * + **执行该函数前必须调用过 [loadRules]**
- * + **执行该函数前必须调用过 [calcEjectValues]**
- */
-export function findCache(url: URL | string): any | null {
-    const {cacheRules} = readRules()
-    const eject = readEjectData()
-    if (typeof url === 'string') url = new URL(url)
-    url = new URL(replaceRequest(url.href))
-    for (let key in cacheRules) {
-        const value = cacheRules[key]
-        if (value.match(url, eject.nodeEject)) return value
-    }
-    return null
-}
-
-/**
- * 替换请求
- *
- * + **执行该函数前必须调用过 [loadRules]**
- * + **执行该函数前必须调用过 [calcEjectValues]**
- */
-export function replaceRequest(url: string): string {
-    const rules = readRules()
-    if (!('modifyRequest' in rules)) return url
-    const {modifyRequest} = rules
-    const request = new Request(url)
-    return modifyRequest?.(request, readEjectData().nodeEject)?.url ?? url
-}

package/src/ServiceWorkerBuilder.ts

@@ -1,156 +0,0 @@
-import {ServiceWorkerConfig} from './SwppConfig'
-import {readRules} from './SwppRules'
-import {getSource, readEjectData} from './Utils'
-import fs from 'fs'
-import nodePath from 'path'
-
-/**
- * 构建 sw
- *
- * + **执行该函数前必须调用过 [loadRules]**
- * + **执行该函数前必须调用过 [calcEjectValues]**
- */
-export function buildServiceWorker(): string {
-    const rules = readRules()
-    const eject = readEjectData()
-    const {
-        modifyRequest,
-        fetchFile,
-        getRaceUrls,
-        getSpareUrls,
-        blockRequest,
-        config
-    } = rules
-    const serviceWorkerConfig = config.serviceWorker as ServiceWorkerConfig
-    const templatePath = nodePath.resolve('./', module.path, 'sw-template.js')
-    // 获取拓展文件
-    let cache = getSource(rules, undefined, [
-        'cacheList', 'modifyRequest', 'getCdnList', 'getSpareUrls', 'blockRequest', 'fetchFile',
-        ...('external' in rules && Array.isArray(rules.external) ? rules.external : [])
-    ], true) + '\n'
-    if (!fetchFile) {
-        if (getRaceUrls)
-            cache += JS_CODE_GET_CDN_LIST
-        else if (getSpareUrls)
-            cache += JS_CODE_GET_SPARE_URLS
-        else
-            cache += JS_CODE_DEF_FETCH_FILE
-    }
-    if (!getSpareUrls) cache += `\nconst getSpareUrls = _ => {}`
-    if ('afterJoin' in rules)
-        cache += `(${getSource(rules['afterJoin'])})()\n`
-    if ('afterTheme' in rules)
-        cache += `(${getSource(rules['afterTheme'])})()\n`
-    const keyword = "const { cacheList, fetchFile, getSpareUrls } = require('../sw-rules')"
-    // noinspection JSUnresolvedVariable
-    let content = fs.readFileSync(templatePath, 'utf8')
-        .replaceAll("// [insertion site] values", eject.strValue ?? '')
-        .replaceAll(keyword, cache)
-        .replaceAll("'@$$[escape]'", (serviceWorkerConfig.escape).toString())
-        .replaceAll("'@$$[cacheName]'", `'${serviceWorkerConfig.cacheName}'`)
-    if (modifyRequest) {
-        content = content.replaceAll('// [modifyRequest call]', `
-                const modify = modifyRequest(request)
-                if (modify) request = modify
-            `).replaceAll('// [modifyRequest else-if]', `
-                else if (modify) event.respondWith(fetch(request))
-            `)
-    }
-    if (blockRequest) {
-        content = content.replace('// [blockRequest call]', `
-                if (blockRequest(url))
-                    return event.respondWith(new Response(null, {status: 208}))
-            `)
-    }
-    // noinspection JSUnresolvedVariable
-    if (serviceWorkerConfig.debug) {
-        content = content.replaceAll('// [debug delete]', `
-                console.debug(\`delete cache: \${url}\`)
-            `).replaceAll('// [debug put]', `
-                console.debug(\`put cache: \${key}\`)
-            `).replaceAll('// [debug message]', `
-                console.debug(\`receive: \${event.data}\`)
-            `).replaceAll('// [debug escape]', `
-                console.debug(\`escape: \${aid}\`)
-            `)
-    }
-    return content
-}
-
-// 缺省的 fetchFile 函数的代码
-const JS_CODE_DEF_FETCH_FILE = `
-    const fetchFile = (request, banCache) => fetch(request, {
-        cache: banCache ? "no-store" : "default",
-        mode: 'cors',
-        credentials: 'same-origin'
-    })
-`
-
-// getCdnList 函数的代码
-const JS_CODE_GET_CDN_LIST = `
-    const fetchFile = (request, banCache) => {
-        const fetchArgs = {
-            cache: banCache ? 'no-store' : 'default',
-            mode: 'cors',
-            credentials: 'same-origin'
-        }
-        const list = getCdnList(request.url)
-        if (!list || !Promise.any) return fetch(request, fetchArgs)
-        const res = list.map(url => new Request(url, request))
-        const controllers = []
-        return Promise.any(res.map(
-            (it, index) => fetch(it, Object.assign(
-                {signal: (controllers[index] = new AbortController()).signal},
-                fetchArgs
-            )).then(response => checkResponse(response) ? {index, response} : Promise.reject())
-        )).then(it => {
-            for (let i in controllers) {
-                if (i != it.index) controllers[i].abort()
-            }
-            return it.response
-        })
-    }
-`
-
-// getSpareUrls 函数的代码
-const JS_CODE_GET_SPARE_URLS = `
-    const fetchFile = (request, banCache, spare = null) => {
-        const fetchArgs = {
-            cache: banCache ? 'no-store' : 'default',
-            mode: 'cors',
-            credentials: 'same-origin'
-        }
-        if (!spare) spare = getSpareUrls(request.url)
-        if (!spare) return fetch(request, fetchArgs)
-        const list = spare.list
-        const controllers = []
-        let error = 0
-        return new Promise((resolve, reject) => {
-            const pull = () => {
-                const flag = controllers.length
-                if (flag === list.length) return
-                const plusError = () => {
-                    if (++error === list.length) reject(\`请求 \${request.url} 失败\`)
-                    else if (flag + 1 === controllers.length) {
-                        clearTimeout(controllers[flag].id)
-                        pull()
-                    }
-                }
-                controllers.push({
-                    ctrl: new AbortController(),
-                    id: setTimeout(pull, spare.timeout)
-                })
-                fetch(new Request(list[flag], request), fetchArgs).then(response => {
-                    if (checkResponse(response)) {
-                        for (let i in controllers) {
-                            if (i !== flag) controllers[i].ctrl.abort()
-                            clearTimeout(controllers[i].id)
-                        }
-                        resolve(response)
-                    } else plusError()
-                }).catch(plusError)
-            }
-            pull()
-        })
-    }
-`