swpp-backends 2.3.12 → 3.0.0-alpha.0

package/README.md

@@ -2,41 +2,4 @@
 
 ## 欢迎使用 SwppBackends
 
-swpp-backends（以下简称 swpp）插件的功能是为网站生成一个高度可用的 ServiceWorker（以下简称 SW），为网站优化二次加载、提供离线体验、提高可靠性，并为此附带了一些其它的功能。
-
-swpp 的全拼为“Service Worker Plus Plus”（或“Service Worker++”），但是其与已有的插件“hexo-service-worker”并没有关系，插件中所有代码均为我个人开发，这一点请不要误解。
-
-swpp 生成的 SW 与其它插件的对比：
-
-|                 |     swpp      | hexo-offline |
-|:---------------:|:-------------:|:------------:|
-|      本地缓存       |      ✔️       |      ✔️      |
-|     缓存增量更新      |      ✔️       |      ❌       |
-|     缓存过期时间      |       ❌       |      ✔️      |
-|     缓存大小限制      |       ❌       |      ✔️      |
-|       预缓存       | ❌<sup>1</sup> |      ✔️      |
-|   Request 篡改    |      ✔️       |      ❌       |
-|     URL 竞速      |      ✔️       |      ❌       |
-|     备用 URL      |      ✔️       |      ❌       |
-|    204 阻塞响应     |      ✔️       |      ❌       |
-|       逃生门       |      ✔️       |      ❌       |
-|      请求合并       |      ✔️       |      ❌       |
-| 跨平台<sup>2</sup> |      ✔️       |      ❌       |
-|      高度自由       |      ✔️       |      ❌       |
-|       更新        |     非常频繁      |   超过两年没有更新   |
-
-+ ✔️：支持
-+ ❌：不支持
-
-1. 预缓存可以在前端实现，SW 实现这个功能容易拖延注册时间
-2. 跨平台是指跨越框架（可在 NodeJS 平台中的任意框架下工作）
-
-目前支持的平台：
-
-|  平台  |     插件名     |                            文档                             |           作者            |
-|:----:|:-----------:|:---------------------------------------------------------:|:-----------------------:|
-| hexo | `hexo-swpp` | [github](https://github.com/EmptyDreams/hexo-swpp#readme) | [空梦](https://kmar.top/) |
-
-如果你为某一个平台做了适配，可以在 gh 上发布 issue 或者在文档页面发布评论~
-
-文档：[Swpp Backends 官方文档 | 山岳库博](https://kmar.top/posts/b70ec88f/)
+本分支是 swpp 的 V3 分支，V3 将重构包括 sw.js 在内的所有代码，目前仍在开发中。

package/dist/index.js

@@ -1,40 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-const Utils_1 = require("./Utils");
-const FileAnalyzer_1 = require("./FileAnalyzer");
-const ServiceWorkerBuilder_1 = require("./ServiceWorkerBuilder");
-const SwppRules_1 = require("./SwppRules");
-const UpdateJsonBuilder_1 = require("./UpdateJsonBuilder");
-const Variant_1 = require("./Variant");
-const VersionAnalyzer_1 = require("./VersionAnalyzer");
-const DomBuilder_1 = require("./DomBuilder");
-// noinspection JSUnusedGlobalSymbols
-exports.default = {
-    version: require('../package.json').version,
-    cache: {
-        readEjectData: Utils_1.readEjectData, readUpdateJson: Variant_1.readUpdateJson,
-        readRules: Variant_1.readRules, readMergeVersionMap: Variant_1.readMergeVersionMap,
-        readOldVersionJson: Variant_1.readOldVersionJson, readNewVersionJson: Variant_1.readNewVersionJson,
-        readAnalyzeResult: Variant_1.readAnalyzeResult
-    },
-    builder: {
-        buildServiceWorker: ServiceWorkerBuilder_1.buildServiceWorker,
-        buildDomJs: DomBuilder_1.buildDomJs,
-        buildVersionJson: FileAnalyzer_1.buildVersionJson,
-        buildUpdateJson: UpdateJsonBuilder_1.buildUpdateJson,
-        calcEjectValues: Utils_1.calcEjectValues,
-        analyzeVersion: VersionAnalyzer_1.analyzeVersion
-    },
-    loader: {
-        loadRules: SwppRules_1.loadRules, loadUpdateJson: UpdateJsonBuilder_1.loadUpdateJson, loadVersionJson: FileAnalyzer_1.loadVersionJson
-    },
-    event: {
-        addRulesMapEvent: SwppRules_1.addRulesMapEvent, refreshUrl: VersionAnalyzer_1.refreshUrl, submitChange: UpdateJsonBuilder_1.submitChange, submitCacheInfo: FileAnalyzer_1.submitCacheInfo, submitExternalUrl: FileAnalyzer_1.submitExternalUrl, registryFileHandler: FileAnalyzer_1.registryFileHandler
-    },
-    utils: {
-        getSource: Utils_1.getSource, getShorthand: UpdateJsonBuilder_1.getShorthand, findCache: FileAnalyzer_1.findCache,
-        fetchFile: Utils_1.fetchFile, replaceDevRequest: Utils_1.replaceDevRequest, replaceRequest: FileAnalyzer_1.replaceRequest,
-        isStable: FileAnalyzer_1.isStable, isExclude: FileAnalyzer_1.isExclude, findFileHandler: FileAnalyzer_1.findFileHandler,
-        eachAllLinkInUrl: FileAnalyzer_1.eachAllLinkInUrl, deepFreeze: Utils_1.deepFreeze, writeVariant: Variant_1.writeVariant, readVariant: Variant_1.readVariant, deleteVariant: Variant_1.deleteVariant
-    }
-};
+exports.version = void 0;
+const untils_1 = require("./swpp/untils");
+/** 版本号 */
+exports.version = require('../package.json').version;
+untils_1.utils.printInfo('INDEX', `欢迎使用 swpp@${exports.version}`);

package/dist/swpp/FileParser.js

@@ -0,0 +1,85 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildFileParser = exports.FileParserRegistry = void 0;
+const path_1 = __importDefault(require("path"));
+const untils_1 = require("./untils");
+class FileParserRegistry {
+    constructor(compilation, obj = {}) {
+        this.compilation = compilation;
+        this.map = new Map();
+        for (let key in obj) {
+            this.map.set(key, obj[key]);
+        }
+    }
+    registry(type, parser) {
+        this.map.set(type, parser);
+    }
+    /** 判断是否支持指定类型 */
+    containsType(type) {
+        return this.map.has(type);
+    }
+    /** 解析本地文件 */
+    async parserLocalFile(path) {
+        const parser = this.map.get(path_1.default.extname(path));
+        if (!parser)
+            return new Set();
+        const content = await parser.readFromLocal(this.compilation, path);
+        return await parser.extractUrls(this.compilation, content);
+    }
+    /** 解析网络文件 */
+    async parserNetworkFile(response, callback) {
+        const fileHandler = this.compilation.compilationEnv.read('FETCH_NETWORK_FILE');
+        const contentType = fileHandler.getUrlContentType(response.url, response);
+        const parser = this.map.get(contentType);
+        if (!parser)
+            return new Set();
+        const content = await parser.readFromNetwork(this.compilation, response);
+        if (callback)
+            await callback(content);
+        return await parser.extractUrls(this.compilation, content);
+    }
+    /**
+     * 解析指定的 URL
+     * @param url 链接
+     * @param isCached 该链接指向的资源是否需要缓存
+     */
+    async parserUrlFile(url, isCached) {
+        const fileHandler = this.compilation.compilationEnv.read('FETCH_NETWORK_FILE');
+        const contentType = fileHandler.getUrlContentType(url);
+        if (!contentType && !isCached)
+            return { file: url, mark: '', urls: new Set() };
+        const parser = this.map.get(contentType);
+        if (parser?.calcUrl) {
+            const result = await parser.calcUrl(url);
+            if (result)
+                return {
+                    file: url,
+                    ...result
+                };
+        }
+        const fetcher = this.compilation.compilationEnv.read('FETCH_NETWORK_FILE');
+        const urls = new Set();
+        let mark = '';
+        await fetcher.fetch(url)
+            .then(response => this.parserNetworkFile(response, content => {
+            mark = untils_1.utils.calcHash(content);
+        }))
+            .then(urls => urls.forEach(it => urls.add(it)));
+        return { file: url, mark, urls };
+    }
+    /** 解析指定类型的文件内容 */
+    async parserContent(type, content) {
+        const parser = this.map.get(type);
+        if (!parser)
+            return new Set();
+        return await parser.extractUrls(this.compilation, content);
+    }
+}
+exports.FileParserRegistry = FileParserRegistry;
+function buildFileParser(parser) {
+    return parser;
+}
+exports.buildFileParser = buildFileParser;

package/dist/swpp/JsonBuilder.js

@@ -0,0 +1,232 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.JsonBuilder = void 0;
+const untils_1 = require("./untils");
+class JsonBuilder {
+    constructor(compilation, urls, headers = new Map(), map = new Map()) {
+        this.compilation = compilation;
+        this.urls = urls;
+        this.headers = headers;
+        this.map = map;
+    }
+    update(key, value) {
+        this.map.set(key, value);
+    }
+    putHeader(key, value) {
+        this.headers.set(key, value);
+    }
+    async buildJson() {
+        const json = await this.compilation.compilationEnv.read('VERSION_FILE')();
+        if (json.info.length == 0) {
+            json.info.push({ version: 1 });
+            return json;
+        }
+        const newChange = createUpdateChangeExps(this.urls, this.map.values());
+        json.info.unshift({
+            version: json.info[0].version + 1,
+            change: [newChange]
+        });
+        this.zipJson(json);
+        this.limitJson(json);
+        return json;
+    }
+    zipJson(json) {
+        const matchUpdateRule = this.compilation.crossDep.read('matchUpdateRule');
+        const htmlMatcher = matchUpdateRule.runOnNode({ flag: 'html' });
+        // 打散新版本的所有规则
+        json.info[0].change = json.info[0].change?.flatMap(exp => {
+            if (!Array.isArray(exp.value))
+                return [exp];
+            return exp.value.map(it => ({
+                flag: exp.flag, value: it
+            }));
+        });
+        // 压缩第一个版本的内容
+        const indexes = (() => {
+            const change = json.info[0].change;
+            if (!change)
+                return new Set();
+            let htmlCount = 0;
+            const indexes = new Set();
+            // 统计每个表达式匹配的资源及刷新的 HTML 总量
+            const indexesArray = change.map(exp => {
+                const matcher = matchUpdateRule.runOnNode(exp);
+                const result = new Set();
+                untils_1.utils.findValueInIterable(this.urls, it => !!matcher(it))
+                    .forEach(item => {
+                    indexes.add(item.index);
+                    result.add(item.index);
+                    if (htmlMatcher(item.value))
+                        ++htmlCount;
+                });
+                return result;
+            });
+            // 如果 HTML 更新数量超过阈值，则直接清除所有 HTML 的缓存
+            if (htmlCount > 0) {
+                const htmlLimit = this.compilation.compilationEnv.read('JSON_HTML_LIMIT');
+                if (htmlLimit > 0 && htmlCount > htmlLimit) {
+                    change.unshift({ flag: 'html' });
+                    const indexes = new Set();
+                    untils_1.utils.findValueInIterable(this.urls, it => !!htmlMatcher(it))
+                        .forEach(({ index }) => indexes.add(index));
+                    indexesArray.unshift(indexes);
+                }
+            }
+            // 分析哪些表达式是冗余的
+            const invalidIndex = new Array(indexesArray.length);
+            for (let i = 0; i < indexesArray.length; i++) {
+                if (invalidIndex[i])
+                    continue;
+                const parent = indexesArray[0];
+                o: for (let k = 0; k < indexesArray.length; k++) {
+                    if (i == k || invalidIndex[k])
+                        continue;
+                    for (let item of indexesArray[k]) {
+                        if (!parent.has(item)) {
+                            continue o;
+                        }
+                    }
+                    invalidIndex[k] = true;
+                }
+            }
+            // 生成新的表达式
+            const validExp = new Map();
+            for (let i = 0; i < invalidIndex.length; i++) {
+                if (invalidIndex[i])
+                    continue;
+                const oldExpList = validExp.get(change[i].flag);
+                const expList = oldExpList ?? [];
+                if (change[i].value) {
+                    console.assert(typeof change[i].value == 'string', `change[${i}].value = ${change[i].value} 应当为字符串`);
+                    expList.push(change[i].value);
+                }
+                if (!oldExpList)
+                    validExp.set(change[i].flag, expList);
+            }
+            const newChange = json.info[0].change = [];
+            validExp.forEach((value, flag) => {
+                switch (value.length) {
+                    case 0:
+                        newChange.push({ flag });
+                        break;
+                    case 1:
+                        newChange.push({ flag, value: value[0] });
+                        break;
+                    default:
+                        newChange.push({ flag, value });
+                }
+            });
+            return indexes;
+        })();
+        // 移除后续表达式中冗余的内容
+        if (indexes.size == 0)
+            return;
+        for (let i = 1; i < json.info.length; i++) {
+            const changes = json.info[i].change;
+            if (!changes)
+                continue;
+            for (let k = changes.length - 1; k >= 0; k--) {
+                const change = json.info[i].change[k];
+                const values = change.value ? (Array.isArray(change.value) ? change.value : [change.value]) : [];
+                const tmpChange = {
+                    flag: change.flag,
+                    value: ''
+                };
+                for (let j = values.length - 1; j >= 0; j--) {
+                    tmpChange.value = values[j];
+                    const matcher = matchUpdateRule.runOnNode(tmpChange);
+                    const matchIndex = untils_1.utils.findValueInIterable(this.urls, url => !!matcher(url));
+                    if (matchIndex.every(it => indexes.has(it.index))) {
+                        values.splice(j, 1);
+                    }
+                }
+                if (values.length == 0)
+                    delete json.info[i].change;
+                else if (values.length == 1)
+                    change.value = values[0];
+                else
+                    change.value = values;
+            }
+        }
+    }
+    limitJson(json) {
+        const lengthLimit = this.compilation.compilationEnv.read('VERSION_LENGTH_LIMIT');
+        if (lengthLimit == 0)
+            return;
+        let sum = 0;
+        for (let i = 0; i < json.info.length; i++) {
+            sum += JSON.stringify(json.info[i]).length;
+            if (sum > lengthLimit) {
+                if (i == 0)
+                    json.info = [{ version: json.info[0].version }];
+                else
+                    json.info.splice(i);
+                return;
+            }
+        }
+    }
+}
+exports.JsonBuilder = JsonBuilder;
+/**
+ * 构建更新表达式
+ *
+ * 具体实现为使用字典树构建最优后缀匹配表达式，时间复杂度 O(N + M)
+ */
+function createUpdateChangeExps(urls, refresh) {
+    function newNode() {
+        return {
+            next: new Array(128),
+            flag: false,
+            isEnd: false
+        };
+    }
+    const head = newNode();
+    const insert = (content, flag) => {
+        let cur = head;
+        for (let i = content.length - 1; i >= 0; i--) {
+            const index = content.charCodeAt(i);
+            if (cur.next[index]) {
+                cur = cur.next[index];
+            }
+            else {
+                cur = cur.next[index] = newNode();
+            }
+        }
+        cur.flag = flag;
+        cur.isEnd = true;
+    };
+    urls.forEach(it => insert(it, false));
+    for (let item of refresh) {
+        insert(item, true);
+    }
+    function dfs(node) {
+        if (node.isEnd)
+            return;
+        node.flag = true;
+        for (let next of node.next) {
+            if (next) {
+                dfs(next);
+                node.flag = node.flag && next.flag;
+            }
+        }
+    }
+    dfs(head);
+    let dfs2S = [];
+    const result = [];
+    function dfs2(node) {
+        if (node.flag) {
+            result.push(dfs2S.reduceRight((a, b) => a + b, ''));
+            return;
+        }
+        for (let i = 0; i < node.next.length; i++) {
+            const next = node.next[i];
+            if (next) {
+                dfs2S.push(String.fromCharCode(i));
+                dfs2(next);
+                dfs2S.pop();
+            }
+        }
+    }
+    dfs2(head);
+    return { flag: 'suf', value: result };
+}

package/dist/swpp/NetworkFileHandler.js

@@ -0,0 +1,68 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FiniteConcurrencyFetcher = void 0;
+const path_1 = __importDefault(require("path"));
+/** 支持并发控制的网络文件拉取工具 */
+class FiniteConcurrencyFetcher {
+    constructor() {
+        this.fetchingCount = 0;
+        this.waitList = [];
+        this.limit = 100;
+        this.referer = 'swpp-backends';
+        this.userAgent = 'swpp-backends';
+        this.headers = {};
+    }
+    fetch(request) {
+        if (this.fetchingCount !== this.limit) {
+            return this.createFetchTask(request);
+        }
+        else {
+            return new Promise((resolve, reject) => {
+                this.waitList.push({ request, resolve, reject });
+            });
+        }
+    }
+    async createFetchTask(url) {
+        ++this.fetchingCount;
+        try {
+            return await fetch(url, {
+                referrer: this.referer,
+                headers: {
+                    ...this.headers,
+                    'User-Agent': this.userAgent
+                }
+            });
+        }
+        finally {
+            --this.fetchingCount;
+            if (this.waitList.length !== 0) {
+                const item = this.waitList.pop();
+                this.createFetchTask(item.request)
+                    .then(response => item.resolve(response))
+                    .catch(err => item.reject(err));
+            }
+        }
+    }
+    getUrlContentType(url, response) {
+        let contentType;
+        if (url.endsWith('/')) {
+            contentType = 'html';
+        }
+        else {
+            contentType = path_1.default.extname(url);
+        }
+        if (!contentType) {
+            if (response)
+                contentType = response.headers.get('content-type') ?? '';
+            if (contentType.startsWith('text/'))
+                contentType = contentType.substring(5);
+            if (contentType === 'javascript')
+                contentType = 'script';
+        }
+        return contentType;
+    }
+}
+exports.FiniteConcurrencyFetcher = FiniteConcurrencyFetcher;