patreon-dl 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +422 -0
- package/bin/patreon-dl.js +5 -0
- package/dist/cli/CLIOptionValidator.d.ts +9 -0
- package/dist/cli/CLIOptionValidator.d.ts.map +1 -0
- package/dist/cli/CLIOptionValidator.js +85 -0
- package/dist/cli/CLIOptionValidator.js.map +1 -0
- package/dist/cli/CLIOptions.d.ts +20 -0
- package/dist/cli/CLIOptions.d.ts.map +1 -0
- package/dist/cli/CLIOptions.js +75 -0
- package/dist/cli/CLIOptions.js.map +1 -0
- package/dist/cli/CommandLineParser.d.ts +11 -0
- package/dist/cli/CommandLineParser.d.ts.map +1 -0
- package/dist/cli/CommandLineParser.js +212 -0
- package/dist/cli/CommandLineParser.js.map +1 -0
- package/dist/cli/ConfigFileParser.d.ts +9 -0
- package/dist/cli/ConfigFileParser.d.ts.map +1 -0
- package/dist/cli/ConfigFileParser.js +163 -0
- package/dist/cli/ConfigFileParser.js.map +1 -0
- package/dist/cli/index.d.ts +7 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +162 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/downloaders/Bootstrap.d.ts +29 -0
- package/dist/downloaders/Bootstrap.d.ts.map +1 -0
- package/dist/downloaders/Bootstrap.js +51 -0
- package/dist/downloaders/Bootstrap.js.map +1 -0
- package/dist/downloaders/Downloader.d.ts +59 -0
- package/dist/downloaders/Downloader.d.ts.map +1 -0
- package/dist/downloaders/Downloader.js +357 -0
- package/dist/downloaders/Downloader.js.map +1 -0
- package/dist/downloaders/DownloaderEvent.d.ts +47 -0
- package/dist/downloaders/DownloaderEvent.d.ts.map +1 -0
- package/dist/downloaders/DownloaderEvent.js +6 -0
- package/dist/downloaders/DownloaderEvent.js.map +1 -0
- package/dist/downloaders/DownloaderOptions.d.ts +39 -0
- package/dist/downloaders/DownloaderOptions.d.ts.map +1 -0
- package/dist/downloaders/DownloaderOptions.js +69 -0
- package/dist/downloaders/DownloaderOptions.js.map +1 -0
- package/dist/downloaders/PostDownloader.d.ts +8 -0
- package/dist/downloaders/PostDownloader.d.ts.map +1 -0
- package/dist/downloaders/PostDownloader.js +428 -0
- package/dist/downloaders/PostDownloader.js.map +1 -0
- package/dist/downloaders/ProductDownloader.d.ts +8 -0
- package/dist/downloaders/ProductDownloader.d.ts.map +1 -0
- package/dist/downloaders/ProductDownloader.js +171 -0
- package/dist/downloaders/ProductDownloader.js.map +1 -0
- package/dist/downloaders/cache/StatusCache.d.ts +43 -0
- package/dist/downloaders/cache/StatusCache.d.ts.map +1 -0
- package/dist/downloaders/cache/StatusCache.js +206 -0
- package/dist/downloaders/cache/StatusCache.js.map +1 -0
- package/dist/downloaders/index.d.ts +7 -0
- package/dist/downloaders/index.d.ts.map +1 -0
- package/dist/downloaders/index.js +6 -0
- package/dist/downloaders/index.js.map +1 -0
- package/dist/downloaders/task/DownloadTask.d.ts +89 -0
- package/dist/downloaders/task/DownloadTask.d.ts.map +1 -0
- package/dist/downloaders/task/DownloadTask.js +240 -0
- package/dist/downloaders/task/DownloadTask.js.map +1 -0
- package/dist/downloaders/task/DownloadTaskBatch.d.ts +45 -0
- package/dist/downloaders/task/DownloadTaskBatch.d.ts.map +1 -0
- package/dist/downloaders/task/DownloadTaskBatch.js +195 -0
- package/dist/downloaders/task/DownloadTaskBatch.js.map +1 -0
- package/dist/downloaders/task/DownloadTaskBatchEvent.d.ts +32 -0
- package/dist/downloaders/task/DownloadTaskBatchEvent.d.ts.map +1 -0
- package/dist/downloaders/task/DownloadTaskBatchEvent.js +2 -0
- package/dist/downloaders/task/DownloadTaskBatchEvent.js.map +1 -0
- package/dist/downloaders/task/DownloadTaskFactory.d.ts +20 -0
- package/dist/downloaders/task/DownloadTaskFactory.d.ts.map +1 -0
- package/dist/downloaders/task/DownloadTaskFactory.js +177 -0
- package/dist/downloaders/task/DownloadTaskFactory.js.map +1 -0
- package/dist/downloaders/task/FFmpegDownloadTask.d.ts +27 -0
- package/dist/downloaders/task/FFmpegDownloadTask.d.ts.map +1 -0
- package/dist/downloaders/task/FFmpegDownloadTask.js +206 -0
- package/dist/downloaders/task/FFmpegDownloadTask.js.map +1 -0
- package/dist/downloaders/task/FetcherDownloadTask.d.ts +21 -0
- package/dist/downloaders/task/FetcherDownloadTask.d.ts.map +1 -0
- package/dist/downloaders/task/FetcherDownloadTask.js +213 -0
- package/dist/downloaders/task/FetcherDownloadTask.js.map +1 -0
- package/dist/downloaders/task/index.d.ts +4 -0
- package/dist/downloaders/task/index.d.ts.map +1 -0
- package/dist/downloaders/task/index.js +3 -0
- package/dist/downloaders/task/index.js.map +1 -0
- package/dist/downloaders/templates/CampaignInfo.d.ts +3 -0
- package/dist/downloaders/templates/CampaignInfo.d.ts.map +1 -0
- package/dist/downloaders/templates/CampaignInfo.js +58 -0
- package/dist/downloaders/templates/CampaignInfo.js.map +1 -0
- package/dist/downloaders/templates/PostInfo.d.ts +4 -0
- package/dist/downloaders/templates/PostInfo.d.ts.map +1 -0
- package/dist/downloaders/templates/PostInfo.js +45 -0
- package/dist/downloaders/templates/PostInfo.js.map +1 -0
- package/dist/downloaders/templates/ProductInfo.d.ts +3 -0
- package/dist/downloaders/templates/ProductInfo.d.ts.map +1 -0
- package/dist/downloaders/templates/ProductInfo.js +20 -0
- package/dist/downloaders/templates/ProductInfo.js.map +1 -0
- package/dist/entities/Attachment.d.ts +7 -0
- package/dist/entities/Attachment.d.ts.map +1 -0
- package/dist/entities/Attachment.js +2 -0
- package/dist/entities/Attachment.js.map +1 -0
- package/dist/entities/Campaign.d.ts +19 -0
- package/dist/entities/Campaign.d.ts.map +1 -0
- package/dist/entities/Campaign.js +2 -0
- package/dist/entities/Campaign.js.map +1 -0
- package/dist/entities/Downloadable.d.ts +6 -0
- package/dist/entities/Downloadable.d.ts.map +1 -0
- package/dist/entities/Downloadable.js +5 -0
- package/dist/entities/Downloadable.js.map +1 -0
- package/dist/entities/MediaItem.d.ts +95 -0
- package/dist/entities/MediaItem.d.ts.map +1 -0
- package/dist/entities/MediaItem.js +2 -0
- package/dist/entities/MediaItem.js.map +1 -0
- package/dist/entities/Post.d.ts +87 -0
- package/dist/entities/Post.d.ts.map +1 -0
- package/dist/entities/Post.js +2 -0
- package/dist/entities/Post.js.map +1 -0
- package/dist/entities/Product.d.ts +17 -0
- package/dist/entities/Product.d.ts.map +1 -0
- package/dist/entities/Product.js +2 -0
- package/dist/entities/Product.js.map +1 -0
- package/dist/entities/Reward.d.ts +14 -0
- package/dist/entities/Reward.d.ts.map +1 -0
- package/dist/entities/Reward.js +2 -0
- package/dist/entities/Reward.js.map +1 -0
- package/dist/entities/User.d.ts +15 -0
- package/dist/entities/User.d.ts.map +1 -0
- package/dist/entities/User.js +2 -0
- package/dist/entities/User.js.map +1 -0
- package/dist/entities/index.d.ts +9 -0
- package/dist/entities/index.d.ts.map +1 -0
- package/dist/entities/index.js +6 -0
- package/dist/entities/index.js.map +1 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -0
- package/dist/parsers/PageParser.d.ts +6 -0
- package/dist/parsers/PageParser.d.ts.map +1 -0
- package/dist/parsers/PageParser.js +23 -0
- package/dist/parsers/PageParser.js.map +1 -0
- package/dist/parsers/Parser.d.ts +43 -0
- package/dist/parsers/Parser.d.ts.map +1 -0
- package/dist/parsers/Parser.js +439 -0
- package/dist/parsers/Parser.js.map +1 -0
- package/dist/parsers/PostParser.d.ts +7 -0
- package/dist/parsers/PostParser.d.ts.map +1 -0
- package/dist/parsers/PostParser.js +259 -0
- package/dist/parsers/PostParser.js.map +1 -0
- package/dist/parsers/ProductParser.d.ts +7 -0
- package/dist/parsers/ProductParser.d.ts.map +1 -0
- package/dist/parsers/ProductParser.js +70 -0
- package/dist/parsers/ProductParser.js.map +1 -0
- package/dist/utils/AttachmentFilenameResolver.d.ts +9 -0
- package/dist/utils/AttachmentFilenameResolver.d.ts.map +1 -0
- package/dist/utils/AttachmentFilenameResolver.js +73 -0
- package/dist/utils/AttachmentFilenameResolver.js.map +1 -0
- package/dist/utils/FSHelper.d.ts +57 -0
- package/dist/utils/FSHelper.d.ts.map +1 -0
- package/dist/utils/FSHelper.js +214 -0
- package/dist/utils/FSHelper.js.map +1 -0
- package/dist/utils/Fetcher.d.ts +45 -0
- package/dist/utils/Fetcher.d.ts.map +1 -0
- package/dist/utils/Fetcher.js +192 -0
- package/dist/utils/Fetcher.js.map +1 -0
- package/dist/utils/FetcherProgressMonitor.d.ts +18 -0
- package/dist/utils/FetcherProgressMonitor.d.ts.map +1 -0
- package/dist/utils/FetcherProgressMonitor.js +56 -0
- package/dist/utils/FetcherProgressMonitor.js.map +1 -0
- package/dist/utils/FilenameFormatHelper.d.ts +44 -0
- package/dist/utils/FilenameFormatHelper.d.ts.map +1 -0
- package/dist/utils/FilenameFormatHelper.js +98 -0
- package/dist/utils/FilenameFormatHelper.js.map +1 -0
- package/dist/utils/FllenameResolver.d.ts +20 -0
- package/dist/utils/FllenameResolver.d.ts.map +1 -0
- package/dist/utils/FllenameResolver.js +55 -0
- package/dist/utils/FllenameResolver.js.map +1 -0
- package/dist/utils/Formatter.d.ts +21 -0
- package/dist/utils/Formatter.d.ts.map +1 -0
- package/dist/utils/Formatter.js +112 -0
- package/dist/utils/Formatter.js.map +1 -0
- package/dist/utils/MediaFilenameResolver.d.ts +9 -0
- package/dist/utils/MediaFilenameResolver.d.ts.map +1 -0
- package/dist/utils/MediaFilenameResolver.js +90 -0
- package/dist/utils/MediaFilenameResolver.js.map +1 -0
- package/dist/utils/Misc.d.ts +14 -0
- package/dist/utils/Misc.d.ts.map +1 -0
- package/dist/utils/Misc.js +4 -0
- package/dist/utils/Misc.js.map +1 -0
- package/dist/utils/ObjectHelper.d.ts +4 -0
- package/dist/utils/ObjectHelper.d.ts.map +1 -0
- package/dist/utils/ObjectHelper.js +30 -0
- package/dist/utils/ObjectHelper.js.map +1 -0
- package/dist/utils/PackageInfo.d.ts +10 -0
- package/dist/utils/PackageInfo.d.ts.map +1 -0
- package/dist/utils/PackageInfo.js +33 -0
- package/dist/utils/PackageInfo.js.map +1 -0
- package/dist/utils/URLHelper.d.ts +40 -0
- package/dist/utils/URLHelper.d.ts.map +1 -0
- package/dist/utils/URLHelper.js +192 -0
- package/dist/utils/URLHelper.js.map +1 -0
- package/dist/utils/index.d.ts +2 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +2 -0
- package/dist/utils/index.js.map +1 -0
- package/dist/utils/logging/ChainLogger.d.ts +11 -0
- package/dist/utils/logging/ChainLogger.d.ts.map +1 -0
- package/dist/utils/logging/ChainLogger.js +50 -0
- package/dist/utils/logging/ChainLogger.js.map +1 -0
- package/dist/utils/logging/ConsoleLogger.d.ts +31 -0
- package/dist/utils/logging/ConsoleLogger.d.ts.map +1 -0
- package/dist/utils/logging/ConsoleLogger.js +126 -0
- package/dist/utils/logging/ConsoleLogger.js.map +1 -0
- package/dist/utils/logging/FileLogger.d.ts +26 -0
- package/dist/utils/logging/FileLogger.d.ts.map +1 -0
- package/dist/utils/logging/FileLogger.js +147 -0
- package/dist/utils/logging/FileLogger.js.map +1 -0
- package/dist/utils/logging/Logger.d.ts +12 -0
- package/dist/utils/logging/Logger.d.ts.map +1 -0
- package/dist/utils/logging/Logger.js +15 -0
- package/dist/utils/logging/Logger.js.map +1 -0
- package/dist/utils/logging/index.d.ts +7 -0
- package/dist/utils/logging/index.d.ts.map +1 -0
- package/dist/utils/logging/index.js +7 -0
- package/dist/utils/logging/index.js.map +1 -0
- package/package.json +78 -0
|
@@ -0,0 +1,428 @@
|
|
|
1
|
+
var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
|
|
2
|
+
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
|
|
3
|
+
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
|
|
4
|
+
return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
|
|
5
|
+
};
|
|
6
|
+
var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
|
|
7
|
+
if (kind === "m") throw new TypeError("Private method is not writable");
|
|
8
|
+
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
|
|
9
|
+
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
|
|
10
|
+
return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
|
|
11
|
+
};
|
|
12
|
+
var _PostDownloader_instances, _PostDownloader_startPromise, _PostDownloader_getInitialPostsAPIUL, _PostDownloader_requestPosts, _PostDownloader_createDownloadTaskBatchForPost;
|
|
13
|
+
import { AbortError } from 'node-fetch';
|
|
14
|
+
import FSHelper from '../utils/FSHelper.js';
|
|
15
|
+
import URLHelper, { PostSortOrder } from '../utils/URLHelper.js';
|
|
16
|
+
import Downloader from './Downloader.js';
|
|
17
|
+
import PageParser from '../parsers/PageParser.js';
|
|
18
|
+
import ObjectHelper from '../utils/ObjectHelper.js';
|
|
19
|
+
import PostParser from '../parsers/PostParser.js';
|
|
20
|
+
import StatusCache from './cache/StatusCache.js';
|
|
21
|
+
import { generatePostEmbedSummary, generatePostSummary } from './templates/PostInfo.js';
|
|
22
|
+
import path from 'path';
|
|
23
|
+
import { TargetSkipReason } from './DownloaderEvent.js';
|
|
24
|
+
export default class PostDownloader extends Downloader {
|
|
25
|
+
constructor() {
|
|
26
|
+
super(...arguments);
|
|
27
|
+
_PostDownloader_instances.add(this);
|
|
28
|
+
this.name = 'PostDownloader';
|
|
29
|
+
_PostDownloader_startPromise.set(this, null);
|
|
30
|
+
}
|
|
31
|
+
start(params) {
|
|
32
|
+
if (__classPrivateFieldGet(this, _PostDownloader_startPromise, "f")) {
|
|
33
|
+
throw Error('Downloader already running');
|
|
34
|
+
}
|
|
35
|
+
__classPrivateFieldSet(this, _PostDownloader_startPromise, new Promise(async (resolve) => {
|
|
36
|
+
const { signal } = params || {};
|
|
37
|
+
const postFetch = this.config.postFetch;
|
|
38
|
+
let batch = null;
|
|
39
|
+
if (this.checkAbortSignal(signal, resolve)) {
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
const abortHandler = async () => {
|
|
43
|
+
this.log('info', 'Abort signal received');
|
|
44
|
+
if (batch) {
|
|
45
|
+
await batch.abort();
|
|
46
|
+
}
|
|
47
|
+
};
|
|
48
|
+
if (signal) {
|
|
49
|
+
signal.addEventListener('abort', abortHandler, { once: true });
|
|
50
|
+
}
|
|
51
|
+
if (postFetch.type === 'byUser') {
|
|
52
|
+
this.log('info', `Targeting posts by '${postFetch.vanity}'`);
|
|
53
|
+
}
|
|
54
|
+
else if (postFetch.type === 'byCollection') {
|
|
55
|
+
this.log('info', `Targeting posts in collection #${postFetch.collectionId}`);
|
|
56
|
+
}
|
|
57
|
+
else { // Single
|
|
58
|
+
this.log('info', `Targeting post #${postFetch.postId}`);
|
|
59
|
+
}
|
|
60
|
+
if ((postFetch.type === 'byUser' || postFetch.type === 'byCollection') && postFetch.filters) {
|
|
61
|
+
const filterStr = Object.entries(postFetch.filters).map(([key, value]) => `${key}=${value}`).join('; ');
|
|
62
|
+
if (filterStr) {
|
|
63
|
+
this.log('info', `Filters: ${filterStr}`);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
// Step 1: Get initial posts (if by user) or target post
|
|
67
|
+
let postsAPIURL;
|
|
68
|
+
try {
|
|
69
|
+
postsAPIURL = await __classPrivateFieldGet(this, _PostDownloader_instances, "m", _PostDownloader_getInitialPostsAPIUL).call(this, signal, resolve, resolve);
|
|
70
|
+
}
|
|
71
|
+
catch (error) {
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
if (postFetch.type === 'byUser' || postFetch.type === 'byCollection') {
|
|
75
|
+
this.log('info', 'Fetch posts');
|
|
76
|
+
this.log('debug', `Request initial posts from API URL "${postsAPIURL}"`);
|
|
77
|
+
this.emit('fetchBegin', { targetType: 'posts' });
|
|
78
|
+
}
|
|
79
|
+
else {
|
|
80
|
+
this.log('info', 'Fetch target post');
|
|
81
|
+
this.log('debug', `Request post #${postFetch.postId} from API URL "${postsAPIURL}`);
|
|
82
|
+
this.emit('fetchBegin', { targetType: 'post' });
|
|
83
|
+
}
|
|
84
|
+
let json;
|
|
85
|
+
try {
|
|
86
|
+
json = await __classPrivateFieldGet(this, _PostDownloader_instances, "m", _PostDownloader_requestPosts).call(this, postsAPIURL, signal, resolve, resolve);
|
|
87
|
+
}
|
|
88
|
+
catch (error) {
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
// Step 2: parse, download and, if targeting posts by user, repeat for next batch
|
|
92
|
+
const postsParser = new PostParser(this.logger);
|
|
93
|
+
let total = null;
|
|
94
|
+
let downloaded = 0;
|
|
95
|
+
let skippedUnviewable = 0;
|
|
96
|
+
let skippedRedundant = 0;
|
|
97
|
+
let campaignSaved = false;
|
|
98
|
+
while (json) {
|
|
99
|
+
const collection = postsParser.parsePostsAPIResponse(json, postsAPIURL);
|
|
100
|
+
if (!campaignSaved && collection.posts[0]?.campaign) {
|
|
101
|
+
await this.saveCampaignInfo(collection.posts[0].campaign, signal);
|
|
102
|
+
campaignSaved = true;
|
|
103
|
+
if (this.checkAbortSignal(signal, resolve)) {
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
total = collection.total;
|
|
108
|
+
if (postFetch.type === 'byUser' || postFetch.type === 'byCollection') {
|
|
109
|
+
this.log('debug', `${collection.posts.length} posts fetched`);
|
|
110
|
+
}
|
|
111
|
+
for (const post of collection.posts) {
|
|
112
|
+
this.emit('targetBegin', { target: post });
|
|
113
|
+
// Step 4.1: post directories
|
|
114
|
+
const postDirs = FSHelper.getPostDirs(post, this.config);
|
|
115
|
+
this.log('debug', 'Post directories:', postDirs);
|
|
116
|
+
// Step 4.2: Check with status cache
|
|
117
|
+
const statusCache = StatusCache.getInstance(postDirs.statusCache, this.logger, this.config.useStatusCache);
|
|
118
|
+
if (statusCache.validate(post, postDirs.root, this.config)) {
|
|
119
|
+
this.log('info', `Skipped downloading post #${post.id}: already downloaded and nothing has changed since last download`);
|
|
120
|
+
this.emit('targetEnd', {
|
|
121
|
+
target: post,
|
|
122
|
+
isSkipped: true,
|
|
123
|
+
skipReason: TargetSkipReason.AlreadyDownloaded,
|
|
124
|
+
skipMessage: 'Target already downloaded and nothing has changed since last download'
|
|
125
|
+
});
|
|
126
|
+
skippedRedundant++;
|
|
127
|
+
continue;
|
|
128
|
+
}
|
|
129
|
+
// Step 4.3: Check viewability
|
|
130
|
+
this.log('info', `Download post #${post.id} (${post.title})`);
|
|
131
|
+
if (!post.isViewable) {
|
|
132
|
+
if (this.config.include.lockedContent) {
|
|
133
|
+
this.log('warn', `Post #${post.id} is not viewable by current user`);
|
|
134
|
+
}
|
|
135
|
+
else {
|
|
136
|
+
this.log('warn', `Skipped downloading post #${post.id}: not viewable by current user`);
|
|
137
|
+
this.emit('targetEnd', {
|
|
138
|
+
target: post,
|
|
139
|
+
isSkipped: true,
|
|
140
|
+
skipReason: TargetSkipReason.Inaccessible,
|
|
141
|
+
skipMessage: 'Target is not viewable by current user'
|
|
142
|
+
});
|
|
143
|
+
skippedUnviewable++;
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
// Step 4.4: Save post info
|
|
148
|
+
if (this.config.include.contentInfo) {
|
|
149
|
+
this.log('info', `Save post info #${post.id}`);
|
|
150
|
+
this.emit('phaseBegin', { target: post, phase: 'saveInfo' });
|
|
151
|
+
FSHelper.createDir(postDirs.info);
|
|
152
|
+
// Post raw data might not be complete or consistent with other posts in the collection.
|
|
153
|
+
// Fetch directly from API.
|
|
154
|
+
// Strictly speaking, we should check for 'error' in results, but since it's not going to be fatal we'll just skip it.
|
|
155
|
+
const { json: fetchedPostAPIData } = await this.commonFetchAPI(URLHelper.constructPostsAPIURL({
|
|
156
|
+
postId: post.id,
|
|
157
|
+
campaignId: post.campaign?.id
|
|
158
|
+
}), signal);
|
|
159
|
+
if (this.checkAbortSignal(signal, resolve)) {
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
162
|
+
// Save summary and raw json
|
|
163
|
+
const summary = generatePostSummary(post);
|
|
164
|
+
const summaryFile = path.resolve(postDirs.info, 'info.txt');
|
|
165
|
+
const saveSummaryResult = await FSHelper.writeTextFile(summaryFile, summary, this.config.fileExistsAction.info);
|
|
166
|
+
this.logWriteTextFileResult(saveSummaryResult, post, 'post summary');
|
|
167
|
+
const postRawFile = path.resolve(postDirs.info, 'post-api.json');
|
|
168
|
+
const savePostRawResult = await FSHelper.writeTextFile(postRawFile, fetchedPostAPIData || post.raw, this.config.fileExistsAction.infoAPI);
|
|
169
|
+
this.logWriteTextFileResult(savePostRawResult, post, 'post API data');
|
|
170
|
+
this.emit('phaseEnd', { target: post, phase: 'saveInfo' });
|
|
171
|
+
// (Downloading of info media items deferred to the next step)
|
|
172
|
+
}
|
|
173
|
+
if (this.config.include.previewMedia || this.config.include.contentMedia) {
|
|
174
|
+
this.emit('phaseBegin', { target: post, phase: 'saveMedia' });
|
|
175
|
+
}
|
|
176
|
+
// Step 4.5: save embed info
|
|
177
|
+
if (post.embed && this.config.include.contentMedia) {
|
|
178
|
+
this.log('info', `Save embed info of post #${post.id}`);
|
|
179
|
+
FSHelper.createDir(postDirs.embed);
|
|
180
|
+
const embedSummary = generatePostEmbedSummary(post.embed);
|
|
181
|
+
let embedFilename;
|
|
182
|
+
switch (post.embed.type) {
|
|
183
|
+
case 'video':
|
|
184
|
+
embedFilename = 'embedded-video.txt';
|
|
185
|
+
break;
|
|
186
|
+
case 'link':
|
|
187
|
+
embedFilename = 'embedded-link.txt';
|
|
188
|
+
break;
|
|
189
|
+
default:
|
|
190
|
+
embedFilename = 'embedded-unknown.txt';
|
|
191
|
+
}
|
|
192
|
+
const embedFile = path.resolve(postDirs.embed, embedFilename);
|
|
193
|
+
const saveSummaryResult = await FSHelper.writeTextFile(embedFile, embedSummary, this.config.fileExistsAction.content);
|
|
194
|
+
this.logWriteTextFileResult(saveSummaryResult, post, 'embed info');
|
|
195
|
+
}
|
|
196
|
+
// Step 4.6: create download tasks
|
|
197
|
+
if (this.config.include.previewMedia ||
|
|
198
|
+
this.config.include.contentMedia ||
|
|
199
|
+
this.config.include.contentInfo) {
|
|
200
|
+
batch = __classPrivateFieldGet(this, _PostDownloader_instances, "m", _PostDownloader_createDownloadTaskBatchForPost).call(this, post, postDirs);
|
|
201
|
+
if (this.config.include.contentInfo) {
|
|
202
|
+
const infoElements = [];
|
|
203
|
+
if (post.coverImage) {
|
|
204
|
+
infoElements.push(post.coverImage);
|
|
205
|
+
}
|
|
206
|
+
if (post.thumbnail) {
|
|
207
|
+
infoElements.push(post.thumbnail);
|
|
208
|
+
}
|
|
209
|
+
if (infoElements.length > 0) {
|
|
210
|
+
this.addToDownloadTaskBatch(batch, {
|
|
211
|
+
target: infoElements,
|
|
212
|
+
targetName: `post #${post.id} -> info elements`,
|
|
213
|
+
destDir: postDirs.info,
|
|
214
|
+
fileExistsAction: this.config.fileExistsAction.info
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
this.log('info', `Download batch created (#${batch.id}): ${batch.getTasks('pending').length} downloads pending`);
|
|
219
|
+
this.emit('phaseBegin', { target: post, phase: 'batchDownload', batch });
|
|
220
|
+
await batch.start();
|
|
221
|
+
// Step 4.7: Update status cache
|
|
222
|
+
statusCache.updateOnDownload(post, postDirs.root, batch.getTasks('error').length > 0, this.config);
|
|
223
|
+
await batch.destroy();
|
|
224
|
+
batch = null;
|
|
225
|
+
this.emit('phaseEnd', { target: post, phase: 'batchDownload' });
|
|
226
|
+
}
|
|
227
|
+
if (this.config.include.previewMedia || this.config.include.contentMedia) {
|
|
228
|
+
this.emit('phaseEnd', { target: post, phase: 'saveMedia' });
|
|
229
|
+
}
|
|
230
|
+
downloaded++;
|
|
231
|
+
this.emit('targetEnd', { target: post, isSkipped: false });
|
|
232
|
+
if (this.checkAbortSignal(signal, resolve)) {
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
if (postFetch.type === 'byUser' || postFetch.type === 'byCollection') {
|
|
237
|
+
if (collection.nextURL) {
|
|
238
|
+
this.log('info', 'Fetch more posts');
|
|
239
|
+
this.log('debug', `Request next batch of posts from API URL "${collection.nextURL}`);
|
|
240
|
+
this.emit('fetchBegin', { targetType: 'posts' });
|
|
241
|
+
try {
|
|
242
|
+
json = await __classPrivateFieldGet(this, _PostDownloader_instances, "m", _PostDownloader_requestPosts).call(this, collection.nextURL, signal, resolve, resolve);
|
|
243
|
+
}
|
|
244
|
+
catch (error) {
|
|
245
|
+
return;
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
else {
|
|
249
|
+
this.log('debug', 'No further posts to fetch');
|
|
250
|
+
json = null;
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
else {
|
|
254
|
+
json = null;
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
if (this.checkAbortSignal(signal, resolve)) {
|
|
258
|
+
return;
|
|
259
|
+
}
|
|
260
|
+
// Done
|
|
261
|
+
if (signal) {
|
|
262
|
+
signal.removeEventListener('abort', abortHandler);
|
|
263
|
+
}
|
|
264
|
+
if (postFetch.type === 'byUser') {
|
|
265
|
+
this.log('info', `Done downloading posts by '${postFetch.vanity}'`);
|
|
266
|
+
}
|
|
267
|
+
else if (postFetch.type === 'byCollection') {
|
|
268
|
+
this.log('info', `Done downloading posts in collection #${postFetch.collectionId}`);
|
|
269
|
+
}
|
|
270
|
+
else {
|
|
271
|
+
this.log('info', `Done downloading post #${postFetch.postId}`);
|
|
272
|
+
}
|
|
273
|
+
if (postFetch.type === 'byUser' || postFetch.type === 'byCollection') {
|
|
274
|
+
const skippedStrParts = [];
|
|
275
|
+
if (skippedUnviewable) {
|
|
276
|
+
skippedStrParts.push(`${skippedUnviewable} unviewable`);
|
|
277
|
+
}
|
|
278
|
+
if (skippedRedundant) {
|
|
279
|
+
skippedStrParts.push(`${skippedRedundant} redundant`);
|
|
280
|
+
}
|
|
281
|
+
const skippedStr = skippedStrParts.length > 0 ? ` (skipped: ${skippedStrParts.join(', ')})` : '';
|
|
282
|
+
this.log('info', `Total ${downloaded} / ${total} posts processed${skippedStr}`);
|
|
283
|
+
}
|
|
284
|
+
this.emit('end', { aborted: false });
|
|
285
|
+
__classPrivateFieldSet(this, _PostDownloader_startPromise, null, "f");
|
|
286
|
+
resolve();
|
|
287
|
+
})
|
|
288
|
+
.finally(async () => {
|
|
289
|
+
if (this.logger) {
|
|
290
|
+
await this.logger.end();
|
|
291
|
+
}
|
|
292
|
+
__classPrivateFieldSet(this, _PostDownloader_startPromise, null, "f");
|
|
293
|
+
}), "f");
|
|
294
|
+
return __classPrivateFieldGet(this, _PostDownloader_startPromise, "f");
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
_PostDownloader_startPromise = new WeakMap(), _PostDownloader_instances = new WeakSet(), _PostDownloader_getInitialPostsAPIUL = async function _PostDownloader_getInitialPostsAPIUL(signal, resolveOnAbort, resolveOnError) {
|
|
298
|
+
const postFetch = this.config.postFetch;
|
|
299
|
+
if (postFetch.type === 'byUser' || postFetch.type === 'byCollection') {
|
|
300
|
+
// Step 1: get initial page data
|
|
301
|
+
const pageURL = postFetch.type === 'byUser' ?
|
|
302
|
+
URLHelper.constructCampaignPageURL(postFetch.vanity) :
|
|
303
|
+
URLHelper.constructCollectionURL(postFetch.collectionId);
|
|
304
|
+
this.log('debug', `Fetch initial data from URL "${pageURL}"`);
|
|
305
|
+
let page, requestPageError;
|
|
306
|
+
try {
|
|
307
|
+
page = await this.fetcher.get({ url: pageURL, type: 'html', maxRetries: this.config.request.maxRetries, signal });
|
|
308
|
+
}
|
|
309
|
+
catch (error) {
|
|
310
|
+
if (error instanceof AbortError) {
|
|
311
|
+
this.log('warn', 'Page request aborted');
|
|
312
|
+
}
|
|
313
|
+
else {
|
|
314
|
+
requestPageError = error;
|
|
315
|
+
}
|
|
316
|
+
page = null;
|
|
317
|
+
}
|
|
318
|
+
if (!page) {
|
|
319
|
+
if (this.checkAbortSignal(signal, resolveOnAbort)) {
|
|
320
|
+
throw Error();
|
|
321
|
+
}
|
|
322
|
+
this.log('error', `Error requesting page "${pageURL}": `, requestPageError);
|
|
323
|
+
this.emit('end', { aborted: false, error: requestPageError });
|
|
324
|
+
resolveOnError();
|
|
325
|
+
throw Error();
|
|
326
|
+
}
|
|
327
|
+
const pageParser = new PageParser(this.logger);
|
|
328
|
+
let initialData, parseInitialDataError;
|
|
329
|
+
try {
|
|
330
|
+
initialData = pageParser.parseInitialData(page, pageURL);
|
|
331
|
+
}
|
|
332
|
+
catch (error) {
|
|
333
|
+
parseInitialDataError = error;
|
|
334
|
+
initialData = null;
|
|
335
|
+
}
|
|
336
|
+
if (!initialData) {
|
|
337
|
+
this.log('error', `Failed to obtain initial page data from "${pageURL}":`, parseInitialDataError);
|
|
338
|
+
this.emit('end', { aborted: false, error: parseInitialDataError });
|
|
339
|
+
resolveOnError();
|
|
340
|
+
throw Error();
|
|
341
|
+
}
|
|
342
|
+
/**
|
|
343
|
+
* Step 2: obtain campaign ID and current user ID (i.e. you, if available in session identified by cookie)
|
|
344
|
+
* from initial data.
|
|
345
|
+
*/
|
|
346
|
+
const campaignId = ObjectHelper.getProperty(initialData, 'bootstrap.campaign.data.id');
|
|
347
|
+
const currentUserId = ObjectHelper.getProperty(initialData, 'bootstrap.currentUser.data.id');
|
|
348
|
+
this.log('debug', `Initial data: campaign ID '${campaignId}'; current user ID '${currentUserId}'`);
|
|
349
|
+
if (!campaignId) {
|
|
350
|
+
const err = Error(`Campaign ID not found in initial data of "${pageURL}"`);
|
|
351
|
+
this.log('error', err);
|
|
352
|
+
this.emit('end', { aborted: false, error: err });
|
|
353
|
+
resolveOnError();
|
|
354
|
+
throw Error();
|
|
355
|
+
}
|
|
356
|
+
let sort;
|
|
357
|
+
if (postFetch.type === 'byCollection') {
|
|
358
|
+
sort = PostSortOrder.CollectionOrder;
|
|
359
|
+
}
|
|
360
|
+
return URLHelper.constructPostsAPIURL({
|
|
361
|
+
campaignId,
|
|
362
|
+
currentUserId: this.config.include.lockedContent ? undefined : currentUserId,
|
|
363
|
+
filters: postFetch.filters,
|
|
364
|
+
sort
|
|
365
|
+
});
|
|
366
|
+
}
|
|
367
|
+
return URLHelper.constructPostsAPIURL({ postId: postFetch.postId });
|
|
368
|
+
}, _PostDownloader_requestPosts = async function _PostDownloader_requestPosts(url, signal, resolveOnAbort, resolveOnError) {
|
|
369
|
+
const { json, error: requestAPIError } = await this.commonFetchAPI(url, signal);
|
|
370
|
+
if (!json) {
|
|
371
|
+
if (this.checkAbortSignal(signal, resolveOnAbort)) {
|
|
372
|
+
throw Error();
|
|
373
|
+
}
|
|
374
|
+
this.log('error', 'Failed to fetch posts');
|
|
375
|
+
this.log('warn', 'End with error');
|
|
376
|
+
this.emit('end', { aborted: false, error: requestAPIError });
|
|
377
|
+
resolveOnError();
|
|
378
|
+
throw Error();
|
|
379
|
+
}
|
|
380
|
+
return json;
|
|
381
|
+
}, _PostDownloader_createDownloadTaskBatchForPost = function _PostDownloader_createDownloadTaskBatchForPost(post, postDirs) {
|
|
382
|
+
const incPreview = this.config.include.previewMedia;
|
|
383
|
+
const incContent = this.config.include.contentMedia;
|
|
384
|
+
const batch = this.createDownloadTaskBatch(`Post #${post.id} (${post.title})`, incPreview && post.audioPreview ? {
|
|
385
|
+
target: [post.audioPreview],
|
|
386
|
+
targetName: `post #${post.id} -> audio preview`,
|
|
387
|
+
destDir: postDirs.audioPreview,
|
|
388
|
+
fileExistsAction: this.config.fileExistsAction.content
|
|
389
|
+
} : null, incPreview && post.videoPreview ? {
|
|
390
|
+
target: [post.videoPreview],
|
|
391
|
+
targetName: `post #${post.id} -> video preview`,
|
|
392
|
+
destDir: postDirs.videoPreview,
|
|
393
|
+
fileExistsAction: this.config.fileExistsAction.content
|
|
394
|
+
} : null,
|
|
395
|
+
/**
|
|
396
|
+
* If post is not viewable by current user, its images will be
|
|
397
|
+
* blurry and we should categorize them as image previews.
|
|
398
|
+
*/
|
|
399
|
+
incPreview && post.images.length > 0 && !post.isViewable ? {
|
|
400
|
+
target: post.images,
|
|
401
|
+
targetName: `post #${post.id} -> image previews`,
|
|
402
|
+
destDir: postDirs.imagePreviews,
|
|
403
|
+
fileExistsAction: this.config.fileExistsAction.content
|
|
404
|
+
} : null, incContent && post.audio ? {
|
|
405
|
+
target: [post.audio],
|
|
406
|
+
targetName: `post #${post.id} -> audio`,
|
|
407
|
+
destDir: postDirs.audio,
|
|
408
|
+
fileExistsAction: this.config.fileExistsAction.content
|
|
409
|
+
} : null, incContent && post.video ? {
|
|
410
|
+
target: [post.video],
|
|
411
|
+
targetName: `post #${post.id} -> video`,
|
|
412
|
+
destDir: postDirs.video,
|
|
413
|
+
fileExistsAction: this.config.fileExistsAction.content
|
|
414
|
+
} : null, incContent && post.images.length > 0 && post.isViewable ? {
|
|
415
|
+
target: post.images,
|
|
416
|
+
targetName: `post #${post.id} -> images`,
|
|
417
|
+
destDir: postDirs.images,
|
|
418
|
+
fileExistsAction: this.config.fileExistsAction.content
|
|
419
|
+
} : null, incContent && post.attachments.length > 0 ? {
|
|
420
|
+
target: post.attachments,
|
|
421
|
+
targetName: `post #${post.id} -> attachments`,
|
|
422
|
+
destDir: postDirs.attachments,
|
|
423
|
+
fileExistsAction: this.config.fileExistsAction.content
|
|
424
|
+
} : null);
|
|
425
|
+
return batch;
|
|
426
|
+
};
|
|
427
|
+
PostDownloader.version = '1.0.0';
|
|
428
|
+
//# sourceMappingURL=PostDownloader.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"PostDownloader.js","sourceRoot":"","sources":["../../src/downloaders/PostDownloader.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,QAAQ,MAAM,sBAAsB,CAAC;AAC5C,OAAO,SAAS,EAAE,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,UAAqC,MAAM,iBAAiB,CAAC;AAEpE,OAAO,UAAU,MAAM,0BAA0B,CAAC;AAClD,OAAO,YAAY,MAAM,0BAA0B,CAAC;AACpD,OAAO,UAAU,MAAM,0BAA0B,CAAC;AAGlD,OAAO,WAAW,MAAM,wBAAwB,CAAC;AACjD,OAAO,EAAE,wBAAwB,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AACxF,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAExD,MAAM,CAAC,OAAO,OAAO,cAAe,SAAQ,UAAkB;IAA9D;;;QAIE,SAAI,GAAG,gBAAgB,CAAC;QAExB,uCAAsC,IAAI,EAAC;IAod7C,CAAC;IAldC,KAAK,CAAC,MAA8B;QAElC,IAAI,uBAAA,IAAI,oCAAc,EAAE;YACtB,MAAM,KAAK,CAAC,4BAA4B,CAAC,CAAC;SAC3C;QAED,uBAAA,IAAI,gCAAiB,IAAI,OAAO,CAAO,KAAK,EAAE,OAAO,EAAE,EAAE;YAEvD,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,EAAE,CAAC;YAChC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;YACxC,IAAI,KAAK,GAA6B,IAAI,CAAC;YAE3C,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE;gBAC1C,OAAO;aACR;YAED,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;gBAC9B,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,uBAAuB,CAAC,CAAC;gBAC1C,IAAI,KAAK,EAAE;oBACT,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;iBACrB;YACH,CAAC,CAAC;YACF,IAAI,MAAM,EAAE;gBACV,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;aAChE;YAED,IAAI,SAAS,CAAC,IAAI,KAAK,QAAQ,EAAE;gBAC/B,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,uBAAuB,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC;aAC9D;iBACI,IAAI,SAAS,CAAC,IAAI,KAAK,cAAc,EAAE;gBAC1C,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,kCAAkC,SAAS,CAAC,YAAY,EAAE,CAAC,CAAC;aAC9E;iBACI,EAAE,SAAS;gBACd,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,mBAAmB,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;aACzD;YACD,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,QAAQ,IAAI,SAAS,CAAC,IAAI,KAAK,cAAc,CAAC,IAAI,SAAS,CAAC,OAAO,EAAE;gBAC3F,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAE,GAAG,EAAE,KAAK,CAAE,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC1G,IAAI,SAAS,EAAE;oBACb,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,SAAS,EAAE,CAAC,CAAC;iBAC3C;aACF;YAED,wDAAwD;YACxD,IAAI,WAAmB,CAAC;YACxB,IAAI;gBACF,WAAW,GAAG,MAAM,uBAAA,IAAI,uEAAsB,MAA1B,IAAI,EAAuB,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;aAC1E;YACD,OAAO,KAAK,EAAE;gBACZ,OAAO;aACR;YACD,IAAI,SAAS,CAAC,IAAI,KAAK,QAAQ,IAAI,SAAS,CAAC,IAAI,KAAK,cAAc,EAAE;gBACpE,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;gBAChC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,uCAAuC,WAAW,GAAG,CAAC,CAAC;gBACzE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC;aAClD;iBACI;gBACH,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC;gBACtC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,iBAAiB,SAAS,CAAC,MAAM,kBAAkB,WAAW,EAAE,CAAC,CAAC;gBACpF,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC;aACjD;YACD,IAAI,IAAI,CAAC;YACT,IAAI;gBACF,IAAI,GAAG,MAAM,uBAAA,IAAI,+DAAc,MAAlB,IAAI,EAAe,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;aACxE;YACD,OAAO,KAAK,EAAE;gBACZ,OAAO;aACR;YAED,iFAAiF;YACjF,MAAM,WAAW,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAChD,IAAI,KAAK,GAAkB,IAAI,CAAC;YAChC,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,IAAI,iBAAiB,GAAG,CAAC,CAAC;YAC1B,IAAI,gBAAgB,GAAG,CAAC,CAAC;YACzB,IAAI,aAAa,GAAG,KAAK,CAAC;YAC1B,OAAO,IAAI,EAAE;gBACX,MAAM,UAAU,GAAG,WAAW,CAAC,qBAAqB,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;gBAExE,IAAI,CAAC,aAAa,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE;oBACnD,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;oBAClE,aAAa,GAAG,IAAI,CAAC;oBACrB,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE;wBAC1C,OAAO;qBACR;iBACF;gBAED,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC;gBACzB,IAAI,SAAS,CAAC,IAAI,KAAK,QAAQ,IAAI,SAAS,CAAC,IAAI,KAAK,cAAc,EAAE;oBACpE,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,gBAAgB,CAAC,CAAC;iBAC/D;gBAED,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,KAAK,EAAE;oBAEnC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;oBAE3C,6BAA6B;oBAC7B,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;oBACzD,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,mBAAmB,EAAE,QAAQ,CAAC,CAAC;oBAEjD,oCAAoC;oBACpC,MAAM,WAAW,GAAG,WAAW,CAAC,WAAW,CAAC,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;oBAC3G,IAAI,WAAW,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE;wBAC1D,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,6BAA6B,IAAI,CAAC,EAAE,kEAAkE,CAAC,CAAC;wBACzH,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;4BACrB,MAAM,EAAE,IAAI;4BACZ,SAAS,EAAE,IAAI;4BACf,UAAU,EAAE,gBAAgB,CAAC,iBAAiB;4BAC9C,WAAW,EAAE,uEAAuE;yBACrF,CAAC,CAAC;wBACH,gBAAgB,EAAE,CAAC;wBACnB,SAAS;qBACV;oBAED,8BAA8B;oBAC9B,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,kBAAkB,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;oBAC9D,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;wBACpB,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE;4BACrC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,IAAI,CAAC,EAAE,kCAAkC,CAAC,CAAC;yBACtE;6BACI;4BACH,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,6BAA6B,IAAI,CAAC,EAAE,gCAAgC,CAAC,CAAC;4BACvF,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;gCACrB,MAAM,EAAE,IAAI;gCACZ,SAAS,EAAE,IAAI;gCACf,UAAU,EAAE,gBAAgB,CAAC,YAAY;gCACzC,WAAW,EAAE,wCAAwC;6BACtD,CAAC,CAAC;4BACH,iBAAiB,EAAE,CAAC;4BACpB,SAAS;yBACV;qBACF;oBAED,2BAA2B;oBAC3B,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE;wBACnC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,mBAAmB,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;wBAC/C,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;wBAC7D,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;wBAClC,wFAAwF;wBACxF,2BAA2B;wBAC3B,sHAAsH;wBACtH,MAAM,EAAE,IAAI,EAAE,kBAAkB,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAC5D,SAAS,CAAC,oBAAoB,CAAC;4BAC7B,MAAM,EAAE,IAAI,CAAC,EAAE;4BACf,UAAU,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE;yBAC9B,CAAC,EACF,MAAM,CACP,CAAC;wBAEF,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE;4BAC1C,OAAO;yBACR;wBAED,4BAA4B;wBAC5B,MAAM,OAAO,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;wBAC1C,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;wBAC5D,MAAM,iBAAiB,GAAG,MAAM,QAAQ,CAAC,aAAa,CAAC,WAAW,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;wBAChH,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;wBAErE,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;wBACjE,MAAM,iBAAiB,GAAG,MAAM,QAAQ,CAAC,aAAa,CACpD,WAAW,EAAE,kBAAkB,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;wBACrF,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,EAAE,IAAI,EAAE,eAAe,CAAC,CAAC;wBACtE,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;wBAE3D,8DAA8D;qBAC/D;oBAED,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,EAAE;wBACxE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;qBAC/D;oBAED,4BAA4B;oBAC5B,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,EAAE;wBAClD,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,4BAA4B,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;wBACxD,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;wBACnC,MAAM,YAAY,GAAG,wBAAwB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBAC1D,IAAI,aAAa,CAAC;wBAClB,QAAQ,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;4BACvB,KAAK,OAAO;gCACV,aAAa,GAAG,oBAAoB,CAAC;gCACrC,MAAM;4BACR,KAAK,MAAM;gCACT,aAAa,GAAG,mBAAmB,CAAC;gCACpC,MAAM;4BACR;gCACE,aAAa,GAAG,sBAAsB,CAAC;yBAC1C;wBACD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;wBAC9D,MAAM,iBAAiB,GAAG,MAAM,QAAQ,CAAC,aAAa,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;wBACtH,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;qBACpE;oBAED,kCAAkC;oBAClC,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY;wBAClC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY;wBAChC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE;wBAEjC,KAAK,GAAG,uBAAA,IAAI,iFAAgC,MAApC,IAAI,EAAiC,IAAI,EAAE,QAAQ,CAAC,CAAC;wBAE7D,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE;4BACnC,MAAM,YAAY,GAAmB,EAAE,CAAC;4BACxC,IAAI,IAAI,CAAC,UAAU,EAAE;gCACnB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;6BACpC;4BACD,IAAI,IAAI,CAAC,SAAS,EAAE;gCAClB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;6BACnC;4BACD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE;gCAC3B,IAAI,CAAC,sBAAsB,CAAC,KAAK,EAC/B;oCACE,MAAM,EAAE,YAAY;oCACpB,UAAU,EAAE,SAAS,IAAI,CAAC,EAAE,mBAAmB;oCAC/C,OAAO,EAAE,QAAQ,CAAC,IAAI;oCACtB,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,IAAI;iCACpD,CACF,CAAC;6BACH;yBACF;wBAED,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,4BAA4B,KAAK,CAAC,EAAE,MAAM,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,MAAM,oBAAoB,CAAC,CAAC;wBACjH,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC;wBAEzE,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;wBAEpB,gCAAgC;wBAChC,WAAW,CAAC,gBAAgB,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;wBAEnG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;wBACtB,KAAK,GAAG,IAAI,CAAC;wBACb,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,eAAe,EAAC,CAAC,CAAC;qBAChE;oBAED,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,EAAE;wBACxE,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;qBAC7D;oBAED,UAAU,EAAE,CAAC;oBACb,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;oBAE3D,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE;wBAC1C,OAAO;qBACR;iBACF;gBAED,IAAI,SAAS,CAAC,IAAI,KAAK,QAAQ,IAAI,SAAS,CAAC,IAAI,KAAK,cAAc,EAAE;oBACpE,IAAI,UAAU,CAAC,OAAO,EAAE;wBACtB,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;wBACrC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,6CAA6C,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;wBACrF,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC;wBACjD,IAAI;4BACF,IAAI,GAAG,MAAM,uBAAA,IAAI,+DAAc,MAAlB,IAAI,EAAe,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;yBAC/E;wBACD,OAAO,KAAK,EAAE;4BACZ,OAAO;yBACR;qBACF;yBACI;wBACH,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,2BAA2B,CAAC,CAAC;wBAC/C,IAAI,GAAG,IAAI,CAAC;qBACb;iBACF;qBACI;oBACH,IAAI,GAAG,IAAI,CAAC;iBACb;aACF;YAED,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE;gBAC1C,OAAO;aACR;YAED,OAAO;YACP,IAAI,MAAM,EAAE;gBACV,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;aACnD;YACD,IAAI,SAAS,CAAC,IAAI,KAAK,QAAQ,EAAE;gBAC/B,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,8BAA8B,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC;aACrE;iBACI,IAAI,SAAS,CAAC,IAAI,KAAK,cAAc,EAAE;gBAC1C,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,yCAAyC,SAAS,CAAC,YAAY,EAAE,CAAC,CAAC;aACrF;iBACI;gBACH,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,0BAA0B,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;aAChE;YACD,IAAI,SAAS,CAAC,IAAI,KAAK,QAAQ,IAAI,SAAS,CAAC,IAAI,KAAK,cAAc,EAAE;gBACpE,MAAM,eAAe,GAAa,EAAE,CAAC;gBACrC,IAAI,iBAAiB,EAAE;oBACrB,eAAe,CAAC,IAAI,CAAC,GAAG,iBAAiB,aAAa,CAAC,CAAC;iBACzD;gBACD,IAAI,gBAAgB,EAAE;oBACpB,eAAe,CAAC,IAAI,CAAC,GAAG,gBAAgB,YAAY,CAAC,CAAC;iBACvD;gBACD,MAAM,UAAU,GAAG,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,UAAU,MAAM,KAAK,mBAAmB,UAAU,EAAE,CAAC,CAAC;aACjF;YACD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YACrC,uBAAA,IAAI,gCAAiB,IAAI,MAAA,CAAC;YAC1B,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;aACC,OAAO,CAAC,KAAK,IAAI,EAAE;YAClB,IAAI,IAAI,CAAC,MAAM,EAAE;gBACf,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;aACzB;YACD,uBAAA,IAAI,gCAAiB,IAAI,MAAA,CAAC;QAC5B,CAAC,CAAC,MAAA,CAAC;QAEL,OAAO,uBAAA,IAAI,oCAAc,CAAC;IAC5B,CAAC;;gIAED,KAAK,+CAAuB,MAA+B,EAAE,cAA0B,EAAE,cAA0B;IACjH,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;IACxC,IAAI,SAAS,CAAC,IAAI,KAAK,QAAQ,IAAI,SAAS,CAAC,IAAI,KAAK,cAAc,EAAE;QACpE,gCAAgC;QAChC,MAAM,OAAO,GACX,SAAS,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;YAC3B,SAAS,CAAC,wBAAwB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;YACtD,SAAS,CAAC,sBAAsB,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QAC7D,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,gCAAgC,OAAO,GAAG,CAAC,CAAC;QAC9D,IAAI,IAAI,EAAE,gBAAgB,CAAC;QAC3B,IAAI;YACF,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC;SACnH;QACD,OAAO,KAAK,EAAE;YACZ,IAAI,KAAK,YAAY,UAAU,EAAE;gBAC/B,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,sBAAsB,CAAC,CAAC;aAC1C;iBACI;gBACH,gBAAgB,GAAG,KAAK,CAAC;aAC1B;YACD,IAAI,GAAG,IAAI,CAAC;SACb;QACD,IAAI,CAAC,IAAI,EAAE;YACT,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,cAAc,CAAC,EAAE;gBACjD,MAAM,KAAK,EAAE,CAAC;aACf;YACD,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,0BAA0B,OAAO,KAAK,EAAE,gBAAgB,CAAC,CAAC;YAC5E,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,CAAC;YAC9D,cAAc,EAAE,CAAC;YACjB,MAAM,KAAK,EAAE,CAAC;SACf;QACD,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/C,IAAI,WAAW,EAAE,qBAAqB,CAAC;QACvC,IAAI;YACF,WAAW,GAAG,UAAU,CAAC,gBAAgB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;SAC1D;QACD,OAAO,KAAK,EAAE;YACZ,qBAAqB,GAAG,KAAK,CAAC;YAC9B,WAAW,GAAG,IAAI,CAAC;SACpB;QACD,IAAI,CAAC,WAAW,EAAE;YAChB,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,4CAA4C,OAAO,IAAI,EAAE,qBAAqB,CAAC,CAAC;YAClG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,qBAAqB,EAAE,CAAC,CAAC;YACnE,cAAc,EAAE,CAAC;YACjB,MAAM,KAAK,EAAE,CAAC;SACf;QAED;;;WAGG;QACH,MAAM,UAAU,GAAG,YAAY,CAAC,WAAW,CAAC,WAAW,EAAE,4BAA4B,CAAC,CAAC;QACvF,MAAM,aAAa,GAAG,YAAY,CAAC,WAAW,CAAC,WAAW,EAAE,+BAA+B,CAAC,CAAC;QAC7F,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,8BAA8B,UAAU,uBAAuB,aAAa,GAAG,CAAC,CAAC;QACnG,IAAI,CAAC,UAAU,EAAE;YACf,MAAM,GAAG,GAAG,KAAK,CAAC,6CAA6C,OAAO,GAAG,CAAC,CAAC;YAC3E,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YACvB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YACjD,cAAc,EAAE,CAAC;YACjB,MAAM,KAAK,EAAE,CAAC;SACf;QAED,IAAI,IAA+B,CAAC;QACpC,IAAI,SAAS,CAAC,IAAI,KAAK,cAAc,EAAE;YACrC,IAAI,GAAG,aAAa,CAAC,eAAe,CAAC;SACtC;QAED,OAAO,SAAS,CAAC,oBAAoB,CAAC;YACpC,UAAU;YACV,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa;YAC5E,OAAO,EAAE,SAAS,CAAC,OAAO;YAC1B,IAAI;SACL,CAAC,CAAC;KACJ;IAED,OAAO,SAAS,CAAC,oBAAoB,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;AAEtE,CAAC,iCAED,KAAK,uCAAe,GAAW,EAAE,MAA+B,EAAE,cAA0B,EAAE,cAA0B;IACtH,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,eAAe,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAChF,IAAI,CAAC,IAAI,EAAE;QACT,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,cAAc,CAAC,EAAE;YACjD,MAAM,KAAK,EAAE,CAAC;SACf;QACD,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,uBAAuB,CAAC,CAAC;QAC3C,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;QACnC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC;QAC7D,cAAc,EAAE,CAAC;QACjB,MAAM,KAAK,EAAE,CAAC;KACf;IACD,OAAO,IAAI,CAAC;AACd,CAAC,2GAE+B,IAAU,EAAE,QAAoD;IAE9F,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC;IACpD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC;IAEpD,MAAM,KAAK,GAAG,IAAI,CAAC,uBAAuB,CACxC,SAAS,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,KAAK,GAAG,EAElC,UAAU,IAAI,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QAChC,MAAM,EAAE,CAAE,IAAI,CAAC,YAAY,CAAE;QAC7B,UAAU,EAAE,SAAS,IAAI,CAAC,EAAE,mBAAmB;QAC/C,OAAO,EAAE,QAAQ,CAAC,YAAY;QAC9B,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO;KACvD,CAAC,CAAC,CAAC,IAAI,EAER,UAAU,IAAI,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QAChC,MAAM,EAAE,CAAE,IAAI,CAAC,YAAY,CAAE;QAC7B,UAAU,EAAE,SAAS,IAAI,CAAC,EAAE,mBAAmB;QAC/C,OAAO,EAAE,QAAQ,CAAC,YAAY;QAC9B,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO;KACvD,CAAC,CAAC,CAAC,IAAI;IAER;;;OAGG;IACH,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QACzD,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,UAAU,EAAE,SAAS,IAAI,CAAC,EAAE,oBAAoB;QAChD,OAAO,EAAE,QAAQ,CAAC,aAAa;QAC/B,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO;KACvD,CAAC,CAAC,CAAC,IAAI,EAER,UAAU,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QACzB,MAAM,EAAE,CAAE,IAAI,CAAC,KAAK,CAAE;QACtB,UAAU,EAAE,SAAS,IAAI,CAAC,EAAE,WAAW;QACvC,OAAO,EAAE,QAAQ,CAAC,KAAK;QACvB,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO;KACvD,CAAC,CAAC,CAAC,IAAI,EAER,UAAU,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QACzB,MAAM,EAAE,CAAE,IAAI,CAAC,KAAK,CAAE;QACtB,UAAU,EAAE,SAAS,IAAI,CAAC,EAAE,WAAW;QACvC,OAAO,EAAE,QAAQ,CAAC,KAAK;QACvB,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO;KACvD,CAAC,CAAC,CAAC,IAAI,EAER,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QACxD,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,UAAU,EAAE,SAAS,IAAI,CAAC,EAAE,YAAY;QACxC,OAAO,EAAE,QAAQ,CAAC,MAAM;QACxB,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO;KACvD,CAAC,CAAC,CAAC,IAAI,EAER,UAAU,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,EAAE,IAAI,CAAC,WAAW;QACxB,UAAU,EAAE,SAAS,IAAI,CAAC,EAAE,iBAAiB;QAC7C,OAAO,EAAE,QAAQ,CAAC,WAAW;QAC7B,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO;KACvD,CAAC,CAAC,CAAC,IAAI,CACT,CAAC;IAEF,OAAO,KAAK,CAAC;AACf,CAAC;AAvdM,sBAAO,GAAG,OAAO,CAAC","sourcesContent":["import { AbortError } from 'node-fetch';\nimport FSHelper from '../utils/FSHelper.js';\nimport URLHelper, { PostSortOrder } from '../utils/URLHelper.js';\nimport Downloader, { DownloaderStartParams } from './Downloader.js';\nimport DownloadTaskBatch from './task/DownloadTaskBatch.js';\nimport PageParser from '../parsers/PageParser.js';\nimport ObjectHelper from '../utils/ObjectHelper.js';\nimport PostParser from '../parsers/PostParser.js';\nimport { Post } from '../entities/Post.js';\nimport { Downloadable } from '../entities/Downloadable.js';\nimport StatusCache from './cache/StatusCache.js';\nimport { generatePostEmbedSummary, generatePostSummary } from './templates/PostInfo.js';\nimport path from 'path';\nimport { TargetSkipReason } from './DownloaderEvent.js';\n\nexport default class PostDownloader extends Downloader<'post'> {\n\n static version = '1.0.0';\n\n name = 'PostDownloader';\n\n #startPromise: Promise<void> | null = null;\n\n start(params?: DownloaderStartParams): Promise<void> {\n\n if (this.#startPromise) {\n throw Error('Downloader already running');\n }\n\n this.#startPromise = new Promise<void>(async (resolve) => {\n\n const { signal } = params || {};\n const postFetch = this.config.postFetch;\n let batch: DownloadTaskBatch | null = null;\n\n if (this.checkAbortSignal(signal, resolve)) {\n return;\n }\n\n const abortHandler = async () => {\n this.log('info', 'Abort signal received');\n if (batch) {\n await batch.abort();\n }\n };\n if (signal) {\n signal.addEventListener('abort', abortHandler, { once: true });\n }\n\n if (postFetch.type === 'byUser') {\n this.log('info', `Targeting posts by '${postFetch.vanity}'`);\n }\n else if (postFetch.type === 'byCollection') {\n this.log('info', `Targeting posts in collection #${postFetch.collectionId}`);\n }\n else { // Single\n this.log('info', `Targeting post #${postFetch.postId}`);\n }\n if ((postFetch.type === 'byUser' || postFetch.type === 'byCollection') && postFetch.filters) {\n const filterStr = Object.entries(postFetch.filters).map(([ key, value ]) => `${key}=${value}`).join('; ');\n if (filterStr) {\n this.log('info', `Filters: ${filterStr}`);\n }\n }\n\n // Step 1: Get initial posts (if by user) or target post\n let postsAPIURL: string;\n try {\n postsAPIURL = await this.#getInitialPostsAPIUL(signal, resolve, resolve);\n }\n catch (error) {\n return;\n }\n if (postFetch.type === 'byUser' || postFetch.type === 'byCollection') {\n this.log('info', 'Fetch posts');\n this.log('debug', `Request initial posts from API URL \"${postsAPIURL}\"`);\n this.emit('fetchBegin', { targetType: 'posts' });\n }\n else {\n this.log('info', 'Fetch target post');\n this.log('debug', `Request post #${postFetch.postId} from API URL \"${postsAPIURL}`);\n this.emit('fetchBegin', { targetType: 'post' });\n }\n let json;\n try {\n json = await this.#requestPosts(postsAPIURL, signal, resolve, resolve);\n }\n catch (error) {\n return;\n }\n\n // Step 2: parse, download and, if targeting posts by user, repeat for next batch\n const postsParser = new PostParser(this.logger);\n let total: number | null = null;\n let downloaded = 0;\n let skippedUnviewable = 0;\n let skippedRedundant = 0;\n let campaignSaved = false;\n while (json) {\n const collection = postsParser.parsePostsAPIResponse(json, postsAPIURL);\n\n if (!campaignSaved && collection.posts[0]?.campaign) {\n await this.saveCampaignInfo(collection.posts[0].campaign, signal);\n campaignSaved = true;\n if (this.checkAbortSignal(signal, resolve)) {\n return;\n }\n }\n\n total = collection.total;\n if (postFetch.type === 'byUser' || postFetch.type === 'byCollection') {\n this.log('debug', `${collection.posts.length} posts fetched`);\n }\n\n for (const post of collection.posts) {\n\n this.emit('targetBegin', { target: post });\n\n // Step 4.1: post directories\n const postDirs = FSHelper.getPostDirs(post, this.config);\n this.log('debug', 'Post directories:', postDirs);\n\n // Step 4.2: Check with status cache\n const statusCache = StatusCache.getInstance(postDirs.statusCache, this.logger, this.config.useStatusCache);\n if (statusCache.validate(post, postDirs.root, this.config)) {\n this.log('info', `Skipped downloading post #${post.id}: already downloaded and nothing has changed since last download`);\n this.emit('targetEnd', {\n target: post,\n isSkipped: true,\n skipReason: TargetSkipReason.AlreadyDownloaded,\n skipMessage: 'Target already downloaded and nothing has changed since last download'\n });\n skippedRedundant++;\n continue;\n }\n\n // Step 4.3: Check viewability\n this.log('info', `Download post #${post.id} (${post.title})`);\n if (!post.isViewable) {\n if (this.config.include.lockedContent) {\n this.log('warn', `Post #${post.id} is not viewable by current user`);\n }\n else {\n this.log('warn', `Skipped downloading post #${post.id}: not viewable by current user`);\n this.emit('targetEnd', {\n target: post,\n isSkipped: true,\n skipReason: TargetSkipReason.Inaccessible,\n skipMessage: 'Target is not viewable by current user'\n });\n skippedUnviewable++;\n continue;\n }\n }\n\n // Step 4.4: Save post info\n if (this.config.include.contentInfo) {\n this.log('info', `Save post info #${post.id}`);\n this.emit('phaseBegin', { target: post, phase: 'saveInfo' });\n FSHelper.createDir(postDirs.info);\n // Post raw data might not be complete or consistent with other posts in the collection.\n // Fetch directly from API.\n // Strictly speaking, we should check for 'error' in results, but since it's not going to be fatal we'll just skip it.\n const { json: fetchedPostAPIData } = await this.commonFetchAPI(\n URLHelper.constructPostsAPIURL({\n postId: post.id,\n campaignId: post.campaign?.id\n }),\n signal\n );\n\n if (this.checkAbortSignal(signal, resolve)) {\n return;\n }\n\n // Save summary and raw json\n const summary = generatePostSummary(post);\n const summaryFile = path.resolve(postDirs.info, 'info.txt');\n const saveSummaryResult = await FSHelper.writeTextFile(summaryFile, summary, this.config.fileExistsAction.info);\n this.logWriteTextFileResult(saveSummaryResult, post, 'post summary');\n\n const postRawFile = path.resolve(postDirs.info, 'post-api.json');\n const savePostRawResult = await FSHelper.writeTextFile(\n postRawFile, fetchedPostAPIData || post.raw, this.config.fileExistsAction.infoAPI);\n this.logWriteTextFileResult(savePostRawResult, post, 'post API data');\n this.emit('phaseEnd', { target: post, phase: 'saveInfo' });\n\n // (Downloading of info media items deferred to the next step)\n }\n\n if (this.config.include.previewMedia || this.config.include.contentMedia) {\n this.emit('phaseBegin', { target: post, phase: 'saveMedia' });\n }\n\n // Step 4.5: save embed info\n if (post.embed && this.config.include.contentMedia) {\n this.log('info', `Save embed info of post #${post.id}`);\n FSHelper.createDir(postDirs.embed);\n const embedSummary = generatePostEmbedSummary(post.embed);\n let embedFilename;\n switch (post.embed.type) {\n case 'video':\n embedFilename = 'embedded-video.txt';\n break;\n case 'link':\n embedFilename = 'embedded-link.txt';\n break;\n default:\n embedFilename = 'embedded-unknown.txt';\n }\n const embedFile = path.resolve(postDirs.embed, embedFilename);\n const saveSummaryResult = await FSHelper.writeTextFile(embedFile, embedSummary, this.config.fileExistsAction.content);\n this.logWriteTextFileResult(saveSummaryResult, post, 'embed info');\n }\n\n // Step 4.6: create download tasks\n if (this.config.include.previewMedia ||\n this.config.include.contentMedia ||\n this.config.include.contentInfo) {\n\n batch = this.#createDownloadTaskBatchForPost(post, postDirs);\n\n if (this.config.include.contentInfo) {\n const infoElements: Downloadable[] = [];\n if (post.coverImage) {\n infoElements.push(post.coverImage);\n }\n if (post.thumbnail) {\n infoElements.push(post.thumbnail);\n }\n if (infoElements.length > 0) {\n this.addToDownloadTaskBatch(batch,\n {\n target: infoElements,\n targetName: `post #${post.id} -> info elements`,\n destDir: postDirs.info,\n fileExistsAction: this.config.fileExistsAction.info\n }\n );\n }\n }\n\n this.log('info', `Download batch created (#${batch.id}): ${batch.getTasks('pending').length} downloads pending`);\n this.emit('phaseBegin', { target: post, phase: 'batchDownload', batch });\n\n await batch.start();\n\n // Step 4.7: Update status cache\n statusCache.updateOnDownload(post, postDirs.root, batch.getTasks('error').length > 0, this.config);\n\n await batch.destroy();\n batch = null;\n this.emit('phaseEnd', { target: post, phase: 'batchDownload'});\n }\n\n if (this.config.include.previewMedia || this.config.include.contentMedia) {\n this.emit('phaseEnd', { target: post, phase: 'saveMedia' });\n }\n\n downloaded++;\n this.emit('targetEnd', { target: post, isSkipped: false });\n\n if (this.checkAbortSignal(signal, resolve)) {\n return;\n }\n }\n\n if (postFetch.type === 'byUser' || postFetch.type === 'byCollection') {\n if (collection.nextURL) {\n this.log('info', 'Fetch more posts');\n this.log('debug', `Request next batch of posts from API URL \"${collection.nextURL}`);\n this.emit('fetchBegin', { targetType: 'posts' });\n try {\n json = await this.#requestPosts(collection.nextURL, signal, resolve, resolve);\n }\n catch (error) {\n return;\n }\n }\n else {\n this.log('debug', 'No further posts to fetch');\n json = null;\n }\n }\n else {\n json = null;\n }\n }\n\n if (this.checkAbortSignal(signal, resolve)) {\n return;\n }\n\n // Done\n if (signal) {\n signal.removeEventListener('abort', abortHandler);\n }\n if (postFetch.type === 'byUser') {\n this.log('info', `Done downloading posts by '${postFetch.vanity}'`);\n }\n else if (postFetch.type === 'byCollection') {\n this.log('info', `Done downloading posts in collection #${postFetch.collectionId}`);\n }\n else {\n this.log('info', `Done downloading post #${postFetch.postId}`);\n }\n if (postFetch.type === 'byUser' || postFetch.type === 'byCollection') {\n const skippedStrParts: string[] = [];\n if (skippedUnviewable) {\n skippedStrParts.push(`${skippedUnviewable} unviewable`);\n }\n if (skippedRedundant) {\n skippedStrParts.push(`${skippedRedundant} redundant`);\n }\n const skippedStr = skippedStrParts.length > 0 ? ` (skipped: ${skippedStrParts.join(', ')})` : '';\n this.log('info', `Total ${downloaded} / ${total} posts processed${skippedStr}`);\n }\n this.emit('end', { aborted: false });\n this.#startPromise = null;\n resolve();\n })\n .finally(async () => {\n if (this.logger) {\n await this.logger.end();\n }\n this.#startPromise = null;\n });\n\n return this.#startPromise;\n }\n\n async #getInitialPostsAPIUL(signal: AbortSignal | undefined, resolveOnAbort: () => void, resolveOnError: () => void) {\n const postFetch = this.config.postFetch;\n if (postFetch.type === 'byUser' || postFetch.type === 'byCollection') {\n // Step 1: get initial page data\n const pageURL =\n postFetch.type === 'byUser' ?\n URLHelper.constructCampaignPageURL(postFetch.vanity) :\n URLHelper.constructCollectionURL(postFetch.collectionId);\n this.log('debug', `Fetch initial data from URL \"${pageURL}\"`);\n let page, requestPageError;\n try {\n page = await this.fetcher.get({ url: pageURL, type: 'html', maxRetries: this.config.request.maxRetries, signal });\n }\n catch (error) {\n if (error instanceof AbortError) {\n this.log('warn', 'Page request aborted');\n }\n else {\n requestPageError = error;\n }\n page = null;\n }\n if (!page) {\n if (this.checkAbortSignal(signal, resolveOnAbort)) {\n throw Error();\n }\n this.log('error', `Error requesting page \"${pageURL}\": `, requestPageError);\n this.emit('end', { aborted: false, error: requestPageError });\n resolveOnError();\n throw Error();\n }\n const pageParser = new PageParser(this.logger);\n let initialData, parseInitialDataError;\n try {\n initialData = pageParser.parseInitialData(page, pageURL);\n }\n catch (error) {\n parseInitialDataError = error;\n initialData = null;\n }\n if (!initialData) {\n this.log('error', `Failed to obtain initial page data from \"${pageURL}\":`, parseInitialDataError);\n this.emit('end', { aborted: false, error: parseInitialDataError });\n resolveOnError();\n throw Error();\n }\n\n /**\n * Step 2: obtain campaign ID and current user ID (i.e. you, if available in session identified by cookie)\n * from initial data.\n */\n const campaignId = ObjectHelper.getProperty(initialData, 'bootstrap.campaign.data.id');\n const currentUserId = ObjectHelper.getProperty(initialData, 'bootstrap.currentUser.data.id');\n this.log('debug', `Initial data: campaign ID '${campaignId}'; current user ID '${currentUserId}'`);\n if (!campaignId) {\n const err = Error(`Campaign ID not found in initial data of \"${pageURL}\"`);\n this.log('error', err);\n this.emit('end', { aborted: false, error: err });\n resolveOnError();\n throw Error();\n }\n\n let sort: PostSortOrder | undefined;\n if (postFetch.type === 'byCollection') {\n sort = PostSortOrder.CollectionOrder;\n }\n\n return URLHelper.constructPostsAPIURL({\n campaignId,\n currentUserId: this.config.include.lockedContent ? undefined : currentUserId,\n filters: postFetch.filters,\n sort\n });\n }\n\n return URLHelper.constructPostsAPIURL({ postId: postFetch.postId });\n\n }\n\n async #requestPosts(url: string, signal: AbortSignal | undefined, resolveOnAbort: () => void, resolveOnError: () => void) {\n const { json, error: requestAPIError } = await this.commonFetchAPI(url, signal);\n if (!json) {\n if (this.checkAbortSignal(signal, resolveOnAbort)) {\n throw Error();\n }\n this.log('error', 'Failed to fetch posts');\n this.log('warn', 'End with error');\n this.emit('end', { aborted: false, error: requestAPIError });\n resolveOnError();\n throw Error();\n }\n return json;\n }\n\n #createDownloadTaskBatchForPost(post: Post, postDirs: ReturnType<typeof FSHelper['getPostDirs']>) {\n\n const incPreview = this.config.include.previewMedia;\n const incContent = this.config.include.contentMedia;\n\n const batch = this.createDownloadTaskBatch(\n `Post #${post.id} (${post.title})`,\n\n incPreview && post.audioPreview ? {\n target: [ post.audioPreview ],\n targetName: `post #${post.id} -> audio preview`,\n destDir: postDirs.audioPreview,\n fileExistsAction: this.config.fileExistsAction.content\n } : null,\n\n incPreview && post.videoPreview ? {\n target: [ post.videoPreview ],\n targetName: `post #${post.id} -> video preview`,\n destDir: postDirs.videoPreview,\n fileExistsAction: this.config.fileExistsAction.content\n } : null,\n\n /**\n * If post is not viewable by current user, its images will be\n * blurry and we should categorize them as image previews.\n */\n incPreview && post.images.length > 0 && !post.isViewable ? {\n target: post.images,\n targetName: `post #${post.id} -> image previews`,\n destDir: postDirs.imagePreviews,\n fileExistsAction: this.config.fileExistsAction.content\n } : null,\n\n incContent && post.audio ? {\n target: [ post.audio ],\n targetName: `post #${post.id} -> audio`,\n destDir: postDirs.audio,\n fileExistsAction: this.config.fileExistsAction.content\n } : null,\n\n incContent && post.video ? {\n target: [ post.video ],\n targetName: `post #${post.id} -> video`,\n destDir: postDirs.video,\n fileExistsAction: this.config.fileExistsAction.content\n } : null,\n\n incContent && post.images.length > 0 && post.isViewable ? {\n target: post.images,\n targetName: `post #${post.id} -> images`,\n destDir: postDirs.images,\n fileExistsAction: this.config.fileExistsAction.content\n } : null,\n\n incContent && post.attachments.length > 0 ? {\n target: post.attachments,\n targetName: `post #${post.id} -> attachments`,\n destDir: postDirs.attachments,\n fileExistsAction: this.config.fileExistsAction.content\n } : null\n );\n\n return batch;\n }\n}\n"]}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import Downloader, { DownloaderStartParams } from './Downloader.js';
|
|
2
|
+
export default class ProductDownloader extends Downloader<'product'> {
|
|
3
|
+
#private;
|
|
4
|
+
static version: string;
|
|
5
|
+
name: string;
|
|
6
|
+
start(params?: DownloaderStartParams): Promise<void>;
|
|
7
|
+
}
|
|
8
|
+
//# sourceMappingURL=ProductDownloader.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ProductDownloader.d.ts","sourceRoot":"","sources":["../../src/downloaders/ProductDownloader.ts"],"names":[],"mappings":"AAGA,OAAO,UAAU,EAAE,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AAOpE,MAAM,CAAC,OAAO,OAAO,iBAAkB,SAAQ,UAAU,CAAC,SAAS,CAAC;;IAElE,MAAM,CAAC,OAAO,SAAW;IAEzB,IAAI,SAAuB;IAI3B,KAAK,CAAC,MAAM,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;CA8KrD"}
|