tt-help-cli-ycl 1.3.11 → 1.3.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -17
- package/cli.js +9 -9
- package/package.json +45 -46
- package/{bat → scripts}/run-explore.bat +68 -68
- package/{bat → scripts}/run-explore.ps1 +81 -81
- package/{bat → scripts}/run-explore.sh +73 -73
- package/scripts/test-captcha-lib.mjs +68 -0
- package/scripts/test-captcha.mjs +81 -0
- package/scripts/test-incognito-lib.mjs +36 -0
- package/scripts/test-login-state.mjs +128 -0
- package/scripts/test-safe-click.mjs +45 -0
- package/src/cli/auto.js +186 -157
- package/src/cli/config.js +116 -0
- package/src/cli/explore-default.js +83 -0
- package/src/cli/explore.js +227 -181
- package/src/cli/progress.js +111 -111
- package/src/cli/refresh.js +216 -0
- package/src/cli/scrape.js +47 -47
- package/src/cli/utils.js +18 -18
- package/src/cli/videos.js +41 -41
- package/src/cli/watch.js +31 -31
- package/src/lib/args.js +456 -391
- package/src/lib/browser/anti-detect.js +23 -23
- package/src/lib/browser/cdp.js +194 -142
- package/src/lib/browser/launch.js +43 -43
- package/src/lib/browser/page.js +146 -87
- package/src/lib/constants.js +119 -119
- package/src/lib/delay.js +54 -54
- package/src/lib/explore-fetch.js +118 -118
- package/src/lib/fetcher.js +45 -45
- package/src/lib/filter.js +66 -66
- package/src/lib/io.js +54 -54
- package/src/lib/output.js +80 -80
- package/src/{scraper/modules/page-error-detector.mjs → lib/page-error-detector.js} +70 -70
- package/src/lib/parser.js +47 -47
- package/src/lib/retry.js +45 -45
- package/src/lib/scrape.js +40 -40
- package/src/{scraper/modules/scroll-collector.mjs → lib/scroll-collector.js} +231 -189
- package/src/lib/url.js +52 -52
- package/src/main.js +48 -0
- package/src/results/user-videos-bar.lar.lar.moeta.json +37 -0
- package/src/scraper/{auto-core.mjs → auto-core.js} +203 -194
- package/src/scraper/{core.mjs → core.js} +211 -190
- package/src/scraper/{explore-core.mjs → explore-core.js} +180 -171
- package/src/scraper/modules/{captcha-handler.mjs → captcha-handler.js} +114 -114
- package/src/scraper/modules/{comment-extractor.mjs → comment-extractor.js} +74 -69
- package/src/scraper/modules/{follow-extractor.mjs → follow-extractor.js} +121 -121
- package/src/scraper/modules/{guess-extractor.mjs → guess-extractor.js} +51 -51
- package/src/scraper/modules/page-error-detector.js +1 -0
- package/src/scraper/modules/{page-helpers.mjs → page-helpers.js} +48 -48
- package/src/scraper/modules/scroll-collector.js +8 -0
- package/src/scraper/refresh-core.js +179 -0
- package/src/videos/{core.mjs → core.js} +126 -126
- package/src/watch/data-store.js +431 -0
- package/src/watch/public/index.html +721 -690
- package/src/watch/{server.mjs → server.js} +484 -349
- package/src/main.mjs +0 -234
- package/src/test-auto-follow.cjs +0 -109
- package/src/test-extractors.cjs +0 -75
- package/src/test-follow.cjs +0 -41
- package/src/watch/data-store.mjs +0 -274
package/src/watch/data-store.mjs
DELETED
|
@@ -1,274 +0,0 @@
|
|
|
1
|
-
import fs from 'fs';
|
|
2
|
-
import path from 'path';
|
|
3
|
-
|
|
4
|
-
function inferStatus(u) {
|
|
5
|
-
if (u.restricted) return 'restricted';
|
|
6
|
-
if (u.error) return 'error';
|
|
7
|
-
if (u.processed) return 'done';
|
|
8
|
-
return 'pending';
|
|
9
|
-
}
|
|
10
|
-
|
|
11
|
-
export function createStore(filePath) {
|
|
12
|
-
let data = [];
|
|
13
|
-
let clientErrors = new Map();
|
|
14
|
-
|
|
15
|
-
let backupTimer = null;
|
|
16
|
-
|
|
17
|
-
if (filePath) {
|
|
18
|
-
const resolved = path.resolve(filePath);
|
|
19
|
-
const backupDir = path.join(path.dirname(resolved), '.backup');
|
|
20
|
-
const maxBackups = 3;
|
|
21
|
-
|
|
22
|
-
if (fs.existsSync(resolved)) {
|
|
23
|
-
try {
|
|
24
|
-
const content = fs.readFileSync(resolved, 'utf-8');
|
|
25
|
-
data = JSON.parse(content);
|
|
26
|
-
if (!Array.isArray(data)) {
|
|
27
|
-
data = [];
|
|
28
|
-
}
|
|
29
|
-
} catch (e) {
|
|
30
|
-
console.error(`[data-store] 读取文件失败: ${e.message}`);
|
|
31
|
-
data = [];
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
function runBackup() {
|
|
36
|
-
if (!fs.existsSync(resolved)) return;
|
|
37
|
-
if (!fs.existsSync(backupDir)) fs.mkdirSync(backupDir, { recursive: true });
|
|
38
|
-
const now = new Date();
|
|
39
|
-
const timestamp = now.toISOString().replace(/[:.]/g, '-').slice(0, 13);
|
|
40
|
-
const backupFile = path.join(backupDir, `data-${timestamp}.json`);
|
|
41
|
-
try {
|
|
42
|
-
fs.copyFileSync(resolved, backupFile);
|
|
43
|
-
const files = fs.readdirSync(backupDir)
|
|
44
|
-
.filter(f => f.startsWith('data-') && f.endsWith('.json'))
|
|
45
|
-
.sort()
|
|
46
|
-
.map(f => path.join(backupDir, f));
|
|
47
|
-
while (files.length > maxBackups) {
|
|
48
|
-
fs.unlinkSync(files.shift());
|
|
49
|
-
}
|
|
50
|
-
} catch (e) {
|
|
51
|
-
console.error(`[data-store] 备份失败: ${e.message}`);
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
backupTimer = setInterval(runBackup, 60 * 60 * 1000);
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
for (const u of data) {
|
|
59
|
-
if (!u.status) u.status = inferStatus(u);
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
function save() {
|
|
63
|
-
if (!filePath) return;
|
|
64
|
-
const resolved = path.resolve(filePath);
|
|
65
|
-
const json = JSON.stringify(data, null, 2);
|
|
66
|
-
fs.writeFileSync(resolved, json, 'utf-8');
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
function stopBackup() {
|
|
70
|
-
if (backupTimer) {
|
|
71
|
-
clearInterval(backupTimer);
|
|
72
|
-
backupTimer = null;
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
function getUser(uid) {
|
|
77
|
-
return data.find(u => u.uniqueId === uid);
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
function hasUser(uid) {
|
|
81
|
-
return getUser(uid) !== undefined;
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
function addUser(user, append) {
|
|
85
|
-
const existing = getUser(user.uniqueId);
|
|
86
|
-
if (existing) {
|
|
87
|
-
for (const key of Object.keys(user)) {
|
|
88
|
-
if (key === 'uniqueId' || key === 'sources') continue;
|
|
89
|
-
if (user[key] !== undefined && user[key] !== null && user[key] !== '') {
|
|
90
|
-
existing[key] = user[key];
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
} else {
|
|
94
|
-
if (!user.status) user.status = inferStatus(user);
|
|
95
|
-
if (user.processed) user.processedAt = user.processedAt || Date.now();
|
|
96
|
-
if (append) data.push(user);
|
|
97
|
-
else data.unshift(user);
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
function getPendingUsers() {
|
|
102
|
-
return data.filter(u => u.status === 'pending');
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
function getProcessedUsers() {
|
|
106
|
-
return data.filter(u => u.status === 'done');
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
function getAllUsers() {
|
|
110
|
-
return data;
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
function claimNextJob(userId, expireMs = 5 * 60 * 1000) {
|
|
114
|
-
let next = data.find(u => u.status === 'pending' && u.pinned);
|
|
115
|
-
|
|
116
|
-
if (!next) {
|
|
117
|
-
const now = Date.now();
|
|
118
|
-
const expired = data.find(u =>
|
|
119
|
-
u.status === 'processing' && u.claimedAt && (now - u.claimedAt) > expireMs
|
|
120
|
-
);
|
|
121
|
-
if (expired) {
|
|
122
|
-
expired.status = 'pending';
|
|
123
|
-
delete expired.claimedAt;
|
|
124
|
-
next = expired;
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
if (!next) {
|
|
129
|
-
next = data.find(u => u.status === 'pending' && u.sources && u.sources.includes('seed'));
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
if (!next) {
|
|
133
|
-
next = data.find(u => u.status === 'pending');
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
if (next) {
|
|
137
|
-
next.status = 'processing';
|
|
138
|
-
next.claimedAt = Date.now();
|
|
139
|
-
next.claimedBy = userId;
|
|
140
|
-
return { uniqueId: next.uniqueId, nickname: next.nickname, claimedAt: next.claimedAt, claimedBy: userId };
|
|
141
|
-
}
|
|
142
|
-
return null;
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
function commitJob(uniqueId, result) {
|
|
146
|
-
const user = getUser(uniqueId);
|
|
147
|
-
if (!user) return { saved: false, error: 'user not found' };
|
|
148
|
-
|
|
149
|
-
if (result.restricted) {
|
|
150
|
-
user.status = 'restricted';
|
|
151
|
-
if (result.userInfo) {
|
|
152
|
-
const info = result.userInfo;
|
|
153
|
-
for (const key of Object.keys(info)) {
|
|
154
|
-
if (key === 'uniqueId' || key === 'sources') continue;
|
|
155
|
-
if (info[key] !== undefined && info[key] !== null && info[key] !== '') {
|
|
156
|
-
user[key] = info[key];
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
user.processed = true;
|
|
161
|
-
user.processedAt = Date.now();
|
|
162
|
-
user.noVideo = true;
|
|
163
|
-
user.sources = [...new Set([...(user.sources || []), 'restricted'])];
|
|
164
|
-
} else if (result.error) {
|
|
165
|
-
user.status = 'error';
|
|
166
|
-
user.error = result.error;
|
|
167
|
-
user.sources = [...new Set([...(user.sources || []), 'error'])];
|
|
168
|
-
} else {
|
|
169
|
-
user.status = 'done';
|
|
170
|
-
user.processed = true;
|
|
171
|
-
user.processedAt = Date.now();
|
|
172
|
-
user.followerCount = result.userInfo?.followerCount ?? user.followerCount;
|
|
173
|
-
user.videoCount = result.userInfo?.videoCount ?? user.videoCount;
|
|
174
|
-
user.nickname = result.userInfo?.nickname || user.nickname;
|
|
175
|
-
user.locationCreated = result.userInfo?.locationCreated || user.locationCreated;
|
|
176
|
-
user.ttSeller = result.userInfo?.ttSeller ?? user.ttSeller;
|
|
177
|
-
user.verified = result.userInfo?.verified ?? user.verified;
|
|
178
|
-
user.region = result.userInfo?.region || user.region;
|
|
179
|
-
user.signature = result.userInfo?.signature ?? user.signature;
|
|
180
|
-
user.followingCount = result.userInfo?.followingCount ?? user.followingCount;
|
|
181
|
-
user.heartCount = result.userInfo?.heartCount ?? user.heartCount;
|
|
182
|
-
if (result.userInfo?.secUid) user.secUid = result.userInfo.secUid;
|
|
183
|
-
const extraFields = ['restricted', 'error', 'userInfo', 'discoveredVideoAuthors',
|
|
184
|
-
'discoveredCommentAuthors', 'discoveredGuessAuthors', 'discoveredFollowing',
|
|
185
|
-
'discoveredFollowers', 'uniqueId', 'sources'];
|
|
186
|
-
for (const key of Object.keys(result)) {
|
|
187
|
-
if (extraFields.includes(key)) continue;
|
|
188
|
-
if (result[key] !== undefined && result[key] !== null && result[key] !== '') {
|
|
189
|
-
user[key] = result[key];
|
|
190
|
-
}
|
|
191
|
-
}
|
|
192
|
-
user.sources = [...new Set([...(user.sources || []), 'processed'])];
|
|
193
|
-
|
|
194
|
-
const discovered = [
|
|
195
|
-
...(result.discoveredVideoAuthors || []).map(v => ({
|
|
196
|
-
uniqueId: v.uniqueId, nickname: v.nickname, locationCreated: v.locationCreated,
|
|
197
|
-
sources: ['video']
|
|
198
|
-
})),
|
|
199
|
-
...(result.discoveredCommentAuthors || []).map(c => {
|
|
200
|
-
const id = typeof c === 'string' ? c.replace(/^@/, '') : c.uniqueId;
|
|
201
|
-
return typeof c === 'string'
|
|
202
|
-
? { uniqueId: id, sources: ['comment'] }
|
|
203
|
-
: { uniqueId: id, ...c, sources: [...new Set([...(c.sources || []), 'comment'])] };
|
|
204
|
-
}),
|
|
205
|
-
...(result.discoveredGuessAuthors || []).map(g => {
|
|
206
|
-
const id = typeof g === 'string' ? g.replace(/^@/, '') : g.uniqueId;
|
|
207
|
-
return typeof g === 'string'
|
|
208
|
-
? { uniqueId: id, sources: ['guess'] }
|
|
209
|
-
: { uniqueId: id, ...g, sources: [...new Set([...(g.sources || []), 'guess'])] };
|
|
210
|
-
}),
|
|
211
|
-
...(result.discoveredFollowing || []).map(([handle, name]) => ({
|
|
212
|
-
uniqueId: handle.replace(/^@/, ''), nickname: name, sources: ['following']
|
|
213
|
-
})),
|
|
214
|
-
...(result.discoveredFollowers || []).map(([handle, name]) => ({
|
|
215
|
-
uniqueId: handle.replace(/^@/, ''), nickname: name, sources: ['follower']
|
|
216
|
-
})),
|
|
217
|
-
];
|
|
218
|
-
|
|
219
|
-
for (const du of discovered) {
|
|
220
|
-
if (!du.uniqueId) continue;
|
|
221
|
-
addUser(du, true);
|
|
222
|
-
}
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
delete user.claimedAt;
|
|
226
|
-
save();
|
|
227
|
-
return { saved: true };
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
function resetJob(uniqueId) {
|
|
231
|
-
const user = getUser(uniqueId);
|
|
232
|
-
if (!user) return { saved: false, error: 'user not found' };
|
|
233
|
-
user.status = 'pending';
|
|
234
|
-
delete user.claimedAt;
|
|
235
|
-
delete user.processedAt;
|
|
236
|
-
delete user.processed;
|
|
237
|
-
delete user.error;
|
|
238
|
-
delete user.restricted;
|
|
239
|
-
delete user.noVideo;
|
|
240
|
-
save();
|
|
241
|
-
return { saved: true };
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
function togglePin(uniqueId) {
|
|
245
|
-
const user = getUser(uniqueId);
|
|
246
|
-
if (!user) return { saved: false, error: 'user not found' };
|
|
247
|
-
user.pinned = !user.pinned;
|
|
248
|
-
save();
|
|
249
|
-
return { saved: true, pinned: user.pinned };
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
function reportClientError(userId, errorType, errorMessage, username) {
|
|
253
|
-
clientErrors.set(userId, {
|
|
254
|
-
userId,
|
|
255
|
-
errorType,
|
|
256
|
-
errorMessage,
|
|
257
|
-
username,
|
|
258
|
-
timestamp: Date.now(),
|
|
259
|
-
});
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
function getClientErrors() {
|
|
263
|
-
return Array.from(clientErrors.values());
|
|
264
|
-
}
|
|
265
|
-
|
|
266
|
-
return {
|
|
267
|
-
save, getUser, hasUser, addUser,
|
|
268
|
-
getPendingUsers, getProcessedUsers, getAllUsers,
|
|
269
|
-
claimNextJob, commitJob, resetJob, togglePin,
|
|
270
|
-
reportClientError, getClientErrors,
|
|
271
|
-
stopBackup,
|
|
272
|
-
data,
|
|
273
|
-
};
|
|
274
|
-
}
|