tt-help-cli-ycl 1.3.11 → 1.3.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/README.md +17 -17
  2. package/cli.js +9 -9
  3. package/package.json +45 -46
  4. package/{bat → scripts}/run-explore.bat +68 -68
  5. package/{bat → scripts}/run-explore.ps1 +81 -81
  6. package/{bat → scripts}/run-explore.sh +73 -73
  7. package/scripts/test-captcha-lib.mjs +68 -0
  8. package/scripts/test-captcha.mjs +81 -0
  9. package/scripts/test-incognito-lib.mjs +36 -0
  10. package/scripts/test-login-state.mjs +128 -0
  11. package/scripts/test-safe-click.mjs +45 -0
  12. package/src/cli/auto.js +186 -157
  13. package/src/cli/config.js +116 -0
  14. package/src/cli/explore-default.js +83 -0
  15. package/src/cli/explore.js +227 -181
  16. package/src/cli/progress.js +111 -111
  17. package/src/cli/refresh.js +216 -0
  18. package/src/cli/scrape.js +47 -47
  19. package/src/cli/utils.js +18 -18
  20. package/src/cli/videos.js +41 -41
  21. package/src/cli/watch.js +31 -31
  22. package/src/lib/args.js +456 -391
  23. package/src/lib/browser/anti-detect.js +23 -23
  24. package/src/lib/browser/cdp.js +194 -142
  25. package/src/lib/browser/launch.js +43 -43
  26. package/src/lib/browser/page.js +146 -87
  27. package/src/lib/constants.js +119 -119
  28. package/src/lib/delay.js +54 -54
  29. package/src/lib/explore-fetch.js +118 -118
  30. package/src/lib/fetcher.js +45 -45
  31. package/src/lib/filter.js +66 -66
  32. package/src/lib/io.js +54 -54
  33. package/src/lib/output.js +80 -80
  34. package/src/{scraper/modules/page-error-detector.mjs → lib/page-error-detector.js} +70 -70
  35. package/src/lib/parser.js +47 -47
  36. package/src/lib/retry.js +45 -45
  37. package/src/lib/scrape.js +40 -40
  38. package/src/{scraper/modules/scroll-collector.mjs → lib/scroll-collector.js} +231 -189
  39. package/src/lib/url.js +52 -52
  40. package/src/main.js +48 -0
  41. package/src/results/user-videos-bar.lar.lar.moeta.json +37 -0
  42. package/src/scraper/{auto-core.mjs → auto-core.js} +203 -194
  43. package/src/scraper/{core.mjs → core.js} +211 -190
  44. package/src/scraper/{explore-core.mjs → explore-core.js} +180 -171
  45. package/src/scraper/modules/{captcha-handler.mjs → captcha-handler.js} +114 -114
  46. package/src/scraper/modules/{comment-extractor.mjs → comment-extractor.js} +74 -69
  47. package/src/scraper/modules/{follow-extractor.mjs → follow-extractor.js} +121 -121
  48. package/src/scraper/modules/{guess-extractor.mjs → guess-extractor.js} +51 -51
  49. package/src/scraper/modules/page-error-detector.js +1 -0
  50. package/src/scraper/modules/{page-helpers.mjs → page-helpers.js} +48 -48
  51. package/src/scraper/modules/scroll-collector.js +8 -0
  52. package/src/scraper/refresh-core.js +179 -0
  53. package/src/videos/{core.mjs → core.js} +126 -126
  54. package/src/watch/data-store.js +431 -0
  55. package/src/watch/public/index.html +721 -690
  56. package/src/watch/{server.mjs → server.js} +484 -349
  57. package/src/main.mjs +0 -234
  58. package/src/test-auto-follow.cjs +0 -109
  59. package/src/test-extractors.cjs +0 -75
  60. package/src/test-follow.cjs +0 -41
  61. package/src/watch/data-store.mjs +0 -274
@@ -1,274 +0,0 @@
1
- import fs from 'fs';
2
- import path from 'path';
3
-
4
- function inferStatus(u) {
5
- if (u.restricted) return 'restricted';
6
- if (u.error) return 'error';
7
- if (u.processed) return 'done';
8
- return 'pending';
9
- }
10
-
11
- export function createStore(filePath) {
12
- let data = [];
13
- let clientErrors = new Map();
14
-
15
- let backupTimer = null;
16
-
17
- if (filePath) {
18
- const resolved = path.resolve(filePath);
19
- const backupDir = path.join(path.dirname(resolved), '.backup');
20
- const maxBackups = 3;
21
-
22
- if (fs.existsSync(resolved)) {
23
- try {
24
- const content = fs.readFileSync(resolved, 'utf-8');
25
- data = JSON.parse(content);
26
- if (!Array.isArray(data)) {
27
- data = [];
28
- }
29
- } catch (e) {
30
- console.error(`[data-store] 读取文件失败: ${e.message}`);
31
- data = [];
32
- }
33
- }
34
-
35
- function runBackup() {
36
- if (!fs.existsSync(resolved)) return;
37
- if (!fs.existsSync(backupDir)) fs.mkdirSync(backupDir, { recursive: true });
38
- const now = new Date();
39
- const timestamp = now.toISOString().replace(/[:.]/g, '-').slice(0, 13);
40
- const backupFile = path.join(backupDir, `data-${timestamp}.json`);
41
- try {
42
- fs.copyFileSync(resolved, backupFile);
43
- const files = fs.readdirSync(backupDir)
44
- .filter(f => f.startsWith('data-') && f.endsWith('.json'))
45
- .sort()
46
- .map(f => path.join(backupDir, f));
47
- while (files.length > maxBackups) {
48
- fs.unlinkSync(files.shift());
49
- }
50
- } catch (e) {
51
- console.error(`[data-store] 备份失败: ${e.message}`);
52
- }
53
- }
54
-
55
- backupTimer = setInterval(runBackup, 60 * 60 * 1000);
56
- }
57
-
58
- for (const u of data) {
59
- if (!u.status) u.status = inferStatus(u);
60
- }
61
-
62
- function save() {
63
- if (!filePath) return;
64
- const resolved = path.resolve(filePath);
65
- const json = JSON.stringify(data, null, 2);
66
- fs.writeFileSync(resolved, json, 'utf-8');
67
- }
68
-
69
- function stopBackup() {
70
- if (backupTimer) {
71
- clearInterval(backupTimer);
72
- backupTimer = null;
73
- }
74
- }
75
-
76
- function getUser(uid) {
77
- return data.find(u => u.uniqueId === uid);
78
- }
79
-
80
- function hasUser(uid) {
81
- return getUser(uid) !== undefined;
82
- }
83
-
84
- function addUser(user, append) {
85
- const existing = getUser(user.uniqueId);
86
- if (existing) {
87
- for (const key of Object.keys(user)) {
88
- if (key === 'uniqueId' || key === 'sources') continue;
89
- if (user[key] !== undefined && user[key] !== null && user[key] !== '') {
90
- existing[key] = user[key];
91
- }
92
- }
93
- } else {
94
- if (!user.status) user.status = inferStatus(user);
95
- if (user.processed) user.processedAt = user.processedAt || Date.now();
96
- if (append) data.push(user);
97
- else data.unshift(user);
98
- }
99
- }
100
-
101
- function getPendingUsers() {
102
- return data.filter(u => u.status === 'pending');
103
- }
104
-
105
- function getProcessedUsers() {
106
- return data.filter(u => u.status === 'done');
107
- }
108
-
109
- function getAllUsers() {
110
- return data;
111
- }
112
-
113
- function claimNextJob(userId, expireMs = 5 * 60 * 1000) {
114
- let next = data.find(u => u.status === 'pending' && u.pinned);
115
-
116
- if (!next) {
117
- const now = Date.now();
118
- const expired = data.find(u =>
119
- u.status === 'processing' && u.claimedAt && (now - u.claimedAt) > expireMs
120
- );
121
- if (expired) {
122
- expired.status = 'pending';
123
- delete expired.claimedAt;
124
- next = expired;
125
- }
126
- }
127
-
128
- if (!next) {
129
- next = data.find(u => u.status === 'pending' && u.sources && u.sources.includes('seed'));
130
- }
131
-
132
- if (!next) {
133
- next = data.find(u => u.status === 'pending');
134
- }
135
-
136
- if (next) {
137
- next.status = 'processing';
138
- next.claimedAt = Date.now();
139
- next.claimedBy = userId;
140
- return { uniqueId: next.uniqueId, nickname: next.nickname, claimedAt: next.claimedAt, claimedBy: userId };
141
- }
142
- return null;
143
- }
144
-
145
- function commitJob(uniqueId, result) {
146
- const user = getUser(uniqueId);
147
- if (!user) return { saved: false, error: 'user not found' };
148
-
149
- if (result.restricted) {
150
- user.status = 'restricted';
151
- if (result.userInfo) {
152
- const info = result.userInfo;
153
- for (const key of Object.keys(info)) {
154
- if (key === 'uniqueId' || key === 'sources') continue;
155
- if (info[key] !== undefined && info[key] !== null && info[key] !== '') {
156
- user[key] = info[key];
157
- }
158
- }
159
- }
160
- user.processed = true;
161
- user.processedAt = Date.now();
162
- user.noVideo = true;
163
- user.sources = [...new Set([...(user.sources || []), 'restricted'])];
164
- } else if (result.error) {
165
- user.status = 'error';
166
- user.error = result.error;
167
- user.sources = [...new Set([...(user.sources || []), 'error'])];
168
- } else {
169
- user.status = 'done';
170
- user.processed = true;
171
- user.processedAt = Date.now();
172
- user.followerCount = result.userInfo?.followerCount ?? user.followerCount;
173
- user.videoCount = result.userInfo?.videoCount ?? user.videoCount;
174
- user.nickname = result.userInfo?.nickname || user.nickname;
175
- user.locationCreated = result.userInfo?.locationCreated || user.locationCreated;
176
- user.ttSeller = result.userInfo?.ttSeller ?? user.ttSeller;
177
- user.verified = result.userInfo?.verified ?? user.verified;
178
- user.region = result.userInfo?.region || user.region;
179
- user.signature = result.userInfo?.signature ?? user.signature;
180
- user.followingCount = result.userInfo?.followingCount ?? user.followingCount;
181
- user.heartCount = result.userInfo?.heartCount ?? user.heartCount;
182
- if (result.userInfo?.secUid) user.secUid = result.userInfo.secUid;
183
- const extraFields = ['restricted', 'error', 'userInfo', 'discoveredVideoAuthors',
184
- 'discoveredCommentAuthors', 'discoveredGuessAuthors', 'discoveredFollowing',
185
- 'discoveredFollowers', 'uniqueId', 'sources'];
186
- for (const key of Object.keys(result)) {
187
- if (extraFields.includes(key)) continue;
188
- if (result[key] !== undefined && result[key] !== null && result[key] !== '') {
189
- user[key] = result[key];
190
- }
191
- }
192
- user.sources = [...new Set([...(user.sources || []), 'processed'])];
193
-
194
- const discovered = [
195
- ...(result.discoveredVideoAuthors || []).map(v => ({
196
- uniqueId: v.uniqueId, nickname: v.nickname, locationCreated: v.locationCreated,
197
- sources: ['video']
198
- })),
199
- ...(result.discoveredCommentAuthors || []).map(c => {
200
- const id = typeof c === 'string' ? c.replace(/^@/, '') : c.uniqueId;
201
- return typeof c === 'string'
202
- ? { uniqueId: id, sources: ['comment'] }
203
- : { uniqueId: id, ...c, sources: [...new Set([...(c.sources || []), 'comment'])] };
204
- }),
205
- ...(result.discoveredGuessAuthors || []).map(g => {
206
- const id = typeof g === 'string' ? g.replace(/^@/, '') : g.uniqueId;
207
- return typeof g === 'string'
208
- ? { uniqueId: id, sources: ['guess'] }
209
- : { uniqueId: id, ...g, sources: [...new Set([...(g.sources || []), 'guess'])] };
210
- }),
211
- ...(result.discoveredFollowing || []).map(([handle, name]) => ({
212
- uniqueId: handle.replace(/^@/, ''), nickname: name, sources: ['following']
213
- })),
214
- ...(result.discoveredFollowers || []).map(([handle, name]) => ({
215
- uniqueId: handle.replace(/^@/, ''), nickname: name, sources: ['follower']
216
- })),
217
- ];
218
-
219
- for (const du of discovered) {
220
- if (!du.uniqueId) continue;
221
- addUser(du, true);
222
- }
223
- }
224
-
225
- delete user.claimedAt;
226
- save();
227
- return { saved: true };
228
- }
229
-
230
- function resetJob(uniqueId) {
231
- const user = getUser(uniqueId);
232
- if (!user) return { saved: false, error: 'user not found' };
233
- user.status = 'pending';
234
- delete user.claimedAt;
235
- delete user.processedAt;
236
- delete user.processed;
237
- delete user.error;
238
- delete user.restricted;
239
- delete user.noVideo;
240
- save();
241
- return { saved: true };
242
- }
243
-
244
- function togglePin(uniqueId) {
245
- const user = getUser(uniqueId);
246
- if (!user) return { saved: false, error: 'user not found' };
247
- user.pinned = !user.pinned;
248
- save();
249
- return { saved: true, pinned: user.pinned };
250
- }
251
-
252
- function reportClientError(userId, errorType, errorMessage, username) {
253
- clientErrors.set(userId, {
254
- userId,
255
- errorType,
256
- errorMessage,
257
- username,
258
- timestamp: Date.now(),
259
- });
260
- }
261
-
262
- function getClientErrors() {
263
- return Array.from(clientErrors.values());
264
- }
265
-
266
- return {
267
- save, getUser, hasUser, addUser,
268
- getPendingUsers, getProcessedUsers, getAllUsers,
269
- claimNextJob, commitJob, resetJob, togglePin,
270
- reportClientError, getClientErrors,
271
- stopBackup,
272
- data,
273
- };
274
- }