tt-help-cli-ycl 1.3.48 → 1.3.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -33
- package/cli.js +9 -9
- package/package.json +52 -52
- package/scripts/run-explore copy.bat +101 -101
- package/scripts/run-explore.bat +134 -134
- package/scripts/run-explore.ps1 +159 -159
- package/scripts/run-explore.sh +121 -121
- package/scripts/test-captcha-lib.mjs +68 -0
- package/scripts/test-captcha.mjs +81 -0
- package/scripts/test-incognito-lib.mjs +36 -0
- package/scripts/test-login-state.mjs +128 -0
- package/scripts/test-safe-click.mjs +45 -0
- package/scripts/test-watch-db-smoke.mjs +246 -0
- package/src/cli/attach.js +331 -331
- package/src/cli/auto.js +265 -265
- package/src/cli/comments.js +620 -620
- package/src/cli/config.js +170 -170
- package/src/cli/db-import.js +51 -51
- package/src/cli/explore.js +555 -555
- package/src/cli/open.js +109 -111
- package/src/cli/progress.js +111 -111
- package/src/cli/refresh.js +288 -288
- package/src/cli/scrape.js +47 -47
- package/src/cli/utils.js +18 -18
- package/src/cli/videos.js +41 -41
- package/src/cli/videostats.js +196 -196
- package/src/cli/watch.js +30 -30
- package/src/lib/api-interceptor.js +161 -161
- package/src/lib/args.js +809 -809
- package/src/lib/browser/anti-detect.js +23 -23
- package/src/lib/browser/cdp.js +261 -261
- package/src/lib/browser/health-checker.js +114 -114
- package/src/lib/browser/launch.js +43 -43
- package/src/lib/browser/page.js +184 -184
- package/src/lib/constants.js +297 -297
- package/src/lib/delay.js +54 -54
- package/src/lib/explore-fetch.js +118 -118
- package/src/lib/fetcher.js +45 -45
- package/src/lib/filter.js +66 -66
- package/src/lib/io.js +54 -54
- package/src/lib/output.js +80 -80
- package/src/lib/page-error-detector.js +109 -109
- package/src/lib/parse-ssr.mjs +69 -69
- package/src/lib/parser.js +47 -47
- package/src/lib/retry.js +45 -45
- package/src/lib/scrape.js +90 -90
- package/src/lib/target-locations.js +61 -61
- package/src/lib/tiktok-scraper.mjs +98 -61
- package/src/lib/url.js +52 -52
- package/src/main.js +73 -73
- package/src/npm-main.js +70 -70
- package/src/results/user-videos-bar.lar.lar.moeta.json +37 -0
- package/src/scraper/auto-core.js +203 -203
- package/src/scraper/core.js +255 -255
- package/src/scraper/explore-core.js +208 -208
- package/src/scraper/modules/captcha-handler.js +114 -114
- package/src/scraper/modules/follow-extractor.js +250 -250
- package/src/scraper/modules/guess-extractor.js +51 -51
- package/src/scraper/modules/page-helpers.js +48 -48
- package/src/scraper/refresh-core.js +213 -213
- package/src/videos/core.js +143 -143
- package/src/watch/data-store.js +2980 -2980
- package/src/watch/public/index.html +2355 -2355
- package/src/watch/server.js +727 -727
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
3
|
+
import os from "node:os";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
|
|
6
|
+
import { createStore, closeStoreDb } from "../src/watch/data-store.js";
|
|
7
|
+
import { startWatchServer } from "../src/watch/server.js";
|
|
8
|
+
|
|
9
|
+
function createTempDbPath(prefix) {
|
|
10
|
+
const dir = mkdtempSync(path.join(os.tmpdir(), `${prefix}-`));
|
|
11
|
+
return {
|
|
12
|
+
dir,
|
|
13
|
+
dbPath: path.join(dir, "result.db"),
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function cleanupTempDir(dir) {
|
|
18
|
+
rmSync(dir, { recursive: true, force: true });
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function seedClaimStore(store) {
|
|
22
|
+
store.addUser({
|
|
23
|
+
uniqueId: "follow-tier2",
|
|
24
|
+
status: "pending",
|
|
25
|
+
followerCount: 100,
|
|
26
|
+
videoCount: 1,
|
|
27
|
+
guessedLocation: "DE",
|
|
28
|
+
sources: ["following"],
|
|
29
|
+
});
|
|
30
|
+
store.addUser({
|
|
31
|
+
uniqueId: "seller-tier2",
|
|
32
|
+
status: "pending",
|
|
33
|
+
followerCount: 50,
|
|
34
|
+
videoCount: 1,
|
|
35
|
+
guessedLocation: "DE",
|
|
36
|
+
ttSeller: true,
|
|
37
|
+
verified: false,
|
|
38
|
+
sources: ["comment"],
|
|
39
|
+
});
|
|
40
|
+
store.addUser({
|
|
41
|
+
uniqueId: "seed-tier1",
|
|
42
|
+
status: "pending",
|
|
43
|
+
followerCount: 10,
|
|
44
|
+
videoCount: 1,
|
|
45
|
+
guessedLocation: "PL",
|
|
46
|
+
sources: ["seed"],
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function seedWatchStore(store) {
|
|
51
|
+
store.addUser({
|
|
52
|
+
uniqueId: "seed-tier1",
|
|
53
|
+
nickname: "Seed User",
|
|
54
|
+
status: "pending",
|
|
55
|
+
followerCount: 10,
|
|
56
|
+
videoCount: 1,
|
|
57
|
+
guessedLocation: "PL",
|
|
58
|
+
ttSeller: false,
|
|
59
|
+
verified: false,
|
|
60
|
+
sources: ["seed"],
|
|
61
|
+
});
|
|
62
|
+
store.addUser({
|
|
63
|
+
uniqueId: "seller-target",
|
|
64
|
+
nickname: "Seller Target",
|
|
65
|
+
status: "pending",
|
|
66
|
+
followerCount: 500,
|
|
67
|
+
videoCount: 1,
|
|
68
|
+
guessedLocation: "DE",
|
|
69
|
+
locationCreated: "DE",
|
|
70
|
+
ttSeller: true,
|
|
71
|
+
verified: false,
|
|
72
|
+
sources: ["comment"],
|
|
73
|
+
});
|
|
74
|
+
store.addUser({
|
|
75
|
+
uniqueId: "follow-tier2",
|
|
76
|
+
nickname: "Follow User",
|
|
77
|
+
status: "pending",
|
|
78
|
+
followerCount: 300,
|
|
79
|
+
videoCount: 1,
|
|
80
|
+
guessedLocation: "FR",
|
|
81
|
+
ttSeller: false,
|
|
82
|
+
verified: false,
|
|
83
|
+
sources: ["following"],
|
|
84
|
+
});
|
|
85
|
+
store.addUser({
|
|
86
|
+
uniqueId: "pending-update",
|
|
87
|
+
nickname: "Needs Update",
|
|
88
|
+
status: "pending",
|
|
89
|
+
guessedLocation: "US",
|
|
90
|
+
ttSeller: null,
|
|
91
|
+
verified: false,
|
|
92
|
+
userUpdateCount: 0,
|
|
93
|
+
sources: ["comment"],
|
|
94
|
+
});
|
|
95
|
+
store.addUser({
|
|
96
|
+
uniqueId: "done-es",
|
|
97
|
+
nickname: "Done ES",
|
|
98
|
+
status: "done",
|
|
99
|
+
followerCount: 800,
|
|
100
|
+
videoCount: 3,
|
|
101
|
+
guessedLocation: "ES",
|
|
102
|
+
locationCreated: "ES",
|
|
103
|
+
ttSeller: true,
|
|
104
|
+
verified: false,
|
|
105
|
+
processedAt: Date.now(),
|
|
106
|
+
processed: true,
|
|
107
|
+
sources: ["processed"],
|
|
108
|
+
});
|
|
109
|
+
store.addUser({
|
|
110
|
+
uniqueId: "restricted-it",
|
|
111
|
+
nickname: "Restricted IT",
|
|
112
|
+
status: "restricted",
|
|
113
|
+
locationCreated: "IT",
|
|
114
|
+
ttSeller: false,
|
|
115
|
+
verified: false,
|
|
116
|
+
sources: ["comment"],
|
|
117
|
+
});
|
|
118
|
+
store.addUser({
|
|
119
|
+
uniqueId: "error-user",
|
|
120
|
+
nickname: "Error User",
|
|
121
|
+
status: "error",
|
|
122
|
+
locationCreated: "US",
|
|
123
|
+
ttSeller: false,
|
|
124
|
+
verified: false,
|
|
125
|
+
sources: ["comment"],
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
async function testClaimPriorityAndRenewal() {
|
|
130
|
+
const { dir, dbPath } = createTempDbPath("tt-watch-claim");
|
|
131
|
+
try {
|
|
132
|
+
const store = createStore(dbPath);
|
|
133
|
+
seedClaimStore(store);
|
|
134
|
+
|
|
135
|
+
const first = store.claimNextJob("worker-a", 5 * 60 * 1000, null, true);
|
|
136
|
+
assert.ok(first, "expected a claim for logged-in worker");
|
|
137
|
+
assert.equal(first.uniqueId, "seed-tier1");
|
|
138
|
+
|
|
139
|
+
const renewed = store.claimNextJob("worker-a", 5 * 60 * 1000, null, true);
|
|
140
|
+
assert.ok(renewed, "expected renewal claim for same worker");
|
|
141
|
+
assert.equal(renewed.uniqueId, "seed-tier1");
|
|
142
|
+
assert.ok(
|
|
143
|
+
renewed.claimedAt >= first.claimedAt,
|
|
144
|
+
"expected renewal claimedAt to advance or stay equal",
|
|
145
|
+
);
|
|
146
|
+
|
|
147
|
+
const loggedOut = store.claimNextJob(
|
|
148
|
+
"worker-b",
|
|
149
|
+
5 * 60 * 1000,
|
|
150
|
+
null,
|
|
151
|
+
false,
|
|
152
|
+
);
|
|
153
|
+
assert.ok(loggedOut, "expected a claim for logged-out worker");
|
|
154
|
+
assert.equal(loggedOut.uniqueId, "follow-tier2");
|
|
155
|
+
} finally {
|
|
156
|
+
closeStoreDb();
|
|
157
|
+
cleanupTempDir(dir);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
async function testWatchHttpEndpoints() {
|
|
162
|
+
const { dir, dbPath } = createTempDbPath("tt-watch-http");
|
|
163
|
+
let server;
|
|
164
|
+
try {
|
|
165
|
+
const store = createStore(dbPath);
|
|
166
|
+
seedWatchStore(store);
|
|
167
|
+
|
|
168
|
+
const started = await startWatchServer(dbPath, 0, store);
|
|
169
|
+
server = started.server;
|
|
170
|
+
const actualPort = server.address().port;
|
|
171
|
+
const baseUrl = `http://127.0.0.1:${actualPort}`;
|
|
172
|
+
|
|
173
|
+
const [statsRes, usersRes, targetRes, lightUsersRes] = await Promise.all([
|
|
174
|
+
fetch(`${baseUrl}/api/stats`),
|
|
175
|
+
fetch(`${baseUrl}/api/users?limit=3`),
|
|
176
|
+
fetch(`${baseUrl}/api/target-users`),
|
|
177
|
+
fetch(`${baseUrl}/api/users?limit=2&view=light`),
|
|
178
|
+
]);
|
|
179
|
+
|
|
180
|
+
assert.equal(statsRes.status, 200);
|
|
181
|
+
assert.equal(usersRes.status, 200);
|
|
182
|
+
assert.equal(targetRes.status, 200);
|
|
183
|
+
assert.equal(lightUsersRes.status, 200);
|
|
184
|
+
|
|
185
|
+
const [stats, users, targets, lightUsers] = await Promise.all([
|
|
186
|
+
statsRes.json(),
|
|
187
|
+
usersRes.json(),
|
|
188
|
+
targetRes.json(),
|
|
189
|
+
lightUsersRes.json(),
|
|
190
|
+
]);
|
|
191
|
+
|
|
192
|
+
assert.equal(stats.totalUsers, 7);
|
|
193
|
+
assert.equal(stats.pendingUsers, 4);
|
|
194
|
+
assert.equal(stats.processedUsers, 1);
|
|
195
|
+
assert.equal(stats.restrictedUsers, 1);
|
|
196
|
+
assert.equal(stats.errorUsers, 1);
|
|
197
|
+
assert.equal(stats.targetUsers, 2);
|
|
198
|
+
assert.equal(stats.userUpdateTasks, 1);
|
|
199
|
+
assert.ok(Array.isArray(stats.countryStats));
|
|
200
|
+
assert.ok(stats.countryStats.some((item) => item.country === "ES"));
|
|
201
|
+
|
|
202
|
+
assert.equal(users.total, 7);
|
|
203
|
+
assert.equal(users.users.length, 3);
|
|
204
|
+
assert.equal(users.users[0].uniqueId, "seller-target");
|
|
205
|
+
|
|
206
|
+
assert.equal(targets.total, 2);
|
|
207
|
+
assert.deepEqual(
|
|
208
|
+
targets.users.map((item) => item.uniqueId),
|
|
209
|
+
["done-es", "seller-target"],
|
|
210
|
+
);
|
|
211
|
+
|
|
212
|
+
assert.equal(lightUsers.total, 7);
|
|
213
|
+
assert.equal(lightUsers.users.length, 2);
|
|
214
|
+
assert.deepEqual(Object.keys(lightUsers.users[0]).sort(), [
|
|
215
|
+
"followerCount",
|
|
216
|
+
"guessedLocation",
|
|
217
|
+
"locationCreated",
|
|
218
|
+
"nickname",
|
|
219
|
+
"pinned",
|
|
220
|
+
"processedAt",
|
|
221
|
+
"sources",
|
|
222
|
+
"status",
|
|
223
|
+
"ttSeller",
|
|
224
|
+
"uniqueId",
|
|
225
|
+
"verified",
|
|
226
|
+
]);
|
|
227
|
+
} finally {
|
|
228
|
+
if (server) {
|
|
229
|
+
await new Promise((resolve) => server.close(resolve));
|
|
230
|
+
}
|
|
231
|
+
closeStoreDb();
|
|
232
|
+
cleanupTempDir(dir);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
async function main() {
|
|
237
|
+
await testClaimPriorityAndRenewal();
|
|
238
|
+
await testWatchHttpEndpoints();
|
|
239
|
+
console.log("watch db smoke test passed");
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
main().catch((error) => {
|
|
243
|
+
console.error("watch db smoke test failed");
|
|
244
|
+
console.error(error);
|
|
245
|
+
process.exitCode = 1;
|
|
246
|
+
});
|