tt-help-cli-ycl 1.3.45 → 1.3.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -33
- package/cli.js +9 -9
- package/package.json +52 -52
- package/scripts/run-explore copy.bat +101 -101
- package/scripts/run-explore.bat +134 -134
- package/scripts/run-explore.ps1 +159 -159
- package/scripts/run-explore.sh +121 -121
- package/src/cli/attach.js +331 -313
- package/src/cli/auto.js +265 -265
- package/src/cli/comments.js +620 -620
- package/src/cli/config.js +170 -170
- package/src/cli/db-import.js +51 -51
- package/src/cli/explore.js +555 -555
- package/src/cli/info.js +10 -16
- package/src/cli/open.js +111 -111
- package/src/cli/progress.js +111 -111
- package/src/cli/refresh.js +288 -288
- package/src/cli/scrape.js +47 -47
- package/src/cli/utils.js +18 -18
- package/src/cli/videos.js +41 -41
- package/src/cli/videostats.js +196 -196
- package/src/cli/watch.js +30 -30
- package/src/cli/webserver.js +19 -0
- package/src/lib/api-interceptor.js +161 -161
- package/src/lib/args.js +809 -778
- package/src/lib/browser/anti-detect.js +23 -23
- package/src/lib/browser/cdp.js +261 -261
- package/src/lib/browser/health-checker.js +114 -114
- package/src/lib/browser/launch.js +43 -43
- package/src/lib/browser/page.js +184 -184
- package/src/lib/constants.js +297 -287
- package/src/lib/delay.js +54 -54
- package/src/lib/explore-fetch.js +118 -118
- package/src/lib/fetcher.js +45 -45
- package/src/lib/filter.js +66 -66
- package/src/lib/io.js +54 -54
- package/src/lib/output.js +80 -80
- package/src/lib/page-error-detector.js +109 -109
- package/src/lib/parse-ssr.mjs +69 -69
- package/src/lib/parser.js +47 -47
- package/src/lib/retry.js +45 -45
- package/src/lib/scrape.js +90 -89
- package/src/lib/target-locations.js +61 -61
- package/src/lib/tiktok-scraper.mjs +173 -105
- package/src/lib/url.js +52 -52
- package/src/main.js +73 -70
- package/src/npm-main.js +70 -69
- package/src/scraper/auto-core.js +203 -203
- package/src/scraper/core.js +255 -255
- package/src/scraper/explore-core.js +208 -208
- package/src/scraper/modules/captcha-handler.js +114 -114
- package/src/scraper/modules/follow-extractor.js +250 -250
- package/src/scraper/modules/guess-extractor.js +51 -51
- package/src/scraper/modules/page-helpers.js +48 -48
- package/src/scraper/refresh-core.js +213 -213
- package/src/videos/core.js +143 -143
- package/src/watch/data-store.js +2980 -2978
- package/src/watch/public/index.html +2355 -2345
- package/src/watch/server.js +727 -727
- package/src/webserver/server.mjs +174 -0
- package/scripts/test-captcha-lib.mjs +0 -68
- package/scripts/test-captcha.mjs +0 -81
- package/scripts/test-incognito-lib.mjs +0 -36
- package/scripts/test-login-state.mjs +0 -128
- package/scripts/test-safe-click.mjs +0 -45
- package/scripts/test-watch-db-smoke.mjs +0 -246
- package/src/results/user-videos-bar.lar.lar.moeta.json +0 -37
|
@@ -1,246 +0,0 @@
|
|
|
1
|
-
import assert from "node:assert/strict";
|
|
2
|
-
import { mkdtempSync, rmSync } from "node:fs";
|
|
3
|
-
import os from "node:os";
|
|
4
|
-
import path from "node:path";
|
|
5
|
-
|
|
6
|
-
import { createStore, closeStoreDb } from "../src/watch/data-store.js";
|
|
7
|
-
import { startWatchServer } from "../src/watch/server.js";
|
|
8
|
-
|
|
9
|
-
function createTempDbPath(prefix) {
|
|
10
|
-
const dir = mkdtempSync(path.join(os.tmpdir(), `${prefix}-`));
|
|
11
|
-
return {
|
|
12
|
-
dir,
|
|
13
|
-
dbPath: path.join(dir, "result.db"),
|
|
14
|
-
};
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
function cleanupTempDir(dir) {
|
|
18
|
-
rmSync(dir, { recursive: true, force: true });
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
function seedClaimStore(store) {
|
|
22
|
-
store.addUser({
|
|
23
|
-
uniqueId: "follow-tier2",
|
|
24
|
-
status: "pending",
|
|
25
|
-
followerCount: 100,
|
|
26
|
-
videoCount: 1,
|
|
27
|
-
guessedLocation: "DE",
|
|
28
|
-
sources: ["following"],
|
|
29
|
-
});
|
|
30
|
-
store.addUser({
|
|
31
|
-
uniqueId: "seller-tier2",
|
|
32
|
-
status: "pending",
|
|
33
|
-
followerCount: 50,
|
|
34
|
-
videoCount: 1,
|
|
35
|
-
guessedLocation: "DE",
|
|
36
|
-
ttSeller: true,
|
|
37
|
-
verified: false,
|
|
38
|
-
sources: ["comment"],
|
|
39
|
-
});
|
|
40
|
-
store.addUser({
|
|
41
|
-
uniqueId: "seed-tier1",
|
|
42
|
-
status: "pending",
|
|
43
|
-
followerCount: 10,
|
|
44
|
-
videoCount: 1,
|
|
45
|
-
guessedLocation: "PL",
|
|
46
|
-
sources: ["seed"],
|
|
47
|
-
});
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
function seedWatchStore(store) {
|
|
51
|
-
store.addUser({
|
|
52
|
-
uniqueId: "seed-tier1",
|
|
53
|
-
nickname: "Seed User",
|
|
54
|
-
status: "pending",
|
|
55
|
-
followerCount: 10,
|
|
56
|
-
videoCount: 1,
|
|
57
|
-
guessedLocation: "PL",
|
|
58
|
-
ttSeller: false,
|
|
59
|
-
verified: false,
|
|
60
|
-
sources: ["seed"],
|
|
61
|
-
});
|
|
62
|
-
store.addUser({
|
|
63
|
-
uniqueId: "seller-target",
|
|
64
|
-
nickname: "Seller Target",
|
|
65
|
-
status: "pending",
|
|
66
|
-
followerCount: 500,
|
|
67
|
-
videoCount: 1,
|
|
68
|
-
guessedLocation: "DE",
|
|
69
|
-
locationCreated: "DE",
|
|
70
|
-
ttSeller: true,
|
|
71
|
-
verified: false,
|
|
72
|
-
sources: ["comment"],
|
|
73
|
-
});
|
|
74
|
-
store.addUser({
|
|
75
|
-
uniqueId: "follow-tier2",
|
|
76
|
-
nickname: "Follow User",
|
|
77
|
-
status: "pending",
|
|
78
|
-
followerCount: 300,
|
|
79
|
-
videoCount: 1,
|
|
80
|
-
guessedLocation: "FR",
|
|
81
|
-
ttSeller: false,
|
|
82
|
-
verified: false,
|
|
83
|
-
sources: ["following"],
|
|
84
|
-
});
|
|
85
|
-
store.addUser({
|
|
86
|
-
uniqueId: "pending-update",
|
|
87
|
-
nickname: "Needs Update",
|
|
88
|
-
status: "pending",
|
|
89
|
-
guessedLocation: "US",
|
|
90
|
-
ttSeller: null,
|
|
91
|
-
verified: false,
|
|
92
|
-
userUpdateCount: 0,
|
|
93
|
-
sources: ["comment"],
|
|
94
|
-
});
|
|
95
|
-
store.addUser({
|
|
96
|
-
uniqueId: "done-es",
|
|
97
|
-
nickname: "Done ES",
|
|
98
|
-
status: "done",
|
|
99
|
-
followerCount: 800,
|
|
100
|
-
videoCount: 3,
|
|
101
|
-
guessedLocation: "ES",
|
|
102
|
-
locationCreated: "ES",
|
|
103
|
-
ttSeller: true,
|
|
104
|
-
verified: false,
|
|
105
|
-
processedAt: Date.now(),
|
|
106
|
-
processed: true,
|
|
107
|
-
sources: ["processed"],
|
|
108
|
-
});
|
|
109
|
-
store.addUser({
|
|
110
|
-
uniqueId: "restricted-it",
|
|
111
|
-
nickname: "Restricted IT",
|
|
112
|
-
status: "restricted",
|
|
113
|
-
locationCreated: "IT",
|
|
114
|
-
ttSeller: false,
|
|
115
|
-
verified: false,
|
|
116
|
-
sources: ["comment"],
|
|
117
|
-
});
|
|
118
|
-
store.addUser({
|
|
119
|
-
uniqueId: "error-user",
|
|
120
|
-
nickname: "Error User",
|
|
121
|
-
status: "error",
|
|
122
|
-
locationCreated: "US",
|
|
123
|
-
ttSeller: false,
|
|
124
|
-
verified: false,
|
|
125
|
-
sources: ["comment"],
|
|
126
|
-
});
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
async function testClaimPriorityAndRenewal() {
|
|
130
|
-
const { dir, dbPath } = createTempDbPath("tt-watch-claim");
|
|
131
|
-
try {
|
|
132
|
-
const store = createStore(dbPath);
|
|
133
|
-
seedClaimStore(store);
|
|
134
|
-
|
|
135
|
-
const first = store.claimNextJob("worker-a", 5 * 60 * 1000, null, true);
|
|
136
|
-
assert.ok(first, "expected a claim for logged-in worker");
|
|
137
|
-
assert.equal(first.uniqueId, "seed-tier1");
|
|
138
|
-
|
|
139
|
-
const renewed = store.claimNextJob("worker-a", 5 * 60 * 1000, null, true);
|
|
140
|
-
assert.ok(renewed, "expected renewal claim for same worker");
|
|
141
|
-
assert.equal(renewed.uniqueId, "seed-tier1");
|
|
142
|
-
assert.ok(
|
|
143
|
-
renewed.claimedAt >= first.claimedAt,
|
|
144
|
-
"expected renewal claimedAt to advance or stay equal",
|
|
145
|
-
);
|
|
146
|
-
|
|
147
|
-
const loggedOut = store.claimNextJob(
|
|
148
|
-
"worker-b",
|
|
149
|
-
5 * 60 * 1000,
|
|
150
|
-
null,
|
|
151
|
-
false,
|
|
152
|
-
);
|
|
153
|
-
assert.ok(loggedOut, "expected a claim for logged-out worker");
|
|
154
|
-
assert.equal(loggedOut.uniqueId, "follow-tier2");
|
|
155
|
-
} finally {
|
|
156
|
-
closeStoreDb();
|
|
157
|
-
cleanupTempDir(dir);
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
async function testWatchHttpEndpoints() {
|
|
162
|
-
const { dir, dbPath } = createTempDbPath("tt-watch-http");
|
|
163
|
-
let server;
|
|
164
|
-
try {
|
|
165
|
-
const store = createStore(dbPath);
|
|
166
|
-
seedWatchStore(store);
|
|
167
|
-
|
|
168
|
-
const started = await startWatchServer(dbPath, 0, store);
|
|
169
|
-
server = started.server;
|
|
170
|
-
const actualPort = server.address().port;
|
|
171
|
-
const baseUrl = `http://127.0.0.1:${actualPort}`;
|
|
172
|
-
|
|
173
|
-
const [statsRes, usersRes, targetRes, lightUsersRes] = await Promise.all([
|
|
174
|
-
fetch(`${baseUrl}/api/stats`),
|
|
175
|
-
fetch(`${baseUrl}/api/users?limit=3`),
|
|
176
|
-
fetch(`${baseUrl}/api/target-users`),
|
|
177
|
-
fetch(`${baseUrl}/api/users?limit=2&view=light`),
|
|
178
|
-
]);
|
|
179
|
-
|
|
180
|
-
assert.equal(statsRes.status, 200);
|
|
181
|
-
assert.equal(usersRes.status, 200);
|
|
182
|
-
assert.equal(targetRes.status, 200);
|
|
183
|
-
assert.equal(lightUsersRes.status, 200);
|
|
184
|
-
|
|
185
|
-
const [stats, users, targets, lightUsers] = await Promise.all([
|
|
186
|
-
statsRes.json(),
|
|
187
|
-
usersRes.json(),
|
|
188
|
-
targetRes.json(),
|
|
189
|
-
lightUsersRes.json(),
|
|
190
|
-
]);
|
|
191
|
-
|
|
192
|
-
assert.equal(stats.totalUsers, 7);
|
|
193
|
-
assert.equal(stats.pendingUsers, 4);
|
|
194
|
-
assert.equal(stats.processedUsers, 1);
|
|
195
|
-
assert.equal(stats.restrictedUsers, 1);
|
|
196
|
-
assert.equal(stats.errorUsers, 1);
|
|
197
|
-
assert.equal(stats.targetUsers, 2);
|
|
198
|
-
assert.equal(stats.userUpdateTasks, 1);
|
|
199
|
-
assert.ok(Array.isArray(stats.countryStats));
|
|
200
|
-
assert.ok(stats.countryStats.some((item) => item.country === "ES"));
|
|
201
|
-
|
|
202
|
-
assert.equal(users.total, 7);
|
|
203
|
-
assert.equal(users.users.length, 3);
|
|
204
|
-
assert.equal(users.users[0].uniqueId, "seller-target");
|
|
205
|
-
|
|
206
|
-
assert.equal(targets.total, 2);
|
|
207
|
-
assert.deepEqual(
|
|
208
|
-
targets.users.map((item) => item.uniqueId),
|
|
209
|
-
["done-es", "seller-target"],
|
|
210
|
-
);
|
|
211
|
-
|
|
212
|
-
assert.equal(lightUsers.total, 7);
|
|
213
|
-
assert.equal(lightUsers.users.length, 2);
|
|
214
|
-
assert.deepEqual(Object.keys(lightUsers.users[0]).sort(), [
|
|
215
|
-
"followerCount",
|
|
216
|
-
"guessedLocation",
|
|
217
|
-
"locationCreated",
|
|
218
|
-
"nickname",
|
|
219
|
-
"pinned",
|
|
220
|
-
"processedAt",
|
|
221
|
-
"sources",
|
|
222
|
-
"status",
|
|
223
|
-
"ttSeller",
|
|
224
|
-
"uniqueId",
|
|
225
|
-
"verified",
|
|
226
|
-
]);
|
|
227
|
-
} finally {
|
|
228
|
-
if (server) {
|
|
229
|
-
await new Promise((resolve) => server.close(resolve));
|
|
230
|
-
}
|
|
231
|
-
closeStoreDb();
|
|
232
|
-
cleanupTempDir(dir);
|
|
233
|
-
}
|
|
234
|
-
}
|
|
235
|
-
|
|
236
|
-
async function main() {
|
|
237
|
-
await testClaimPriorityAndRenewal();
|
|
238
|
-
await testWatchHttpEndpoints();
|
|
239
|
-
console.log("watch db smoke test passed");
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
main().catch((error) => {
|
|
243
|
-
console.error("watch db smoke test failed");
|
|
244
|
-
console.error(error);
|
|
245
|
-
process.exitCode = 1;
|
|
246
|
-
});
|
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"user": {
|
|
3
|
-
"uniqueId": "bar.lar.lar.moeta",
|
|
4
|
-
"secUid": "MS4wLjABAAAA3cgKTWvKfga0JAWeakAzx3zQ-aFAC8RuQvxD4HQFraKKsc_TbOIyMo3_ofVlXofV",
|
|
5
|
-
"nickname": "Bar Lar Lar Moetain",
|
|
6
|
-
"ttSeller": false,
|
|
7
|
-
"verified": false,
|
|
8
|
-
"followerCount": 24000,
|
|
9
|
-
"videoCount": 749,
|
|
10
|
-
"followingCount": 4293,
|
|
11
|
-
"heartCount": 254300,
|
|
12
|
-
"signature": ""
|
|
13
|
-
},
|
|
14
|
-
"totalVideos": 5,
|
|
15
|
-
"videos": [
|
|
16
|
-
{
|
|
17
|
-
"id": "7638231799084158228",
|
|
18
|
-
"url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7638231799084158228"
|
|
19
|
-
},
|
|
20
|
-
{
|
|
21
|
-
"id": "7638162444698914068",
|
|
22
|
-
"url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7638162444698914068"
|
|
23
|
-
},
|
|
24
|
-
{
|
|
25
|
-
"id": "7638116251767819541",
|
|
26
|
-
"url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7638116251767819541"
|
|
27
|
-
},
|
|
28
|
-
{
|
|
29
|
-
"id": "7638069637321690388",
|
|
30
|
-
"url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7638069637321690388"
|
|
31
|
-
},
|
|
32
|
-
{
|
|
33
|
-
"id": "7637927171025112341",
|
|
34
|
-
"url": "https://www.tiktok.com/@bar.lar.lar.moeta/video/7637927171025112341"
|
|
35
|
-
}
|
|
36
|
-
]
|
|
37
|
-
}
|