tt-help-cli-ycl 1.3.45 → 1.3.47

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/README.md +33 -33
  2. package/cli.js +9 -9
  3. package/package.json +52 -52
  4. package/scripts/run-explore copy.bat +101 -101
  5. package/scripts/run-explore.bat +134 -134
  6. package/scripts/run-explore.ps1 +159 -159
  7. package/scripts/run-explore.sh +121 -121
  8. package/src/cli/attach.js +331 -313
  9. package/src/cli/auto.js +265 -265
  10. package/src/cli/comments.js +620 -620
  11. package/src/cli/config.js +170 -170
  12. package/src/cli/db-import.js +51 -51
  13. package/src/cli/explore.js +555 -555
  14. package/src/cli/info.js +10 -16
  15. package/src/cli/open.js +111 -111
  16. package/src/cli/progress.js +111 -111
  17. package/src/cli/refresh.js +288 -288
  18. package/src/cli/scrape.js +47 -47
  19. package/src/cli/utils.js +18 -18
  20. package/src/cli/videos.js +41 -41
  21. package/src/cli/videostats.js +196 -196
  22. package/src/cli/watch.js +30 -30
  23. package/src/cli/webserver.js +19 -0
  24. package/src/lib/api-interceptor.js +161 -161
  25. package/src/lib/args.js +809 -778
  26. package/src/lib/browser/anti-detect.js +23 -23
  27. package/src/lib/browser/cdp.js +261 -261
  28. package/src/lib/browser/health-checker.js +114 -114
  29. package/src/lib/browser/launch.js +43 -43
  30. package/src/lib/browser/page.js +184 -184
  31. package/src/lib/constants.js +297 -287
  32. package/src/lib/delay.js +54 -54
  33. package/src/lib/explore-fetch.js +118 -118
  34. package/src/lib/fetcher.js +45 -45
  35. package/src/lib/filter.js +66 -66
  36. package/src/lib/io.js +54 -54
  37. package/src/lib/output.js +80 -80
  38. package/src/lib/page-error-detector.js +109 -109
  39. package/src/lib/parse-ssr.mjs +69 -69
  40. package/src/lib/parser.js +47 -47
  41. package/src/lib/retry.js +45 -45
  42. package/src/lib/scrape.js +90 -89
  43. package/src/lib/target-locations.js +61 -61
  44. package/src/lib/tiktok-scraper.mjs +173 -105
  45. package/src/lib/url.js +52 -52
  46. package/src/main.js +73 -70
  47. package/src/npm-main.js +70 -69
  48. package/src/scraper/auto-core.js +203 -203
  49. package/src/scraper/core.js +255 -255
  50. package/src/scraper/explore-core.js +208 -208
  51. package/src/scraper/modules/captcha-handler.js +114 -114
  52. package/src/scraper/modules/follow-extractor.js +250 -250
  53. package/src/scraper/modules/guess-extractor.js +51 -51
  54. package/src/scraper/modules/page-helpers.js +48 -48
  55. package/src/scraper/refresh-core.js +213 -213
  56. package/src/videos/core.js +143 -143
  57. package/src/watch/data-store.js +2980 -2978
  58. package/src/watch/public/index.html +2355 -2345
  59. package/src/watch/server.js +727 -727
  60. package/src/webserver/server.mjs +174 -0
  61. package/scripts/test-captcha-lib.mjs +0 -68
  62. package/scripts/test-captcha.mjs +0 -81
  63. package/scripts/test-incognito-lib.mjs +0 -36
  64. package/scripts/test-login-state.mjs +0 -128
  65. package/scripts/test-safe-click.mjs +0 -45
  66. package/scripts/test-watch-db-smoke.mjs +0 -246
  67. package/src/results/user-videos-bar.lar.lar.moeta.json +0 -37
@@ -1,161 +1,161 @@
1
- import { delay } from "./delay.js";
2
- import { assertPageUrl } from "./browser/page.js";
3
- import {
4
- detectPageError,
5
- detectPageErrorWithWait,
6
- } from "./page-error-detector.js";
7
-
8
- async function processAPIResponse(
9
- data,
10
- username,
11
- maxVideos,
12
- items,
13
- page,
14
- apiRequestUrl,
15
- log,
16
- ) {
17
- const firstPageItems = data.itemList || [];
18
- for (const item of firstPageItems) {
19
- if (items.length >= maxVideos) break;
20
- const href = `https://www.tiktok.com/@${username}/video/${item.id}`;
21
- items.push({ id: item.id, href });
22
- }
23
-
24
- let cursor = data.cursor;
25
- let hasMore = data.hasMore;
26
-
27
- while (hasMore && cursor && items.length < maxVideos) {
28
- const reqUrl = apiRequestUrl || data.apiRequestUrl;
29
- if (!reqUrl) break;
30
-
31
- const newUrl = reqUrl.replace(/cursor=\d+/, `cursor=${cursor}`);
32
-
33
- try {
34
- const pageData = await (() => {
35
- // 重试包装:处理页面导航导致的执行上下文销毁
36
- const tryEval = async (retries = 3) => {
37
- for (let i = 0; i < retries; i++) {
38
- try {
39
- return await page.evaluate(async (u) => {
40
- const res = await fetch(u);
41
- return await res.json();
42
- }, newUrl);
43
- } catch (e) {
44
- if (
45
- e.message.includes("Execution context was destroyed") &&
46
- i < retries - 1
47
- ) {
48
- await delay(500 * (i + 1), 500 * (i + 1));
49
- } else {
50
- throw e;
51
- }
52
- }
53
- }
54
- };
55
- return tryEval();
56
- })();
57
-
58
- if (pageData && pageData.itemList) {
59
- for (const item of pageData.itemList) {
60
- if (items.length >= maxVideos) break;
61
- const href = `https://www.tiktok.com/@${username}/video/${item.id}`;
62
- items.push({ id: item.id, href });
63
- }
64
- }
65
-
66
- cursor = pageData.cursor;
67
- hasMore = pageData.hasMore;
68
- } catch {
69
- break;
70
- }
71
-
72
- await delay(300, 600);
73
- }
74
-
75
- log(` [API拦截] 共 ${items.length} 条视频`);
76
-
77
- const videoMap = new Map();
78
- for (const v of items) {
79
- if (!videoMap.has(v.id)) videoMap.set(v.id, v);
80
- }
81
-
82
- return videoMap;
83
- }
84
-
85
- async function fetchUserVideosAPI(page, username, maxVideos, log) {
86
- log(` [API拦截] 获取 @${username} 视频 ...`);
87
-
88
- let apiRequestUrl = null;
89
- let sawApiRequest = false;
90
-
91
- const requestHandler = (request) => {
92
- if (request.url().includes("/api/post/item_list/")) {
93
- sawApiRequest = true;
94
- if (!apiRequestUrl) apiRequestUrl = request.url();
95
- }
96
- };
97
-
98
- page.on("request", requestHandler);
99
-
100
- const t0 = Date.now();
101
- let data = null;
102
- let interceptionError = null;
103
-
104
- try {
105
- await page.goto(`https://www.tiktok.com/@${username}`, {
106
- waitUntil: "domcontentloaded",
107
- timeout: 30000,
108
- });
109
- await assertPageUrl(page, `@${username}`);
110
-
111
- const response = await page.waitForResponse(
112
- (res) => res.url().includes("/api/post/item_list/"),
113
- { timeout: 30000 },
114
- );
115
-
116
- data = await response.json();
117
- } catch (e) {
118
- interceptionError = e.message;
119
- } finally {
120
- page.off("request", requestHandler);
121
- }
122
-
123
- const elapsed = Date.now() - t0;
124
-
125
- if (data && Array.isArray(data.itemList) && data.itemList.length > 0) {
126
- log(` [API拦截] 成功,耗时 ${elapsed}ms`);
127
- return await processAPIResponse(
128
- data,
129
- username,
130
- maxVideos,
131
- [],
132
- page,
133
- apiRequestUrl,
134
- log,
135
- );
136
- }
137
-
138
- let pageError = await detectPageError(page);
139
- if (!pageError) {
140
- pageError = await detectPageErrorWithWait(page, 5000);
141
- }
142
-
143
- if (pageError === "service_error") {
144
- throw new Error(`被封: ${username}`);
145
- }
146
-
147
- if (pageError) {
148
- log(` [API拦截] 数据为空,识别到页面错误: ${pageError}`);
149
- return new Map();
150
- }
151
-
152
- if (interceptionError) {
153
- log(` [API拦截] 超时/拦截失败: ${interceptionError},耗时 ${elapsed}ms`);
154
- throw new Error(`@${username} 页面异常(API拦截超时,未识别到错误类型)`);
155
- }
156
-
157
- log(` [API拦截] 数据为空,未识别到错误类型,当作无视频处理`);
158
- return new Map();
159
- }
160
-
161
- export { fetchUserVideosAPI };
1
+ import { delay } from "./delay.js";
2
+ import { assertPageUrl } from "./browser/page.js";
3
+ import {
4
+ detectPageError,
5
+ detectPageErrorWithWait,
6
+ } from "./page-error-detector.js";
7
+
8
+ async function processAPIResponse(
9
+ data,
10
+ username,
11
+ maxVideos,
12
+ items,
13
+ page,
14
+ apiRequestUrl,
15
+ log,
16
+ ) {
17
+ const firstPageItems = data.itemList || [];
18
+ for (const item of firstPageItems) {
19
+ if (items.length >= maxVideos) break;
20
+ const href = `https://www.tiktok.com/@${username}/video/${item.id}`;
21
+ items.push({ id: item.id, href });
22
+ }
23
+
24
+ let cursor = data.cursor;
25
+ let hasMore = data.hasMore;
26
+
27
+ while (hasMore && cursor && items.length < maxVideos) {
28
+ const reqUrl = apiRequestUrl || data.apiRequestUrl;
29
+ if (!reqUrl) break;
30
+
31
+ const newUrl = reqUrl.replace(/cursor=\d+/, `cursor=${cursor}`);
32
+
33
+ try {
34
+ const pageData = await (() => {
35
+ // 重试包装:处理页面导航导致的执行上下文销毁
36
+ const tryEval = async (retries = 3) => {
37
+ for (let i = 0; i < retries; i++) {
38
+ try {
39
+ return await page.evaluate(async (u) => {
40
+ const res = await fetch(u);
41
+ return await res.json();
42
+ }, newUrl);
43
+ } catch (e) {
44
+ if (
45
+ e.message.includes("Execution context was destroyed") &&
46
+ i < retries - 1
47
+ ) {
48
+ await delay(500 * (i + 1), 500 * (i + 1));
49
+ } else {
50
+ throw e;
51
+ }
52
+ }
53
+ }
54
+ };
55
+ return tryEval();
56
+ })();
57
+
58
+ if (pageData && pageData.itemList) {
59
+ for (const item of pageData.itemList) {
60
+ if (items.length >= maxVideos) break;
61
+ const href = `https://www.tiktok.com/@${username}/video/${item.id}`;
62
+ items.push({ id: item.id, href });
63
+ }
64
+ }
65
+
66
+ cursor = pageData.cursor;
67
+ hasMore = pageData.hasMore;
68
+ } catch {
69
+ break;
70
+ }
71
+
72
+ await delay(300, 600);
73
+ }
74
+
75
+ log(` [API拦截] 共 ${items.length} 条视频`);
76
+
77
+ const videoMap = new Map();
78
+ for (const v of items) {
79
+ if (!videoMap.has(v.id)) videoMap.set(v.id, v);
80
+ }
81
+
82
+ return videoMap;
83
+ }
84
+
85
+ async function fetchUserVideosAPI(page, username, maxVideos, log) {
86
+ log(` [API拦截] 获取 @${username} 视频 ...`);
87
+
88
+ let apiRequestUrl = null;
89
+ let sawApiRequest = false;
90
+
91
+ const requestHandler = (request) => {
92
+ if (request.url().includes("/api/post/item_list/")) {
93
+ sawApiRequest = true;
94
+ if (!apiRequestUrl) apiRequestUrl = request.url();
95
+ }
96
+ };
97
+
98
+ page.on("request", requestHandler);
99
+
100
+ const t0 = Date.now();
101
+ let data = null;
102
+ let interceptionError = null;
103
+
104
+ try {
105
+ await page.goto(`https://www.tiktok.com/@${username}`, {
106
+ waitUntil: "domcontentloaded",
107
+ timeout: 30000,
108
+ });
109
+ await assertPageUrl(page, `@${username}`);
110
+
111
+ const response = await page.waitForResponse(
112
+ (res) => res.url().includes("/api/post/item_list/"),
113
+ { timeout: 30000 },
114
+ );
115
+
116
+ data = await response.json();
117
+ } catch (e) {
118
+ interceptionError = e.message;
119
+ } finally {
120
+ page.off("request", requestHandler);
121
+ }
122
+
123
+ const elapsed = Date.now() - t0;
124
+
125
+ if (data && Array.isArray(data.itemList) && data.itemList.length > 0) {
126
+ log(` [API拦截] 成功,耗时 ${elapsed}ms`);
127
+ return await processAPIResponse(
128
+ data,
129
+ username,
130
+ maxVideos,
131
+ [],
132
+ page,
133
+ apiRequestUrl,
134
+ log,
135
+ );
136
+ }
137
+
138
+ let pageError = await detectPageError(page);
139
+ if (!pageError) {
140
+ pageError = await detectPageErrorWithWait(page, 5000);
141
+ }
142
+
143
+ if (pageError === "service_error") {
144
+ throw new Error(`被封: ${username}`);
145
+ }
146
+
147
+ if (pageError) {
148
+ log(` [API拦截] 数据为空,识别到页面错误: ${pageError}`);
149
+ return new Map();
150
+ }
151
+
152
+ if (interceptionError) {
153
+ log(` [API拦截] 超时/拦截失败: ${interceptionError},耗时 ${elapsed}ms`);
154
+ throw new Error(`@${username} 页面异常(API拦截超时,未识别到错误类型)`);
155
+ }
156
+
157
+ log(` [API拦截] 数据为空,未识别到错误类型,当作无视频处理`);
158
+ return new Map();
159
+ }
160
+
161
+ export { fetchUserVideosAPI };