rssany 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/README.md +22 -22
  2. package/app/plugins/builtin/agi-eval-evaluation.rssany.js +6 -7
  3. package/app/plugins/builtin/amii-research-talent.rssany.js +6 -7
  4. package/app/plugins/builtin/anthropic-research.rssany.js +6 -8
  5. package/app/plugins/builtin/appen-resources.rssany.js +6 -7
  6. package/app/plugins/builtin/baai-wudao-paper-article.rssany.js +9 -10
  7. package/app/plugins/builtin/baaidata-csdn.rssany.js +6 -7
  8. package/app/plugins/builtin/baidu-research.rssany.js +5 -8
  9. package/app/plugins/builtin/brightdata-blog.rssany.js +6 -11
  10. package/app/plugins/builtin/bytedance-seed-research.rssany.js +5 -7
  11. package/app/plugins/builtin/email.rssany.js +9 -9
  12. package/app/plugins/builtin/five-radar.rssany.js +9 -11
  13. package/app/plugins/builtin/flageval-news.rssany.js +5 -7
  14. package/app/plugins/builtin/google-deepmind-research.rssany.js +6 -8
  15. package/app/plugins/builtin/google-research-datasets.rssany.js +6 -8
  16. package/app/plugins/builtin/google-research.rssany.js +6 -8
  17. package/app/plugins/builtin/hacker-news-newest.rssany.js +7 -9
  18. package/app/plugins/builtin/harvard-dataverse.rssany.js +6 -8
  19. package/app/plugins/builtin/huaweicloud-bbs-blogs.rssany.js +7 -9
  20. package/app/plugins/builtin/lingowhale.rssany.js +7 -9
  21. package/app/plugins/builtin/meituan-tech.rssany.js +7 -10
  22. package/app/plugins/builtin/meta-ai-publications.rssany.js +6 -11
  23. package/app/plugins/builtin/mila-quebec.rssany.js +6 -8
  24. package/app/plugins/builtin/mit-csail-research.rssany.js +7 -9
  25. package/app/plugins/builtin/moonshot.rssany.js +6 -8
  26. package/app/plugins/builtin/opendatalab-news.rssany.js +6 -7
  27. package/app/plugins/builtin/opendatalab.rssany.js +5 -6
  28. package/app/plugins/builtin/opendrivelab-autonomous-driving.rssany.js +6 -7
  29. package/app/plugins/builtin/opendrivelab-embodiedai.rssany.js +7 -8
  30. package/app/plugins/builtin/opendrivelab-publications.rssany.js +6 -8
  31. package/app/plugins/builtin/opendrivelab.rssany.js +7 -8
  32. package/app/plugins/builtin/paperswithcode.rssany.js +6 -8
  33. package/app/plugins/builtin/pjlab-adg-publications.rssany.js +7 -9
  34. package/app/plugins/builtin/rss.rssany.js +11 -12
  35. package/app/plugins/builtin/selectdataset.rssany.js +6 -8
  36. package/app/plugins/builtin/sensetime-tech-achievements.rssany.js +7 -8
  37. package/app/plugins/builtin/supervisely-blog.rssany.js +6 -8
  38. package/app/plugins/builtin/theinformation-briefings.rssany.js +7 -13
  39. package/app/plugins/builtin/uci-ml-repository.rssany.js +6 -7
  40. package/app/plugins/builtin/venturebeat.rssany.js +7 -9
  41. package/app/plugins/builtin/worldlabs.rssany.js +6 -8
  42. package/app/plugins/builtin/x.rssany.js +7 -9
  43. package/app/plugins/builtin/xiaohongshu.rssany.js +119 -56
  44. package/app/plugins/builtin/zhipu-research.rssany.js +5 -8
  45. package/app/plugins/site.rssany.js +25 -26
  46. package/{statics → app/statics}/README.md +7 -7
  47. package/bin/rssany.js +226 -6
  48. package/dist/index.js +209 -152
  49. package/dist/index.js.map +1 -1
  50. package/package.json +16 -9
  51. package/scripts/dev.mjs +114 -0
  52. package/scripts/reset.mjs +1 -1
  53. package/init/config.json +0 -17
  54. package/init/sources.json +0 -353
  55. package/statics/401.html +0 -56
  56. package/statics/404.html +0 -12
  57. package/statics/image.png +0 -0
  58. package/webui/build/200.html +0 -49
  59. package/webui/build/_app/env.js +0 -1
  60. package/webui/build/_app/immutable/assets/0.BB88QFoe.css +0 -1
  61. package/webui/build/_app/immutable/assets/10.Dj8_pmut.css +0 -1
  62. package/webui/build/_app/immutable/assets/11.qYZMiTb0.css +0 -1
  63. package/webui/build/_app/immutable/assets/12.DfJcfUWl.css +0 -1
  64. package/webui/build/_app/immutable/assets/13.BhO9zvFi.css +0 -1
  65. package/webui/build/_app/immutable/assets/14.CujIhjQK.css +0 -1
  66. package/webui/build/_app/immutable/assets/15.nNGjXhCQ.css +0 -1
  67. package/webui/build/_app/immutable/assets/16.PP9XLDf7.css +0 -1
  68. package/webui/build/_app/immutable/assets/4.9wPHhVwv.css +0 -1
  69. package/webui/build/_app/immutable/assets/5.B-dPiwB7.css +0 -1
  70. package/webui/build/_app/immutable/assets/6.B27N7pdA.css +0 -1
  71. package/webui/build/_app/immutable/assets/7.CrNxmd8B.css +0 -1
  72. package/webui/build/_app/immutable/assets/8.Cgji2b15.css +0 -1
  73. package/webui/build/_app/immutable/assets/9.BsCIAvn3.css +0 -1
  74. package/webui/build/_app/immutable/assets/BackToParentRoute.DGk-X5ow.css +0 -1
  75. package/webui/build/_app/immutable/assets/SourcesList.yTBBi3_m.css +0 -1
  76. package/webui/build/_app/immutable/assets/homeFeedPanelStore.CSvlNcpm.css +0 -1
  77. package/webui/build/_app/immutable/chunks/5LVkDJzw.js +0 -1
  78. package/webui/build/_app/immutable/chunks/B-OsL1Ct.js +0 -1
  79. package/webui/build/_app/immutable/chunks/B2Q1a1-H.js +0 -2
  80. package/webui/build/_app/immutable/chunks/BK3WtZwv.js +0 -1
  81. package/webui/build/_app/immutable/chunks/BQqoDzLx.js +0 -1
  82. package/webui/build/_app/immutable/chunks/BUApaBEI.js +0 -1
  83. package/webui/build/_app/immutable/chunks/BbWUOQ_m.js +0 -1
  84. package/webui/build/_app/immutable/chunks/Bfc47y5P.js +0 -1
  85. package/webui/build/_app/immutable/chunks/Bns1MuyM.js +0 -36
  86. package/webui/build/_app/immutable/chunks/Bp63qm3L.js +0 -1
  87. package/webui/build/_app/immutable/chunks/Bu9HsS-V.js +0 -1
  88. package/webui/build/_app/immutable/chunks/CBY2biv-.js +0 -1
  89. package/webui/build/_app/immutable/chunks/CVzlFH44.js +0 -1
  90. package/webui/build/_app/immutable/chunks/CWNeClHp.js +0 -6
  91. package/webui/build/_app/immutable/chunks/Cihqbfi5.js +0 -1
  92. package/webui/build/_app/immutable/chunks/CmjOpds-.js +0 -1
  93. package/webui/build/_app/immutable/chunks/D5GvRCv7.js +0 -1
  94. package/webui/build/_app/immutable/chunks/DEDI7Ecm.js +0 -1
  95. package/webui/build/_app/immutable/chunks/DFuhmi31.js +0 -1
  96. package/webui/build/_app/immutable/chunks/DMWEh-Ek.js +0 -2
  97. package/webui/build/_app/immutable/chunks/DjNLq3TF.js +0 -1
  98. package/webui/build/_app/immutable/chunks/Dt2CddFe.js +0 -1
  99. package/webui/build/_app/immutable/chunks/Dw782Tjs.js +0 -1
  100. package/webui/build/_app/immutable/chunks/Xy_fhzQq.js +0 -1
  101. package/webui/build/_app/immutable/chunks/bvuf_jZd.js +0 -36
  102. package/webui/build/_app/immutable/chunks/hp4PFHFv.js +0 -1
  103. package/webui/build/_app/immutable/chunks/lk5LaiqA.js +0 -1
  104. package/webui/build/_app/immutable/chunks/mW5RwvnK.js +0 -13
  105. package/webui/build/_app/immutable/chunks/tB7QMF3U.js +0 -1
  106. package/webui/build/_app/immutable/chunks/xtNWTdbD.js +0 -1
  107. package/webui/build/_app/immutable/entry/app.BVkrDt5l.js +0 -2
  108. package/webui/build/_app/immutable/entry/start.D3Q-BMMd.js +0 -1
  109. package/webui/build/_app/immutable/nodes/0.I1lQdWMl.js +0 -11
  110. package/webui/build/_app/immutable/nodes/1.BiQQfx2j.js +0 -1
  111. package/webui/build/_app/immutable/nodes/10.CvfUsqsw.js +0 -1
  112. package/webui/build/_app/immutable/nodes/11.B4LHPNL6.js +0 -1
  113. package/webui/build/_app/immutable/nodes/12.DVFJuIWI.js +0 -1
  114. package/webui/build/_app/immutable/nodes/13.nT3SOzEB.js +0 -1
  115. package/webui/build/_app/immutable/nodes/14.DfaAf0f8.js +0 -1
  116. package/webui/build/_app/immutable/nodes/15.CMzkX9OK.js +0 -1
  117. package/webui/build/_app/immutable/nodes/16.zPgTQNze.js +0 -24
  118. package/webui/build/_app/immutable/nodes/17.BtYZF6FM.js +0 -1
  119. package/webui/build/_app/immutable/nodes/18.BIzqhTqv.js +0 -1
  120. package/webui/build/_app/immutable/nodes/2.BYWOpaxy.js +0 -1
  121. package/webui/build/_app/immutable/nodes/3.B8Viux9S.js +0 -1
  122. package/webui/build/_app/immutable/nodes/4.DTSxpKm7.js +0 -2
  123. package/webui/build/_app/immutable/nodes/5.B6fR3n6J.js +0 -2
  124. package/webui/build/_app/immutable/nodes/6.j2O5Mwjv.js +0 -1
  125. package/webui/build/_app/immutable/nodes/7.Bd2USIrl.js +0 -1
  126. package/webui/build/_app/immutable/nodes/8.Bw_d63B_.js +0 -1
  127. package/webui/build/_app/immutable/nodes/9.pMMi5PP6.js +0 -1
  128. package/webui/build/_app/version.json +0 -1
package/bin/rssany.js CHANGED
@@ -1,6 +1,226 @@
1
- #!/usr/bin/env node
2
- if (process.argv[2] === "reset") {
3
- await import(new URL("../scripts/reset.mjs", import.meta.url));
4
- } else {
5
- await import(new URL("../dist/index.js", import.meta.url));
6
- }
1
+ #!/usr/bin/env node
2
+ import { spawn } from "node:child_process";
3
+ import { closeSync, openSync } from "node:fs";
4
+ import { access, mkdir, readFile, rm, writeFile } from "node:fs/promises";
5
+ import http from "node:http";
6
+ import { homedir, networkInterfaces } from "node:os";
7
+ import { dirname, join } from "node:path";
8
+ import { fileURLToPath } from "node:url";
9
+
10
+ const command = process.argv[2];
11
+ const binDir = dirname(fileURLToPath(import.meta.url));
12
+ const packageRoot = join(binDir, "..");
13
+ const userDir = process.env.RSSANY_USER_DIR?.trim() || join(homedir(), ".rssany");
14
+ const pidPath = join(userDir, "rssany.pid");
15
+ const logPath = join(userDir, "rssany.log");
16
+ const port = Number(process.env.PORT) || 18473;
17
+ const serverOrigin = `http://127.0.0.1:${port}`;
18
+
19
+ async function pathExists(path) {
20
+ try {
21
+ await access(path);
22
+ return true;
23
+ } catch {
24
+ return false;
25
+ }
26
+ }
27
+
28
+ async function readPid() {
29
+ try {
30
+ const raw = await readFile(pidPath, "utf-8");
31
+ const pid = Number(raw.trim());
32
+ return Number.isInteger(pid) && pid > 0 ? pid : null;
33
+ } catch {
34
+ return null;
35
+ }
36
+ }
37
+
38
+ function isProcessRunning(pid) {
39
+ try {
40
+ process.kill(pid, 0);
41
+ return true;
42
+ } catch {
43
+ return false;
44
+ }
45
+ }
46
+
47
+ function getLanUrl() {
48
+ const lanIp = Object.values(networkInterfaces())
49
+ .flat()
50
+ .find((iface) => iface?.family === "IPv4" && !iface.internal)?.address;
51
+ return lanIp ? `http://${lanIp}:${port}/` : null;
52
+ }
53
+
54
+ function printAddress(prefix = "RssAny 已启动") {
55
+ console.log(`${prefix}: http://127.0.0.1:${port}/`);
56
+ const lanUrl = getLanUrl();
57
+ if (lanUrl) console.log(`局域网访问: ${lanUrl}`);
58
+ }
59
+
60
+ function printUsage() {
61
+ console.log("用法: rssany <start|stop|reset|crawl>");
62
+ console.log(" rssany start 后台启动服务并输出访问地址");
63
+ console.log(" rssany stop 关闭后台服务并输出执行状态");
64
+ console.log(" rssany reset 重置本地数据");
65
+ console.log(" rssany crawl <ref> 按内部抓取链路拉取指定信源");
66
+ }
67
+
68
+ async function canConnectToServer() {
69
+ return new Promise((resolve) => {
70
+ const req = http.get(`${serverOrigin}/api/server-info`, (res) => {
71
+ res.resume();
72
+ resolve(true);
73
+ });
74
+ req.setTimeout(500, () => {
75
+ req.destroy();
76
+ resolve(false);
77
+ });
78
+ req.on("error", () => resolve(false));
79
+ });
80
+ }
81
+
82
+ async function waitForServer(timeoutMs = 5000) {
83
+ const startTime = Date.now();
84
+ while (Date.now() - startTime < timeoutMs) {
85
+ if (await canConnectToServer()) return true;
86
+ await new Promise((resolve) => setTimeout(resolve, 250));
87
+ }
88
+ return false;
89
+ }
90
+
91
+ async function start() {
92
+ await mkdir(userDir, { recursive: true });
93
+
94
+ const currentPid = await readPid();
95
+ if (currentPid && isProcessRunning(currentPid)) {
96
+ printAddress(`RssAny 已在运行 (pid ${currentPid})`);
97
+ return;
98
+ }
99
+
100
+ const entry = join(packageRoot, "dist", "index.js");
101
+ if (!(await pathExists(entry))) {
102
+ console.error("未找到 dist/index.js,请先构建项目或重新安装 rssany。");
103
+ process.exitCode = 1;
104
+ return;
105
+ }
106
+
107
+ const logFd = openSync(logPath, "a");
108
+ const child = spawn(process.execPath, [entry], {
109
+ cwd: process.cwd(),
110
+ detached: true,
111
+ env: process.env,
112
+ stdio: ["ignore", logFd, logFd],
113
+ });
114
+ closeSync(logFd);
115
+
116
+ await writeFile(pidPath, `${child.pid}\n`, "utf-8");
117
+ console.log(`日志: ${logPath}`);
118
+ if (await waitForServer()) {
119
+ child.unref();
120
+ printAddress(`RssAny 已启动 (pid ${child.pid})`);
121
+ return;
122
+ }
123
+
124
+ child.unref();
125
+ console.error(`RssAny 启动未完成,请查看日志: ${logPath}`);
126
+ process.exitCode = 1;
127
+ }
128
+
129
+ async function stop() {
130
+ const pid = await readPid();
131
+ if (!pid) {
132
+ console.log("RssAny 未运行:没有找到 pid 文件。");
133
+ return;
134
+ }
135
+
136
+ if (!isProcessRunning(pid)) {
137
+ await rm(pidPath, { force: true });
138
+ console.log(`RssAny 未运行:已清理失效 pid ${pid}。`);
139
+ return;
140
+ }
141
+
142
+ process.kill(pid, "SIGTERM");
143
+ await rm(pidPath, { force: true });
144
+ console.log(`RssAny 已发送停止信号 (pid ${pid})。`);
145
+ }
146
+
147
+ function readCrawlRef(args) {
148
+ const refFlagIndex = args.findIndex((arg) => arg === "--ref");
149
+ if (refFlagIndex >= 0) return args[refFlagIndex + 1]?.trim() || "";
150
+ const refEquals = args.find((arg) => arg.startsWith("--ref="));
151
+ if (refEquals) return refEquals.slice("--ref=".length).trim();
152
+ return args.find((arg) => !arg.startsWith("-"))?.trim() || "";
153
+ }
154
+
155
+ async function postJson(path, body) {
156
+ const res = await fetch(`${serverOrigin}${path}`, {
157
+ method: "POST",
158
+ headers: { "Content-Type": "application/json" },
159
+ body: JSON.stringify(body),
160
+ });
161
+ const data = await res.json().catch(() => ({}));
162
+ if (!res.ok) {
163
+ throw new Error(data.error || `HTTP ${res.status}`);
164
+ }
165
+ return data;
166
+ }
167
+
168
+ async function getJson(path) {
169
+ const res = await fetch(`${serverOrigin}${path}`);
170
+ const data = await res.json().catch(() => ({}));
171
+ if (!res.ok) {
172
+ throw new Error(data.error || `HTTP ${res.status}`);
173
+ }
174
+ return data;
175
+ }
176
+
177
+ async function pollTask(taskId, timeoutMs = 120000) {
178
+ const start = Date.now();
179
+ while (Date.now() - start < timeoutMs) {
180
+ const task = await getJson(`/api/tasks/${encodeURIComponent(taskId)}`);
181
+ if (task.status === "done") return task;
182
+ if (task.status === "error") {
183
+ throw new Error(task.error || "抓取失败");
184
+ }
185
+ await new Promise((resolve) => setTimeout(resolve, 800));
186
+ }
187
+ throw new Error("抓取超时");
188
+ }
189
+
190
+ async function crawl() {
191
+ const ref = readCrawlRef(process.argv.slice(3));
192
+ if (!ref) {
193
+ console.error("ref 不能为空。用法: rssany crawl <ref>");
194
+ process.exitCode = 1;
195
+ return;
196
+ }
197
+ if (!(await canConnectToServer())) {
198
+ console.error(`RssAny 服务未运行,请先执行 rssany start。目标: ${serverOrigin}`);
199
+ process.exitCode = 1;
200
+ return;
201
+ }
202
+ try {
203
+ const { taskId } = await postJson("/api/tasks", { type: "source-pull", ref });
204
+ if (!taskId) throw new Error("后端未返回 taskId");
205
+ console.log(`crawl 已提交: ${ref}`);
206
+ console.log(`task: ${taskId}`);
207
+ await pollTask(taskId);
208
+ console.log("crawl 完成");
209
+ } catch (err) {
210
+ console.error(err instanceof Error ? err.message : String(err));
211
+ process.exitCode = 1;
212
+ }
213
+ }
214
+
215
+ if (command === "reset") {
216
+ await import(new URL("../scripts/reset.mjs", import.meta.url));
217
+ } else if (command === "start") {
218
+ await start();
219
+ } else if (command === "stop") {
220
+ await stop();
221
+ } else if (command === "crawl") {
222
+ await crawl();
223
+ } else {
224
+ printUsage();
225
+ if (command) process.exitCode = 1;
226
+ }