rsshub 1.0.0-master.ff487f3 → 1.0.0-master.ff76aa7

package/lib/middleware/parameter.ts

@@ -32,7 +32,7 @@ const resolveRelativeLink = ($: CheerioAPI, elem: Element, attr: string, baseUrl
     }
 };
 
-const summarizeArticle = async (articleText: string) => {
+const getAiCompletion = async (prompt: string, text: string) => {
     const apiUrl = `${config.openai.endpoint}/chat/completions`;
     const response = await ofetch(apiUrl, {
         method: 'POST',
@@ -40,8 +40,8 @@ const summarizeArticle = async (articleText: string) => {
             model: config.openai.model,
             max_tokens: config.openai.maxTokens,
             messages: [
-                { role: 'system', content: config.openai.prompt },
-                { role: 'user', content: articleText },
+                { role: 'system', content: prompt },
+                { role: 'user', content: text },
             ],
             temperature: config.openai.temperature,
         },
@@ -327,23 +327,53 @@ const middleware: MiddlewareHandler = async (ctx, next) => {
         if (ctx.req.query('chatgpt') && config.openai.apiKey) {
             data.item = await Promise.all(
                 data.item.map(async (item) => {
-                    if (item.description) {
-                        try {
-                            const summary = await cache.tryGet(`openai:${item.link}`, async () => {
-                                const text = convert(item.description!);
-                                if (text.length < 300) {
-                                    return '';
-                                }
-                                const summary_md = await summarizeArticle(text);
-                                return md.render(summary_md);
+                    try {
+                        // handle description
+                        if (config.openai.inputOption === 'description' && item.description) {
+                            const description = await cache.tryGet(`openai:description:${item.link}`, async () => {
+                                const description = convert(item.description!);
+                                const descriptionMd = await getAiCompletion(config.openai.promptDescription, description);
+                                return md.render(descriptionMd);
                             });
-                            // 将总结结果添加到文章数据中
-                            if (summary !== '') {
-                                item.description = summary + '<hr/><br/>' + item.description;
+                            // add it to the description
+                            if (description !== '') {
+                                item.description = description + '<hr/><br/>' + item.description;
+                            }
+                        }
+                        // handle title
+                        else if (config.openai.inputOption === 'title' && item.title) {
+                            const title = await cache.tryGet(`openai:title:${item.link}`, async () => {
+                                const title = convert(item.title!);
+                                return await getAiCompletion(config.openai.promptTitle, title);
+                            });
+                            // replace the title
+                            if (title !== '') {
+                                item.title = title + '';
+                            }
+                        }
+                        // handle both
+                        else if (config.openai.inputOption === 'both' && item.title && item.description) {
+                            const title = await cache.tryGet(`openai:title:${item.link}`, async () => {
+                                const title = convert(item.title!);
+                                return await getAiCompletion(config.openai.promptTitle, title);
+                            });
+                            // replace the title
+                            if (title !== '') {
+                                item.title = title + '';
+                            }
+
+                            const description = await cache.tryGet(`openai:description:${item.link}`, async () => {
+                                const description = convert(item.description!);
+                                const descriptionMd = await getAiCompletion(config.openai.promptDescription, description);
+                                return md.render(descriptionMd);
+                            });
+                            // add it to the description
+                            if (description !== '') {
+                                item.description = description + '<hr/><br/>' + item.description;
                             }
-                        } catch {
-                            // when openai failed, return default description and not write cache
                         }
+                    } catch {
+                        // when openai failed, return default content and not write cache
                     }
                     return item;
                 })

package/lib/middleware/template.tsx

@@ -74,8 +74,16 @@ const middleware: MiddlewareHandler = async (ctx, next) => {
                 }
 
                 if (outputType !== 'rss') {
-                    item.pubDate && (item.pubDate = convertDateToISO8601(item.pubDate) || '');
-                    item.updated && (item.updated = convertDateToISO8601(item.updated) || '');
+                    try {
+                        item.pubDate && (item.pubDate = convertDateToISO8601(item.pubDate) || '');
+                    } catch {
+                        item.pubDate = '';
+                    }
+                    try {
+                        item.updated && (item.updated = convertDateToISO8601(item.updated) || '');
+                    } catch {
+                        item.updated = '';
+                    }
                 }
             }
         }

package/lib/registry.test.ts

@@ -29,4 +29,12 @@ describe('registry', () => {
         const response = await app.request('/favicon.ico');
         expect(response.status).toBe(200);
     });
+
+    // healthz
+    it('/healthz', async () => {
+        const response = await app.request('/healthz');
+        expect(response.status).toBe(200);
+        expect(response.headers.get('cache-control')).toBe('no-cache');
+        expect(await response.text()).toBe('ok');
+    });
 });

package/lib/registry.ts

@@ -7,6 +7,7 @@ import { serveStatic } from '@hono/node-server/serve-static';
 import { config } from '@/config';
 
 import index from '@/routes/index';
+import healthz from '@/routes/healthz';
 import robotstxt from '@/routes/robots.txt';
 import metrics from '@/routes/metrics';
 
@@ -100,6 +101,7 @@ for (const namespace in namespaces) {
 }
 
 app.get('/', index);
+app.get('/healthz', healthz);
 app.get('/robots.txt', robotstxt);
 if (config.debugInfo) {
     // Only enable tracing in debug mode

package/lib/router.js

@@ -47,14 +47,6 @@ router.get('/huya/live/:id', lazyloadRouteHandler('./routes/huya/live'));
 // f-droid
 // router.get('/fdroid/apprelease/:app', lazyloadRouteHandler('./routes/fdroid/apprelease'));
 
-// konachan
-router.get('/konachan/post/popular_recent', lazyloadRouteHandler('./routes/konachan/post-popular-recent'));
-router.get('/konachan.com/post/popular_recent', lazyloadRouteHandler('./routes/konachan/post-popular-recent'));
-router.get('/konachan.net/post/popular_recent', lazyloadRouteHandler('./routes/konachan/post-popular-recent'));
-router.get('/konachan/post/popular_recent/:period', lazyloadRouteHandler('./routes/konachan/post-popular-recent'));
-router.get('/konachan.com/post/popular_recent/:period', lazyloadRouteHandler('./routes/konachan/post-popular-recent'));
-router.get('/konachan.net/post/popular_recent/:period', lazyloadRouteHandler('./routes/konachan/post-popular-recent'));
-
 // PornHub
 // router.get('/pornhub/category/:caty', lazyloadRouteHandler('./routes/pornhub/category'));
 // router.get('/pornhub/search/:keyword', lazyloadRouteHandler('./routes/pornhub/search'));
@@ -361,7 +353,7 @@ router.get('/ltaaa/:category?', lazyloadRouteHandler('./routes/ltaaa/index'));
 router.get('/autotrader/:query', lazyloadRouteHandler('./routes/autotrader'));
 
 // 极客公园
-router.get('/geekpark/breakingnews', lazyloadRouteHandler('./routes/geekpark/breakingnews'));
+// router.get('/geekpark/breakingnews', lazyloadRouteHandler('./routes/geekpark/breakingnews'));
 
 // 搜狗
 // router.get('/sogou/doodles', lazyloadRouteHandler('./routes/sogou/doodles'));

package/lib/routes/4ksj/forum.ts

@@ -3,13 +3,13 @@ import { getCurrentPath } from '@/utils/helpers';
 const __dirname = getCurrentPath(import.meta.url);
 
 import cache from '@/utils/cache';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
 import { load } from 'cheerio';
 import timezone from '@/utils/timezone';
 import { parseDate } from '@/utils/parse-date';
 import { art } from '@/utils/render';
 import path from 'node:path';
-import iconv from 'iconv-lite';
+import md5 from '@/utils/md5';
 
 export const route: Route = {
     path: '/:id?',
@@ -27,6 +27,16 @@ export const route: Route = {
     categories: ['multimedia'],
 };
 
+function stringtoHex(acSTR) {
+    let val = '';
+    for (let i = 0; i <= acSTR.length - 1; i++) {
+        const str = acSTR.charAt(i);
+        const code = str.codePointAt();
+        val += code;
+    }
+    return val;
+}
+
 async function handler(ctx) {
     const { id = '4k-uhd-1' } = ctx.req.param();
     const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 25;
@@ -34,11 +44,13 @@ async function handler(ctx) {
     const rootUrl = 'https://www.4ksj.com';
     const currentUrl = new URL(`${id}.html`, rootUrl).href;
 
-    const { data: response } = await got(currentUrl, {
-        responseType: 'buffer',
+    const response = await ofetch(currentUrl, {
+        responseType: 'arrayBuffer',
     });
 
-    const $ = load(iconv.decode(response, 'gbk'));
+    const decoder = new TextDecoder('gbk');
+
+    const $ = load(decoder.decode(response));
 
     const language = 'zh';
     const image = $('div.nexlogo img').prop('src');
@@ -54,14 +66,42 @@ async function handler(ctx) {
             };
         });
 
+    const getCookie = () =>
+        cache.tryGet('4ksj:cookie', async () => {
+            const response = await ofetch(items[0].link);
+            const $ = load(response);
+            const scriptPath = $('script').attr('src')!;
+            const scriptUrl = new URL(scriptPath, rootUrl).href;
+
+            const scriptResponse = await ofetch(scriptUrl);
+            const key = scriptResponse.match(/{var key="(.*?)"/)?.[1];
+            const value = scriptResponse.match(/",value="(.*?)"/)?.[1];
+            const getPath = scriptResponse.match(/\.get\("(.*?&key=)"/)?.[1];
+
+            if (!key || !value || !getPath) {
+                throw new Error('Failed to get cookie');
+            }
+
+            const cookieResponse = await ofetch.raw(`${rootUrl}${getPath}${key}&value=${md5(stringtoHex(value))}`);
+            return cookieResponse.headers
+                .getSetCookie()
+                .map((c) => c.split(';')[0])
+                .join('; ');
+        });
+
+    const cookie = await getCookie();
+
     items = await Promise.all(
         items.map((item) =>
             cache.tryGet(item.link, async () => {
-                const { data: detailResponse } = await got(item.link, {
-                    responseType: 'buffer',
+                const detailResponse = await ofetch(item.link, {
+                    responseType: 'arrayBuffer',
+                    headers: {
+                        Cookie: cookie as string,
+                    },
                 });
 
-                const $$ = load(iconv.decode(detailResponse, 'gbk'));
+                const $$ = load(decoder.decode(detailResponse));
 
                 $$('div.nex_drama_intros em').first().remove();
                 $$('strong font').each((_, el) => {
@@ -86,14 +126,8 @@ async function handler(ctx) {
                         const value = li.find('span').length === 0 ? li.contents().last().text().trim() : li.find('span').text().trim();
 
                         return { [key]: value };
-                    })
-                    .reduce(
-                        (obj, item) => ({
-                            ...obj,
-                            ...item,
-                        }),
-                        {}
-                    );
+                    });
+                const mergedDetails = Object.assign({}, ...details);
 
                 const links =
                     $$('td.t_f ignore_js_op').length === 0
@@ -154,17 +188,17 @@ async function handler(ctx) {
                           ]
                         : undefined,
                     title,
-                    keys: Object.keys(details),
-                    details,
+                    keys: Object.keys(mergedDetails),
+                    details: mergedDetails,
                     description,
                     info: $$('div.nex_drama_sums').html(),
                     links,
                 });
                 item.pubDate = timezone(parseDate(pubDate, 'YYYY-M-D HH:mm:ss'), +8);
-                item.category = Object.values(details)
+                item.category = Object.values(mergedDetails)
                     .flatMap((c) => c.split(/\s/))
                     .filter(Boolean);
-                item.author = details['导演'];
+                item.author = mergedDetails['导演'];
                 item.content = {
                     html: description,
                     text: $$('div.nex_drama_intros').text(),

package/lib/routes/500px/tribe-set.ts

@@ -1,4 +1,4 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import { getCurrentPath } from '@/utils/helpers';
 const __dirname = getCurrentPath(import.meta.url);
 
@@ -10,7 +10,8 @@ import { baseUrl, getTribeDetail, getTribeSets } from './utils';
 
 export const route: Route = {
     path: '/tribe/set/:id',
-    categories: ['picture'],
+    categories: ['picture', 'popular'],
+    view: ViewType.Pictures,
     example: '/500px/tribe/set/f5de0b8aa6d54ec486f5e79616418001',
     parameters: { id: '部落 ID' },
     name: '部落影集',

package/lib/routes/51cto/recommend.ts

@@ -2,6 +2,10 @@ import { Route } from '@/types';
 import { parseDate } from '@/utils/parse-date';
 import got from '@/utils/got';
 import { getToken, sign } from './utils';
+import { load } from 'cheerio';
+import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
+import logger from '@/utils/logger';
 
 export const route: Route = {
     path: '/index/recommend',
@@ -13,11 +17,46 @@ export const route: Route = {
         },
     ],
     name: '推荐',
-    maintainers: ['cnkmmk'],
+    maintainers: ['cnkmmk', 'ovo-tim'],
     handler,
     url: '51cto.com/',
 };
 
+const pattern = /'(WTKkN|bOYDu|wyeCN)':\s*(\d+)/g;
+
+async function getFullcontent(item, cookie = '') {
+    let fullContent: null | string = null;
+    const articleResponse = await ofetch(item.url, {
+        headers: {
+            cookie,
+        },
+    });
+    const $ = load(articleResponse);
+
+    fullContent = new URL(item.url).host === 'ost.51cto.com' ? $('.posts-content').html() : $('article').html();
+
+    if (!fullContent && cookie === '') {
+        // If fullContent is null and haven't tried to request with cookie, try to get fullContent with cookie
+        try {
+            // More details: https://github.com/DIYgod/RSSHub/pull/16583#discussion_r1738643033
+            const _matches = articleResponse!.match(pattern)!.slice(0, 3);
+            const matches = _matches.map((str) => Number(str.split(':')[1]));
+            const [v1, v2, v3] = matches;
+            const cookie = '__tst_status=' + (v1 + v2 + v3) + '#;';
+            return await getFullcontent(item, cookie);
+        } catch (error) {
+            logger.error(error);
+        }
+    }
+
+    return {
+        title: item.title,
+        link: item.url,
+        pubDate: parseDate(item.pubdate, +8),
+        description: fullContent || item.abstract, // Return item.abstract if fullContent is null
+    };
+}
+
 async function handler(ctx) {
     const url = 'https://api-media.51cto.com';
     const requestPath = 'index/index/recommend';
@@ -29,6 +68,7 @@ async function handler(ctx) {
         limit_time: 0,
         name_en: '',
     };
+
     const response = await got(`${url}/${requestPath}`, {
         searchParams: {
             ...params,
@@ -38,15 +78,13 @@ async function handler(ctx) {
         },
     });
     const list = response.data.data.data.list;
+
+    const items = await Promise.all(list.map((item) => cache.tryGet(item.url, async () => await getFullcontent(item))));
+
     return {
         title: '51CTO',
         link: 'https://www.51cto.com/',
         description: '51cto - 推荐',
-        item: list.map((item) => ({
-            title: item.title,
-            link: item.url,
-            pubDate: parseDate(item.pubdate, +8),
-            description: item.abstract,
-        })),
+        item: items,
     };
 }

package/lib/routes/5eplay/index.ts

@@ -1,11 +1,8 @@
 import { Route } from '@/types';
 import cache from '@/utils/cache';
 import { load } from 'cheerio';
-import zlib from 'zlib';
 import got from '@/utils/got';
 import { parseDate } from '@/utils/parse-date';
-import { config } from '@/config';
-import { getAcwScV2ByArg1 } from './utils';
 
 export const route: Route = {
     path: '/article',
@@ -34,70 +31,23 @@ export const route: Route = {
 async function handler() {
     const rootUrl = 'https://csgo.5eplay.com/';
     const apiUrl = `${rootUrl}api/article?page=1&type_id=0&time=0&order_by=0`;
-    const articleUrl = `${rootUrl}article`;
 
     const { data: response } = await got({
         method: 'get',
         url: apiUrl,
     });
 
-    // get acw_sc__v2
-    const acw_sc__v2 = await cache.tryGet(
-        articleUrl,
-        async () => {
-            // Zlib Z_BUF_ERROR: unexpected end of file, should close decompress
-            const detailResponse = await got(
-                {
-                    method: 'get',
-                    url: articleUrl,
-                },
-                {
-                    decompress: false,
-                }
-            );
-
-            const unzipData = zlib.createUnzip({
-                chunkSize: 20 * 1024,
-            });
-            unzipData.write(detailResponse.body);
-
-            let acw_sc__v2 = '';
-            for await (const data of unzipData) {
-                const strData = data.toString();
-                const matches = strData.match(/var arg1='(.*?)';/);
-                if (matches) {
-                    acw_sc__v2 = getAcwScV2ByArg1(matches[1]);
-                    break;
-                }
-            }
-            return acw_sc__v2;
-        },
-        Math.min(config.cache.routeExpire, 25 * 60),
-        false
-    );
-
     const items = await Promise.all(
         response.data.list.map((item) =>
             cache.tryGet(item.jump_link, async () => {
-                if (!acw_sc__v2) {
-                    return {
-                        title: item.title,
-                        description: item.title + (item.images?.[0] ? `<img src="${item.images[0]}" />` : ''),
-                        pubDate: parseDate(item.dateline * 1000),
-                        link: item.jump_link,
-                    };
-                }
                 const { data: detailResponse } = await got({
                     method: 'get',
                     url: item.jump_link,
-                    headers: {
-                        cookie: `acw_sc__v2=${acw_sc__v2}`,
-                    },
                 });
                 const $ = load(detailResponse);
 
                 const content = $('.article-text');
-                const res = [];
+                const res: string[] = [];
 
                 content.find('> p, > blockquote').each((i, e) => {
                     res.push($(e).text());

package/lib/routes/5eplay/utils.ts

@@ -13,10 +13,12 @@ const getAcwScV2ByArg1 = (arg1) => {
         }
         return res;
     };
-    const unsbox = function (str) {
+    const unsbox = function (str: string) {
         const code = [15, 35, 29, 24, 33, 16, 1, 38, 10, 9, 19, 31, 40, 27, 22, 23, 25, 13, 6, 11, 39, 18, 20, 8, 14, 21, 32, 26, 2, 30, 7, 4, 17, 5, 3, 28, 34, 37, 12, 36];
-        const res = [];
-        for (const [i, cur] of str.entries()) {
+        const res: string[] = [];
+        // eslint-disable-next-line unicorn/no-for-loop
+        for (let i = 0; i < str.length; i++) {
+            const cur = str[i];
             for (const [j, element] of code.entries()) {
                 if (element === i + 1) {
                     res[j] = cur;