rsshub 1.0.0-master.f71451d → 1.0.0-master.f72af1b

package/lib/routes/423down/index.ts

@@ -1,123 +1,280 @@
 import { Route } from '@/types';
+import { getCurrentPath } from '@/utils/helpers';
+const __dirname = getCurrentPath(import.meta.url);
+
 import cache from '@/utils/cache';
 import got from '@/utils/got';
 import { load } from 'cheerio';
 import { parseDate } from '@/utils/parse-date';
+import { art } from '@/utils/render';
+import path from 'node:path';
 
-const rootUrl = 'https://www.423down.com';
+export const handler = async (ctx) => {
+    const { category = '' } = ctx.req.param();
+    const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 18;
 
-const categeoryMap = {
-    index: {
-        all: '',
-    },
-    android: {
-        apk: 'apk',
-    },
-    computer: {
-        originalsoft: 'zd423',
-        multimedia: 'multimedia',
-        browser: 'browser',
-        image: 'image',
-        im: 'im',
-        work: 'work',
-        down: 'down',
-        systemsoft: 'systemsoft',
-        systemplus: 'systemplus',
-        security: 'security',
-        patch: 'patch',
-        hardware: 'hardware',
-    },
-    os: {
-        win11: 'win11',
-        win10: 'win10',
-        win7: 'win7',
-        winxp: 'winxp',
-        winpe: 'pe-system',
-    },
-};
+    const domain = '423down.com';
+    const rootUrl = `https://www.${domain}`;
+    const currentUrl = new URL(category, rootUrl).href;
 
-const titleMap = {
-    index: {
-        all: '首页',
-    },
-    android: {
-        apk: '安卓软件',
-    },
-    computer: {
-        originalsoft: '原创软件',
-        multimedia: '媒体播放',
-        browser: '网页浏览',
-        image: '图形图像',
-        im: '聊天软件',
-        work: '办公软件',
-        down: '上传下载',
-        systemsoft: '系统辅助',
-        systemplus: '系统必备',
-        security: '安全软件',
-        patch: '补丁相关',
-        hardwork: '硬件相关',
-    },
-    os: {
-        win11: 'windows 11',
-        win10: 'Windows 10',
-        win7: 'Windows 7',
-        winxp: 'Windows XP',
-        winpe: 'Windows PE',
-    },
-};
+    const { data: response } = await got(currentUrl);
 
-export const route: Route = {
-    path: '/:category/:type?',
-    name: 'Unknown',
-    maintainers: [],
-    handler,
-};
+    const $ = load(response);
 
-async function handler(ctx) {
-    const { category, type } = ctx.req.param();
+    const language = $('html').prop('lang');
 
-    const url = `${rootUrl}/${categeoryMap[category][type]}`;
+    let items = $('ul.excerpt li')
+        .toArray()
+        .filter((item) => {
+            item = $(item);
 
-    const response = await got.get(url);
-    const $ = load(response.data);
-    const list = $('div.content-wrap > div > ul > li > a')
-        .filter((_, item) => {
-            const notAnotherWebPage = $(item).attr('style') !== 'display: none !important;';
-            return notAnotherWebPage;
+            const link = item.find('h2 a').prop('href');
+
+            return new RegExp(domain).test(link);
         })
-        .map((_, item) => ({
-            link: $(item).attr('href'),
-        }))
-        .get();
-
-    const items = await Promise.all(
-        list.map(async (item) => {
-            item = await cache.tryGet(item.link, async () => {
-                const detailResponse = await got.get(item.link);
-                const $ = load(detailResponse.data);
-
-                const title = $('div.content-wrap > div > div.meta > h1 > a').text();
-                const pageContent = $('div.content-wrap > div > div.entry').html();
-                const pageComments = $('#postcomments > ol').html();
-                const desc = pageContent + pageComments;
-                const date = $('div.content-wrap > div > div.meta > p').text();
-                const categeory = $('div.content-wrap > div > div.meta > p > a:not(.comm)').text();
+        .slice(0, limit)
+        .map((item) => {
+            item = $(item);
+
+            const title = item.find('h2').text();
+            const image = item.find('a.pic img').prop('src');
+            const description = art(path.join(__dirname, 'templates/description.art'), {
+                images: image
+                    ? [
+                          {
+                              src: image,
+                              alt: title,
+                          },
+                      ]
+                    : undefined,
+                intro: item.find('div.note').text(),
+            });
+
+            return {
+                title,
+                description,
+                pubDate: parseDate(item.find('span.time').text(), 'MM-DD'),
+                link: item.find('h2 a').prop('href'),
+                category: item
+                    .find('span.cat a')
+                    .toArray()
+                    .map((c) => $(c).text()),
+                content: {
+                    html: description,
+                    text: item.find('div.note').text(),
+                },
+                image,
+                banner: image,
+                language,
+                enclosure_url: image,
+                enclosure_type: image ? `image/${image.split(/\./).pop()}` : undefined,
+                enclosure_title: title,
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            cache.tryGet(item.link, async () => {
+                const { data: detailResponse } = await got(item.link);
+
+                const $$ = load(detailResponse);
+
+                const title = $$('h1.meta-tit a').text();
+                const description =
+                    item.description +
+                    art(path.join(__dirname, 'templates/description.art'), {
+                        description: $$('div.entry').html(),
+                    });
 
                 item.title = title;
-                item.description = desc;
-                item.categeory = categeory;
-                item.pubDate = parseDate(date.split(' ')[0], 'YYYY-MM-DD');
+                item.description = description;
+                item.pubDate = parseDate($$('p.meta-info').contents().first().text().trim().split(/\s/)[0], 'YYYY-MM-DD');
+                item.category = $$('p.meta-info a[rel="category tag"]')
+                    .toArray()
+                    .map((c) => $$(c).text());
+                item.content = {
+                    html: description,
+                    text: $$('div.entry').text(),
+                };
+                item.language = language;
 
                 return item;
-            });
-
-            return item;
-        })
+            })
+        )
     );
 
+    const title = $('title').first().text();
+    const image = new URL('wp-content/themes/D7/img/423Down.png', rootUrl).href;
+
     return {
-        title: `423down-${titleMap[category][type]}`,
-        link: url,
+        title,
+        description: $('title').last().text(),
+        link: currentUrl,
         item: items,
+        allowEmpty: true,
+        image,
+        author: title.split(/-/).pop()?.trim(),
+        language,
     };
-}
+};
+
+export const route: Route = {
+    path: '/:category{.+}?',
+    name: '423Down',
+    url: '423down.com',
+    maintainers: ['nczitzk'],
+    handler,
+    example: '/423down',
+    parameters: { category: '分类，默认为首页，可在对应分类页 URL 中找到' },
+    description: `:::tip
+  若订阅 [Android - 423Down](https://www.423down.com/apk)，网址为 \`https://www.423down.com/apk\`。截取 \`https://www.423down.com/\` 到末尾的部分 \`apk\` 作为参数填入，此时路由为 [\`/423down/apk\`](https://rsshub.app/423down/apk)。
+  :::
+
+  #### [安卓软件](https://www.423down.com/apk)
+
+  | [安卓软件](https://www.423down.com/apk) |
+  | --------------------------------------- |
+  | [apk](https://rsshub.app/423down/apk)   |
+  
+  #### 电脑软件
+  
+  | [原创软件](https://www.423down.com/zd423) | [媒体播放](https://www.423down.com/multimedia)      | [网页浏览](https://www.423down.com/browser)   | [图形图像](https://www.423down.com/image) | [聊天软件](https://www.423down.com/im) |
+  | ----------------------------------------- | --------------------------------------------------- | --------------------------------------------- | ----------------------------------------- | -------------------------------------- |
+  | [zd423](https://rsshub.app/423down/zd423) | [multimedia](https://rsshub.app/423down/multimedia) | [browser](https://rsshub.app/423down/browser) | [image](https://rsshub.app/423down/image) | [im](https://rsshub.app/423down/im)    |
+  
+  | [办公软件](https://www.423down.com/work) | [上传下载](https://www.423down.com/down) | [实用软件](https://www.423down.com/softtool)    | [系统辅助](https://www.423down.com/systemsoft)      | [系统必备](https://www.423down.com/systemplus)      |
+  | ---------------------------------------- | ---------------------------------------- | ----------------------------------------------- | --------------------------------------------------- | --------------------------------------------------- |
+  | [work](https://rsshub.app/423down/work)  | [down](https://rsshub.app/423down/down)  | [softtool](https://rsshub.app/423down/softtool) | [systemsoft](https://rsshub.app/423down/systemsoft) | [systemplus](https://rsshub.app/423down/systemplus) |
+  
+  | [安全软件](https://www.423down.com/security)    | [补丁相关](https://www.423down.com/patch) | [硬件相关](https://www.423down.com/hardware)    |
+  | ----------------------------------------------- | ----------------------------------------- | ----------------------------------------------- |
+  | [security](https://rsshub.app/423down/security) | [patch](https://rsshub.app/423down/patch) | [hardware](https://rsshub.app/423down/hardware) |
+  
+  #### 操作系统
+  
+  | [Windows 11](https://www.423down.com/win11) | [Windows 10](https://www.423down.com/win10) | [Windows 7](https://www.423down.com/win7) | [Windows XP](https://www.423down.com/win7/winxp)    | [WinPE](https://www.423down.com/pe-system)        |
+  | ------------------------------------------- | ------------------------------------------- | ----------------------------------------- | --------------------------------------------------- | ------------------------------------------------- |
+  | [win11](https://rsshub.app/423down/win11)   | [win10](https://rsshub.app/423down/win10)   | [win7](https://rsshub.app/423down/win7)   | [win7/winxp](https://rsshub.app/423down/win7/winxp) | [pe-system](https://rsshub.app/423down/pe-system) |
+  `,
+    categories: ['program-update'],
+
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportRadar: true,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['423down.com/:category', '423down.com'],
+            target: (params) => {
+                const category = params.category;
+
+                return `/423down${category ? `/${category}` : ''}`;
+            },
+        },
+        {
+            title: '首页',
+            source: ['www.423down.com'],
+            target: '/',
+        },
+        {
+            title: '安卓软件',
+            source: ['www.423down.com/apk'],
+            target: '/apk',
+        },
+        {
+            title: '电脑软件 - 原创软件',
+            source: ['www.423down.com/zd423'],
+            target: '/zd423',
+        },
+        {
+            title: '电脑软件 - 媒体播放',
+            source: ['www.423down.com/multimedia'],
+            target: '/multimedia',
+        },
+        {
+            title: '电脑软件 - 网页浏览',
+            source: ['www.423down.com/browser'],
+            target: '/browser',
+        },
+        {
+            title: '电脑软件 - 图形图像',
+            source: ['www.423down.com/image'],
+            target: '/image',
+        },
+        {
+            title: '电脑软件 - 聊天软件',
+            source: ['www.423down.com/im'],
+            target: '/im',
+        },
+        {
+            title: '电脑软件 - 办公软件',
+            source: ['www.423down.com/work'],
+            target: '/work',
+        },
+        {
+            title: '电脑软件 - 上传下载',
+            source: ['www.423down.com/down'],
+            target: '/down',
+        },
+        {
+            title: '电脑软件 - 实用软件',
+            source: ['www.423down.com/softtool'],
+            target: '/softtool',
+        },
+        {
+            title: '电脑软件 - 系统辅助',
+            source: ['www.423down.com/systemsoft'],
+            target: '/systemsoft',
+        },
+        {
+            title: '电脑软件 - 系统必备',
+            source: ['www.423down.com/systemplus'],
+            target: '/systemplus',
+        },
+        {
+            title: '电脑软件 - 安全软件',
+            source: ['www.423down.com/security'],
+            target: '/security',
+        },
+        {
+            title: '电脑软件 - 补丁相关',
+            source: ['www.423down.com/patch'],
+            target: '/patch',
+        },
+        {
+            title: '电脑软件 - 硬件相关',
+            source: ['www.423down.com/hardware'],
+            target: '/hardware',
+        },
+        {
+            title: '操作系统 - Windows 11',
+            source: ['www.423down.com/win11'],
+            target: '/win11',
+        },
+        {
+            title: '操作系统 - Windows 10',
+            source: ['www.423down.com/win10'],
+            target: '/win10',
+        },
+        {
+            title: '操作系统 - Windows 7',
+            source: ['www.423down.com/win7'],
+            target: '/win7',
+        },
+        {
+            title: '操作系统 - Windows XP',
+            source: ['www.423down.com/win7/winxp'],
+            target: '/win7/winxp',
+        },
+        {
+            title: '操作系统 - WinPE',
+            source: ['www.423down.com/pe-system'],
+            target: '/pe-system',
+        },
+    ],
+};

package/lib/routes/423down/namespace.ts

@@ -2,5 +2,7 @@ import type { Namespace } from '@/types';
 
 export const namespace: Namespace = {
     name: '423Down',
-    url: 'www.423down.com',
+    url: '423down.com',
+    categories: ['program-update'],
+    description: '',
 };

package/lib/routes/423down/templates/description.art

@@ -0,0 +1,21 @@
+{{ if images }}
+  {{ each images image }}
+    {{ if image?.src }}
+      <figure>
+        <img
+          {{ if image.alt }}
+            alt="{{ image.alt }}"
+          {{ /if }}
+        src="{{ image.src }}">
+      </figure>
+    {{ /if }}
+  {{ /each }}
+{{ /if }}
+
+{{ if intro }}
+  <blockquote>{{ intro }}</blockquote>
+{{ /if }}
+
+{{ if description }}
+  {{@ description }}
+{{ /if }}

package/lib/routes/4ksj/forum.ts

@@ -3,13 +3,13 @@ import { getCurrentPath } from '@/utils/helpers';
 const __dirname = getCurrentPath(import.meta.url);
 
 import cache from '@/utils/cache';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
 import { load } from 'cheerio';
 import timezone from '@/utils/timezone';
 import { parseDate } from '@/utils/parse-date';
 import { art } from '@/utils/render';
 import path from 'node:path';
-import iconv from 'iconv-lite';
+import md5 from '@/utils/md5';
 
 export const route: Route = {
     path: '/:id?',
@@ -27,6 +27,16 @@ export const route: Route = {
     categories: ['multimedia'],
 };
 
+function stringtoHex(acSTR) {
+    let val = '';
+    for (let i = 0; i <= acSTR.length - 1; i++) {
+        const str = acSTR.charAt(i);
+        const code = str.codePointAt();
+        val += code;
+    }
+    return val;
+}
+
 async function handler(ctx) {
     const { id = '4k-uhd-1' } = ctx.req.param();
     const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 25;
@@ -34,11 +44,13 @@ async function handler(ctx) {
     const rootUrl = 'https://www.4ksj.com';
     const currentUrl = new URL(`${id}.html`, rootUrl).href;
 
-    const { data: response } = await got(currentUrl, {
-        responseType: 'buffer',
+    const response = await ofetch(currentUrl, {
+        responseType: 'arrayBuffer',
     });
 
-    const $ = load(iconv.decode(response, 'gbk'));
+    const decoder = new TextDecoder('gbk');
+
+    const $ = load(decoder.decode(response));
 
     const language = 'zh';
     const image = $('div.nexlogo img').prop('src');
@@ -54,14 +66,42 @@ async function handler(ctx) {
             };
         });
 
+    const getCookie = () =>
+        cache.tryGet('4ksj:cookie', async () => {
+            const response = await ofetch(items[0].link);
+            const $ = load(response);
+            const scriptPath = $('script').attr('src')!;
+            const scriptUrl = new URL(scriptPath, rootUrl).href;
+
+            const scriptResponse = await ofetch(scriptUrl);
+            const key = scriptResponse.match(/{var key="(.*?)"/)?.[1];
+            const value = scriptResponse.match(/",value="(.*?)"/)?.[1];
+            const getPath = scriptResponse.match(/\.get\("(.*?&key=)"/)?.[1];
+
+            if (!key || !value || !getPath) {
+                throw new Error('Failed to get cookie');
+            }
+
+            const cookieResponse = await ofetch.raw(`${rootUrl}${getPath}${key}&value=${md5(stringtoHex(value))}`);
+            return cookieResponse.headers
+                .getSetCookie()
+                .map((c) => c.split(';')[0])
+                .join('; ');
+        });
+
+    const cookie = await getCookie();
+
     items = await Promise.all(
         items.map((item) =>
             cache.tryGet(item.link, async () => {
-                const { data: detailResponse } = await got(item.link, {
-                    responseType: 'buffer',
+                const detailResponse = await ofetch(item.link, {
+                    responseType: 'arrayBuffer',
+                    headers: {
+                        Cookie: cookie as string,
+                    },
                 });
 
-                const $$ = load(iconv.decode(detailResponse, 'gbk'));
+                const $$ = load(decoder.decode(detailResponse));
 
                 $$('div.nex_drama_intros em').first().remove();
                 $$('strong font').each((_, el) => {
@@ -86,14 +126,8 @@ async function handler(ctx) {
                         const value = li.find('span').length === 0 ? li.contents().last().text().trim() : li.find('span').text().trim();
 
                         return { [key]: value };
-                    })
-                    .reduce(
-                        (obj, item) => ({
-                            ...obj,
-                            ...item,
-                        }),
-                        {}
-                    );
+                    });
+                const mergedDetails = Object.assign({}, ...details);
 
                 const links =
                     $$('td.t_f ignore_js_op').length === 0
@@ -154,17 +188,17 @@ async function handler(ctx) {
                           ]
                         : undefined,
                     title,
-                    keys: Object.keys(details),
-                    details,
+                    keys: Object.keys(mergedDetails),
+                    details: mergedDetails,
                     description,
                     info: $$('div.nex_drama_sums').html(),
                     links,
                 });
                 item.pubDate = timezone(parseDate(pubDate, 'YYYY-M-D HH:mm:ss'), +8);
-                item.category = Object.values(details)
+                item.category = Object.values(mergedDetails)
                     .flatMap((c) => c.split(/\s/))
                     .filter(Boolean);
-                item.author = details['导演'];
+                item.author = mergedDetails['导演'];
                 item.content = {
                     html: description,
                     text: $$('div.nex_drama_intros').text(),

package/lib/routes/500px/tribe-set.ts

@@ -1,4 +1,4 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import { getCurrentPath } from '@/utils/helpers';
 const __dirname = getCurrentPath(import.meta.url);
 
@@ -10,7 +10,8 @@ import { baseUrl, getTribeDetail, getTribeSets } from './utils';
 
 export const route: Route = {
     path: '/tribe/set/:id',
-    categories: ['picture'],
+    categories: ['picture', 'popular'],
+    view: ViewType.Pictures,
     example: '/500px/tribe/set/f5de0b8aa6d54ec486f5e79616418001',
     parameters: { id: '部落 ID' },
     name: '部落影集',

package/lib/routes/51cto/recommend.ts

@@ -2,6 +2,10 @@ import { Route } from '@/types';
 import { parseDate } from '@/utils/parse-date';
 import got from '@/utils/got';
 import { getToken, sign } from './utils';
+import { load } from 'cheerio';
+import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
+import logger from '@/utils/logger';
 
 export const route: Route = {
     path: '/index/recommend',
@@ -13,11 +17,46 @@ export const route: Route = {
         },
     ],
     name: '推荐',
-    maintainers: ['cnkmmk'],
+    maintainers: ['cnkmmk', 'ovo-tim'],
     handler,
     url: '51cto.com/',
 };
 
+const pattern = /'(WTKkN|bOYDu|wyeCN)':\s*(\d+)/g;
+
+async function getFullcontent(item, cookie = '') {
+    let fullContent: null | string = null;
+    const articleResponse = await ofetch(item.url, {
+        headers: {
+            cookie,
+        },
+    });
+    const $ = load(articleResponse);
+
+    fullContent = new URL(item.url).host === 'ost.51cto.com' ? $('.posts-content').html() : $('article').html();
+
+    if (!fullContent && cookie === '') {
+        // If fullContent is null and haven't tried to request with cookie, try to get fullContent with cookie
+        try {
+            // More details: https://github.com/DIYgod/RSSHub/pull/16583#discussion_r1738643033
+            const _matches = articleResponse!.match(pattern)!.slice(0, 3);
+            const matches = _matches.map((str) => Number(str.split(':')[1]));
+            const [v1, v2, v3] = matches;
+            const cookie = '__tst_status=' + (v1 + v2 + v3) + '#;';
+            return await getFullcontent(item, cookie);
+        } catch (error) {
+            logger.error(error);
+        }
+    }
+
+    return {
+        title: item.title,
+        link: item.url,
+        pubDate: parseDate(item.pubdate, +8),
+        description: fullContent || item.abstract, // Return item.abstract if fullContent is null
+    };
+}
+
 async function handler(ctx) {
     const url = 'https://api-media.51cto.com';
     const requestPath = 'index/index/recommend';
@@ -29,6 +68,7 @@ async function handler(ctx) {
         limit_time: 0,
         name_en: '',
     };
+
     const response = await got(`${url}/${requestPath}`, {
         searchParams: {
             ...params,
@@ -38,15 +78,13 @@ async function handler(ctx) {
         },
     });
     const list = response.data.data.data.list;
+
+    const items = await Promise.all(list.map((item) => cache.tryGet(item.url, async () => await getFullcontent(item))));
+
     return {
         title: '51CTO',
         link: 'https://www.51cto.com/',
         description: '51cto - 推荐',
-        item: list.map((item) => ({
-            title: item.title,
-            link: item.url,
-            pubDate: parseDate(item.pubdate, +8),
-            description: item.abstract,
-        })),
+        item: items,
     };
 }