rsshub 1.0.0-master.f7cdf8b → 1.0.0-master.f7f8b7a

package/lib/v2/picnob/templates/desc.art

@@ -3,11 +3,11 @@
         <source src="{{ item.video }}" type="video/mp4">
     </video>
 {{ else if item.type === 'img_multi' }}
-    {{ each images i }}
-        <a href="{{ i.ori }}"><img src="{{ i.url }}"></a>
+    {{ each item.images i }}
+        <img src="{{ i.url }}">
     {{ /each }}
 {{ else if item.type === 'img_sig' }}
-    <a href="{{ item.pic }}"><img src="{{ item.pic }}"></a>
+    <img src="{{ item.pic }}">
 {{ /if }}
 <br>
 {{@ item.sum }}

package/lib/v2/picnob/user.js

@@ -3,53 +3,90 @@ const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
 const { art } = require('@/utils/render');
 const path = require('path');
+const { puppeteerGet } = require('./utils');
 
 module.exports = async (ctx) => {
     const baseUrl = 'https://www.picnob.com';
     const { id } = ctx.params;
     const url = `${baseUrl}/profile/${id}/`;
-    const { data: response } = await got(url);
-    const $ = cheerio.load(response);
 
+    const browser = await require('@/utils/puppeteer')();
+    // TODO: can't bypass cloudflare 403 error without puppeteer
+    let html;
+    let usePuppeteer = false;
+    try {
+        const { data } = await got(url, {
+            headers: {
+                accept: 'text/html',
+                referer: 'https://www.google.com/',
+            },
+        });
+        html = data;
+    } catch (e) {
+        if (e.message.includes('code 403')) {
+            html = await puppeteerGet(url, browser);
+            usePuppeteer = true;
+        }
+    }
+    const $ = cheerio.load(html);
     const profileName = $('h1.fullname').text();
     const userId = $('input[name=userid]').attr('value');
 
-    const { data } = await got(`${baseUrl}/api/posts`, {
-        searchParams: {
-            userid: userId,
-        },
-    });
+    let posts;
+    if (!usePuppeteer) {
+        const { data } = await got(`${baseUrl}/api/posts`, {
+            headers: {
+                accept: 'application/json',
+            },
+            searchParams: {
+                userid: userId,
+            },
+        });
+        posts = data.posts;
+    } else {
+        const data = await puppeteerGet(`${baseUrl}/api/posts?userid=${userId}`, browser);
+        posts = data.posts;
+    }
 
-    const list = data.posts.items.map(async (item) => {
-        const { shortcode, type } = item;
-        const link = `${baseUrl}/post/${shortcode}/`;
-        let images = [];
-        if (type === 'img_multi') {
-            images = await ctx.cache.tryGet(link, async () => {
-                const { data } = await got(link);
-                const $ = cheerio.load(data);
-                return [
-                    ...new Set(
-                        $('.post_slide a')
-                            .toArray()
-                            .map((a) => {
-                                a = $(a);
-                                return {
-                                    ori: a.attr('href'),
-                                    url: a.find('img').attr('data-src'),
-                                };
-                            })
-                    ),
-                ];
-            });
-        }
-        return {
-            title: item.sum_pure,
-            description: art(path.join(__dirname, 'templates/desc.art'), { item, images }),
-            link,
-            pubDate: parseDate(item.time, 'X'),
-        };
-    });
+    const list = await Promise.all(
+        posts.items.map(async (item) => {
+            const { shortcode, type } = item;
+            const link = `${baseUrl}/post/${shortcode}/`;
+            if (type === 'img_multi') {
+                item.images = await ctx.cache.tryGet(link, async () => {
+                    let html;
+                    if (!usePuppeteer) {
+                        const { data } = await got(link);
+                        html = data;
+                    } else {
+                        html = await puppeteerGet(link, browser);
+                    }
+                    const $ = cheerio.load(html);
+                    return [
+                        ...new Set(
+                            $('.post_slide a')
+                                .toArray()
+                                .map((a) => {
+                                    a = $(a);
+                                    return {
+                                        ori: a.attr('href'),
+                                        url: a.find('img').attr('data-src'),
+                                    };
+                                })
+                        ),
+                    ];
+                });
+            }
+
+            return {
+                title: item.sum_pure,
+                description: art(path.join(__dirname, 'templates/desc.art'), { item }),
+                link,
+                pubDate: parseDate(item.time, 'X'),
+            };
+        })
+    );
+    await browser.close();
 
     ctx.state.data = {
         title: `${profileName} (@${id}) - Picnob`,

package/lib/v2/picnob/utils.js

@@ -0,0 +1,24 @@
+const puppeteerGet = async (url, browser) => {
+    let data;
+    const page = await browser.newPage();
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' ? request.continue() : request.abort();
+    });
+    page.on('response', async (response) => {
+        if (response.request().url().includes('/api/posts')) {
+            data = await response.json();
+        } else {
+            data = await response.text();
+        }
+    });
+    await page.goto(url, {
+        waitUntil: 'domcontentloaded',
+    });
+    await page.close();
+    return data;
+};
+
+module.exports = {
+    puppeteerGet,
+};

package/lib/v2/qbitai/category.js

@@ -0,0 +1,27 @@
+const parser = require('@/utils/rss-parser');
+
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const { category } = ctx.params;
+    const link = encodeURI(`https://www.qbitai.com/category/${category}/feed`);
+    const feed = await parser.parseURL(link);
+
+    const items = feed.items.map((item) => ({
+        title: item.title,
+        pubDate: parseDate(item.pubDate),
+        link: item.link,
+        author: '量子位',
+        category: item.categories,
+        description: item['content:encoded'],
+    }));
+
+    ctx.state.data = {
+        // 源标题
+        title: `量子位-${category}`,
+        // 源链接
+        link: `https://www.qbitai.com/category/${category}`,
+        // 源文章
+        item: items,
+    };
+};

package/lib/v2/qbitai/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/category/:category': ['FuryMartin'],
+    '/tag/:tag': ['FuryMartin'],
+};

package/lib/v2/qbitai/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'qbitai.com': {
+        _name: '量子位',
+        '.': [
+            {
+                title: '分类',
+                docs: 'https://docs.rsshub.app/routes/new-media#liang-zi-wei',
+                source: ['/category/:category'],
+                target: '/qbitai/category/:category',
+            },
+            {
+                title: '标签',
+                docs: 'https://docs.rsshub.app/routes/new-media#liang-zi-wei',
+                source: ['/tag/:tag'],
+                target: '/qbitai/tag/:tag',
+            },
+        ],
+    },
+};

package/lib/v2/qbitai/router.js

@@ -0,0 +1,4 @@
+module.exports = (router) => {
+    router.get('/category/:category', require('./category'));
+    router.get('/tag/:tag', require('./tag'));
+};

package/lib/v2/qbitai/tag.js

@@ -0,0 +1,27 @@
+const parser = require('@/utils/rss-parser');
+
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const { tag } = ctx.params;
+    const link = encodeURI(`https://www.qbitai.com/tag/${tag}/feed`);
+    const feed = await parser.parseURL(link);
+
+    const items = feed.items.map((item) => ({
+        title: item.title,
+        pubDate: parseDate(item.pubDate),
+        link: item.link,
+        author: '量子位',
+        category: item.categories,
+        description: item['content:encoded'],
+    }));
+
+    ctx.state.data = {
+        // 源标题
+        title: `量子位-${tag}`,
+        // 源链接
+        link: `https://www.qbitai.com/tag/${tag}`,
+        // 源文章
+        item: items,
+    };
+};

package/lib/v2/questmobile/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/report/:industry?/:label?': ['nczitzk'],
+};

package/lib/v2/questmobile/radar.js

@@ -0,0 +1,18 @@
+module.exports = {
+    'questmobile.com.cn': {
+        _name: 'QuestMobile',
+        '.': [
+            {
+                title: '行业研究报告',
+                docs: 'https://docs.rsshub.app/routes/new-media#questmobile-hang-ye-yan-jiu-bao-gao',
+                source: ['/research/reports/:industry/:label'],
+                target: (params) => {
+                    const industry = params.industry;
+                    const label = params.label;
+
+                    return `/questmobile/report/${industry}/${label}`;
+                },
+            },
+        ],
+    },
+};

package/lib/v2/questmobile/report.js

@@ -0,0 +1,127 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+/**
+ * Parses a tree array and returns an array of objects containing the key-value pairs.
+ * @param {Array} tree - The tree to parse.
+ * @param {Array} result - The result array to store the parsed key-value pairs. Default is an empty array.
+ *
+ * @returns {Array} - An array of objects containing the key-value pairs.
+ */
+const parseTree = (tree, result = []) => {
+    tree.forEach((obj) => {
+        const { key, value, children } = obj;
+        result.push({ key, value });
+
+        if (children && children.length > 0) {
+            parseTree(children, result);
+        }
+    });
+
+    return result;
+};
+
+module.exports = async (ctx) => {
+    const { industry, label } = ctx.params;
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit, 10) : 50;
+
+    const rootUrl = 'https://www.questmobile.com.cn';
+    const apiUrl = new URL('api/v2/report/article-list', rootUrl).href;
+    const apiTreeUrl = new URL('api/v2/report/industry-label-tree', rootUrl).href;
+
+    const {
+        data: {
+            data: { industryTree, labelTree },
+        },
+    } = await got(apiTreeUrl);
+
+    const industries = parseTree(industryTree);
+    const labels = parseTree(labelTree);
+
+    const industryObj = industry ? industries.find((i) => i.key === industry || i.value === industry) : undefined;
+    const labelObj = label ? labels.find((i) => i.key === label || i.value === label) : industryObj ? undefined : labels.find((i) => i.key === industry || i.value === industry);
+
+    const industryId = industryObj?.key ?? -1;
+    const labelId = labelObj?.key ?? -1;
+
+    const currentUrl = new URL(`research/reports/${industryObj?.key ?? -1}/${labelObj?.key ?? -1}`, rootUrl).href;
+
+    const { data: response } = await got(apiUrl, {
+        searchParams: {
+            version: 0,
+            pageSize: limit,
+            pageNo: 1,
+            industryId,
+            labelId,
+        },
+    });
+
+    let items = response.data.slice(0, limit).map((item) => ({
+        title: item.title,
+        link: new URL(`research/report/${item.id}`, rootUrl).href,
+        description: art(path.join(__dirname, 'templates/description.art'), {
+            image: {
+                src: item.coverImgUrl,
+                alt: item.title,
+            },
+            introduction: item.introduction,
+            description: item.content,
+        }),
+        category: [...(item.industryList ?? []), ...(item.labelList ?? [])],
+        guid: `questmobile-report#${item.id}`,
+        pubDate: parseDate(item.publishTime),
+    }));
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: detailResponse } = await got(item.link);
+
+                const content = cheerio.load(detailResponse);
+
+                content('div.text div.daoyu').remove();
+
+                item.title = content('div.title h1').text();
+                item.description += art(path.join(__dirname, 'templates/description.art'), {
+                    description: content('div.text').html(),
+                });
+                item.author = content('div.source')
+                    .text()
+                    .replace(/^.*?：/, '');
+                item.category = content('div.hy, div.keyword')
+                    .find('span')
+                    .toArray()
+                    .map((c) => content(c).text());
+                item.pubDate = parseDate(content('div.data span').prop('datetime'));
+
+                return item;
+            })
+        )
+    );
+
+    const { data: currentResponse } = await got(currentUrl);
+
+    const $ = cheerio.load(currentResponse);
+
+    const author = $('meta[property="og:title"]').prop('content').split(/-/)[0];
+    const categories = [industryObj?.value, labelObj?.value].filter((c) => c);
+    const image = $(`img[alt="${author}"]`).prop('src');
+    const icon = $('link[rel="shortcut icon"]').prop('href');
+
+    ctx.state.data = {
+        item: items,
+        title: `${author}${categories.length === 0 ? '' : ` - ${categories.join(' - ')}`}`,
+        link: currentUrl,
+        description: $('meta[property="og:description"]').prop('content'),
+        language: 'zh',
+        image,
+        icon,
+        logo: icon,
+        subtitle: $('meta[name="keywords"]').prop('content'),
+        author,
+        allowEmpty: true,
+    };
+};

package/lib/v2/questmobile/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/report/:industry?/:label?', require('./report'));
+};

package/lib/v2/questmobile/templates/description.art

@@ -0,0 +1,17 @@
+{{ if image?.src }}
+  <figure>
+    <img
+      {{ if image.alt }}
+        alt="{{ image.alt }}"
+      {{ /if }}
+    src="{{ image.src }}">
+  </figure>
+{{ /if }}
+
+{{ if introduction }}
+  <blockquote>{{ introduction }}</blockquote>
+{{ /if }}
+
+{{ if description }}
+  {{@ description }}
+{{ /if }}

package/lib/v2/readhub/daily.js

@@ -0,0 +1,43 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+const { rootUrl, processItems } = require('./util');
+
+module.exports = async (ctx) => {
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit, 10) : 11;
+
+    const currentUrl = new URL('daily', rootUrl).href;
+
+    const { data: currentResponse } = await got(currentUrl);
+
+    const dailyItems = JSON.parse(currentResponse.match(/\{\\"daily\\":(.*?),\\"ts\\":\d+/)[1].replace(/\\"/g, '"'));
+
+    let items = dailyItems.slice(0, limit).map((item) => ({
+        title: item.title,
+        link: new URL(`topic/${item.uid}`, rootUrl).href,
+        guid: item.uid,
+    }));
+
+    items = await processItems(items, ctx.cache.tryGet);
+
+    const $ = cheerio.load(currentResponse);
+
+    const author = $('meta[name="application-name"]').prop('content');
+    const subtitle = $('meta[property="og:title"]').prop('content');
+    const image = 'https://readhub-oss.nocode.com/static/readhub.png';
+    const icon = new URL($('link[rel="apple-touch-icon"]').prop('href'), rootUrl);
+
+    ctx.state.data = {
+        item: items,
+        title: `${author} - ${subtitle}`,
+        link: currentUrl,
+        description: $('meta[name="description"]').prop('content'),
+        language: 'zh',
+        image,
+        icon,
+        logo: icon,
+        subtitle,
+        author,
+        allowEmpty: true,
+    };
+};

package/lib/v2/readhub/index.js

@@ -1,54 +1,29 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
-const timezone = require('@/utils/timezone');
 const { parseDate } = require('@/utils/parse-date');
-const { art } = require('@/utils/render');
 const path = require('path');
-const dayjs = require('dayjs');
+
+const { rootUrl, apiTopicUrl, art, processItems } = require('./util');
 
 module.exports = async (ctx) => {
-    const { category } = ctx.params;
-    const limit = ctx.query.limit ? parseInt(ctx.query.limit, 10) : 20;
+    const { category = '' } = ctx.params;
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit, 10) : 30;
 
-    const domain = 'readhub.cn';
-    const rootUrl = `https://${domain}`;
-    const apiRootUrl = `https://api.${domain}`;
     const currentUrl = new URL(category ?? '', rootUrl).href;
 
-    const isCategory = !category || category === 'daily';
-
-    const formatDate = (date, format) => dayjs(date).format(format);
-    const toTopicUrl = (id) => new URL(`topic/${id}`, rootUrl).href;
-
-    art.defaults.imports = {
-        ...art.defaults.imports,
-        ...{
-            formatDate,
-            toTopicUrl,
-        },
-    };
-
     const { data: currentResponse } = await got(currentUrl);
 
-    const buildId = currentResponse.match(/"buildId":"(\w+)"/)[1];
-
-    const apiUrl = new URL(category ? (category === 'daily' ? `_next/data/${buildId}/daily.json` : 'news/list') : 'topic/list', category === 'daily' ? rootUrl : apiRootUrl).href;
+    const type = currentResponse.match(/\[\\"type\\",\\"(\d+)\\",\\"d\\"\]/)?.[1] ?? '1';
 
-    const { data: response } = await got(apiUrl, {
+    const { data: response } = await got(apiTopicUrl, {
         searchParams: {
-            ...(isCategory
-                ? {}
-                : {
-                      type: currentResponse.match(/"query":\{"type":"(\d+)"\}/)[1],
-                  }),
-            ...{
-                page: 1,
-                size: limit,
-            },
+            type,
+            page: 1,
+            size: limit,
         },
     });
 
-    let items = (response.data?.items ?? response.pageProps.daily).slice(0, limit).map((item) => ({
+    let items = response.data.items.slice(0, limit).map((item) => ({
         title: item.title,
         link: item.url ?? new URL(`topic/${item.uid}`, rootUrl).href,
         description: art(path.join(__dirname, 'templates/description.art'), {
@@ -57,55 +32,27 @@ module.exports = async (ctx) => {
             timeline: item.timeline,
         }),
         author: item.siteNameDisplay,
-        category: item.tagList?.map((c) => c.name) ?? [],
-        guid: isCategory ? item.uid : `readhub-${item.guid}`,
-        pubDate: timezone(parseDate(item.publishDate), +8),
+        category: [...(item.entityList.map((c) => c.name) ?? []), ...(item.tagList.map((c) => c.name) ?? [])],
+        guid: item.uid,
+        pubDate: parseDate(item.publishDate),
     }));
 
-    if (isCategory) {
-        items = await Promise.all(
-            items.map((item) =>
-                ctx.cache.tryGet(`readhub-${item.guid}`, async () => {
-                    const { data: detailResponse } = await got(new URL(`_next/data/${buildId}/topic/${item.guid}.json`, rootUrl).href);
-
-                    const data = detailResponse.pageProps.topic;
-
-                    item.title = data.title;
-                    item.link = data.url ?? new URL(`topic/${data.uid}`, rootUrl).href;
-                    item.description = art(path.join(__dirname, 'templates/description.art'), {
-                        description: data.summary,
-                        news: data.newsAggList,
-                        timeline: data.timeline,
-                    });
-                    item.author = data.entityList.map((a) => a.name).join('/');
-                    item.category = data.tagList.map((c) => c.name);
-                    item.guid = `readhub-${data.uid}`;
-                    item.pubDate = timezone(parseDate(data.publishDate), +8);
-
-                    return item;
-                })
-            )
-        );
-    }
+    items = await processItems(items, ctx.cache.tryGet);
 
     const $ = cheerio.load(currentResponse);
 
     const author = $('meta[property="og:site_name"]').prop('content');
-    const subtitle = $(
-        $('nav div a')
-            .toArray()
-            .filter((a) => /style_active.*/.test($(a).prop('class')))
-            .pop()
-    ).text();
-    const icon = new URL($('link[rel="apple-touch-icon"]').prop('href'), rootUrl).href;
+    const subtitle = $(`a[data-path="/${category}"]`).text();
+    const image = $('link[rel="preload"][as="image"]').prop('href');
+    const icon = $('meta[property="og:image"]').prop('content');
 
     ctx.state.data = {
         item: items,
         title: `${author} - ${subtitle}`,
         link: currentUrl,
         description: $('meta[property="og:description"]').prop('content'),
-        language: $('html').prop('lang'),
-        image: $('meta[property="og:image"]').prop('content'),
+        language: 'zh',
+        image,
         icon,
         logo: icon,
         subtitle,

package/lib/v2/readhub/maintainer.js

@@ -1,3 +1,4 @@
 module.exports = {
+    '/daily': ['nczitzk'],
     '/:category?': ['WhiteWorld', 'nczitzk', 'Fatpandac'],
 };

package/lib/v2/readhub/radar.js

@@ -12,6 +12,12 @@ module.exports = {
                     return `/readhub${category ? `/${category}` : ''}`;
                 },
             },
+            {
+                title: '每日早报',
+                docs: 'https://docs.rsshub.app/routes/new-media#readhub-mei-ri-zao-bao',
+                source: ['/daily'],
+                target: '/readhub/daily',
+            },
         ],
     },
 };

package/lib/v2/readhub/router.js

@@ -1,3 +1,4 @@
 module.exports = (router) => {
+    router.get('/daily', require('./daily'));
     router.get('/:category?', require('./'));
 };