rsshub 1.0.0-master.f7cdf8b → 1.0.0-master.f812e91

package/lib/v2/woshipm/radar.js

@@ -8,24 +8,12 @@ module.exports = {
                 source: ['/'],
                 target: '/woshipm/popular',
             },
-            {
-                title: '用户收藏',
-                docs: 'https://docs.rsshub.app/routes/new-media#ren-ren-dui-shi-chan-pin-jing-li',
-                source: ['/u/:id'],
-                target: '/woshipm/bookmarks/:id',
-            },
             {
                 title: '用户文章',
                 docs: 'https://docs.rsshub.app/routes/new-media#ren-ren-dui-shi-chan-pin-jing-li',
                 source: ['/u/:id'],
                 target: '/woshipm/user_article/:id',
             },
-            {
-                title: '最新文章',
-                docs: 'https://docs.rsshub.app/routes/new-media#ren-ren-dui-shi-chan-pin-jing-li',
-                source: ['/'],
-                target: '/woshipm/latest',
-            },
         ],
         wen: [
             {

package/lib/v2/woshipm/router.js

@@ -1,6 +1,4 @@
 module.exports = (router) => {
-    router.get('/bookmarks/:id', require('./bookmarks'));
-    router.get('/latest', require('./latest'));
     router.get('/popular/:range?', require('./popular'));
     router.get('/user_article/:id', require('./user_article'));
     router.get('/wen', require('./wen'));

package/lib/v2/xiaohongshu/util.js

@@ -14,20 +14,30 @@ const getUser = (url, cache) =>
                 page.on('request', (request) => {
                     request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' || request.resourceType() === 'other' ? request.continue() : request.abort();
                 });
-                logger.debug(`Requesting ${url}`);
+                logger.http(`Requesting ${url}`);
                 await page.goto(url, {
                     waitUntil: 'domcontentloaded',
                 });
                 await page.waitForSelector('div.reds-tab-item:nth-child(2)');
-                await page.click('div.reds-tab-item:nth-child(2)');
-                const response = await page.waitForResponse((res) => {
-                    const req = res.request();
-                    return req.url().includes('/api/sns/web/v2/note/collect/page') && req.method() === 'GET' && req.resourceType() === 'xhr';
-                });
-                collect = await response.json();
 
                 const initialState = await page.evaluate(() => window.__INITIAL_STATE__);
 
+                if (!(await page.$('.lock-icon'))) {
+                    await page.click('div.reds-tab-item:nth-child(2)');
+                    try {
+                        const response = await page.waitForResponse(
+                            (res) => {
+                                const req = res.request();
+                                return req.url().includes('/api/sns/web/v2/note/collect/page') && req.method() === 'GET' && req.resourceType() === 'xhr';
+                            },
+                            { timeout: 5000 }
+                        );
+                        collect = await response.json();
+                    } catch (e) {
+                        //
+                    }
+                }
+
                 let { userPageData, notes } = initialState.user;
                 userPageData = userPageData._rawValue || userPageData;
                 notes = notes._rawValue || notes;

package/lib/v2/xmnn/maintainer.js

@@ -1,3 +1,4 @@
 module.exports = {
     '/epaper/:id?': ['nczitzk'],
+    '/news/:category?': ['nczitzk'],
 };

package/lib/v2/xmnn/news.js

@@ -0,0 +1,65 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const { category = 'xmxw' } = ctx.params;
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit, 10) : 30;
+
+    const rootUrl = 'https://news.xmnn.cn';
+    const currentUrl = new URL(`${category}/`, rootUrl).href;
+
+    const { data: response } = await got(currentUrl);
+
+    const $ = cheerio.load(response);
+
+    let items = $('div#sort_body ul li a')
+        .slice(0, limit)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            return {
+                title: item.find('h1').text().trim(),
+                link: item.prop('href'),
+                description: item.find('div.abstract').html(),
+                author: item.find('div.source').text(),
+                pubDate: timezone(parseDate(item.find('div.time').text()), +8),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: detailResponse } = await got(item.link);
+
+                const content = cheerio.load(detailResponse);
+
+                item.title = content('div.cont-h, div.tip h1').text().trim();
+                item.description = content('div.TRS_Editor').html();
+                item.author = content('span.cont-a-src a')
+                    .toArray()
+                    .map((a) => content(a).text());
+                item.pubDate = timezone(parseDate(content('span.time, div.pubtime div.w').contents().first().text().trim()), +8);
+
+                return item;
+            })
+        )
+    );
+
+    const title = $('title').text();
+    const icon = new URL($('link[rel="icon"]').prop('href'), rootUrl).href;
+
+    ctx.state.data = {
+        item: items,
+        title,
+        link: currentUrl,
+        description: $('meta[name="description"]').prop('content'),
+        language: 'zh',
+        icon,
+        logo: icon,
+        subtitle: $('div.h').text(),
+        author: title.split(/_/).pop(),
+    };
+};

package/lib/v2/xmnn/radar.js

@@ -5,9 +5,75 @@ module.exports = {
             {
                 title: '数字媒体',
                 docs: 'https://docs.rsshub.app/routes/traditional-media#xia-men-wang-shu-zi-mei-ti',
-                source: ['/:id', '/'],
+                source: ['/:id'],
                 target: '/xmnn/epaper/:id',
             },
         ],
+        news: [
+            {
+                title: '新闻',
+                docs: 'https://docs.rsshub.app/routes/traditional-media#xia-men-wang-xin-wen',
+                source: ['/:category*'],
+                target: (params) => {
+                    const category = params.category;
+
+                    return `/xmnn/news/${category ? `/${category}` : ''}`;
+                },
+            },
+            {
+                title: '厦门新闻发布',
+                docs: 'https://docs.rsshub.app/routes/traditional-media#xia-men-wang-xin-wen',
+                source: ['/xmxwfb'],
+                target: '/xmnn/news/xmxwfb',
+            },
+            {
+                title: '厦门新闻',
+                docs: 'https://docs.rsshub.app/routes/traditional-media#xia-men-wang-xin-wen',
+                source: ['/xmxw'],
+                target: '/xmnn/news/xmxw',
+            },
+            {
+                title: '本网快报',
+                docs: 'https://docs.rsshub.app/routes/traditional-media#xia-men-wang-xin-wen',
+                source: ['/bwkb'],
+                target: '/xmnn/news/bwkb',
+            },
+            {
+                title: '厦门网眼',
+                docs: 'https://docs.rsshub.app/routes/traditional-media#xia-men-wang-xin-wen',
+                source: ['/xmwy'],
+                target: '/xmnn/news/xmwy',
+            },
+            {
+                title: '福建新闻',
+                docs: 'https://docs.rsshub.app/routes/traditional-media#xia-men-wang-xin-wen',
+                source: ['/fjxw'],
+                target: '/xmnn/news/fjxw',
+            },
+            {
+                title: '国内新闻',
+                docs: 'https://docs.rsshub.app/routes/traditional-media#xia-men-wang-xin-wen',
+                source: ['/gnxw'],
+                target: '/xmnn/news/gnxw',
+            },
+            {
+                title: '国际新闻',
+                docs: 'https://docs.rsshub.app/routes/traditional-media#xia-men-wang-xin-wen',
+                source: ['/gjxw'],
+                target: '/xmnn/news/gjxw',
+            },
+            {
+                title: '台海新闻',
+                docs: 'https://docs.rsshub.app/routes/traditional-media#xia-men-wang-xin-wen',
+                source: ['/thxw'],
+                target: '/xmnn/news/thxw',
+            },
+            {
+                title: '社会新闻',
+                docs: 'https://docs.rsshub.app/routes/traditional-media#xia-men-wang-xin-wen',
+                source: ['/shxw'],
+                target: '/xmnn/news/shxw',
+            },
+        ],
     },
 };

package/lib/v2/xmnn/router.js

@@ -1,3 +1,4 @@
 module.exports = function (router) {
     router.get('/epaper/:id?', require('./epaper'));
+    router.get('/news/:category*', require('./news'));
 };

package/lib/v2/xsijishe/maintainer.js

@@ -1,3 +1,4 @@
 module.exports = {
     '/forum/:fid': ['akynazh'],
+    '/rank/:type': ['akynazh'],
 };

package/lib/v2/xsijishe/radar.js

@@ -14,6 +14,18 @@ module.exports = {
                     }
                 },
             },
+            {
+                title: '周排行榜',
+                docs: 'https://docs.rsshub.app/routes/bbs#si-ji-she',
+                source: ['/*'],
+                target: '/xsijishe/rank/weekly',
+            },
+            {
+                title: '月排行榜',
+                docs: 'https://docs.rsshub.app/routes/bbs#si-ji-she',
+                source: ['/*'],
+                target: '/xsijishe/rank/monthly',
+            },
         ],
     },
 };

package/lib/v2/xsijishe/rank.js

@@ -0,0 +1,60 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const baseUrl = 'https://xsijishe.com';
+
+module.exports = async (ctx) => {
+    const rankType = ctx.params.type;
+    let title;
+    let rankId;
+    if (rankType === 'weekly') {
+        title = '司机社综合周排行榜';
+        rankId = 'nex_recons_demens';
+    } else if (rankType === 'monthly') {
+        title = '司机社综合月排行榜';
+        rankId = 'nex_recons_demens1';
+    } else {
+        throw Error('Invalid rank type');
+    }
+    const url = `${baseUrl}/portal.php`;
+    const resp = await got(url);
+    const $ = cheerio.load(resp.data);
+    let items = $(`#${rankId} dd`)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const title = item.find('h5').text().trim();
+            const link = item.find('a').attr('href');
+            return {
+                title,
+                link: `${baseUrl}/${link}`,
+            };
+        });
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const resp = await got(item.link);
+                const $ = cheerio.load(resp.data);
+                const firstViewBox = $('.t_f').first();
+
+                firstViewBox.find('img').each((_, img) => {
+                    img = $(img);
+                    if (img.attr('zoomfile')) {
+                        img.attr('src', img.attr('zoomfile'));
+                        img.removeAttr('zoomfile');
+                        img.removeAttr('file');
+                    }
+                    img.removeAttr('onmouseover');
+                });
+
+                item.description = firstViewBox.html();
+                return item;
+            })
+        )
+    );
+    ctx.state.data = {
+        title,
+        link: url,
+        description: title,
+        item: items,
+    };
+};

package/lib/v2/xsijishe/router.js

@@ -1,3 +1,4 @@
 module.exports = function (router) {
     router.get('/forum/:fid', require('./forum'));
+    router.get('/rank/:type', require('./rank'));
 };

package/lib/v2/yahoo/maintainer.js

@@ -1,3 +1,5 @@
 module.exports = {
     '/news/:region/:category?': ['KeiLongW'],
+    '/news/provider/:region/:providerId': ['TonyRL'],
+    '/news/providers/:region': ['TonyRL'],
 };

package/lib/v2/yahoo/news/tw/index.js

@@ -0,0 +1,24 @@
+const { getArchive, getCategories, parseList, parseItem } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { region, category } = ctx.params;
+    if (!['hk', 'tw'].includes(region)) {
+        throw Error(`Unknown region: ${region}`);
+    }
+
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit, 10) : 20;
+    const categoryMap = await getCategories(region, ctx.cache.tryGet);
+    const tag = category ? categoryMap[category].yctMap : null;
+
+    const response = await getArchive(region, limit, tag);
+    const list = parseList(region, response);
+
+    const items = await Promise.all(list.map((item) => parseItem(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: `Yahoo 新聞 - ${category ? categoryMap[category].name : '所有類別'}`,
+        link: `https://${region}.news.yahoo.com/${category ? `${category}/` : ''}archive`,
+        image: 'https://s.yimg.com/cv/apiv2/social/images/yahoo_default_logo-1200x1200.png',
+        item: items,
+    };
+};

package/lib/v2/yahoo/news/tw/provider-helper.js

@@ -0,0 +1,22 @@
+const { getProviderList } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { region } = ctx.params;
+    if (!['hk', 'tw'].includes(region)) {
+        throw Error(`Unknown region: ${region}`);
+    }
+
+    const providerList = await getProviderList(region, ctx.cache.tryGet);
+
+    const items = providerList.map((provider) => ({
+        ...provider,
+        description: provider.key,
+    }));
+
+    ctx.state.data = {
+        title: 'Yahoo 新聞 - 新聞來源列表',
+        link: `https://${region}.news.yahoo.com`,
+        image: 'https://s.yimg.com/cv/apiv2/social/images/yahoo_default_logo-1200x1200.png',
+        item: items,
+    };
+};

package/lib/v2/yahoo/news/tw/provider.js

@@ -0,0 +1,24 @@
+const { getArchive, getProviderList, parseList, parseItem } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { region, providerId } = ctx.params;
+    if (!['hk', 'tw'].includes(region)) {
+        throw Error(`Unknown region: ${region}`);
+    }
+
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit, 10) : 20;
+    const providerList = await getProviderList(region, ctx.cache.tryGet);
+    const provider = providerList.find((p) => p.key === providerId);
+
+    const response = await getArchive(region, limit, null, providerId);
+    const list = parseList(region, response);
+
+    const items = await Promise.all(list.map((item) => parseItem(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: `Yahoo 新聞 - ${provider?.title ?? ''}`,
+        link: provider?.link ?? `https://${region}.news.yahoo.com`,
+        image: 'https://s.yimg.com/cv/apiv2/social/images/yahoo_default_logo-1200x1200.png',
+        item: items,
+    };
+};

package/lib/v2/yahoo/news/tw/utils.js

@@ -0,0 +1,129 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { join } = require('path');
+const { art } = require('@/utils/render');
+
+const getArchive = async (region, limit, tag, providerId) => {
+    const { data: response } = await got(
+        `https://${region}.news.yahoo.com/_td-news/api/resource/NCPListService;api=archive;ncpParams=${encodeURIComponent(
+            JSON.stringify({
+                query: {
+                    count: limit,
+                    // imageSizes: '220x128',
+                    start: 0,
+                    providerid: providerId,
+                    tag,
+                },
+            })
+        )}`
+    );
+    return response;
+};
+
+const getCategories = (region, tryGet) =>
+    tryGet(`yahoo:${region}:categoryMap`, async () => {
+        const { PageStore } = await getStores(region, tryGet);
+
+        const { Col1: col1 } = PageStore.pagesConfigRaw.base.section.regions;
+        const { categoryMap } = col1.find((c) => c.name === 'ArchiveFilterBar').props;
+        for (const [key, value] of Object.entries(categoryMap)) {
+            categoryMap[key] = {
+                name: value,
+                yctMap: col1.find((c) => c.name === 'StreamContainerArchive').props.yctMap[key],
+            };
+        }
+
+        return categoryMap;
+    });
+
+const getProviderList = (region, tryGet) =>
+    tryGet(`yahoo:${region}:providerList`, async () => {
+        const { ProviderListStore } = await getStores(region, tryGet);
+
+        return ProviderListStore.providerList.flatMap((list) =>
+            list.providers.map((provider) => ({
+                title: `${list.title} - ${provider.title}`,
+                key: provider.key,
+                link: new URL(provider.url, `https://${region}.news.yahoo.com`).href,
+            }))
+        );
+    });
+
+const getStores = (region, tryGet) =>
+    tryGet(`yahoo:${region}:stores`, async () => {
+        const { data: response } = await got(`https://${region}.news.yahoo.com/archive`);
+        const $ = cheerio.load(response);
+
+        const appData = JSON.parse(
+            $('script:contains("root.App.main")')
+                .text()
+                .match(/root.App.main\s+=\s+(\{.+\});/)[1]
+        );
+
+        return appData.context.dispatcher.stores;
+    });
+
+const parseList = (region, response) =>
+    response.map((item) => ({
+        title: item.title,
+        link: new URL(item.url, `https://${region}.news.yahoo.com`).href,
+        description: item.summary,
+        pubDate: parseDate(item.published_at, 'X'),
+    }));
+
+const parseItem = (item, tryGet) =>
+    tryGet(item.link, async () => {
+        const { data: response } = await got(item.link);
+        const $ = cheerio.load(response);
+
+        const ldJson = JSON.parse($('script[type="application/ld+json"]').first().text());
+        const body = $('.caas-body');
+
+        body.find('noscript').remove();
+        // remove padding
+        body.find('.caas-figure-with-pb, .caas-img-container').each((_, ele) => {
+            ele = $(ele);
+            ele.removeAttr('style');
+        });
+
+        body.find('img').each((_, ele) => {
+            ele = $(ele);
+            let dataSrc = ele.data('src');
+
+            if (dataSrc) {
+                const match = dataSrc.match(/.*--\/.*--\/(.*)/);
+                if (match?.[1]) {
+                    dataSrc = match?.[1];
+                }
+                ele.attr('src', dataSrc);
+                ele.removeAttr('data-src');
+            }
+        });
+        // fix blockquote iframe
+        body.find('.caas-iframe').each((_, ele) => {
+            ele = $(ele);
+            if (ele.data('type') === 'youtube') {
+                ele.replaceWith(
+                    art(join(__dirname, '../../templates/youtube.art'), {
+                        id: ele.find('blockquote').data('src').split('/').pop()?.split('?')?.[0],
+                    })
+                );
+            }
+        });
+
+        item.description = body.html();
+        item.category = ldJson.keywords;
+        item.updated = parseDate(ldJson.dateModified);
+
+        return item;
+    });
+
+module.exports = {
+    getArchive,
+    getCategories,
+    getProviderList,
+    getStores,
+    parseList,
+    parseItem,
+};

package/lib/v2/yahoo/radar.js

@@ -1,10 +1,38 @@
 module.exports = {
     'yahoo.com': {
         _name: 'Yahoo',
+        'hk.news': [
+            {
+                title: '分類新聞',
+                docs: 'https://docs.rsshub.app/routes/new-media#yahoo',
+                source: ['/:category/archive', '/archive'],
+                target: '/yahoo/news/hk/:category?',
+            },
+            {
+                title: '新聞來源',
+                docs: 'https://docs.rsshub.app/routes/new-media#yahoo',
+                source: ['/:providerId/archive', '/archive'],
+                target: '/yahoo/news/hk/:providerId',
+            },
+        ],
+        'tw.news': [
+            {
+                title: '分類新聞',
+                docs: 'https://docs.rsshub.app/routes/new-media#yahoo',
+                source: ['/:category/archive', '/archive'],
+                target: '/yahoo/news/tw/:category?',
+            },
+            {
+                title: '新聞來源',
+                docs: 'https://docs.rsshub.app/routes/new-media#yahoo',
+                source: ['/:providerId/archive', '/archive'],
+                target: '/yahoo/news/tw/:providerId',
+            },
+        ],
         '.': [
             {
                 title: '新闻',
-                docs: 'https://docs.rsshub.app/routes/traditional-media#yahoo',
+                docs: 'https://docs.rsshub.app/routes/new-media#yahoo',
                 source: ['/'],
                 target: '/yahoo/news/:region/:category?',
             },

package/lib/v2/yahoo/router.js

@@ -1,3 +1,6 @@
 module.exports = function (router) {
-    router.get('/news/:region/:category?', require('./news/index'));
+    router.get('/news/en/:category?', require('./news/us/index'));
+    router.get('/news/provider/:region/:providerId', require('./news/tw/provider'));
+    router.get('/news/providers/:region', require('./news/tw/provider-helper'));
+    router.get('/news/:region/:category?', require('./news/tw/index'));
 };

package/lib/v2/yahoo/templates/youtube.art

@@ -0,0 +1 @@
+<iframe width="560" height="315" src="https://www.youtube-nocookie.com/embed/{{ id }}" frameborder="0" allow="encrypted-media; picture-in-picture; web-share" allowfullscreen></iframe>

package/lib/v2/yuque/book.js

@@ -2,15 +2,19 @@ const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
 const { card2Html } = require('./utils');
+const { CookieJar } = require('tough-cookie');
 
 const APP_DATA_REGEX = /window\.appData = JSON\.parse\(decodeURIComponent\("(.+?)"\)\);/;
 const baseUrl = 'https://www.yuque.com';
 
 module.exports = async (ctx) => {
+    const cookieJar = new CookieJar();
     const { name, book } = ctx.params;
     const bookUrl = `${baseUrl}/${name}/${book}`;
 
-    const { data: bookHtml } = await got(bookUrl);
+    const { data: bookHtml } = await got(bookUrl, {
+        cookieJar,
+    });
     const $ = cheerio.load(bookHtml);
     const appData = JSON.parse(decodeURIComponent($('script').text().match(APP_DATA_REGEX)[1]));
 
@@ -22,6 +26,7 @@ module.exports = async (ctx) => {
         searchParams: {
             book_id: bookId,
         },
+        cookieJar,
     });
 
     const list = docs.map((item) => ({

package/lib/v2/yuque/utils.js

@@ -14,6 +14,7 @@ const card2Html = (elem, link) => {
             break;
         case 'bookmarkInline':
         case 'bookmarklink':
+        case 'yuqueinline':
             html = `<a href='${value.src}'>${value.detail.title}</a>`;
             break;
         case 'checkbox':
@@ -32,6 +33,9 @@ const card2Html = (elem, link) => {
         case 'hr':
             html = '<hr>';
             break;
+        case 'label':
+            html = `<b>${value.label}</b>`;
+            break;
         case 'mention':
             html = `<a href='https://www.yuque.com/${value.login}'>${value.name}</a>`;
             break;
@@ -46,14 +50,26 @@ const card2Html = (elem, link) => {
                 html = `<iframe frameborder="no" border="0" marginwidth="0" marginheight="0" height=66 src="${value.src}"></iframe>`;
             } else if (elem.attr('alias') === 'bilibili' || elem.attr('alias') === undefined) {
                 html = `<iframe src="${value.src}&high_quality=1&autoplay=0" width="650" height="477" scrolling="no" border="0" frameborder="no" framespacing="0" allowfullscreen="true"></iframe>`;
+            } else if (value.type === 'codepen') {
+                html = `<iframe height="265" style="width: 100%;" scrolling="no" title="codepen" src="${value.url}" frameborder="no" allowtransparency="true" allowfullscreen="true"></iframe>`;
             } else {
                 throw Error(`Unhandled thirdparty on ${link}: ${elem.attr('alias')}`);
             }
             break;
+        case 'yuque':
+            if (value.mode === 'card') {
+                html = `<a href='${value.src}'>${value.detail.title}</a>`;
+            } else if (value.mode === 'embed') {
+                html = `<iframe src="${value.url}" width="100%" height="518"></iframe>`;
+            } else {
+                throw Error(`Unhandled mode on ${link}: ${value.mode}`);
+            }
+            break;
         case 'video':
             // fake video src
             html = `<video src='${value.videoId}'></video>`;
             break;
+
         default:
             throw Error(`Unhandled card on ${link}: ${name}`);
     }

package/lib/v2/zagg/maintainer.js

@@ -1,3 +1,3 @@
 module.exports = {
-    '/new-arrivals/:query?': ['NavePnow'],
+    '/new-arrivals/:query?': ['EthanWng97'],
 };