rsshub 1.0.0-master.fc5a2f6 → 1.0.0-master.fc648b5

package/lib/routes/weibo/user.js

@@ -62,7 +62,7 @@ module.exports = async (ctx) => {
         false
     );
 
-    const resultItem = await Promise.all(
+    let resultItems = await Promise.all(
         cards
             .filter((item) => item.mblog)
             .map(async (item) => {
@@ -131,27 +131,18 @@ module.exports = async (ctx) => {
                     link,
                     pubDate,
                     author: item.mblog.user.screen_name,
+                    isPinned: item.profile_type_id?.startsWith('proweibotop'),
                 };
             })
     );
 
-    // remove pinned weibo if it is too old (older than all rest weibo).
-    // if pinned weibo exists, it will always be the first item and the total item count will be 11 (otherwise 10).
-    // if < 11, either there is no pinned weibo or the user sent < 11 weibo in total, no need to remove anything.
-    // there is still another characteristic of a pinned weibo: card.profile_type_id.startsWith('proweibotop'),
-    // but it can be changed in the future, so here we do not rely on it.
-    if (
-        resultItem.length >= 11 &&
-        resultItem[0].pubDate &&
-        resultItem[0].pubDate <
-            Math.min(
-                ...resultItem
-                    .slice(1)
-                    .map((item) => item.pubDate)
-                    .filter((item) => item)
-            )
-    ) {
-        resultItem.shift();
+    // remove pinned weibo if they are too old (older than all the rest weibo)
+    // the character of pinned weibo is `card.profile_type_id.startsWith('proweibotop')`
+    // there can be 1 or 2 (WHAT A FANTASTIC BRAIN THE PM HAS?) pinned weibo at the same time
+    const pinnedItems = resultItems.filter((item) => item.isPinned);
+    const ordinaryItems = resultItems.filter((item) => !item.isPinned);
+    if (pinnedItems.length > 0 && ordinaryItems.length > 0 && Math.max(...pinnedItems.map((i) => i.pubDate).filter(Boolean)) < Math.min(...ordinaryItems.map((i) => i.pubDate).filter(Boolean))) {
+        resultItems = ordinaryItems;
     }
 
     ctx.state.data = {
@@ -159,6 +150,6 @@ module.exports = async (ctx) => {
         link: `https://weibo.com/${uid}/`,
         description,
         image: profileImageUrl,
-        item: resultItem,
+        item: resultItems,
     };
 };

package/lib/v2/19lou/index.js

@@ -0,0 +1,74 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+const iconv = require('iconv-lite');
+
+const setCookie = function (cookieName, cookieValue, seconds, path, domain, secure) {
+    let expires = null;
+    if (seconds !== -1) {
+        expires = new Date();
+        expires.setTime(expires.getTime() + seconds);
+    }
+    return [encodeURI(cookieName), '=', encodeURI(cookieValue), expires ? '; expires=' + expires.toGMTString() : '', path ? '; path=' + path : '/', domain ? '; domain=' + domain : '', secure ? '; secure' : ''].join('');
+};
+
+module.exports = async (ctx) => {
+    const city = ctx.params.city ?? 'www';
+
+    const rootUrl = `https://${city}.19lou.com`;
+
+    const response = await got({
+        method: 'get',
+        url: rootUrl,
+        responseType: 'buffer',
+    });
+
+    const $ = cheerio.load(iconv.decode(response.data, 'gbk'));
+
+    $('.title-more').remove();
+
+    let items = $('.center-center-jiazi')
+        .find('a[title]')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            return {
+                title: item.attr('title'),
+                link: `https:${item.attr('href')}`,
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                    responseType: 'buffer',
+                    headers: {
+                        cookie: setCookie('_Z3nY0d4C_', '37XgPK9h', 365, '/', '19lou.com'),
+                        referer: rootUrl,
+                    },
+                });
+
+                const content = cheerio.load(iconv.decode(detailResponse.data, 'gbk'));
+
+                content('.name-lz, .postView-pk-mod').remove();
+
+                item.author = content('.uname, .user-name').first().text();
+                item.description = content('.post-cont').first().html() || content('.thread-cont').html();
+                item.pubDate = timezone(parseDate(content('.cont-top-left meta').first().attr('content')), +8);
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title').text().split('-')[0],
+        link: rootUrl,
+        item: items,
+    };
+};

package/lib/v2/19lou/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:city?': ['nczitzk'],
+};

package/lib/v2/19lou/radar.js

@@ -0,0 +1,29 @@
+module.exports = {
+    '19lou.com': {
+        _name: '19 楼',
+        '.': [
+            {
+                title: '头条',
+                docs: 'https://docs.rsshub.app/bbs.html#19-lou-tou-tiao',
+                source: ['/'],
+                target: (params, url) => `/19lou/${new URL(url).toString().match(/\/\/(.*?)\.19lou/)[1]}`,
+            },
+        ],
+        www: [
+            {
+                title: '头条',
+                docs: 'https://docs.rsshub.app/bbs.html#19-lou-tou-tiao',
+                source: ['/'],
+                target: '/19lou/www',
+            },
+        ],
+        jiaxing: [
+            {
+                title: '头条',
+                docs: 'https://docs.rsshub.app/bbs.html#19-lou-tou-tiao',
+                source: ['/'],
+                target: '/19lou/jiaxing',
+            },
+        ],
+    },
+};

package/lib/v2/19lou/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:city?', require('./index'));
+};

package/lib/v2/36kr/router.js

@@ -3,5 +3,5 @@ module.exports = (router) => {
     router.get('/news/:category?', require('./news'));
     router.get('/newsflashes', require('./newsflashes'));
     router.get('/search/article/:keyword', require('./search/article'));
-    router.get('/user/:uid', require('./user'));
+    router.get('/user/:id', require('./user'));
 };

package/lib/v2/36kr/user.js

@@ -2,35 +2,60 @@ const got = require('@/utils/got');
 const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
-    const userUrl = `https://36kr.com/user/${ctx.params.uid}`;
+    const id = ctx.params.id;
+
+    const rootUrl = 'https://36kr.com';
+    const apiRootUrl = 'https://gateway.36kr.com';
+    const currentUrl = `${rootUrl}/user/${id}`;
+
+    const infoResponse = await got({
+        method: 'post',
+        url: `${apiRootUrl}/api/mis/user/info`,
+        json: {
+            param: {
+                userId: id,
+                siteId: 1,
+                platformId: 2,
+            },
+            platformId: 2,
+            siteId: 1,
+            userId: id,
+            partner_id: 'web',
+        },
+    });
+
     const response = await got({
-        method: 'get',
-        url: userUrl,
+        method: 'post',
+        url: `${apiRootUrl}/api/mis/me/article`,
+        json: {
+            param: {
+                userId: id,
+                pageEvent: 0,
+                pageSize: 20,
+                pageCallback: '',
+                siteId: 1,
+                platformId: 2,
+            },
+            pageCallback: '',
+            pageEvent: 0,
+            pageSize: 20,
+            platformId: 2,
+            siteId: 1,
+            userId: id,
+            partner_id: 'web',
+        },
     });
-    const data = JSON.parse(response.data.match(/"authorDetailData":(.*?),"channel":/)[1]);
 
-    const authorInfo = data.authorInfo.data;
-    const authorFlowList = data.authorFlowList.data.itemList;
-    const articleList = authorFlowList.map((item) => ({
+    const items = response.data.data.itemList.map((item) => ({
         title: item.templateMaterial.widgetTitle,
-        link: `https://36kr.com/${item.route.split('?')[0] === 'detail_video' ? 'video' : 'p'}/${item.itemId}`,
+        link: `${rootUrl}/p/${item.itemId}`,
         pubDate: parseDate(item.templateMaterial.publishTime),
+        description: `<p>${item.templateMaterial.widgetContent}</p>`,
     }));
 
     ctx.state.data = {
-        title: `36氪用户 - ${authorInfo.userNick}`,
-        link: userUrl,
-        item: await Promise.all(
-            articleList.map((item) =>
-                ctx.cache.tryGet(item.link, async () => {
-                    const contentResponse = await got({ method: 'get', url: item.link });
-                    const result = contentResponse.data.match(/"articleDetailData":(.*?),"articleRecommendData":/) || contentResponse.data.match(/"videoDetail":(.*?),"authorVideos":/);
-                    const contentData = JSON.parse(result[1]);
-                    item.description = contentData.data.widgetContent;
-                    return item;
-                })
-            )
-        ),
-        description: `${authorInfo.label} | ${authorInfo.summary}`,
+        title: `36kr - ${infoResponse.data.data.userNick}`,
+        link: currentUrl,
+        item: items,
     };
 };

package/lib/v2/acfun/article.js

@@ -0,0 +1,100 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+const baseUrl = 'https://www.acfun.cn';
+const categoryMap = {
+    184: {
+        title: '二次元画师',
+        realmId: 'realmId=18' + '&realmId=14' + '&realmId=51',
+    },
+    110: {
+        title: '综合',
+        realmId: 'realmId=5' + '&realmId=22' + '&realmId=28' + '&realmId=3' + '&realmId=4',
+    },
+    73: {
+        title: '生活情感',
+        realmId: 'realmId=50' + '&realmId=25' + '&realmId=34' + '&realmId=7' + '&realmId=6' + '&realmId=17' + '&realmId=1' + '&realmId=2' + '&realmId=49',
+    },
+    164: {
+        title: '游戏',
+        realmId: 'realmId=8' + '&realmId=53' + '&realmId=52' + '&realmId=11' + '&realmId=43' + '&realmId=44' + '&realmId=45' + '&realmId=46' + '&realmId=47',
+    },
+    74: {
+        title: '动漫文化',
+        realmId: 'realmId=13' + '&realmId=31' + '&realmId=48',
+    },
+    75: {
+        title: '漫画文学',
+        realmId: 'realmId=15' + '&realmId=23' + '&realmId=16',
+    },
+};
+const sortTypeEnum = ['createTime', 'lastCommentTime', 'hotScore'];
+const timeRangeEnum = ['all', 'oneDay', 'threeDay', 'oneWeek', 'oneMonth'];
+
+module.exports = async (ctx) => {
+    const { categoryId, sortType = 'createTime', timeRange = 'all' } = ctx.params;
+    if (!categoryMap[categoryId]) {
+        throw `Invalid category Id: ${categoryId}`;
+    }
+    if (!sortTypeEnum.includes(sortType)) {
+        throw `Invalid sort type: ${sortType}`;
+    }
+    if (!timeRangeEnum.includes(timeRange)) {
+        throw `Invalid time range: ${timeRange}`;
+    }
+
+    const url = `${baseUrl}/v/list${categoryId}/index.htm`;
+    const response = await got.post(
+        `${baseUrl}/rest/pc-direct/article/feed` +
+            '?cursor=first_page' +
+            '&onlyOriginal=false' +
+            '&limit=10' +
+            `&sortType=${sortType}` +
+            `&timeRange=${sortType === 'hotScore' ? timeRange : 'all'}` +
+            `&${categoryMap[categoryId].realmId}`,
+        {
+            headers: {
+                referer: url,
+            },
+        }
+    );
+
+    const list = response.data.data.map((item) => ({
+        title: item.title,
+        link: `${baseUrl}/a/ac${item.articleId}`,
+        author: item.userName,
+        pubDate: parseDate(item.createTime, 'x'),
+        category: item.realmName,
+    }));
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got(item.link, {
+                    headers: {
+                        referer: url,
+                    },
+                });
+                const $ = cheerio.load(response.data);
+                const articleInfo = $('.main script')
+                    .text()
+                    .match(/window.articleInfo = (.*);\n\s*window.likeDomain/)[1];
+                const data = JSON.parse(articleInfo);
+
+                item.description = data.parts[0].content;
+                if (data.tagList) {
+                    item.category = [item.category, ...data.tagList.map((tag) => tag.name)];
+                }
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: categoryMap[categoryId].title,
+        link: url,
+        item: items,
+    };
+};

package/lib/{routes → v2}/acfun/bangumi.js

@@ -1,14 +1,13 @@
 const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const { id } = ctx.params;
-    const url = `http://www.acfun.cn/bangumi/aa${id}`;
+    const url = `https://www.acfun.cn/bangumi/aa${id}`;
 
-    const bangumiPage = await got({
-        method: 'get',
-        url,
+    const bangumiPage = await got(url, {
         headers: {
-            Referer: 'http://www.acfun.cn',
+            Referer: 'https://www.acfun.cn',
         },
     });
     const bangumiData = JSON.parse(bangumiPage.data.match(/window.bangumiData = (.*?);\n/)[1]);
@@ -18,11 +17,12 @@ module.exports = async (ctx) => {
         title: bangumiData.bangumiTitle,
         link: url,
         description: bangumiData.bangumiIntro,
+        image: bangumiData.belongResource.coverImageV,
         item: bangumiList.items.map((item) => ({
             title: `${item.episodeName} - ${item.title}`,
-            description: `<img src="${item.image.split('?')[0]}">`,
+            description: `<img src="${item.imgInfo.thumbnailImage.cdnUrls[0].url.split('?')[0]}">`,
             link: `http://www.acfun.cn/bangumi/aa${id}_36188_${item.itemId}`,
-            pubDate: new Date(item.updateTime).toUTCString(),
+            pubDate: parseDate(item.updateTime, 'x'),
         })),
     };
 };

package/lib/v2/acfun/maintainer.js

@@ -0,0 +1,5 @@
+module.exports = {
+    '/article/:categoryId/:sortType?/:timeRange?': ['TonyRL'],
+    '/bangumi/:id': ['xyqfer'],
+    '/user/video/:uid': ['wdssmq'],
+};

package/lib/v2/acfun/radar.js

@@ -0,0 +1,25 @@
+module.exports = {
+    'acfun.cn': {
+        _name: 'AcFun',
+        www: [
+            {
+                tilte: '番剧',
+                docs: 'https://docs.rsshub.app/anime.html#acfun-fan-ju',
+                source: '/bangumi/:id',
+                target: (params) => `/acfun/bangumi/${params.id.replace('aa', '')}`,
+            },
+            {
+                title: '用户投稿',
+                docs: 'https://docs.rsshub.app/anime.html#acfun-yong-hu-tou-gao',
+                source: '/u/:id',
+                target: '/acfun/user/video/:id',
+            },
+            {
+                title: '文章',
+                docs: 'https://docs.rsshub.app/anime.html#acfun',
+                source: '/v/:categoryId/index.htm',
+                target: (params) => `/acfun/article/${params.categoryId.replace('list', '')}`,
+            },
+        ],
+    },
+};

package/lib/v2/acfun/router.js

@@ -0,0 +1,5 @@
+module.exports = (router) => {
+    router.get('/article/:categoryId/:sortType?/:timeRange?', require('./article'));
+    router.get('/bangumi/:id', require('./bangumi'));
+    router.get('/user/video/:uid', require('./video'));
+};

package/lib/v2/acfun/video.js

@@ -0,0 +1,45 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const uid = ctx.params.uid;
+    const url = `https://www.acfun.cn/u/${uid}`;
+    const host = 'https://www.acfun.cn';
+    const response = await got(url, {
+        headers: {
+            Referer: host,
+        },
+    });
+    const data = response.data;
+
+    const $ = cheerio.load(data);
+    const title = $('title').text();
+    const description = $('.signature .complete').text();
+    const list = $('#ac-space-video-list a').get();
+    const image = $('head style')
+        .text()
+        .match(/.user-photo{\n\s*background:url\((.*)\) 0% 0% \/ 100% no-repeat;/)[1];
+
+    ctx.state.data = {
+        title,
+        link: url,
+        description,
+        image,
+        item: list.map((item) => {
+            item = $(item);
+
+            const itemTitle = item.find('p.title').text();
+            const itemImg = item.find('figure img').attr('src');
+            const itemUrl = item.attr('href');
+            const itemDate = item.find('.date').text();
+
+            return {
+                title: itemTitle,
+                description: `<img src="${itemImg.split('?')[0]}">`,
+                link: host + itemUrl,
+                pubDate: parseDate(itemDate, 'YYYY/MM/DD'),
+            };
+        }),
+    };
+};

package/lib/v2/agora0/index.js

@@ -0,0 +1,54 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category ?? 'initium';
+
+    const rootUrl = 'https://agora0.gitlab.io';
+    const currentUrl = `${rootUrl}/blog/${category}`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('.card span:not(.comments) a')
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 50)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            return {
+                title: item.text(),
+                link: item.attr('href'),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                item.author = content('meta[name="author"]').attr('content');
+                item.pubDate = parseDate(content('meta[property="article:published_time"]').attr('content'));
+                item.description = content('.post-content').html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/agora0/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:category?': ['nczitzk'],
+};

package/lib/v2/agora0/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'gitlab.io': {
+        _name: 'GitLab',
+        agora0: [
+            {
+                title: '零博客',
+                docs: 'https://docs.rsshub.app/blog.html#ling-bo-ke-fen-lei',
+                source: ['/blog/:category', '/'],
+                target: '/agora0/:category',
+            },
+        ],
+    },
+};

package/lib/v2/agora0/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:category?', require('./index'));
+};

package/lib/v2/apnews/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/topics/:topic?': ['zoenglinghou', 'mjysci', 'TonyRL'],
+};

package/lib/v2/apnews/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'apnews.com': {
+        _name: 'AP News',
+        '.': [
+            {
+                title: '话题',
+                docs: 'https://docs.rsshub.app/traditional-media.html#ap-news',
+                source: '/hub/:topic',
+                target: '/apnews/topics/:topic',
+            },
+        ],
+    },
+};

package/lib/v2/apnews/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/topics/:topic?', require('./topics'));
+};

package/lib/v2/apnews/templates/description.art

@@ -0,0 +1,14 @@
+{{ if media }}
+    {{ each media }}
+        {{ if $value.type === 'Photo' }}
+            <figure>
+                <img src="{{ $value.gcsBaseUrl + $value.imageRenderedSizes[0] + $value.imageFileExtension }}" alt="{{ $value.flattenedCaption }}">
+                <figcaption>{{@ $value.caption }}</figcaption>
+            </figure>
+        {{ else if $value.type === 'YouTube' }}
+            <iframe id="ytplayer" type="text/html" width="640" height="360" src="https://www.youtube-nocookie.com/embed/{{ $value.externalId }}" frameborder="0" allowfullscreen></iframe>
+            {{ if $value.caption }}{{@ $value.caption }}{{ /if }}
+        {{ /if }}
+    {{ /each }}
+{{ /if }}
+{{@ description }}

package/lib/v2/apnews/topics.js

@@ -0,0 +1,44 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+const HOME_PAGE = 'https://apnews.com';
+
+module.exports = async (ctx) => {
+    const { topic = 'trending-news' } = ctx.params;
+    const urlLink = `${HOME_PAGE}/hub/${topic}`;
+    const response = await got(urlLink);
+    const $ = cheerio.load(response.data);
+    const data = JSON.parse(
+        $('script')
+            .not('[src], [type]')
+            .text()
+            .match(/window\['titanium-state'\] = (.*)\nwindow\['titanium-cacheConfig'\]/)[1]
+    );
+    const meta = data.hub.data[`/${topic}`];
+
+    const items = meta.cards.map((item) => {
+        const description = cheerio.load(item.contents[0].storyHTML, null, false);
+        description('.ad-placeholder').remove();
+        return {
+            title: item.contents[0].headline,
+            description: art(path.join(__dirname, 'templates/description.art'), {
+                media: item.contents[0].media,
+                description: description.html(),
+            }),
+            link: `${HOME_PAGE}/article/${item.contents[0].canonicalUrl}-${item.contents[0].shortId}`,
+            author: item.contents[0].bylines ? item.contents[0].bylines.slice(3) : null,
+            pubDate: parseDate(item.contents[0].published),
+            category: item.contents[0].tagObjs.map((tag) => tag.name),
+        };
+    });
+
+    ctx.state.data = {
+        title: meta.tagObjs[0].seoTitle,
+        description: meta.tagObjs[0].seoDescription,
+        link: urlLink,
+        item: items,
+        language: $('html').attr('lang'),
+    };
+};