rsshub 1.0.0-master.fe50f9c → 1.0.0-master.fe560d2

package/lib/v2/twitter/web-api/twitter-api.js

@@ -115,24 +115,28 @@ function gatherLegacyFromData(entries, filter = 'tweet-') {
     const tweets = [];
     entries.forEach((entry) => {
         if (entry.entryId && entry.entryId.includes(filter)) {
-            const tweet = entry.content.itemContent.tweet_results.result;
-            const retweet = tweet.legacy.retweeted_status_result?.result;
-            for (const t of [tweet, retweet]) {
-                if (!t?.legacy) {
-                    continue;
+            const tweet = entry.content.itemContent.tweet_results?.result;
+            if (tweet) {
+                const retweet = tweet.legacy?.retweeted_status_result?.result;
+                for (const t of [tweet, retweet]) {
+                    if (!t?.legacy) {
+                        continue;
+                    }
+                    t.legacy.user = t.core.user_results.result.legacy;
+                    const quote = t.quoted_status_result?.result;
+                    if (quote) {
+                        t.legacy.quoted_status = quote.legacy;
+                        t.legacy.quoted_status.user = quote.core.user_results.result.legacy;
+                    }
                 }
-                t.legacy.user = t.core.user_results.result.legacy;
-                const quote = t.quoted_status_result?.result;
-                if (quote) {
-                    t.legacy.quoted_status = quote.legacy;
-                    t.legacy.quoted_status.user = quote.core.user_results.result.legacy;
+                const legacy = tweet.legacy;
+                if (legacy) {
+                    if (retweet) {
+                        legacy.retweeted_status = retweet.legacy;
+                    }
+                    tweets.push(legacy);
                 }
             }
-            const legacy = tweet.legacy;
-            if (retweet) {
-                legacy.retweeted_status = retweet.legacy;
-            }
-            tweets.push(legacy);
         }
     });
     return tweets;

package/lib/v2/twitter/web-api/twitter-got.js

@@ -6,8 +6,8 @@ const config = require('@/config').value;
 // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L716-L726
 const headers = {
     authorization: config.twitter.authorization,
-    // every single user and guest share the same token:
-    // Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA
+    // Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw
+    // reference: https://github.com/dangeredwolf/FixTweet/blob/f3082bbb0d69798687481a605f6760b2eb7558e0/src/constants.ts#L23-L25
     'x-guest-token': undefined,
     'x-twitter-auth-type': undefined,
     'x-twitter-client-language': 'en',

package/lib/v2/twreporter/category.js

@@ -8,16 +8,25 @@ module.exports = async (ctx) => {
     const url = baseURL + `/categories/${ctx.params.cid}`;
     const res = await got(url);
     const $ = cheerio.load(res.data);
-    const list = $('.lnKPLr').get();
-    const category = $('.kCfkTU').text();
+    const regexp = /^window\.__REDUX_STATE__=(.*);$/gm;
+    const raw = $('script[charset="UTF-8"]').html().replace(regexp, '$1');
+    const list = JSON.parse(raw);
+    const ids = list.entities.posts.allIds;
+
+    const links = [];
+    const titles = [];
+    const category = list.entities.posts.byId[ids[0]].categories[0].name;
+
+    for (const id of ids) {
+        links.push(baseURL + '/a/' + list.entities.posts.byId[id].slug);
+        titles.push(list.entities.posts.byId[id].title);
+    }
 
     const out = await Promise.all(
-        list.map((item) => {
-            const $ = cheerio.load(item);
-            const address = baseURL + $('a').attr('href');
-            const title = $('.list-item__Title-sc-1dx5lew-5').text();
-            return ctx.cache.tryGet(address, async () => {
-                const single = await fetch(address);
+        links.map((item, index) => {
+            const title = titles[index];
+            return ctx.cache.tryGet(item, async () => {
+                const single = await fetch(item);
                 single.title = title;
                 return single;
             });

package/lib/v2/uber/blog.js

@@ -0,0 +1,59 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+
+const rootURL = 'https://www.uber.com';
+const apiURL = 'https://blogapi.uber.com';
+
+module.exports = async (ctx) => {
+    let maxPage = Number(ctx.params.maxPage);
+    if (Number.isNaN(maxPage)) {
+        maxPage = 1;
+    }
+
+    let pages = await Promise.all(
+        [...Array(maxPage).keys()].map((pageIdx) =>
+            got(`${apiURL}/wp-json/blog/v1/data`, {
+                searchParams: {
+                    page: pageIdx + 1,
+                    parent: 'pittsburgh',
+                    slug: 'engineering',
+                },
+            })
+        )
+    );
+    pages = pages.map((page) => page.data);
+
+    const result = await Promise.all(
+        pages.map(
+            async (page) =>
+                await Promise.all(
+                    page.posts.map((post) =>
+                        ctx.cache.tryGet(`${rootURL}${post.link}`, async () => {
+                            let { data: article } = await got(`${apiURL}/wp-json/blog/v1/data`, {
+                                searchParams: {
+                                    slug: post.link.replace(/\//g, '').replace('blog', ''),
+                                },
+                            });
+                            article = article.article;
+
+                            return {
+                                link: article.link,
+                                title: article.title,
+                                description: article.content,
+                                pubDate: parseDate(article.created),
+                                author: article.author,
+                                category: article.categories.map((category) => category.category_name),
+                            };
+                        })
+                    )
+                )
+        )
+    );
+
+    ctx.state.data = {
+        title: `Uber Engineering Blog`,
+        link: rootURL + '/blog/engineering',
+        description: 'The technology behind Uber Engineering',
+        item: result.flat(),
+    };
+};

package/lib/v2/uber/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/blog/:maxPage?': ['hulb'],
+};

package/lib/v2/uber/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'uber.com': {
+        _name: 'Uber Blog',
+        www: [
+            {
+                title: 'Engineering',
+                docs: 'https://docs.rsshub.app/blog.html#you-bu',
+                source: ['/blog/pittsburgh/engineering'],
+                target: '/uber/blog',
+            },
+        ],
+    },
+};

package/lib/v2/uber/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/blog/:maxPage?', require('./blog'));
+};

package/lib/v2/udn/global/index.js

@@ -0,0 +1,67 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category ?? '';
+
+    const start = category === 'hot' ? 6 : 0;
+    const end = category === 'new' ? 6 : 12;
+
+    const rootUrl = 'https://global.udn.com';
+    const currentUrl = `${rootUrl}/global_vision/index${category ? `/${category}` : ''}`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    $('.topic').remove();
+
+    let items = $('.news_cards ul li a')
+        .toArray()
+        .slice(start, end)
+        .concat(category === '' ? $('.last24, h2').find('a').toArray() : [])
+        .map((item) => {
+            item = $(item);
+
+            const link = item.attr('href');
+
+            return {
+                title: item.find('h3').text() || item.text(),
+                link: /^http/.test(link) ? link : `${rootUrl}${item.attr('href')}`,
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                content('#story_art_title, #story_bady_info, #story_also').remove();
+                content('.social_bar, .photo_pop, .only_mobile, .area').remove();
+
+                item.description = content('#tags').prev().html();
+                item.author = content('#story_author_name').text();
+                item.pubDate = timezone(parseDate(content('meta[name="date"]').attr('content')), +8);
+                item.category = content('meta[name="news_keywords"]').attr('content').split(',');
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/udn/global/tag.js

@@ -0,0 +1,51 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const tag = ctx.params.tag ?? '過去24小時';
+
+    const rootUrl = 'https://global.udn.com';
+    const currentUrl = `${rootUrl}/search/tagging/1020/${tag}`;
+    const apiUrl = `${rootUrl}/search/ajax_tag/1020/${tag}`;
+
+    const response = await got({
+        method: 'get',
+        url: apiUrl,
+    });
+
+    let items = response.data.articles.map((item) => ({
+        title: item.TITLE,
+        author: item.AUTHOR,
+        pubDate: parseDate(item.TIMESTAMP * 1000),
+        category: item.TAG.map((t) => t.tag),
+        link: `${rootUrl}/global_vision/story/${item.CATE_ID}/${item.ART_ID}`,
+    }));
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                content('#story_art_title, #story_bady_info, #story_also').remove();
+                content('.social_bar, .photo_pop, .only_mobile, .area').remove();
+
+                item.author = content('#story_author_name').text();
+                item.description = content('#tags').prev().html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `轉角國際 udn Global - ${tag}`,
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/udn/maintainer.js

@@ -1,3 +1,5 @@
 module.exports = {
+    '/global/:category?': ['nczitzk'],
+    '/global/tag/:tag?': ['emdoe', 'nczitzk'],
     '/news/breakingnews/:id': ['miles170'],
 };

package/lib/v2/udn/radar.js

@@ -5,9 +5,23 @@ module.exports = {
             {
                 title: '即時新聞',
                 docs: 'https://docs.rsshub.app/new-media.html#lian-he-xin-wen-wang-ji-shi-xin-wen',
-                source: '/news/breaknews/1/:id',
+                source: ['/news/breaknews/1/:id', '/'],
                 target: '/udn/news/breakingnews/:id',
             },
         ],
+        global: [
+            {
+                title: '轉角國際 - 首頁',
+                docs: 'https://docs.rsshub.app/new-media.html#lian-he-xin-wen-wang-zhuan-jiao-guo-ji-shou-ye',
+                source: ['/global_vision/index/:category', '/'],
+                target: '/udn/global/:category?',
+            },
+            {
+                title: '轉角國際 - 標籤',
+                docs: 'https://docs.rsshub.app/new-media.html#lian-he-xin-wen-wang-zhuan-jiao-guo-ji-biao-qian',
+                source: ['/search/tagging/1020/:tag', '/'],
+                target: '/udn/global/tag/:tag?',
+            },
+        ],
     },
 };

package/lib/v2/udn/router.js

@@ -1,3 +1,5 @@
 module.exports = (router) => {
+    router.get('/global/tag/:tag?', require('./global/tag'));
+    router.get('/global/:category?', require('./global/index'));
     router.get('/news/breakingnews/:id', require('./breaking-news'));
 };

package/lib/v2/unusualwhales/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/news': ['TonyRL'],
+};

package/lib/v2/unusualwhales/news.js

@@ -0,0 +1,24 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+
+const apiBase = 'https://phx.unusualwhales.com';
+
+module.exports = async (ctx) => {
+    const { data } = await got(`${apiBase}/api/fj_articles`);
+
+    const items = data.map((item) => ({
+        title: item.title,
+        description: item.description,
+        link: item.url,
+        pubDate: parseDate(item.publish_date),
+    }));
+
+    ctx.state.data = {
+        title: 'Flow - News',
+        description: 'Explore unusual options, options flow, dark pools, short activity, and stock activity on unusualwhales.com. Unusual whales has a full news service available!',
+        link: 'https://unusualwhales.com/news-feed',
+        image: 'https://unusualwhales.com/android-icon-192x192.png',
+        language: 'en-US',
+        item: items,
+    };
+};

package/lib/v2/unusualwhales/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'unusualwhales.com': {
+        _name: 'Unusual Whales',
+        '.': [
+            {
+                title: 'News Flow',
+                docs: 'https://docs.rsshub.app/en/finance.html#unusual-whales',
+                source: ['/news', '/'],
+                target: '/unusualwhales/news',
+            },
+        ],
+    },
+};

package/lib/v2/unusualwhales/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/news', require('./news'));
+};

package/lib/v2/usenix/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/usenix-security-sympoium': ['ZeddYu'],
+};

package/lib/v2/usenix/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'usenix.org': {
+        _name: 'USENIX',
+        '.': [
+            {
+                title: 'Security Symposia',
+                docs: 'https://docs.rsshub.app/journal.html#usenix',
+                source: ['/conferences/all', '/conferences', '/'],
+                target: '/usenix/usenix-security-sympoium',
+            },
+        ],
+    },
+};

package/lib/v2/usenix/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/usenix-security-sympoium', require('./usenix'));
+};

package/lib/v2/usenix/usenix.js

@@ -0,0 +1,63 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const url = 'https://www.usenix.org';
+const { parseDate } = require('@/utils/parse-date');
+
+const seasons = ['spring', 'summer', 'fall', 'winter'];
+
+module.exports = async (ctx) => {
+    const last = new Date().getFullYear() + 1;
+    const urlList = Array.from({ length: last - 2020 }, (_, v) => `${url}/conference/usenixsecurity${v + 20}`)
+        .map((url) => seasons.map((season) => `${url}/${season}-accepted-papers`))
+        .flat();
+    const responses = await got.all(
+        urlList.map(async (url) => {
+            let res;
+            try {
+                res = await got(url);
+            } catch (e) {
+                // ignore 404
+            }
+            return res;
+        })
+    );
+
+    const list = responses
+        .filter((element) => element)
+        .map((response) => {
+            const $ = cheerio.load(response.data);
+            const pubDate = parseDate($('meta[property=article:modified_time]').attr('content'));
+            return $('article.node-paper')
+                .toArray()
+                .map((item) => {
+                    item = $(item);
+                    return {
+                        title: item.find('h2.node-title > a').text().trim(),
+                        link: `${url}${item.find('h2.node-title > a').attr('href')}`,
+                        author: item.find('div.field.field-name-field-paper-people-text.field-type-text-long.field-label-hidden p').text().trim(),
+                        pubDate,
+                    };
+                });
+        })
+        .flat();
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got(item.link);
+                const $ = cheerio.load(response.data);
+                item.description = $('.content').html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: 'USENIX',
+        link: url,
+        description: 'USENIX Security Symposium Accpeted Papers',
+        allowEmpty: true,
+        item: items,
+    };
+};

package/lib/v2/usepanda/index.js

@@ -0,0 +1,26 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const feedId = ctx.params.id;
+    const limit = ctx.query.limit ?? 30; // 默认30条
+
+    const rootUrl = 'https://api-panda.com/v4/';
+    const apiUrl = `${rootUrl}articles?feeds=${feedId}&limit=${limit}&page=1&sort=popular`;
+
+    const { data } = await got(apiUrl);
+
+    const items = data.map((item) => ({
+        title: `${item.title} 🌟 ${item.source.likesCount}`,
+        author: item.source.username,
+        pubDate: parseDate(item.source.createdAt),
+        link: item.source.targetUrl,
+        description: item.description,
+    }));
+
+    ctx.state.data = {
+        title: 'Panda Feeds',
+        link: 'https://usepanda.com/',
+        item: items,
+    };
+};

package/lib/v2/usepanda/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/feeds/:id': ['lyrl'],
+};

package/lib/v2/usepanda/radar.js

@@ -0,0 +1,11 @@
+module.exports = {
+    'usepanda.com': {
+        _name: 'Panda',
+        '.': [
+            {
+                title: 'Feeds',
+                docs: 'https://docs.rsshub.app/other.html#panda',
+            },
+        ],
+    },
+};

package/lib/v2/usepanda/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/feeds/:id', require('./index'));
+};

package/lib/v2/uw/gix/news.js

@@ -0,0 +1,67 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+const gixBaseURL = 'https://gixnetwork.org';
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category;
+
+    let newsURL = gixBaseURL + '/news';
+    let feedTitle = 'UW GIX News - ';
+    let listSelector = 'body > div.site > div.site-content > div.content-area > main.site-main > ';
+
+    switch (category) {
+        case 'blog':
+            newsURL += '/blog/';
+            feedTitle += 'Blogs';
+            listSelector += 'div.blog-wrapper > section.blog-list > article';
+            break;
+        case 'inthenews':
+            newsURL += '/inthenews/';
+            feedTitle += 'In The News';
+            listSelector += 'div.news-wrapper > section.news-list > article';
+            break;
+    }
+
+    const response = await got(newsURL);
+
+    const data = response.data;
+
+    const $ = cheerio.load(data);
+
+    const list = $(listSelector)
+        .map((index, item) => {
+            item = $(item);
+            const content = item.find('header').find('h2').find('a');
+            const time = item.find('header').find('span.h4').text();
+
+            return {
+                // title: content.text(),
+                time,
+                link: content.attr('href'),
+            };
+        })
+        .get();
+
+    const itemContent = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const descriptionResponse = await got(item.link);
+
+                const content = cheerio.load(descriptionResponse.data);
+
+                item.title = content('header.entry-header').find('h1').text();
+                item.description = content('div.entry-content').html();
+                item.pubDate = Date.parse(item.time);
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: feedTitle,
+        link: newsURL,
+        item: itemContent,
+    };
+};

package/lib/v2/uw/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/gix/news/:category': ['dykderrick'],
+};

package/lib/v2/uw/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'gixnetwork.org': {
+        _name: 'University of Washington',
+        '.': [
+            {
+                title: 'Global Innovation Exchange News',
+                docs: 'https://docs.rsshub.app/university.html#university-of-washington',
+                source: ['/news/:category'],
+                target: '/uw/gix/news/:category',
+            },
+        ],
+    },
+};

package/lib/v2/uw/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/gix/news/:category', require('./gix/news'));
+};

package/lib/v2/verse/articles.js

@@ -0,0 +1,42 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category;
+    const rootUrl = 'https://www.verse.com.tw/articles';
+    const currentUrl = category ? `${rootUrl}/${category}/` : rootUrl;
+    const response = await got(currentUrl);
+    const $ = cheerio.load(response.data);
+    const items = await Promise.all(
+        $('.templateA__list .templateA__listTitle a')
+            .get()
+            .map((item) => {
+                item = $(item);
+                const link = item.attr('href');
+                return ctx.cache.tryGet(link, async () => {
+                    const response = await got(link);
+                    const $ = cheerio.load(response.data);
+
+                    return {
+                        title: $('title').text(),
+                        author: $('#__author').first().text(),
+                        description: $('.article__template').html(),
+                        category: $('.article__infoType .category a')
+                            .toArray()
+                            .map((item) => $(item).text()),
+                        pubDate: timezone(parseDate($('.article__data').text(), '日期 YYYY/MM/DD'), +8),
+                        link,
+                    };
+                });
+            })
+    );
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: currentUrl,
+        description: $('meta[name="description"]').attr('content'),
+        item: items,
+    };
+};